diff --git a/cmd/mesh-controller/batches.go b/cmd/mesh-controller/batches.go index 291a6718..a0db5821 100644 --- a/cmd/mesh-controller/batches.go +++ b/cmd/mesh-controller/batches.go @@ -367,7 +367,13 @@ func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, } closes, latest := windowOf(merges, b.Created, window, atMost) b.Delivery.Merges = named - b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath()} + var closed bool + var withheld []string + if was := b.Delivery.Batch; was != nil { + closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299) + } + b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath(), + Closed: closed, Withheld: withheld} b.State = inventory.PlanAssembling if behind != "" && windowClosed(*b, now) { b.State = inventory.PlanQueued @@ -402,7 +408,7 @@ func windowClosed(b inventory.Plan, now time.Time) bool { return true } w := b.Delivery.Batch - return !now.Before(w.ClosesAt) || !now.Before(w.AtMost) + return w.Closed || !now.Before(w.ClosesAt) || !now.Before(w.AtMost) } // carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and @@ -763,6 +769,15 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [ sort.Strings(also) ordered := groupOrderEdges(members, entries, read, edges) plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...)) + // **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the + // merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it + // would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change, + // and the stop named it. + if batch.Delivery != nil && batch.Delivery.Batch != nil { + if out := withhold(&plan, batch.Delivery.Batch.Withheld, moved); len(out) > 0 { + fmt.Printf(" withheld, a stopped merge's modules built only as dependents: %s\n", strings.Join(out, ", ")) + } + } plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch plan.Commits = commits newest := newestCommit(commits) @@ -823,6 +838,39 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [ return nil } +// withhold takes from a plan every module named that its merges do not move, and the tiers left empty: the +// modules taken. +func withhold(plan *inventory.Plan, names, moved []string) []string { + var out []string + for _, n := range names { + if _, in := plan.Modules[n]; in && !slices.Contains(moved, n) { + delete(plan.Modules, n) + out = append(out, n) + } + } + if len(out) == 0 { + return nil + } + var tiers [][]string + for _, t := range plan.Tiers { + var kept []string + for _, n := range t { + if !slices.Contains(out, n) { + kept = append(kept, n) + } + } + if len(kept) > 0 { + tiers = append(tiers, kept) + } + } + if tiers == nil { + tiers = [][]string{} + } + plan.Tiers = tiers + sort.Strings(out) + return out +} + // walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window // closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its // class. A merge walked alone had no window. diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index 1a4d7400..f19ee10b 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -121,22 +121,35 @@ func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) return p, err } -// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the walk stopped, the -// batch that walks its other merges again, those merges as `@`, and every merge left out. +// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the record stopped, the +// batch that walks its other merges again with those merges, every merge left out, the modules the stopped walk +// already sent that stay on the machines named, the modules the batch's walk withholds, and every merge elsewhere +// that will deliver a stopped change when it is walked — a later merge of its repository, or one moving its modules. type stopAnswer struct { - Stopped string `json:"stopped"` - WalkedAgainBy string `json:"walked_again_by,omitempty"` - WalkedAgain []string `json:"walked_again,omitempty"` - LeftOut []inventory.PlanLeftOut `json:"left_out,omitempty"` + Stopped string `json:"stopped"` + WalkedAgainBy string `json:"walked_again_by,omitempty"` + WalkedAgain []string `json:"walked_again,omitempty"` + LeftOut []inventory.PlanLeftOut `json:"left_out,omitempty"` + Kept []inventory.PlanKept `json:"kept,omitempty"` + Withheld []string `json:"withheld,omitempty"` + StillCarriedBy []string `json:"still_carried_by,omitempty"` } // stopWalkWithout ends a walk on its delivery's word and, given merges to leave out, walks every other merge it -// answered again in a fresh batch without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its -// batch, so ending it for one group member ended every other delivery on it. Left out are the merges named and -// each later merge of their repository's branch, which contains one of them and cannot be walked without -// delivering it. The fresh batch's window is closed — its merges were heard before — and it is cut by the usual -// rules, as a failed walk's merges are walked again (ADR 0276 decision 3). With nothing left to walk, or no merge -// named, the walk is only ended. A merge named that the walk does not answer refuses the stop whole. +// answered again without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its batch, so ending +// it for one group member ended every other delivery on it. +// +// - Left out are the merges named, each later merge of their repository's branch, which contains one, and each +// merge moving a module a left-out merge moves: a module is built at its branch, which holds the stopped merge. +// - The others join the open batch of their kind — merges heard while the walk ran wait there, and an older +// commit never walks after a newer one — or a new batch when none is open; its window is closed, and its walk +// withholds the left-out merges' modules where it would build them only as dependents. +// - Given a batch not yet cut, the merges left out leave it for a stopped record of their own, and the batch +// goes on without them; with nothing left in it, the batch itself is stopped. +// - The stopped record, the batch and every merge moved are written in one transaction (decision 7). +// +// With nothing left to walk, or no merge named, the walk is only ended. A merge named that the record does not +// answer refuses the stop whole. func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why string, without []string, now time.Time) (inventory.Plan, stopAnswer, error) { said := stopAnswer{Stopped: id} @@ -149,94 +162,158 @@ func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why if err != nil { return inventory.Plan{}, said, err } - if !p.Open() { + batch := p.Batch() + if !p.Open() && !(batch && len(without) > 0) { return p, said, fmt.Errorf("%s is already %s", p.ID, p.State) } - var again []inventory.BatchedMerge + var merges, again []inventory.BatchedMerge if len(without) > 0 { - merges, err := inv.MergesOf(ctx, p.ID) - if err != nil { + if merges, err = inv.MergesOf(ctx, p.ID); err != nil { return p, said, err } if again, said.LeftOut, err = leaveOut(merges, without); err != nil { return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err) } } - if p.Delivery == nil { - p.Delivery = &inventory.PlanDelivery{} - } - p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why - p.Delivery.LeftOut = said.LeftOut - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why) - if len(again) > 0 { - batch, err := walkAgain(ctx, inv, p, again, now) - if err != nil { - return p, said, fmt.Errorf("%s is not stopped: its other merges could not be batched again: %w", p.ID, err) + stopped := func(r *inventory.Plan) { + if r.Delivery == nil { + r.Delivery = &inventory.PlanDelivery{} } - said.WalkedAgainBy = batch.ID - for _, m := range again { - said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit) - } - p.Delivery.WalkedAgainBy = batch.ID - p.Note += fmt.Sprintf("; its other merges are walked again without %s, by %s: %s", mergesLeftOutWords(said.LeftOut), - batch.ID, strings.Join(said.WalkedAgain, ", ")) - if err := inv.SavePlan(ctx, &p); err != nil { - // The merges go back to the walk that still answers them, and the batch holds none: it is never walked. - for _, m := range again { - _ = inv.AnswerMerge(ctx, m.Repository, m.Commit, p.ID, m.Alone) - } - return p, said, err - } - fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, batch.ID, - strings.Join(said.WalkedAgain, ", ")) - return p, said, nil + r.Delivery.Stopped, r.Delivery.StoppedWhy, r.Delivery.LeftOut = by, why, said.LeftOut + r.State = inventory.PlanFailed + r.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, r.Tier, why) } - if len(said.LeftOut) > 0 { - p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut)) - } - if err := inv.SavePlan(ctx, &p); err != nil { + withheld := movesOf(leftOutMerges(merges, said.LeftOut)) + said.Withheld = withheld + if said.StillCarriedBy, err = stillCarried(ctx, inv, p.ID, merges, said.LeftOut, withheld); err != nil { return p, said, err } + if batch { + return p, said, stopInBatch(ctx, inv, &p, merges, again, &said, stopped, now) + } + said.Kept = keptOf(p, movesOf(again)) + stopped(&p) + p.Delivery.Kept = said.Kept + if len(again) == 0 { + if len(said.LeftOut) > 0 { + p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut)) + } + return p, said, inv.SavePlan(ctx, &p) + } + target, err := batchToWalkAgain(ctx, inv, again, now) + if err != nil { + return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err) + } + target.Delivery.Batch.Closed = true + target.Delivery.Batch.Withheld = union(target.Delivery.Batch.Withheld, withheld) + sort.Strings(target.Delivery.Batch.Withheld) + said.WalkedAgainBy = target.ID + var moves []inventory.MergeMove + for _, m := range again { + said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit) + moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: target.ID, Alone: m.Alone}) + } + p.Delivery.WalkedAgainBy = target.ID + p.Note += fmt.Sprintf("; its other merges are walked again by %s: %s", target.ID, strings.Join(said.WalkedAgain, ", ")) + if len(said.LeftOut) > 0 { + p.Note += "; left out " + mergesLeftOutWords(said.LeftOut) + } + if len(said.Kept) > 0 { + p.Note += "; " + keptWords(said.Kept) + } + // The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7). + if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&target, &p}, moves); err != nil { + return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err) + } + // What the batch holds and names now: written again by the cutter's next look should this fail. + if err := keepBatch(ctx, inv, &target, now, ""); err != nil { + fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err) + } + fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, target.ID, + strings.Join(said.WalkedAgain, ", ")) return p, said, nil } -// walkAgain puts merges of a stopped walk into a fresh batch of their kind, its window closed: kept before its -// first merge names it, then each merge moved to it. -func walkAgain(ctx context.Context, inv *inventory.Inventory, stopped inventory.Plan, again []inventory.BatchedMerge, +// stopInBatch leaves merges out of a batch not yet cut (ADR 0299): they go to a stopped record of their own, which +// names the batch as walking the others, and the batch withholds their modules; a batch left with none is stopped +// itself. +func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, merges, again []inventory.BatchedMerge, + said *stopAnswer, stopped func(*inventory.Plan), now time.Time) error { + if len(again) == 0 { + stopped(b) + b.Delivery.Batch = nil + b.Note = "a batch whose every merge was left out by a stop: " + b.Note + return inv.SavePlan(ctx, b) + } + left := leftOutMerges(merges, said.LeftOut) + first := left[0] + r := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch, + Commit: first.Commit, Merged: first.Merged, Created: now, Tiers: [][]string{}, + Modules: map[string]*inventory.PlanModule{}} + if r.ID == b.ID { + r.ID += "-left" + } + stopped(&r) + r.Delivery.WalkedAgainBy = b.ID + r.Note += fmt.Sprintf("; left out of %s before it was walked: %s; %s walks the others", b.ID, + mergesLeftOutWords(said.LeftOut), b.ID) + b.Delivery.Batch.Withheld = union(b.Delivery.Batch.Withheld, said.Withheld) + sort.Strings(b.Delivery.Batch.Withheld) + var moves []inventory.MergeMove + for _, m := range left { + moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: r.ID, Alone: m.Alone}) + } + for _, m := range again { + said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit) + } + said.Stopped, said.WalkedAgainBy = r.ID, b.ID + if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&r, b}, moves); err != nil { + return fmt.Errorf("%s is not changed, and none of its merges moved: %w", b.ID, err) + } + if err := keepBatch(ctx, inv, b, now, ""); err != nil { + fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", b.ID, err) + } + fmt.Printf("%s: %s left out by a stop, kept by %s\n", b.ID, mergesLeftOutWords(said.LeftOut), r.ID) + return nil +} + +// batchToWalkAgain is the batch a stopped walk's merges join: the open batch of their kind, or a new one, not yet +// written. The kind is read from what the merges moved. +func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, again []inventory.BatchedMerge, now time.Time) (inventory.Plan, error) { own := false - for _, m := range again { - for _, n := range announcedOf(m).Moves { - if _, on := onTheControllersPath[n]; on { - own = true + for _, n := range movesOf(again) { + if _, on := onTheControllersPath[n]; on { + own = true + } + } + batches, err := inv.Batches(ctx) + if err != nil { + return inventory.Plan{}, err + } + for _, b := range batches { + if b.OwnPath() == own { + if b.Delivery == nil { + b.Delivery = &inventory.PlanDelivery{} } + if b.Delivery.Batch == nil { + b.Delivery.Batch = &inventory.PlanBatch{Own: own} + } + return b, nil } } first := again[0] - b := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch, + return inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch, Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}, - Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}} - if b.ID == stopped.ID { - b.ID += "-again" - } - if err := inv.SavePlan(ctx, &b); err != nil { - return b, err - } - for _, m := range again { - if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, b.ID, m.Alone); err != nil { - return b, err - } - } - // Its merges were heard before: the window, measured from the last of them, is closed, and the cutter cuts it - // once no walk of its kind is open. - return b, keepBatch(ctx, inv, &b, now, "") + Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil } // leaveOut splits a walk's merges by the merges named to leave out, each `@` (a commit's -// first characters will do): the merges walked again, and those left out — each named one, and each later merge -// of its repository's branch, which contains it. A name the walk does not answer is refused. +// first characters will do): the merges walked again, and those left out — each named one; each later merge of +// its repository's branch, which contains it; and each merge moving a module a left-out merge moves, which is +// built at its branch and so holds the stopped merge — until none more is. A name the walk does not answer is +// refused. func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.BatchedMerge, []inventory.PlanLeftOut, error) { var stopped []inventory.BatchedMerge for _, w := range without { @@ -252,40 +329,181 @@ func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.Ba } stopped = append(stopped, merges[i]) } - // The earliest stopped merge of each repository's branch: every merge of that branch after it contains it. - earliest := map[string]inventory.BatchedMerge{} - for _, s := range stopped { - k := strings.ToLower(s.Repository) + "\x00" + s.Branch - if e, ok := earliest[k]; !ok || laterMerge(e, s) { - earliest[k] = s + same := func(a, b inventory.BatchedMerge) bool { + return strings.EqualFold(a.Repository, b.Repository) && a.Commit == b.Commit + } + out := map[int]inventory.PlanLeftOut{} + for i, m := range merges { + if slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool { return same(s, m) }) { + out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit} + } + } + for grew := true; grew; { + grew = false + for i, m := range merges { + if _, left := out[i]; left { + continue + } + for j, o := range out { + l := merges[j] + contains := o.Contains + if contains == "" { + contains = l.Commit + } + if strings.EqualFold(l.Repository, m.Repository) && l.Branch == m.Branch && laterMerge(m, l) { + out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains} + } else if shared := sharedMoves(m, l); len(shared) > 0 { + out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains, + Shares: shared} + } else { + continue + } + grew = true + break + } } } var again []inventory.BatchedMerge - var out []inventory.PlanLeftOut - for _, m := range merges { - named := slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool { - return strings.EqualFold(s.Repository, m.Repository) && s.Commit == m.Commit - }) - e, has := earliest[strings.ToLower(m.Repository)+"\x00"+m.Branch] - switch { - case named: - out = append(out, inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit}) - case has && laterMerge(m, e): - out = append(out, inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: e.Commit}) - default: + var left []inventory.PlanLeftOut + for i, m := range merges { + if o, is := out[i]; is { + left = append(left, o) + } else { again = append(again, m) } } - sort.SliceStable(out, func(i, j int) bool { return out[i].Contains == "" && out[j].Contains != "" }) - return again, out, nil + sort.SliceStable(left, func(i, j int) bool { return left[i].Contains == "" && left[j].Contains != "" }) + return again, left, nil } -// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one says so. +// sharedMoves are the modules two kept merges both moved when they were heard. +func sharedMoves(a, b inventory.BatchedMerge) []string { + var out []string + theirs := announcedOf(b).Moves + for _, n := range announcedOf(a).Moves { + if slices.Contains(theirs, n) && !slices.Contains(out, n) { + out = append(out, n) + } + } + sort.Strings(out) + return out +} + +// leftOutMerges are the kept merges a stop left out. +func leftOutMerges(merges []inventory.BatchedMerge, out []inventory.PlanLeftOut) []inventory.BatchedMerge { + var left []inventory.BatchedMerge + for _, m := range merges { + if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { + return strings.EqualFold(o.Repository, m.Repository) && o.Commit == m.Commit + }) { + left = append(left, m) + } + } + return left +} + +// movesOf is every module kept merges moved when they were heard, sorted. +func movesOf(merges []inventory.BatchedMerge) []string { + var out []string + for _, m := range merges { + out = union(out, announcedOf(m).Moves) + } + sort.Strings(out) + return out +} + +// keptOf are the modules a stopped walk sent that no merge walked again moves, and the machines each reached: +// those keep running the stopped walk's build. +func keptOf(p inventory.Plan, walkedAgain []string) []inventory.PlanKept { + var out []inventory.PlanKept + for name, m := range p.Modules { + if m == nil || slices.Contains(walkedAgain, name) { + continue + } + var machines []string + if m.FirstAt != nil || m.SentAt != nil { + machines = union(machines, m.First) + } + if m.Gate != nil && m.Gate.Since != nil { + machines = union(machines, m.Gate.Machines) + } + for machine := range m.Rest { + machines = union(machines, []string{machine}) + } + if len(machines) == 0 { + continue + } + sort.Strings(machines) + out = append(out, inventory.PlanKept{Module: name, Machines: machines}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) + return out +} + +// keptWords is what a stopped walk leaves on the machines, as a person reads it. +func keptWords(kept []inventory.PlanKept) string { + var words []string + for _, k := range kept { + words = append(words, k.Module+" on "+strings.Join(k.Machines, ", ")) + } + return "left running the stopped walk's build, which nothing walks again: " + strings.Join(words, "; ") +} + +// stillCarried are the merges outside the stopped record, in a batch or an open walk, that will deliver a stopped +// change when they are walked: a later merge of its repository's branch, which contains it, or one that moves a +// module it moves. Holding a merged change back for good is a revert's (ADR 0299). +func stillCarried(ctx context.Context, inv *inventory.Inventory, id string, merges []inventory.BatchedMerge, + out []inventory.PlanLeftOut, withheld []string) ([]string, error) { + var named []inventory.BatchedMerge + for _, m := range leftOutMerges(merges, out) { + if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { return o.Commit == m.Commit && o.Contains == "" }) { + named = append(named, m) + } + } + if len(named) == 0 { + return nil, nil + } + open, err := inv.OpenPlans(ctx) + if err != nil { + return nil, err + } + batches, err := inv.Batches(ctx) + if err != nil { + return nil, err + } + var carried []string + for _, p := range append(open, batches...) { + if p.ID == id { + continue + } + ms, err := inv.MergesOf(ctx, p.ID) + if err != nil { + return nil, err + } + for _, m := range ms { + later := slices.ContainsFunc(named, func(s inventory.BatchedMerge) bool { + return strings.EqualFold(s.Repository, m.Repository) && s.Branch == m.Branch && laterMerge(m, s) + }) + shares := slices.ContainsFunc(announcedOf(m).Moves, func(n string) bool { return slices.Contains(withheld, n) }) + if later || shares { + carried = append(carried, fmt.Sprintf("%s@%s (in %s)", m.Repository, m.Commit, p.ID)) + } + } + } + sort.Strings(carried) + return carried, nil +} + +// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one, or +// shares its modules, says so. func mergesLeftOutWords(out []inventory.PlanLeftOut) string { var words []string for _, o := range out { s := o.Repository + "@" + short(o.Commit) - if o.Contains != "" { + switch { + case len(o.Shares) > 0: + s += " (which moves " + strings.Join(o.Shares, ", ") + " as " + short(o.Contains) + " does)" + case o.Contains != "": s += " (which contains " + short(o.Contains) + ")" } words = append(words, s) diff --git a/cmd/mesh-controller/issue459_test.go b/cmd/mesh-controller/issue459_test.go index c5591b3d..b57accc3 100644 --- a/cmd/mesh-controller/issue459_test.go +++ b/cmd/mesh-controller/issue459_test.go @@ -1,7 +1,10 @@ package main import ( + "encoding/json" + "errors" "reflect" + "slices" "strings" "testing" "time" @@ -52,6 +55,49 @@ func TestLeftOutAreTheStoppedMergeAndTheMergesThatContainIt(t *testing.T) { } } +// moving is a kept merge that moved modules when it was heard. +func moving(m inventory.BatchedMerge, modules ...string) inventory.BatchedMerge { + m.Event, _ = json.Marshal(keptMerge{Moves: modules}) + return m +} + +// A merge of another repository that moves a module the stopped merge moves is left out too, naming the modules +// it shares: the module is built at its branch, which holds the stopped merge (review of #232). And so on, until +// none more is: a merge sharing a module with that one is left out as well. +func TestAMergeSharingAStoppedMergesModuleIsLeftOut(t *testing.T) { + stopped := moving(leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0), "app") + reader := moving(leaved("novox/one", "o1bbbbbbbbbb", t0.Add(time.Second)), "app", "one") + next := moving(leaved("novox/two", "t1cccccccccc", t0.Add(2*time.Second)), "one") + free := moving(leaved("novox/three", "h1dddddddddd", t0.Add(3*time.Second)), "three") + again, out, err := leaveOut([]inventory.BatchedMerge{stopped, reader, next, free}, []string{"novox/mesh-catalog@c1aaaaaaa"}) + if err != nil { + t.Fatal(err) + } + if len(again) != 1 || again[0].Commit != free.Commit { + t.Fatalf("walked again: %+v; want the merge that shares nothing alone", again) + } + want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: stopped.Commit}, + {Repository: "novox/one", Commit: reader.Commit, Contains: stopped.Commit, Shares: []string{"app"}}, + {Repository: "novox/two", Commit: next.Commit, Contains: stopped.Commit, Shares: []string{"one"}}} + if !reflect.DeepEqual(out, want) { + t.Fatalf("left out %+v; want %+v", out, want) + } +} + +// A walk withholds a module a stopped merge moved where it would build it only as a dependent; one a merge of its +// own batch moves is built, and empty tiers go. +func TestAWalkWithholdsAStoppedMergesDependents(t *testing.T) { + plan := inventory.Plan{Tiers: [][]string{{"one"}, {"app"}, {"notes", "app2"}}, + Modules: map[string]*inventory.PlanModule{"one": {}, "app": {}, "notes": {}, "app2": {}}} + got := withhold(&plan, []string{"app", "notes", "absent"}, []string{"notes"}) + if !reflect.DeepEqual(got, []string{"app"}) || !reflect.DeepEqual(plan.Tiers, [][]string{{"one"}, {"notes", "app2"}}) { + t.Fatalf("withheld %v, tiers %v", got, plan.Tiers) + } + if _, in := plan.Modules["app"]; in { + t.Fatal("a withheld module is still in the plan") + } +} + // The seat verb passes the merges to leave out to the command. func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) { got, err := argvFor("delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen", @@ -175,3 +221,173 @@ func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) { t.Fatalf("a batch of nothing was made: %+v", bs) } } + +// A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge +// walked again joins that batch, never a batch of its own that would walk the older commit after the newer one +// (review of #232): its window closes, and its walk is at the newer commit, answering both. +func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) { + open := windowed(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one1, t0.Add(6*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + walk := ws[0] + one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute)) + hear(t, open, one2, t0.Add(3*time.Minute)) + _, bs := walks(t, open) + if len(bs) != 1 { + t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs)) + } + waiting := bs[0] + p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) + if err != nil { + t.Fatal(err) + } + if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID { + t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID) + } + if _, bs := walks(t, open); len(bs) != 1 { + t.Fatalf("%d batches; want the open one alone", len(bs)) + } + b, _ := open.inventory.PlanByID(ctx, waiting.ID) + if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") { + t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch) + } + // Its window closed by the stop, not by the clock: cut at once. + cutAt(t, open, t0.Add(3*time.Minute+11*time.Second)) + fresh, _ := open.inventory.PlanByID(ctx, waiting.ID) + if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit { + t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one")) + } + var answered []string + for _, m := range fresh.Delivery.Merges { + answered = append(answered, m.Commit+">"+m.Carried) + } + if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") { + t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered) + } +} + +// A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of +// the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk +// that walked its merges again is refused a retry, saying why. +func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) { + open := windowed(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one1, t0.Add(6*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + walk := ws[0] + sent := t0.Add(150 * time.Second) + for name, m := range walk.Modules { + if name == "app" || name == "one" { + m.First, m.FirstAt = []string{"anchor"}, &sent + } + } + if err := open.inventory.SavePlan(ctx, &walk); err != nil { + t.Fatal(err) + } + cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute)) + hear(t, open, cat2, t0.Add(3*time.Minute)) + _, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) { + t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept) + } + p, _ := open.inventory.PlanByID(ctx, walk.ID) + if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") { + t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note) + } + if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) { + t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy) + } + if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") { + t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err) + } +} + +// A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on +// with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped. +func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) { + open := windowed(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one, t0.Add(6*time.Second)) + _, bs := walks(t, open) + b := bs[0] + r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second)) + if err != nil { + t.Fatal(err) + } + if said.WalkedAgainBy != b.ID || said.Stopped == b.ID { + t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said) + } + if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit { + t.Fatalf("the stopped record holds %+v", left) + } + if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed || + stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID { + t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery) + } + _ = r + cutAt(t, open, t0.Add(3*time.Minute)) + w, _ := open.inventory.PlanByID(ctx, b.ID) + if _, in := w.Modules["app"]; in || w.Batch() { + t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules) + } + // A batch whose every merge is left out is stopped itself. + hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) + _, bs = walks(t, open) + last := bs[0] + if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x", + []string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID { + t.Fatalf("a batch left with nothing: %+v %v", said, err) + } + if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() { + t.Fatalf("the emptied batch is %s", p.State) + } +} + +// The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody +// else since it was read refuses all of it, and no merge moves, no plan is written. +func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) { + open := windowed(t) + ctx := t.Context() + hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second)) + _, bs := walks(t, open) + stale := bs[0] + moved := stale + moved.Note = "written by somebody else" + if err := open.inventory.SavePlan(ctx, &moved); err != nil { + t.Fatal(err) + } + fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0, + State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}} + err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale}, + []inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}}) + if !errors.Is(err, inventory.ErrPlanMoved) { + t.Fatalf("a stale plan in the act: %v", err) + } + if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil { + t.Fatal("the new plan was written though the act was refused") + } + if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID { + t.Fatalf("the merge moved to %q though the act was refused", m.Plan) + } +} diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index df28926e..a41acb69 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -278,6 +278,13 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { plans = append(plans, recent...) // **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for // the merge that brought the failure answers them now, and a retried walk would name them twice. + // **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped, + // and the others are another walk's now. + if p.Delivery != nil && (p.Delivery.WalkedAgainBy != "" || len(p.Delivery.LeftOut) > 0) { + return "", fmt.Errorf("%s was stopped without %s: those merges were stopped, and %s walks the others; a "+ + "newer merge, or `rebuild `, builds again", p.ID, mergesLeftOutWords(p.Delivery.LeftOut), + orWords(p.Delivery.WalkedAgainBy, "nothing")) + } if p.Delivery != nil && len(p.Delivery.Merges) > 0 { kept, err := inv.MergesOf(ctx, p.ID) if err != nil { diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 965f06bf..40bba5f7 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -161,9 +161,11 @@ var ControllerVerbs = []Verb{ []string{"plan"})}, {Name: "delivery-stop", Description: "End a walk on its delivery's word (novox/hq ADR 0239): failed, said as stopped " + "by whom and why; what it asked still builds and registers, nothing further is asked or sent. Given merges to " + - "leave out, every other merge the walk answered is walked again in a fresh batch without them and without each " + - "later merge of their repository, which contains one and is left out too, named (novox/hq ADR 0299); the " + - "answer names the batch, the merges walked again and the merges left out.", + "leave out, every other merge the walk answered is walked again — in the open batch of its kind, or a new one — " + + "without them, without each later merge of their repository, which contains one, and without each merge moving " + + "a module one moves, each left out too and named (novox/hq ADR 0299); a batch not yet cut drops the merges. The " + + "answer names the batch, the merges walked again, the merges left out, the modules the stopped walk sent that " + + "stay on their machines, and the merges elsewhere that will still deliver a stopped change.", Input: schema(map[string]string{"plan": "the walk's id", "why": "why", "by": "who stopped the delivery", "without": "merges to leave out, @, comma-separated"}, []string{"plan", "why"})}, diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index cbc20f3f..70304f78 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -111,6 +111,12 @@ type PlanBatch struct { // a merge never shares a batch with one that waits for mesh-delivery's (ADR 0276, decided during the // build, 2026-10-10). Own bool `json:"own,omitempty"` + // Closed says the window was closed by a stop that walks a stopped walk's merges again in this batch: they + // were heard before, and an older commit never waits behind a newer one (novox/hq ADR 0299). Withheld are + // the modules a stopped merge moves: built at the branch, which holds that merge, they would deliver it, so + // the batch's walk does not build them. + Closed bool `json:"closed,omitempty"` + Withheld []string `json:"withheld,omitempty"` } // OwnPath says the record is a batch of merges on the controller's own path. @@ -192,6 +198,9 @@ type PlanDelivery struct { // ends stopped with the walk, one of a merge walked again follows that batch's walk. WalkedAgainBy string `json:"walked_again_by,omitempty"` LeftOut []PlanLeftOut `json:"left_out,omitempty"` + // Kept are the modules the stopped walk already sent and nothing walks again: the machines named keep running + // the stopped walk's build (novox/hq ADR 0299). + Kept []PlanKept `json:"kept,omitempty"` // Batch is the window of a batch; nil once it is cut into a walk. Batch *PlanBatch `json:"batch,omitempty"` // Alone says the walk walks one merge on its own commit, after a failed walk carried it in a later one: @@ -205,8 +214,18 @@ type PlanDelivery struct { type PlanLeftOut struct { Repository string `json:"repository"` Commit string `json:"commit"` - // Contains is the stopped merge a later merge contains; empty for a merge stopped itself. + // Contains is the stopped merge a later merge contains, or whose modules it shares; empty for a merge + // stopped itself. Contains string `json:"contains,omitempty"` + // Shares are the modules it moves that a stopped merge moves too: built at the branch, which holds the stopped + // merge, they would deliver it. + Shares []string `json:"shares,omitempty"` +} + +// PlanKept is a module a stopped walk sent and nothing walks again, and the machines it reached. +type PlanKept struct { + Module string `json:"module"` + Machines []string `json:"machines"` } // Waiting is whether the walk waits for its delivery's word. @@ -400,39 +419,28 @@ func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRol // with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records. // On success p's revision and epoch are the ones written, so the caller may save it again. func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error { + return i.SavePlansMoving(ctx, []*Plan{p}, nil) +} + +// MergeMove is a merge given to the plan that answers it now (novox/hq ADR 0276). +type MergeMove struct { + Repository, Commit, Plan string + Alone bool +} + +// SavePlansMoving writes plans, as SavePlan does each, and gives merges to the plans that answer them, all in one +// transaction: every write and move, or none (novox/hq ADR 0299 decision 7 — a stop that walks merges again never +// leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by +// somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once +// the transaction is committed, in order. +func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error { epoch, err := i.actingEpoch(ctx) if err != nil { - return fmt.Errorf("the plan for %s %s is not written: %w", p.Repository, p.Commit, err) - } - tiers, err := json.Marshal(p.Tiers) - if err != nil { + if len(plans) > 0 { + return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err) + } return err } - modules, err := json.Marshal(p.Modules) - if err != nil { - return err - } - var release, delivery, commits, times []byte - if p.Times != nil { - if times, err = json.Marshal(p.Times); err != nil { - return err - } - } - if len(p.Commits) > 0 { - if commits, err = json.Marshal(p.Commits); err != nil { - return err - } - } - if p.Release != nil { - if release, err = json.Marshal(p.Release); err != nil { - return err - } - } - if p.Delivery != nil { - if delivery, err = json.Marshal(p.Delivery); err != nil { - return err - } - } // **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the // plan moves, in the same transaction as the move, so no save can move a tier unmeasured or // measure one twice. @@ -441,9 +449,77 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error { return err } defer func() { _ = tx.Rollback(ctx) }() + type written struct { + revision int64 + entered time.Time + } + done := make([]written, len(plans)) + for n, p := range plans { + revision, entered, err := writePlan(ctx, tx, p, epoch) + if err != nil { + return err + } + done[n] = written{revision, entered} + } + for _, m := range moves { + if _, err := tx.Exec(ctx, + `update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`, + m.Repository, m.Commit, m.Plan, m.Alone); err != nil { + return err + } + } + if err := tx.Commit(ctx); err != nil { + return err + } + for n, p := range plans { + p.Revision, p.TierEntered = done[n].revision, done[n].entered + if epoch != nil { + p.Epoch = uint64(*epoch) + } else { + p.Epoch = 0 + } + if PlanSaved != nil { + PlanSaved(*p) + } + } + return nil +} + +// writePlan writes one plan within a transaction, by compare-and-set on its revision: its new revision, and when +// it entered its tier. +func writePlan(ctx context.Context, tx pgx.Tx, p *Plan, epoch *int64) (int64, time.Time, error) { + tiers, err := json.Marshal(p.Tiers) + if err != nil { + return 0, time.Time{}, err + } + modules, err := json.Marshal(p.Modules) + if err != nil { + return 0, time.Time{}, err + } + var release, delivery, commits, times []byte + if p.Times != nil { + if times, err = json.Marshal(p.Times); err != nil { + return 0, time.Time{}, err + } + } + if len(p.Commits) > 0 { + if commits, err = json.Marshal(p.Commits); err != nil { + return 0, time.Time{}, err + } + } + if p.Release != nil { + if release, err = json.Marshal(p.Release); err != nil { + return 0, time.Time{}, err + } + } + if p.Delivery != nil { + if delivery, err = json.Marshal(p.Delivery); err != nil { + return 0, time.Time{}, err + } + } entered, err := planTierLeft(ctx, tx, *p, time.Now()) if err != nil { - return err + return 0, time.Time{}, err } var revision int64 err = tx.QueryRow(ctx, @@ -464,24 +540,9 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error { // The row is there and at another revision — moved since this was read, or there already // when this one is new: either way not this writer's to overwrite. (A plan saved before plans // had revisions is at zero, and its first save here is from a read at zero.) - return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) + return 0, time.Time{}, fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) } - if err != nil { - return err - } - if err := tx.Commit(ctx); err != nil { - return err - } - p.Revision, p.TierEntered = revision, entered - if epoch != nil { - p.Epoch = uint64(*epoch) - } else { - p.Epoch = 0 - } - if PlanSaved != nil { - PlanSaved(*p) - } - return nil + return revision, entered, err } // short is a commit as a person reads it. diff --git a/merge-check.sh b/merge-check.sh index 296fe271..775baa06 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -32,3 +32,18 @@ else echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector" CGO_ENABLED=0 go test -count=1 -timeout 30m ./... fi + +# **The store tests of novox/hq ADR 0299 ran** (review of #232): a test that needs the store skips without one, +# and a skip passes unseen. With the store this check is given, each must have run and passed, said by name; +# without one, that is said. +adr0299='^(TestAStopWithoutAMergeWalksTheOthersAgain|TestAStopWithoutAMergeEndsTheMergesThatContainIt|TestAStopWalksTheOthersAgainInTheOpenBatch|TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt|TestAStopWithoutAMergeLeavesItOutOfABatch|TestSavingPlansAndMovingMergesIsOneAct)$' +if [ -n "${MESH_TEST_POSTGRES:-}" ]; then + ran=$(CGO_ENABLED=0 go test -count=1 -v -run "$adr0299" ./cmd/mesh-controller 2>&1 | grep -E '^--- (PASS|FAIL|SKIP)' || true) + printf 'ADR 0299 store tests:\n%s\n' "$ran" + if [ "$(printf '%s\n' "$ran" | grep -c '^--- PASS')" -ne 6 ]; then + echo "ADR 0299's store tests did not all run and pass against the store" + exit 1 + fi +else + echo "NOT STORE-CHECKED: no MESH_TEST_POSTGRES, so ADR 0299's store tests skipped" +fi