682 lines
33 KiB
Go
682 lines
33 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// novox/hq issues 319 and 316: a held delivery whose change already runs, or that builds nothing, is not a
|
|
// question for a person. A merge that moves nothing is delivered at once; a merge whose every module a later
|
|
// merge on the same trunk delivered is covered — never stopped, never an error on its pull request; and a
|
|
// group member held for a composed check that had not answered goes on once it passes.
|
|
|
|
var t0 = time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
|
|
// mergedAt is the forge's `pull.merged` at a moment.
|
|
func mergedAt(p PullEvent, merge string, at time.Time) PullEvent {
|
|
p.MergeCommit = merge
|
|
p.MergedAt = at.UTC().Format(time.RFC3339)
|
|
return p
|
|
}
|
|
|
|
// verdictMoving is a passing verdict whose plan moves these modules — or builds nothing, given none.
|
|
func verdictMoving(repo string, number int, sha string, modules ...string) CheckedEvent {
|
|
c := verdict(repo, number, sha, "pass")
|
|
c.Plan = planMoving(repo, modules...)
|
|
return c
|
|
}
|
|
|
|
func planMoving(repo string, modules ...string) *DeliveryPlan {
|
|
if len(modules) == 0 {
|
|
return &DeliveryPlan{Repository: repo, Base: "main", Summary: "builds nothing: the change touches no module of the mesh's graph"}
|
|
}
|
|
return &DeliveryPlan{Repository: repo, Base: "main", Moved: modules, Summary: "builds " + strings.Join(modules, ", ") + " → anchor"}
|
|
}
|
|
|
|
// deliverLater is a later merge into a trunk, walked: its walk opened for its merge commit, let go, and left
|
|
// in the state given, every module of it sent.
|
|
func deliverLater(w *world, repo string, number int, base, headSHA, merge string, at time.Time, state string,
|
|
modules ...string) string {
|
|
w.t.Helper()
|
|
p := pr(repo, number, headSHA, fmt.Sprintf("feat/later-%d", number))
|
|
p.Base = base
|
|
w.h.PullUpdated(p)
|
|
w.h.Checked(verdictMoving(repo, number, headSHA, modules...))
|
|
w.h.PullMerged(mergedAt(p, merge, at))
|
|
walk := aWalk("plan-"+merge[:6], repo, merge, true)
|
|
walk.Branch = base
|
|
walk.Modules = map[string]*WalkModule{}
|
|
for _, m := range modules {
|
|
walk.Modules[m] = &WalkModule{}
|
|
}
|
|
w.h.WalkMoved(w.ctl.put(walk))
|
|
w.settleAll()
|
|
walk = w.ctl.walk(walk.ID)
|
|
sent := at.Add(5 * time.Minute)
|
|
for _, m := range modules {
|
|
walk.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
|
|
}
|
|
walk.State = state
|
|
w.h.WalkMoved(w.ctl.put(walk))
|
|
w.settleAll()
|
|
return IDOf(repo, headSHA)
|
|
}
|
|
|
|
func statusOn(w *world, repo, commit string) string {
|
|
w.forge.mu.Lock()
|
|
defer w.forge.mu.Unlock()
|
|
return w.forge.statuses[repo+"@"+commit+" mesh/delivery"]
|
|
}
|
|
|
|
// **A merge that moves nothing is delivered at once** (issue 319): there is nothing for a walk to move and
|
|
// nothing for a person to decide — whether its check passed, failed or never answered, and whether its plan
|
|
// builds nothing at all or only a new module the controller builds and sends nowhere (issue 300).
|
|
func TestAMergeThatMovesNothingIsDeliveredAtOnce(t *testing.T) {
|
|
for _, c := range []struct {
|
|
name string
|
|
verdict string // "" is a merge before any verdict
|
|
plan *DeliveryPlan
|
|
}{
|
|
{"builds nothing, merged before its check answered", "", planMoving("novox/hq")},
|
|
{"builds nothing, merged on a failed check", "fail", planMoving("novox/hq")},
|
|
{"builds nothing, merged on a passing check", "pass", planMoving("novox/hq")},
|
|
{"a new module, sent nowhere", "pass", &DeliveryPlan{Repository: "novox/hq", New: []string{"modules/zfs"},
|
|
Summary: "builds new: modules/zfs, sent nowhere; no bus step"}},
|
|
{"a new module, sent nowhere, merged before its check answered", "", &DeliveryPlan{Repository: "novox/hq",
|
|
New: []string{"modules/zfs"}, Summary: "builds new: modules/zfs, sent nowhere; no bus step"}},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
w := newWorld(t)
|
|
w.ctl.planned = c.plan
|
|
p := pr("novox/hq", 7, head, "docs/x", "02-DECISIONS/x.md")
|
|
w.h.PullUpdated(p)
|
|
if c.verdict != "" {
|
|
v := verdict("novox/hq", 7, head, c.verdict)
|
|
v.Plan = c.plan
|
|
w.h.Checked(v)
|
|
}
|
|
w.later(31 * time.Minute)
|
|
w.h.PullMerged(mergedAt(p, merge, t0.Add(30*time.Minute)))
|
|
w.settleAll()
|
|
// Past the bound a merge waits for its walk: nothing holds it then either.
|
|
w.later(15 * time.Minute)
|
|
if err := w.h.Reconcile(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.settleAll()
|
|
d := w.delivery(IDOf("novox/hq", head))
|
|
if d.State != Delivered {
|
|
t.Fatalf("a merge whose plan is %q is %s (%s), not delivered", c.plan.Summary, d.State, d.HeldWhy)
|
|
}
|
|
if s := statusOn(w, "novox/hq", head); !strings.HasPrefix(s, "success") {
|
|
t.Fatalf("its pull request's mesh/delivery says %q", s)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// **A held delivery whose every module a later merge on its trunk delivered is covered** (issue 319): the
|
|
// later merge commit contains it, its walk is done, and it sent each module this one moves. Anything short
|
|
// of that — a module not carried, an earlier merge, another trunk, a walk that did not finish — leaves it held
|
|
// for a person, as before.
|
|
func TestAMergeALaterDeliveredWalkCarriedIsCovered(t *testing.T) {
|
|
const ctl = "novox/mesh-controller"
|
|
for _, c := range []struct {
|
|
name string
|
|
// The later merge: its trunk, minutes after t0 it merged (this one merged at 30), its walk's end, and
|
|
// what it sent.
|
|
base string
|
|
at int
|
|
state string
|
|
modules []string
|
|
// Whether this one was held for a person before the later merge was delivered.
|
|
heldFirst bool
|
|
// What the forge says of the later merge containing this one: yes, no, or away until asked again.
|
|
forge string
|
|
want State
|
|
}{
|
|
{"carried every module, delivered after this one was held", "main", 35, walkDone,
|
|
[]string{"build-agent", "mesh-controller"}, true, "yes", Covered},
|
|
{"carried every module, delivered before this one was due to be held", "main", 35, walkDone,
|
|
[]string{"build-agent", "mesh-controller", "route-proxy"}, false, "yes", Covered},
|
|
{"carried one of its two modules", "main", 35, walkDone, []string{"mesh-controller"}, true, "yes", Held},
|
|
{"merged before this one", "main", 25, walkDone, []string{"build-agent", "mesh-controller"}, true, "yes", Held},
|
|
{"merged into another branch", "release", 35, walkDone, []string{"build-agent", "mesh-controller"}, true, "yes", Held},
|
|
{"its walk failed", "main", 35, walkFailed, []string{"build-agent", "mesh-controller"}, true, "yes", Held},
|
|
{"the forge says the later merge does not contain it", "main", 35, walkDone,
|
|
[]string{"build-agent", "mesh-controller"}, true, "no", Held},
|
|
{"the forge could not say at first, then said it does", "main", 35, walkDone,
|
|
[]string{"build-agent", "mesh-controller"}, true, "away", Covered},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
w := newWorld(t)
|
|
pair := merge + " in nnnnnnnnnnnn8888"
|
|
switch c.forge {
|
|
case "yes", "away":
|
|
w.forge.contains[pair] = true
|
|
case "no":
|
|
w.forge.contains[pair] = false
|
|
}
|
|
w.forge.containsDown = c.forge == "away"
|
|
p := pr(ctl, 1, head, "feat/one", "cmd/a.go")
|
|
w.h.PullUpdated(p)
|
|
w.h.Checked(verdictMoving(ctl, 1, head, "mesh-controller", "build-agent"))
|
|
w.later(31 * time.Minute)
|
|
w.h.PullMerged(mergedAt(p, merge, t0.Add(30*time.Minute)))
|
|
w.settleAll()
|
|
if c.heldFirst {
|
|
w.later(15 * time.Minute)
|
|
if err := w.h.Reconcile(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.settleAll()
|
|
if s := w.state(IDOf(ctl, head)); s != Held {
|
|
t.Fatalf("a merge no walk opened for is %s before the later one", s)
|
|
}
|
|
}
|
|
later := deliverLater(w, ctl, 2, c.base, "bbbbbbbbbbbb2222", "nnnnnnnnnnnn8888", t0.Add(time.Duration(c.at)*time.Minute),
|
|
c.state, c.modules...)
|
|
if c.state == walkDone && w.state(later) != Delivered {
|
|
t.Fatalf("the later merge is %s", w.state(later))
|
|
}
|
|
w.later(20 * time.Minute)
|
|
if err := w.h.Reconcile(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.settleAll()
|
|
if c.forge == "away" {
|
|
// A forge that cannot say is "not yet covered": held, and asked again a minute on.
|
|
if s := w.state(IDOf(ctl, head)); s != Held {
|
|
t.Fatalf("with the forge away it is %s", s)
|
|
}
|
|
w.forge.mu.Lock()
|
|
w.forge.containsDown = false
|
|
w.forge.mu.Unlock()
|
|
w.later(2 * time.Minute)
|
|
w.settleAll()
|
|
}
|
|
d := w.delivery(IDOf(ctl, head))
|
|
if d.State != c.want {
|
|
t.Fatalf("it is %s (%s), want %s", d.State, d.HeldWhy, c.want)
|
|
}
|
|
s := statusOn(w, ctl, head)
|
|
if c.want == Covered {
|
|
if !strings.HasPrefix(s, "success covered: a later merge that contains it was delivered") ||
|
|
!strings.Contains(s, "nnnnnnnn") {
|
|
t.Fatalf("its pull request's mesh/delivery says %q", s)
|
|
}
|
|
last := d.Transitions[len(d.Transitions)-1]
|
|
if !strings.Contains(last.Why, IDOf(ctl, "bbbbbbbbbbbb2222")) {
|
|
t.Fatalf("its transition does not name what covered it: %q", last.Why)
|
|
}
|
|
}
|
|
if strings.HasPrefix(s, "error") {
|
|
t.Fatalf("its pull request's mesh/delivery is an error: %q", s)
|
|
}
|
|
// Asked once for the pair, however many ticks read it — once more only after the forge was away.
|
|
asked := 0
|
|
for _, a := range w.forge.containsAsks {
|
|
if a == pair {
|
|
asked++
|
|
}
|
|
}
|
|
candidate := c.base == "main" && c.at > 30 && c.state == walkDone && len(c.modules) >= 2
|
|
switch {
|
|
case !candidate && asked != 0:
|
|
t.Fatalf("the forge was asked whether %s for a merge order and walk that cannot cover it", pair)
|
|
case candidate && c.forge != "away" && asked != 1:
|
|
t.Fatalf("the forge was asked %d time(s) whether %s, want once", asked, pair)
|
|
case candidate && c.forge == "away" && asked < 2:
|
|
t.Fatalf("the forge, away, was asked %d time(s) whether %s: never again", asked, pair)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A covered member of a group counts as delivered: the member after it is let go, not stopped.
|
|
func TestACoveredGroupMemberLetsTheNextOneGo(t *testing.T) {
|
|
w := newWorld(t)
|
|
const ctl, lab = "novox/mesh-controller", "novox/mesh-lab"
|
|
const ctlHead, labHead = "c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2"
|
|
ctlID, labID := IDOf(ctl, ctlHead), IDOf(lab, labHead)
|
|
pc, pl := pr(ctl, 1, ctlHead, "feat/g", "cmd/a.go"), pr(lab, 2, labHead, "feat/g", "replays/a.go")
|
|
w.h.PullUpdated(pc)
|
|
w.h.PullUpdated(pl)
|
|
w.h.Checked(verdictMoving(ctl, 1, ctlHead, "mesh-controller"))
|
|
w.h.Checked(verdictMoving(lab, 2, labHead, "mesh-lab"))
|
|
w.settleAll()
|
|
w.h.Checked(CheckedEvent{Group: "feat/g", ID: "check-1", Verdict: "pass", Summary: "both compose"})
|
|
w.later(31 * time.Minute)
|
|
w.h.PullMerged(mergedAt(pc, "m1m1m1m1m1m1", t0.Add(30*time.Minute)))
|
|
w.h.PullMerged(mergedAt(pl, "m2m2m2m2m2m2", t0.Add(31*time.Minute)))
|
|
w.h.WalkMoved(w.ctl.put(aWalk("plan-lab", lab, "m2m2m2m2m2m2", true)))
|
|
w.settleAll()
|
|
w.forge.contains["m1m1m1m1m1m1 in m3m3m3m3m3m3"] = true
|
|
deliverLater(w, ctl, 3, "main", "e3e3e3e3e3e3e3e3", "m3m3m3m3m3m3", t0.Add(35*time.Minute), walkDone, "mesh-controller")
|
|
w.later(20 * time.Minute)
|
|
if err := w.h.Reconcile(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.settleAll()
|
|
if w.state(ctlID) != Covered {
|
|
t.Fatalf("the controller's member is %s", w.state(ctlID))
|
|
}
|
|
if w.state(labID) != Delivering || !containsString(w.ctl.delivers, "plan-lab") {
|
|
t.Fatalf("the member after it is %s, let go %v", w.state(labID), w.ctl.delivers)
|
|
}
|
|
lw := w.ctl.walk("plan-lab")
|
|
lw.State = walkDone
|
|
w.h.WalkMoved(w.ctl.put(lw))
|
|
w.settleAll()
|
|
if state, why := GroupState(w.h.groups["feat/g"], w.h.membersOf(w.h.groups["feat/g"])); state != "delivered" {
|
|
t.Fatalf("the group is %s: %s", state, why)
|
|
}
|
|
}
|
|
|
|
// **On rollout, the deliveries already held close by themselves** (issue 319's held ones), and a restart
|
|
// after that moves nothing again: what was read back is where it stays.
|
|
func TestHeldDeliveriesReadBackCloseByThemselvesOnce(t *testing.T) {
|
|
const ctl, cat = "novox/mesh-controller", "novox/mesh-catalog"
|
|
merged1, merged2 := t0, t0.Add(2*time.Hour)
|
|
coverer := func(repo, module string) *Delivery {
|
|
return &Delivery{ID: IDOf(repo, "c0c0c0c0c0c0c0c0"), Repository: repo, Commit: "c0c0c0c0c0c0c0c0", Number: 90,
|
|
Base: "main", State: Delivered, Since: merged2.Add(time.Hour), Created: merged2.Add(-time.Hour),
|
|
MergedAs: "f0f0f0f0f0f0f0f0", MergedAt: merged2, Plan: planMoving(repo, module), Check: &Verdict{Gate: "pass"},
|
|
Walk: &WalkSeen{ID: "plan-90", Commit: "f0f0f0f0f0f0f0f0", State: walkDone},
|
|
Steps: []Step{{Machine: "anchor", Module: module, State: StepPassed}, {Machine: "the rest running it", Module: module, State: StepSent}}}
|
|
}
|
|
held := func(repo string, plan *DeliveryPlan, why string) *Delivery {
|
|
return &Delivery{ID: IDOf(repo, head), Repository: repo, Commit: head, Number: 7, Base: "main", State: Held,
|
|
Since: merged1.Add(10 * time.Minute), Created: merged1.Add(-time.Hour), MergedAs: merge, MergedAt: merged1,
|
|
HeldWhy: why, Plan: plan, Transitions: []Transition{{At: merged1, From: Ready, To: Held, Event: EvHold, Why: why}}}
|
|
}
|
|
const unwalkedWhy = "merged, and the controller opened no walk for it within 10m0s — nothing it holds follows that branch, " +
|
|
"or the merge was not heard: a person decides"
|
|
withCoverer := func(d *Delivery, change func(c *Delivery)) []*Delivery {
|
|
c := coverer(d.Repository, d.Plan.Moved[0])
|
|
change(c)
|
|
return []*Delivery{d, c}
|
|
}
|
|
const uncheckedWhy = "merged without a passing check: only a person decides that it goes on"
|
|
for _, c := range []struct {
|
|
name string
|
|
kept []*Delivery
|
|
group *Group
|
|
want State
|
|
}{
|
|
{"merged without a passing check, builds nothing",
|
|
[]*Delivery{held("novox/hq", planMoving("novox/hq"), "merged without a passing check: only a person decides that it goes on")},
|
|
nil, Delivered},
|
|
{"a new module sent nowhere, no walk opened",
|
|
[]*Delivery{held(cat, &DeliveryPlan{New: []string{"modules/zfs"}, Summary: "builds new: modules/zfs, sent nowhere"}, unwalkedWhy)},
|
|
nil, Delivered},
|
|
{"no walk opened, a later delivered walk carried its module",
|
|
[]*Delivery{held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), coverer(ctl, "mesh-controller")}, nil, Covered},
|
|
{"merged without a passing check, a later delivered walk whose own check passed carried its module",
|
|
withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), uncheckedWhy), func(*Delivery) {}), nil, Covered},
|
|
{"merged without a passing check, the later delivery's own check did not pass",
|
|
withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), uncheckedWhy), func(c *Delivery) {
|
|
c.Check = &Verdict{Gate: "fail"}
|
|
}), nil, Held},
|
|
{"no walk opened, the later delivery's walk was of another commit",
|
|
withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), func(c *Delivery) {
|
|
c.Walk.Commit = "e0e0e0e0e0e0e0e0"
|
|
}), nil, Held},
|
|
{"no walk opened, the later walk sent its module to its first machine only",
|
|
withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), func(c *Delivery) {
|
|
c.Steps = c.Steps[:1]
|
|
}), nil, Held},
|
|
{"no walk opened, the later walk's branch is another",
|
|
withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), func(c *Delivery) {
|
|
c.Walk.Branch = "release"
|
|
}), nil, Held},
|
|
{"no walk opened, a later delivered walk carried another module",
|
|
[]*Delivery{held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), coverer(ctl, "route-proxy")}, nil, Held},
|
|
{"held for its group, its own walk superseded, a later delivered walk carried its module",
|
|
func() []*Delivery {
|
|
d := held(cat, planMoving(cat, "systemd"), "its group feat/j's composed check did not pass for the heads that "+
|
|
"merged; a person decides that it goes on")
|
|
d.Group = "feat/j"
|
|
d.Walk = &WalkSeen{ID: "plan-7", Commit: merge, State: walkSuperseded, Waits: true, WaitedFor: "mesh-delivery"}
|
|
other := &Delivery{ID: IDOf("novox/mesh-host", "abababababababab"), Repository: "novox/mesh-host",
|
|
Commit: "abababababababab", Number: 3, Base: "main", Group: "feat/j", State: Delivered,
|
|
MergedAs: "acacacacacacacac", MergedAt: merged1}
|
|
return []*Delivery{d, other, coverer(cat, "systemd")}
|
|
}(),
|
|
&Group{ID: "feat/j", Members: []string{IDOf(cat, head), IDOf("novox/mesh-host", "abababababababab")}, Closed: true,
|
|
Check: &GroupCheck{Asked: "check-1", Verdict: "pass", Commits: map[string]string{IDOf(cat, head): head,
|
|
IDOf("novox/mesh-host", "abababababababab"): "abababababababab"}}},
|
|
Covered},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
store := newMemStore()
|
|
for _, d := range c.kept {
|
|
if err := store.PutDelivery(d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if c.group != nil {
|
|
if err := store.PutGroup(c.group); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
now := merged2.Add(30 * time.Hour)
|
|
forge := newFakeForge()
|
|
forge.contains[merge+" in f0f0f0f0f0f0f0f0"] = true
|
|
w := newWorldOver(t, store, newFakeController(), forge, now)
|
|
if err := w.h.Reconcile(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.settleAll()
|
|
id := c.kept[0].ID
|
|
d := w.delivery(id)
|
|
if d.State != c.want {
|
|
t.Fatalf("read back held, it is %s, want %s", d.State, c.want)
|
|
}
|
|
if s := statusOn(w, d.Repository, d.Commit); c.want != Held && !strings.HasPrefix(s, "success") {
|
|
t.Fatalf("its pull request's mesh/delivery says %q", s)
|
|
}
|
|
// A restart over what it kept: nothing moves again.
|
|
again := newWorldOver(t, store, newFakeController(), forge, now.Add(time.Hour))
|
|
if err := again.h.Reconcile(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.settleAll()
|
|
if d2 := again.delivery(id); d2.State != d.State || len(d2.Transitions) != len(d.Transitions) {
|
|
t.Fatalf("a restart moved it again: %s with %d transitions, was %s with %d", d2.State, len(d2.Transitions),
|
|
d.State, len(d.Transitions))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// **A group member is not held for a composed check that has not answered** (issue 316): it waits published,
|
|
// and goes on once the check passes for the heads that merged.
|
|
func TestAGroupMemberWaitsForItsComposedCheckToAnswer(t *testing.T) {
|
|
w := newWorld(t)
|
|
const a, b = "novox/a", "novox/b"
|
|
const aHead, bHead = "a1a1a1a1a1a1a1a1", "b2b2b2b2b2b2b2b2"
|
|
aID := IDOf(a, aHead)
|
|
pa, pb := pr(a, 1, aHead, "feat/k", "x"), pr(b, 2, bHead, "feat/k", "y")
|
|
w.h.PullUpdated(pa)
|
|
w.h.PullUpdated(pb)
|
|
w.h.Checked(verdictMoving(a, 1, aHead, "a"))
|
|
w.h.Checked(verdictMoving(b, 2, bHead, "b"))
|
|
w.settleAll()
|
|
if len(w.ctl.checks) != 1 {
|
|
t.Fatalf("the composed check was asked %v", w.ctl.checks)
|
|
}
|
|
w.h.PullMerged(mergedAt(pa, "m1m1m1m1m1m1", t0))
|
|
w.h.WalkMoved(w.ctl.put(aWalk("plan-a", a, "m1m1m1m1m1m1", true)))
|
|
w.settleAll()
|
|
if s := w.state(aID); s != Published {
|
|
t.Fatalf("merged while its group's composed check had not answered, it is %s (%s)", s, w.delivery(aID).HeldWhy)
|
|
}
|
|
w.h.Checked(CheckedEvent{Group: "feat/k", ID: "check-1", Verdict: "pass", Summary: "both compose"})
|
|
w.settleAll()
|
|
if s := w.state(aID); s != Published || len(w.ctl.delivers) != 0 {
|
|
t.Fatalf("its composed check passed, but the other member is not merged: %s, let go %v", s, w.ctl.delivers)
|
|
}
|
|
// The other member merges: every member is on the trunk, and the walk is let go (ADR 0418 §2).
|
|
w.h.PullMerged(mergedAt(pb, "m2m2m2m2m2m2", t0))
|
|
w.h.WalkMoved(w.ctl.put(aWalk("plan-b", b, "m2m2m2m2m2m2", true)))
|
|
w.settleAll()
|
|
if s := w.state(aID); s != Delivering || !containsString(w.ctl.delivers, "plan-a") {
|
|
t.Fatalf("every member merged, and it is %s, let go %v", s, w.ctl.delivers)
|
|
}
|
|
}
|
|
|
|
// **A member already held for its group's composed check goes on when that check passes for the heads that
|
|
// merged** (issue 316); a check that failed, or passed for other heads, keeps it held for a person.
|
|
func TestAMemberHeldForItsComposedCheckIsLiftedWhenItPasses(t *testing.T) {
|
|
const a, b = "novox/a", "novox/b"
|
|
const aHead, bHead = "a1a1a1a1a1a1a1a1", "b2b2b2b2b2b2b2b2"
|
|
aID, bID := IDOf(a, aHead), IDOf(b, bHead)
|
|
for _, c := range []struct {
|
|
name string
|
|
verdict string
|
|
commits map[string]string
|
|
want State
|
|
}{
|
|
{"passed for the heads that merged", "pass", map[string]string{aID: aHead, bID: bHead}, Delivering},
|
|
{"warned for the heads that merged", "warning", map[string]string{aID: aHead, bID: bHead}, Delivering},
|
|
{"failed", "fail", map[string]string{aID: aHead, bID: bHead}, Held},
|
|
{"passed for other heads", "pass", map[string]string{aID: "a9a9a9a9a9a9a9a9", bID: bHead}, Held},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
store := newMemStore()
|
|
why := "its group feat/k's composed check did not pass for the heads that merged; a person decides that it goes on"
|
|
da := &Delivery{ID: aID, Repository: a, Commit: aHead, Number: 1, Base: "main", Branch: "feat/k", Group: "feat/k",
|
|
State: Held, Since: t0, Created: t0.Add(-time.Hour), MergedAs: "m1m1m1m1m1m1", MergedAt: t0, HeldWhy: why,
|
|
Plan: planMoving(a, "a"), Check: &Verdict{Gate: "pass"},
|
|
Walk: &WalkSeen{ID: "plan-a", Commit: "m1m1m1m1m1m1", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"}}
|
|
db := &Delivery{ID: bID, Repository: b, Commit: bHead, Number: 2, Base: "main", Branch: "feat/k", Group: "feat/k",
|
|
State: Published, Since: t0, Created: t0.Add(-time.Hour), MergedAs: "m2m2m2m2m2m2", MergedAt: t0,
|
|
Plan: planMoving(b, "b"), Check: &Verdict{Gate: "pass"},
|
|
Walk: &WalkSeen{ID: "plan-b", Commit: "m2m2m2m2m2m2", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"}}
|
|
g := &Group{ID: "feat/k", Members: []string{aID, bID}, Closed: true,
|
|
Check: &GroupCheck{Asked: "check-old", At: t0, Commits: c.commits}}
|
|
for _, d := range []*Delivery{da, db} {
|
|
_ = store.PutDelivery(d)
|
|
}
|
|
_ = store.PutGroup(g)
|
|
ctl := newFakeController()
|
|
ctl.put(aWalk("plan-a", a, "m1m1m1m1m1m1", true))
|
|
ctl.put(aWalk("plan-b", b, "m2m2m2m2m2m2", true))
|
|
w := newWorldOver(t, store, ctl, newFakeForge(), t0.Add(time.Minute))
|
|
w.settleAll()
|
|
if s := w.state(aID); s != Held {
|
|
t.Fatalf("before its composed verdict it is %s", s)
|
|
}
|
|
w.h.Checked(CheckedEvent{Group: "feat/k", ID: "check-old", Verdict: c.verdict, Summary: "composed: " + c.verdict})
|
|
w.settleAll()
|
|
if s := w.state(aID); s != c.want {
|
|
t.Fatalf("its composed check %s, and it is %s (%s), want %s", c.name, s, w.delivery(aID).HeldWhy, c.want)
|
|
}
|
|
if c.want == Delivering && !containsString(ctl.delivers, "plan-a") {
|
|
t.Fatalf("lifted, its walk was not let go: %v", ctl.delivers)
|
|
}
|
|
// A composed check of other heads is asked again for these heads (novox/hq ADR 0323 §6).
|
|
if c.name == "passed for other heads" && !containsString(ctl.checks, "feat/k:"+aID+","+bID) {
|
|
t.Fatalf("the composed check was not asked again for the heads that merged: %v", ctl.checks)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// **A module a release carried to every machine the held delivery sends it to covers it** (novox/hq issue
|
|
// 351). A build the controller held back for a person and released on their word reaches each machine by a
|
|
// backlog walk, which credits the delivery that published the build one step per machine — never the walk's
|
|
// own "every machine running it" or "the rest running it". The held delivery's plan says which machines
|
|
// receive the module: a passed step on each of them is the module sent everywhere. A machine still judging,
|
|
// one that failed, or a plan that names no machines leaves it held, as before.
|
|
func TestAModuleAReleaseCarriedToEveryMachineCovers(t *testing.T) {
|
|
const cat = "novox/mesh-catalog"
|
|
merged1, merged2 := t0, t0.Add(2*time.Hour)
|
|
laptop, desktop := "laptop", "desktop"
|
|
plan := func(machines ...string) *DeliveryPlan {
|
|
p := planMoving(cat, "openrazer")
|
|
for _, m := range machines {
|
|
p.Machines = append(p.Machines, MachinePlan{Machine: m, Receives: []string{"openrazer"}})
|
|
}
|
|
return p
|
|
}
|
|
held := func(p *DeliveryPlan) *Delivery {
|
|
why := "its group feat/j's composed check did not pass for the heads that merged; a person decides that it goes on"
|
|
return &Delivery{ID: IDOf(cat, head), Repository: cat, Commit: head, Number: 7, Base: "main", State: Held,
|
|
Since: merged1.Add(10 * time.Minute), Created: merged1.Add(-time.Hour), MergedAs: merge, MergedAt: merged1,
|
|
HeldWhy: why, Plan: p, Walk: &WalkSeen{ID: "plan-7", Commit: merge, State: walkSuperseded, Waits: true, WaitedFor: "mesh-delivery"},
|
|
Transitions: []Transition{{At: merged1, From: Published, To: Held, Event: EvHold, Why: why}}}
|
|
}
|
|
step := func(machine string, state StepState) Step {
|
|
return Step{Machine: machine, Module: "openrazer", Build: "build-1", State: state, Why: "carried by release-1"}
|
|
}
|
|
coverer := func(steps ...Step) *Delivery {
|
|
return &Delivery{ID: IDOf(cat, "c0c0c0c0c0c0c0c0"), Repository: cat, Commit: "c0c0c0c0c0c0c0c0", Number: 90,
|
|
Base: "main", State: Delivered, Since: merged2.Add(time.Hour), Created: merged2.Add(-time.Hour),
|
|
MergedAs: "f0f0f0f0f0f0f0f0", MergedAt: merged2, Plan: planMoving(cat, "power"), Check: &Verdict{Gate: "pass"},
|
|
Walk: &WalkSeen{ID: "plan-90", Commit: "f0f0f0f0f0f0f0f0", State: walkDone},
|
|
Steps: append([]Step{{Machine: "anchor", Module: "power", State: StepPassed},
|
|
{Machine: "the rest running it", Module: "power", State: StepSent}}, steps...)}
|
|
}
|
|
for _, c := range []struct {
|
|
name string
|
|
held *Delivery
|
|
by *Delivery
|
|
want State
|
|
}{
|
|
{"passed on every machine its plan sends it to", held(plan(laptop, desktop)),
|
|
coverer(step(laptop, StepPassed), step(desktop, StepPassed)), Covered},
|
|
{"still judging on one of them", held(plan(laptop, desktop)),
|
|
coverer(step(laptop, StepPassed), step(desktop, StepJudging)), Held},
|
|
{"failed on one of them", held(plan(laptop, desktop)),
|
|
coverer(step(laptop, StepPassed), step(desktop, StepFailed)), Held},
|
|
{"never sent to one of them", held(plan(laptop, desktop)), coverer(step(laptop, StepPassed)), Held},
|
|
{"its plan names no machines", held(plan()), coverer(step(laptop, StepPassed), step(desktop, StepPassed)), Held},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
store := newMemStore()
|
|
for _, d := range []*Delivery{c.held, c.by} {
|
|
if err := store.PutDelivery(d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
forge := newFakeForge()
|
|
forge.contains[merge+" in f0f0f0f0f0f0f0f0"] = true
|
|
w := newWorldOver(t, store, newFakeController(), forge, merged2.Add(30*time.Hour))
|
|
if err := w.h.Reconcile(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.settleAll()
|
|
d := w.delivery(c.held.ID)
|
|
if d.State != c.want {
|
|
t.Fatalf("it is %s (%s), want %s", d.State, d.HeldWhy, c.want)
|
|
}
|
|
if c.want == Covered {
|
|
if s := statusOn(w, cat, head); !strings.HasPrefix(s, "success covered: a later merge that contains it was delivered") {
|
|
t.Fatalf("its pull request's mesh/delivery says %q", s)
|
|
}
|
|
if last := d.Transitions[len(d.Transitions)-1]; !strings.Contains(last.Why, "openrazer") {
|
|
t.Fatalf("its transition does not name the module: %q", last.Why)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// **A finished release moves the steps it carried to passed** (novox/hq issue 351): a backlog walk's gate
|
|
// credits a step per machine while it judges, and its last record keeps no gate — only which machines it is
|
|
// done on. A step credited to it, on a machine it is done on, is passed; a step it failed on, one on a machine
|
|
// it is not done on, and a step another walk carried are left as they are.
|
|
func TestAFinishedReleaseMovesTheStepsItCarriedToPassed(t *testing.T) {
|
|
const cat = "novox/mesh-catalog"
|
|
carried := func(machine string, state StepState, by string) Step {
|
|
return Step{Machine: machine, Module: "openrazer", Build: "build-1", State: state, Why: "carried by " + by}
|
|
}
|
|
for _, c := range []struct {
|
|
name string
|
|
steps []Step
|
|
done []string
|
|
want map[string]StepState
|
|
}{
|
|
{"done on both", []Step{carried("laptop", StepJudging, "release-1"), carried("desktop", StepJudging, "release-1")},
|
|
[]string{"laptop", "desktop"}, map[string]StepState{"laptop": StepPassed, "desktop": StepPassed}},
|
|
{"done on one, the other still to come", []Step{carried("laptop", StepJudging, "release-1"), carried("desktop", StepJudging, "release-1")},
|
|
[]string{"laptop"}, map[string]StepState{"laptop": StepPassed, "desktop": StepJudging}},
|
|
{"a step it failed on stays failed", []Step{carried("laptop", StepFailed, "release-1")},
|
|
[]string{"laptop"}, map[string]StepState{"laptop": StepFailed}},
|
|
{"a step another walk carried is not its", []Step{carried("laptop", StepJudging, "release-2")},
|
|
[]string{"laptop"}, map[string]StepState{"laptop": StepJudging}},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
d := &Delivery{ID: IDOf(cat, "c0c0c0c0c0c0c0c0"), Repository: cat, Commit: "c0c0c0c0c0c0c0c0", Number: 90,
|
|
Base: "main", State: Delivered, Since: t0.Add(time.Hour), Created: t0, MergedAs: "f0f0f0f0f0f0f0f0", MergedAt: t0,
|
|
Plan: planMoving(cat, "power"), Check: &Verdict{Gate: "pass"},
|
|
Walk: &WalkSeen{ID: "plan-90", Commit: "f0f0f0f0f0f0f0f0", State: walkDone}, Steps: c.steps}
|
|
store := newMemStore()
|
|
if err := store.PutDelivery(d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w := newWorldOver(t, store, newFakeController(), newFakeForge(), t0.Add(2*time.Hour))
|
|
w.h.WalkMoved(Walk{ID: "release-1", Repository: "the builds waiting for a gate", State: walkDone,
|
|
Tiers: [][]string{{"openrazer"}}, Modules: map[string]*WalkModule{},
|
|
Release: &WalkRelease{Order: []string{"laptop", "desktop"}, Next: len(c.done), Done: c.done}})
|
|
got := map[string]StepState{}
|
|
for _, s := range w.delivery(d.ID).Steps {
|
|
got[s.Machine] = s.State
|
|
}
|
|
for machine, want := range c.want {
|
|
if got[machine] != want {
|
|
t.Fatalf("openrazer on %s is %s, want %s (steps %v)", machine, got[machine], want, got)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The day's case whole (novox/hq issue 351): a member held for its group's check, its own walk taken over by
|
|
// later merges; the one later walk that carried its module failed its gate; a later merge's walk was delivered
|
|
// without the module; the controller then held the module's build back for a person, who released it, and the
|
|
// release reached both machines. The held delivery is covered by that later merge once the release is done —
|
|
// read by its id, since a walk that ended before the holder's restart may be older than the ended walks the
|
|
// controller lists back.
|
|
func TestAHeldDeliveryWhoseModuleAReleaseCarriedIsCoveredOnceTheReleaseIsDone(t *testing.T) {
|
|
const cat = "novox/mesh-catalog"
|
|
merged1, merged2 := t0, t0.Add(2*time.Hour)
|
|
plan := planMoving(cat, "openrazer")
|
|
plan.Machines = []MachinePlan{{Machine: "laptop", Receives: []string{"openrazer"}}, {Machine: "desktop", Receives: []string{"openrazer"}}}
|
|
why := "its group feat/j's composed check did not pass for the heads that merged; a person decides that it goes on"
|
|
held := &Delivery{ID: IDOf(cat, head), Repository: cat, Commit: head, Number: 7, Base: "main", State: Held,
|
|
Since: merged1.Add(10 * time.Minute), Created: merged1.Add(-time.Hour), MergedAs: merge, MergedAt: merged1,
|
|
HeldWhy: why, Plan: plan, Walk: &WalkSeen{ID: "plan-7", Commit: merge, State: walkSuperseded, Waits: true, WaitedFor: "mesh-delivery"},
|
|
Transitions: []Transition{{At: merged1, From: Published, To: Held, Event: EvHold, Why: why}}}
|
|
later := &Delivery{ID: IDOf(cat, "c0c0c0c0c0c0c0c0"), Repository: cat, Commit: "c0c0c0c0c0c0c0c0", Number: 90,
|
|
Base: "main", State: Delivered, Since: merged2.Add(time.Hour), Created: merged2.Add(-time.Hour),
|
|
MergedAs: "f0f0f0f0f0f0f0f0", MergedAt: merged2, Plan: planMoving(cat, "power"), Check: &Verdict{Gate: "pass"},
|
|
Walk: &WalkSeen{ID: "plan-90", Commit: "f0f0f0f0f0f0f0f0", State: walkDone},
|
|
Steps: []Step{{Machine: "anchor", Module: "power", State: StepPassed}, {Machine: "the rest running it", Module: "power", State: StepSent}}}
|
|
store := newMemStore()
|
|
for _, d := range []*Delivery{held, later} {
|
|
if err := store.PutDelivery(d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
forge := newFakeForge()
|
|
forge.contains[merge+" in f0f0f0f0f0f0f0f0"] = true
|
|
w := newWorldOver(t, store, newFakeController(), forge, merged2.Add(3*time.Hour))
|
|
if err := w.h.Reconcile(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.settleAll()
|
|
if s := w.state(held.ID); s != Held {
|
|
t.Fatalf("before any release it is %s", s)
|
|
}
|
|
// The release judges the build on each machine in turn: its gate names what it carried.
|
|
release := Walk{ID: "release-1", Repository: "the builds waiting for a gate", State: walkRolling,
|
|
Tiers: [][]string{{"openrazer"}}, Modules: map[string]*WalkModule{},
|
|
Release: &WalkRelease{Order: []string{"laptop", "desktop"}, Gate: &WalkGate{Machines: []string{"laptop"},
|
|
Carried: []WalkCarry{{Module: "openrazer", Node: "laptop", To: "f0f0f0f0f0f0f0f0", Build: "build-1"},
|
|
{Module: "openrazer", Node: "desktop", To: "f0f0f0f0f0f0f0f0", Build: "build-1"}}}}}
|
|
w.h.WalkMoved(release)
|
|
w.settleAll()
|
|
if s := w.state(held.ID); s != Held {
|
|
t.Fatalf("while the release judges it is %s", s)
|
|
}
|
|
// Done on both: its last record keeps no gate. It ended while this holder was down, and is older than the
|
|
// ended walks the controller lists back: the holder asks for it by its id, since a step it carried still judges.
|
|
release.State, release.Release.Gate, release.Release.Done, release.Release.Next = walkDone, nil, []string{"laptop", "desktop"}, 2
|
|
w.ctl.put(release)
|
|
w.later(time.Hour) // a walk a step still judges is asked for once an hour; at a start, at once
|
|
w.settleAll()
|
|
d := w.delivery(held.ID)
|
|
if d.State != Covered {
|
|
t.Fatalf("once the release is done it is %s (%s), want covered", d.State, d.HeldWhy)
|
|
}
|
|
if last := d.Transitions[len(d.Transitions)-1]; !strings.Contains(last.Why, IDOf(cat, "c0c0c0c0c0c0c0c0")) ||
|
|
!strings.Contains(last.Why, "openrazer") {
|
|
t.Fatalf("its transition does not name what covered it: %q", last.Why)
|
|
}
|
|
}
|