diff --git a/cmd/mesh-controller/issue648_test.go b/cmd/mesh-controller/issue648_test.go new file mode 100644 index 00000000..45261cc0 --- /dev/null +++ b/cmd/mesh-controller/issue648_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A tier whose module is no move — made from the source every machine running it runs, so nothing is +// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the +// send that last carried that build before the walk got there, and nothing will make them report again: +// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever. +func TestANoMoveTierWaitsForNoNewReport(t *testing.T) { + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + + // app is the build machine, so a module of the next tier is built by it: the tier gates the next on app + // running on its machines. + for _, m := range []catalogue.Manifest{ + {Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}, + {Module: "web", Version: "w1"}, + } { + if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil { + t.Fatal(err) + } + } + // What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is. + for _, b := range []inventory.Build{ + {ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app", + Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)}, + {ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()}, + } { + b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + // Each machine answered that send, applied, before the walk's tier was built. + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n, + Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1, + At: time.Now().Add(-10 * time.Second)}); err != nil { + t.Fatal(err) + } + } + built := time.Now().UTC() + p := g.plan(t) + p.Tiers = [][]string{{"app"}, {"web"}} + p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + + advancePlans(ctx, g.open) + + if len(g.sent) != 0 { + t.Fatalf("a no-move build was sent: %v", g.sent) + } + p = g.plan(t) + if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") { + t.Fatalf("the build was not marked no move: %+v", s) + } + if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") { + t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note) + } + + // A machine that has not answered its last send is still waited on: no move is not "never sent". + p.Tier, p.State, p.Note = 0, inventory.PlanRolling, "" + if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, g.open) + p = g.plan(t) + if p.Tier != 0 || !strings.Contains(p.Note, "waiting for app on laptop") { + t.Fatalf("a machine that has not answered its last send was not waited on: tier %d, %q", p.Tier, p.Note) + } +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 68fdca5d..f28516fa 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -251,6 +251,20 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) return out } +// noMoveWhy is what a plan says of a module of its tier whose build was made from the source every machine +// running it runs (novox/hq issue 280): nothing is sent for it. +// noMovePrefix opens it, and is what noMove reads: one prefix for the writer and the reader. +const ( + noMovePrefix = "no move" + noMoveWhy = noMovePrefix + ": made from the source every machine running it runs" +) + +// noMove is whether the plan marked the module no move and sent it nothing. Read from what the plan says, +// so a plan kept before the mark had a name of its own is read the same (novox/hq issue 648). +func noMove(s *inventory.PlanModule) bool { + return s != nil && s.FirstAt == nil && s.Gate == nil && strings.HasPrefix(s.Why, noMovePrefix) +} + // applied says whether every machine running the module has reported since the module was built. // appliedEach is applied with a moment of its own for each machine: the reports that count are the ones // after that machine was sent the build (novox/hq issue 256). @@ -805,7 +819,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, } else if same { now := time.Now().UTC() state.SentAt = &now - state.Why = "no move: made from the source every machine running it runs" + state.Why = noMoveWhy fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m) continue } @@ -995,6 +1009,25 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, if state.BuiltAt != nil { since = *state.BuiltAt } + // **A build that is no move is waited on by the send that last carried it** (novox/hq issue 648). + // Nothing was sent for it, so no report follows its marking or its build: a machine that does not + // happen to report again held the walk for ever. Each machine runs it once it answered its last + // send — the one whose builds made it no move — whenever that was. + if noMove(state) { + byNode := map[string]inventory.Reported{} + for _, r := range reports { + byNode[r.Node] = r + } + if ok, on := appliedEach(m, func(node string) time.Time { + if s := byNode[node].Sent; s != nil { + return *s + } + return time.Time{} + }, running, reports); !ok { + waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", "))) + } + continue + } // **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218) // a module is sent twice — the first machine, then the rest — and SentAt is the second. // Asked of every machine, the first machine's report, made between the two sends, read as diff --git a/testdata/store-tests b/testdata/store-tests index 48692606..ff17eedf 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -69,3 +69,4 @@ ./cmd/mesh-controller TestAFirstMachineThatFailsStillStopsTheWalk ./cmd/mesh-controller TestTheControlNodeIsSentLastInAStep ./cmd/mesh-controller TestARestMachineSilentPastItsBoundIsLeftAndSaid +./cmd/mesh-controller TestANoMoveTierWaitsForNoNewReport