diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 8679101..6c390c5 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -282,6 +282,22 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) } // applied says whether every machine running the module has reported since the module was built. +// appliedEach is applied with a moment of its own for each machine: the reports that count are the ones +// after that machine was sent the build (novox/hq issue 256). +func appliedEach(module string, since func(node string) time.Time, running []string, reports []inventory.Reported) (bool, []string) { + at := map[string]*time.Time{} + for _, r := range reports { + at[r.Node] = r.At + } + var waiting []string + for _, n := range running { + if t := at[n]; t == nil || t.Before(since(n)) { + waiting = append(waiting, n) + } + } + return len(waiting) == 0, waiting +} + func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) { at := map[string]*time.Time{} for _, r := range reports { @@ -637,10 +653,24 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, if state.BuiltAt != nil { since = *state.BuiltAt } - if state.SentAt != nil && state.SentAt.After(since) { - since = *state.SentAt + // **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218) + // a module is sent twice — the first machine, then the rest — and SentAt is the second. + // Asked of every machine, the first machine's report, made between the two sends, read as + // older than the build, and the gate waited for a report it already had, for ever. + sinceFor := func(node string) time.Time { + at := since + sent := state.SentAt + for _, n := range state.First { + if n == node { + sent = state.FirstAt + } + } + if sent != nil && sent.After(at) { + at = *sent + } + return at } - if ok, on := applied(m, since, running, reports); !ok { + if ok, on := appliedEach(m, sinceFor, running, reports); !ok { waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", "))) } } @@ -791,7 +821,7 @@ func planLine(p inventory.Plan, now time.Time) string { case inventory.PlanSuperseded: return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note) } - since := now.Sub(p.Updated).Round(time.Minute) + since := now.Sub(p.Updated).Round(time.Second) late := "" if since > planWaitBound { late = " — LATE" diff --git a/cmd/mesh-controller/release_plan_test.go b/cmd/mesh-controller/release_plan_test.go index 93a2288..365cdfc 100644 --- a/cmd/mesh-controller/release_plan_test.go +++ b/cmd/mesh-controller/release_plan_test.go @@ -181,3 +181,34 @@ func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) { t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"]) } } + +// The first machine's report, made between the send to it and the send to the rest, opens the gate for it: +// each machine is judged from its own send, not from the last one (novox/hq issue 256). +func TestTheGateJudgesEachMachineFromItsOwnSend(t *testing.T) { + built := time.Date(2026, 10, 5, 18, 22, 0, 0, time.UTC) + firstSent := built.Add(31 * time.Second) + firstReported := built.Add(43 * time.Second) + restSent := built.Add(58 * time.Second) + restReported := built.Add(74 * time.Second) + reports := []inventory.Reported{ + {Node: "ace", At: &firstReported}, + {Node: "g14", At: &restReported}, + } + since := func(node string) time.Time { + if node == "ace" { + return firstSent + } + return restSent + } + if ok, waiting := appliedEach("build-agent", since, []string{"ace", "g14"}, reports); !ok { + t.Fatalf("the gate still waits on %v, though each reported after its own send", waiting) + } + // The old reading, every machine from the last send, is what held the plan. + if ok, _ := applied("build-agent", restSent, []string{"ace", "g14"}, reports); ok { + t.Fatal("the single-moment reading should hold the first machine back") + } + early := built.Add(10 * time.Second) + if ok, waiting := appliedEach("build-agent", since, []string{"ace"}, []inventory.Reported{{Node: "ace", At: &early}}); ok || waiting[0] != "ace" { + t.Fatal("a report from before the machine was sent opened the gate") + } +}