package main import ( "context" "slices" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/inventory" ) // novox/hq ADR 0360 §4, §5: a walk ends on its machines' reports — every machine sent has reported on its send, // or is silent past ApplySilentAfter — and names a failed or refused apply; it never waits for a health reading. func TestAWalkEndsOnItsMachinesReports(t *testing.T) { now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) sent := now.Add(-time.Minute) long := now.Add(-inventory.ApplySilentAfter - time.Minute) at := now.Add(-10 * time.Second) p := inventory.Plan{Modules: map[string]*inventory.PlanModule{ "app": {Rest: map[string]inventory.SentDeclaration{ "ace": {Digest: "d-ace", At: sent}, "novox": {Digest: "d-novox", At: sent}, "shanks": {Digest: "d-shanks", At: sent}, "laptop": {Digest: "d-laptop", At: long}, }}, "proxy": {Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: sent}}}, }} reports := []inventory.Reported{ {Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}, {Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"}, {Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"}, } waiting, failed, silent := walkReports(p, reports, now, nil) if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) || !slices.Equal(silent, []string{"laptop"}) { t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent) } reports[2].Declared = "d-shanks" if waiting, _, _ := walkReports(p, reports, now, nil); len(waiting) != 0 { t.Fatalf("the walk still waits once every machine reported: %v", waiting) } } // withTheGate turns the first-machine gate on for a test of its own path: kept for the walks a controller before // novox/hq ADR 0360 started. func withTheGate(t *testing.T) { t.Helper() was := gateFirstMachine gateFirstMachine = true t.Cleanup(func() { gateFirstMachine = was }) } // Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7); // before, the first machine was judged before the rest were sent (ADR 0353 §1). func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { judgeLeavesAsTheCore(t) b := aBacklogOnThree(t) ctx := t.Context() said := events(t) for i := 0; i < 12; i++ { advancePlans(ctx, b.open) } p := b.release(t) if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) } for _, n := range []string{"desk", "laptop", "anchor"} { if !slices.Contains(*said, "send "+n) { t.Fatalf("%s was not sent: %v", n, *said) } } if slices.Contains(*said, "judge") { t.Fatalf("a machine was judged by its health: %v", *said) } if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) } } // A machine that reports its apply failed is named and fails the release, with nothing put back, and every other // machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3). func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { judgeLeavesAsTheCore(t) b := aBacklogOnThree(t) ctx := t.Context() inv := b.open.inventory b.broken["desk"] = true // desk fails what it is sent advancePlans(ctx, b.open) for i := 0; i < 12 && b.release(t).Open(); i++ { advancePlans(ctx, b.open) } p := b.release(t) if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) } if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") { t.Fatalf("the others have no verdict of their own: %+v", p.Release) } if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) } if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" { t.Fatalf("desk was put back: %v", sent) } } // A release's send kept before it was made, and made, is judged by the controller that takes over without being // made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7). func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) { judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() inv := b.open.inventory p, err := releaseBacklog(ctx, b.open, "") if err != nil || p == nil { t.Fatalf("no release opened: %v", err) } if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending { t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) } if err := inv.SavePlan(ctx, p); err != nil { t.Fatal(err) } // The send is made, and the controller is replaced before it keeps it. if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { t.Fatal(err) } advancePlans(ctx, b.open) if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 { t.Fatalf("anchor was sent again: %v", b.sent) } g := b.release(t).Release.Gates["anchor"] if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { t.Fatalf("anchor is not judged on the send made: %+v", g) } } // A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put // back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7). func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) { judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() inv := b.open.inventory reporting := sendRollout sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing b.sent = append(b.sent, names) current, _ := open.inventory.CurrentBuilds(ctx) return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) } return reporting(ctx, open, names) } t.Cleanup(func() { sendRollout = reporting }) for i := 0; i < 8; i++ { advancePlans(ctx, b.open) } p := b.release(t) g := p.Release.Gates["laptop"] if !p.Open() || g == nil || g.Verdict != "" || g.Silent { t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) } since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) g.Since = &since if err := inv.SavePlan(ctx, &p); err != nil { t.Fatal(err) } advancePlans(ctx, b.open) p = b.release(t) // anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7). if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) } if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { t.Fatal("a silent machine's build was marked failed") } if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { t.Fatalf("laptop was put back: %v", sent) } obs, err := silentObservation(ctx, inv) if err != nil { t.Fatal(err) } if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { t.Fatalf("the silence is not said: %+v", obs) } } // Issue 840 (novox/hq ADR 0362 §1-5): a walk's end counts a machine's send of a provisioned module applied only once // the engine and the machine's provisioner client both said so. func TestAWalkEndWaitsForTheProvisioner(t *testing.T) { now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) at := now.Add(-10 * time.Second) plan := func(sentAt time.Time) inventory.Plan { return inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{ "ace": {Digest: "d-ace", At: sentAt, Provisioned: map[string]string{"app": "t1"}}, "g14": {Digest: "d-g14", At: sentAt, Provisioned: map[string]string{"app": "t1"}}, }}}} } reports := []inventory.Reported{ {Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}, {Node: "g14", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-g14"}, } says := map[string]inventory.ProvReport{} prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: at} // (a) the engine applied, the client has not said: the end waits on both. waiting, failed, silent := walkReports(plan(now.Add(-time.Minute)), reports, now, prov) if !slices.Equal(waiting, []string{"ace", "g14"}) || len(failed)+len(silent) != 0 { t.Fatalf("a: waiting %v, failed %v, silent %v", waiting, failed, silent) } // ace's client applied, g14's failed on that build (b): ace done, g14 a failed apply naming the reason. says["ace/app"] = applied says["g14/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: at} waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov) if len(waiting) != 0 || len(silent) != 0 || len(failed) != 1 || !strings.HasPrefix(failed[0], "g14 (") || !strings.Contains(failed[0], "no value") { t.Fatalf("b: waiting %v, failed %v, silent %v", waiting, failed, silent) } // (c) no client report past ApplySilentAfter: named silent, never applied. delete(says, "g14/app") waiting, failed, silent = walkReports(plan(now.Add(-inventory.ApplySilentAfter-time.Minute)), reports, now, prov) if len(waiting) != 0 || len(failed) != 0 || !slices.Equal(silent, []string{"g14"}) { t.Fatalf("c: waiting %v, failed %v, silent %v", waiting, failed, silent) } // both said it: nothing waits. says["g14/app"] = applied if waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { t.Fatalf("both said: waiting %v, failed %v, silent %v", waiting, failed, silent) } // (d) a send handing nothing to the provisioner is the engine's report alone, as before. bare := plan(now.Add(-time.Minute)) bare.Modules["app"].Rest = map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: now.Add(-time.Minute)}} if waiting, failed, silent = walkReports(bare, reports, now, func(string, string) (inventory.ProvReport, bool) { return inventory.ProvReport{}, false }); len(waiting)+len(failed)+len(silent) != 0 { t.Fatalf("d: waiting %v, failed %v, silent %v", waiting, failed, silent) } }