Builds of one module in flight together finish in any order, and the mesh took whatever it heard last as what the module is: RegisterModule overwrote the module's manifest unconditionally, and Held/BuiltAgainst/ReadRepositories ordered builds by when they were recorded. A postgres build asked before the mesh-tools runtime fix finished after the one asked after it, and the next push deployed the stale image (novox/hq issue 219). A build is now ordered by when it was asked, read from the build-<nanos> id the controller writes: build.asked and module.built_asked (migration 0055). A registration from an earlier request than the module's current one is recorded and refused as superseded. A plan takes as its outcome only a build asked at or after its own ask, so an earlier plan's leftover build cannot settle a later plan. Ids of any other shape keep the old order.
184 lines
9.2 KiB
Go
184 lines
9.2 KiB
Go
package main
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A merge produces a tiered plan (novox/hq ADR 0162): what moved and everything reachable from it,
|
|
// sorted so a tier depends only on earlier ones — with the three kinds of dependency told apart.
|
|
func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
|
edges := []inventory.Edge{
|
|
// build dependencies: images on the runtime, a plugin on one of them
|
|
{From: "shop", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
|
{From: "postgres", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
|
{From: "shop-plugin", To: "shop", Kind: inventory.EdgeDeclared},
|
|
// a code dependency: the proxy packages the controller's source — same tier
|
|
{From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages},
|
|
{From: "builder", To: "mesh-controller", Kind: inventory.EdgePackages},
|
|
// runtime dependencies: everything source-built is built by the builder
|
|
{From: "shop", To: "builder", Kind: inventory.EdgeBuiltBy},
|
|
{From: "postgres", To: "builder", Kind: inventory.EdgeBuiltBy},
|
|
{From: "shop-plugin", To: "builder", Kind: inventory.EdgeBuiltBy},
|
|
{From: "route-proxy", To: "builder", Kind: inventory.EdgeBuiltBy},
|
|
{From: "mesh-controller", To: "builder", Kind: inventory.EdgeBuiltBy},
|
|
{From: "mesh-tools", To: "builder", Kind: inventory.EdgeBuiltBy},
|
|
{From: "builder", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
|
{From: "unrelated", To: "alpine", Kind: inventory.EdgeStandsOn},
|
|
}
|
|
// The runtime image moved: everything on it, and what is built by what is on it.
|
|
set := reachableFrom([]string{"mesh-tools"}, edges)
|
|
// What stands on the runtime, and the builder that stands on it; not the controller, which the
|
|
// builder merely builds, nor the proxy that packages the controller.
|
|
want := []string{"builder", "mesh-tools", "postgres", "shop", "shop-plugin"}
|
|
if len(set) != len(want) {
|
|
t.Fatalf("reachable from the runtime: %v, want %v", set, want)
|
|
}
|
|
tiers := tiersOf(set, edges)
|
|
pos := map[string]int{}
|
|
for i, tier := range tiers {
|
|
for _, m := range tier {
|
|
pos[m] = i
|
|
}
|
|
}
|
|
if pos["mesh-tools"] != 0 || pos["builder"] != 1 {
|
|
t.Fatalf("the runtime then the builder: %v", tiers)
|
|
}
|
|
if !(pos["shop"] > pos["builder"] && pos["postgres"] > pos["builder"]) {
|
|
t.Fatalf("what the builder builds comes after the builder: %v", tiers)
|
|
}
|
|
if pos["shop-plugin"] <= pos["shop"] {
|
|
t.Fatalf("a plugin after what it is declared on: %v", tiers)
|
|
}
|
|
if hasCycle(tiers, edges) {
|
|
t.Fatalf("no cycle here: %v", tiers)
|
|
}
|
|
|
|
// The controller alone moved: the proxy with it, nothing else.
|
|
small := reachableFrom([]string{"mesh-controller"}, edges)
|
|
if len(small) != 3 {
|
|
t.Fatalf("a controller merge rebuilds the controller and what packages it: %v", small)
|
|
}
|
|
// The builder packages the controller's source (same tier by that edge) and the controller is
|
|
// built by the builder (next tier by that one): the builder first, then the controller and the
|
|
// proxy together — a code dependency in one tier, a runtime dependency across tiers.
|
|
smallTiers := tiersOf(small, edges)
|
|
if len(smallTiers) != 2 || smallTiers[0][0] != "builder" || len(smallTiers[1]) != 2 {
|
|
t.Fatalf("the builder, then the controller and the proxy together: %v", smallTiers)
|
|
}
|
|
// The builder alone moved: the builder, and nothing it builds.
|
|
if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 {
|
|
t.Fatalf("a build machine change rebuilds the build machine alone: %v", only)
|
|
}
|
|
|
|
// Only a runtime dependency gates on deployment, and only when the module rolls out.
|
|
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges)
|
|
p.Tier = pos["builder"]
|
|
rollsOut := func(m string) bool { return m == "builder" }
|
|
if g := gates(p, edges, rollsOut); len(g) != 1 || g[0] != "builder" {
|
|
t.Fatalf("the builder gates the tier after it: %v", g)
|
|
}
|
|
p.Tier = 0
|
|
if g := gates(p, edges, rollsOut); len(g) != 0 {
|
|
t.Fatalf("the runtime image is a build dependency and gates nothing: %v", g)
|
|
}
|
|
if g := gates(p, edges, func(string) bool { return false }); len(g) != 0 {
|
|
t.Fatalf("a module that only records its upgrade gates nothing: %v", g)
|
|
}
|
|
}
|
|
|
|
// A gate is open once every machine running the module has reported after it was built.
|
|
func TestAGateOpensWhenTheMachinesHaveReportedSinceTheBuild(t *testing.T) {
|
|
built := time.Date(2026, 10, 1, 15, 0, 0, 0, time.UTC)
|
|
before, after := built.Add(-time.Minute), built.Add(time.Minute)
|
|
reports := []inventory.Reported{{Node: "anchor", At: &after}, {Node: "home-server", At: &before}}
|
|
ok, waiting := applied("builder", built, []string{"anchor", "home-server"}, reports)
|
|
if ok || len(waiting) != 1 || waiting[0] != "home-server" {
|
|
t.Fatalf("one machine has not reported since the build: ok=%v waiting=%v", ok, waiting)
|
|
}
|
|
if ok, _ := applied("builder", built, []string{"anchor"}, reports); !ok {
|
|
t.Fatal("the machine that reported after the build holds the gate open")
|
|
}
|
|
if ok, _ := applied("builder", built, nil, reports); !ok {
|
|
t.Fatal("a module running nowhere gates nothing")
|
|
}
|
|
}
|
|
|
|
// A cycle is not lost: what remains is one last tier, and the caller says so.
|
|
func TestACycleIsOneLastTierAndSaidSo(t *testing.T) {
|
|
edges := []inventory.Edge{{From: "a", To: "b", Kind: inventory.EdgeStandsOn}, {From: "b", To: "a", Kind: inventory.EdgeStandsOn}}
|
|
tiers := tiersOf([]string{"a", "b"}, edges)
|
|
if len(tiers) != 1 || len(tiers[0]) != 2 || !hasCycle(tiers, edges) {
|
|
t.Fatalf("a cycle should be one tier of two, said: %v", tiers)
|
|
}
|
|
}
|
|
|
|
// novox/hq 04-ISSUES/211: a merge moving the toolchain and a bundle compiled in it builds the
|
|
// bundle a tier after the toolchain, not beside it.
|
|
func TestABundleIsPlannedAfterTheToolchainItIsCompiledIn(t *testing.T) {
|
|
edges := []inventory.Edge{{From: "node-tools", To: "mesh-tools", Kind: inventory.EdgeStandsOn}}
|
|
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"},
|
|
[]string{"mesh-tools", "node-tools"}, edges)
|
|
if len(p.Tiers) != 2 || p.Tiers[0][0] != "mesh-tools" || p.Tiers[1][0] != "node-tools" {
|
|
t.Fatalf("the toolchain, then the bundle: %v", p.Tiers)
|
|
}
|
|
}
|
|
|
|
// novox/hq 04-ISSUES/214: a plan whose build outcome was recorded while no controller followed it —
|
|
// the controller rebuilding itself — settles from the build records instead of waiting for ever.
|
|
func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
|
|
asked := time.Date(2026, 10, 3, 19, 20, 0, 0, time.UTC)
|
|
p := inventory.Plan{ID: "plan-1", Tiers: [][]string{{"mesh-controller", "builder"}, {"route-proxy"}},
|
|
Modules: map[string]*inventory.PlanModule{
|
|
"mesh-controller": {State: "asked", AskedAt: &asked},
|
|
"builder": {State: "asked", AskedAt: &asked},
|
|
}}
|
|
records := map[string][]inventory.Build{
|
|
// Newest first, as Builds answers: the build after the ask is the outcome.
|
|
"mesh-controller": {
|
|
{ID: "build-2", Commit: "2ebbb799", At: asked.Add(4 * time.Minute)},
|
|
{ID: "build-1", Commit: "06ea2168", At: asked.Add(-10 * time.Minute)},
|
|
},
|
|
// Only a build from before the ask: not this ask's outcome.
|
|
"builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}},
|
|
}
|
|
if !settleFromRecords(&p, p.Tiers[0], records) {
|
|
t.Fatal("nothing settled, though the controller's build is recorded after the ask")
|
|
}
|
|
if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil {
|
|
t.Errorf("the controller's ask is %+v, want built from 2ebbb799", s)
|
|
}
|
|
if s := p.Modules["builder"]; s.State != "asked" {
|
|
t.Errorf("an ask with no record after it was settled: %+v", s)
|
|
}
|
|
|
|
// novox/hq 04-ISSUES/219: a build recorded after the ask but asked before it — an earlier
|
|
// plan's late outcome — is not this ask's, built or failed.
|
|
r := inventory.Plan{ID: "plan-3", Tiers: [][]string{{"postgres"}},
|
|
Modules: map[string]*inventory.PlanModule{"postgres": {State: "asked", AskedAt: &asked}}}
|
|
late := map[string][]inventory.Build{"postgres": {
|
|
{ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)},
|
|
}}
|
|
if settleFromRecords(&r, r.Tiers[0], late) || r.Modules["postgres"].State != "asked" {
|
|
t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"])
|
|
}
|
|
// Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it.
|
|
late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73",
|
|
Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)})
|
|
if !settleFromRecords(&r, r.Tiers[0], late) || r.Modules["postgres"].State != "built" ||
|
|
r.Modules["postgres"].Commit != "4bcd5f73" {
|
|
t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"])
|
|
}
|
|
|
|
// A failure recorded after the ask fails the plan, as hearing it would have.
|
|
q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}},
|
|
Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}}
|
|
settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}})
|
|
if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" {
|
|
t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"])
|
|
}
|
|
}
|