jschoubben
da8a5e9bc0
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate head da8a5e9bc0f7 merged into trunk 76304d9db817 — pass: builds mesh-controller → novox; no bus step; every machine composes with the chang…
mesh/repo-check head da8a5e9bc0f7 merged into trunk 76304d9db817 — fail: its merge-check.sh failed: --- FAIL: TestAReusedCheckBuildIsRegisteredByItsDigestA…
The operator chose the leanest rollout: no gate, no one machine first, failures loud and fixed forward. A walk sends every rolled-out module to every machine running it in one step, built-by orders no tier, and a walk ends on its machines' reports. The backlog walk sends every machine at once and passes each on its applied report.
265 lines
14 KiB
Go
265 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"slices"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// A merge produces a tiered plan (novox/hq ADR 0162): what moved and everything reachable from it,
|
|
// sorted so a tier depends only on earlier ones — with the three kinds of dependency told apart.
|
|
func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
|
edges := []inventory.Edge{
|
|
// build dependencies: images on the runtime, a plugin on one of them
|
|
{From: "shop", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
|
{From: "postgres", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
|
{From: "shop-plugin", To: "shop", Kind: inventory.EdgeDeclared},
|
|
// a code dependency: the proxy packages the controller's source — same tier
|
|
{From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages},
|
|
{From: "builder", To: "mesh-controller", Kind: inventory.EdgePackages},
|
|
// runtime dependencies: everything source-built is built by the builder
|
|
{From: "shop", To: "builder", Kind: inventory.EdgeBuiltBy},
|
|
{From: "postgres", To: "builder", Kind: inventory.EdgeBuiltBy},
|
|
{From: "shop-plugin", To: "builder", Kind: inventory.EdgeBuiltBy},
|
|
{From: "route-proxy", To: "builder", Kind: inventory.EdgeBuiltBy},
|
|
{From: "mesh-controller", To: "builder", Kind: inventory.EdgeBuiltBy},
|
|
{From: "mesh-tools", To: "builder", Kind: inventory.EdgeBuiltBy},
|
|
{From: "builder", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
|
// the build seat's holder follows the controller that defines its worker (hq issue 206)
|
|
{From: "builder", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
|
|
{From: "unrelated", To: "alpine", Kind: inventory.EdgeStandsOn},
|
|
}
|
|
// The runtime image moved: everything on it, and what is built by what is on it.
|
|
set := reachableFrom([]string{"mesh-tools"}, edges)
|
|
// What stands on the runtime, and the builder that stands on it; not the controller, which the
|
|
// builder merely builds, nor the proxy that packages the controller.
|
|
want := []string{"builder", "mesh-tools", "postgres", "shop", "shop-plugin"}
|
|
if len(set) != len(want) {
|
|
t.Fatalf("reachable from the runtime: %v, want %v", set, want)
|
|
}
|
|
tiers := tiersOf(set, edges)
|
|
pos := map[string]int{}
|
|
for i, tier := range tiers {
|
|
for _, m := range tier {
|
|
pos[m] = i
|
|
}
|
|
}
|
|
if pos["mesh-tools"] != 0 || pos["builder"] != 1 {
|
|
t.Fatalf("the runtime then the builder: %v", tiers)
|
|
}
|
|
// Built-by orders nothing (novox/hq ADR 0360 §3): what the builder builds is beside it, after the runtime it
|
|
// stands on.
|
|
if pos["shop"] != 1 || pos["postgres"] != 1 {
|
|
t.Fatalf("what the builder builds waits for the builder: %v", tiers)
|
|
}
|
|
if pos["shop-plugin"] <= pos["shop"] {
|
|
t.Fatalf("a plugin after what it is declared on: %v", tiers)
|
|
}
|
|
if hasCycle(tiers, edges) {
|
|
t.Fatalf("no cycle here: %v", tiers)
|
|
}
|
|
|
|
// The controller alone moved: the controller alone — what packages its repository moves only when the
|
|
// change is in its own build source (novox/hq ADR 0267), so a packages edge widens nothing.
|
|
if alone := reachableFrom([]string{"mesh-controller"}, edges); len(alone) != 1 {
|
|
t.Fatalf("a controller merge rebuilds the controller alone: %v", alone)
|
|
}
|
|
// A change to a package all three build from moves all three. The builder holds
|
|
// the build seat, whose worker the controller defines, so it follows the controller (worker-of,
|
|
// novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the
|
|
// build machine that is running. The proxy is built by whichever builder runs, so it goes beside the
|
|
// controller (novox/hq ADR 0360 §3; it read "the controller, the builder, the proxy" before).
|
|
small := reachableFrom([]string{"mesh-controller", "builder", "route-proxy"}, edges)
|
|
smallTiers := tiersOf(small, edges)
|
|
if got := tiered(smallTiers); got != "mesh-controller,route-proxy | builder" {
|
|
t.Fatalf("the controller and the proxy, then the builder: %v", smallTiers)
|
|
}
|
|
// The builder alone moved: the builder, and nothing it builds.
|
|
if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 {
|
|
t.Fatalf("a build machine change rebuilds the build machine alone: %v", only)
|
|
}
|
|
|
|
// Nothing waits on deployment for a built-by edge (novox/hq ADR 0360 §3): the builder gates no tier after it.
|
|
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges)
|
|
p.Tier = pos["builder"]
|
|
rollsOut := func(m string) bool { return m == "builder" }
|
|
if g := gates(p, edges, rollsOut); len(g) != 0 {
|
|
t.Fatalf("the builder gates the tier after it: %v", g)
|
|
}
|
|
p.Tier = 0
|
|
if g := gates(p, edges, rollsOut); len(g) != 0 {
|
|
t.Fatalf("the runtime image is a build dependency and gates nothing: %v", g)
|
|
}
|
|
if g := gates(p, edges, func(string) bool { return false }); len(g) != 0 {
|
|
t.Fatalf("a module that only records its upgrade gates nothing: %v", g)
|
|
}
|
|
}
|
|
|
|
// A gate is open once every machine running the module has reported after it was built.
|
|
func TestAGateOpensWhenTheMachinesHaveReportedSinceTheBuild(t *testing.T) {
|
|
built := time.Date(2026, 10, 1, 15, 0, 0, 0, time.UTC)
|
|
before, after := built.Add(-time.Minute), built.Add(time.Minute)
|
|
reports := []inventory.Reported{{Node: "anchor", At: &after}, {Node: "home-server", At: &before}}
|
|
ok, waiting := applied("builder", built, []string{"anchor", "home-server"}, reports)
|
|
if ok || len(waiting) != 1 || waiting[0] != "home-server" {
|
|
t.Fatalf("one machine has not reported since the build: ok=%v waiting=%v", ok, waiting)
|
|
}
|
|
if ok, _ := applied("builder", built, []string{"anchor"}, reports); !ok {
|
|
t.Fatal("the machine that reported after the build holds the gate open")
|
|
}
|
|
if ok, _ := applied("builder", built, nil, reports); !ok {
|
|
t.Fatal("a module running nowhere gates nothing")
|
|
}
|
|
}
|
|
|
|
// A cycle is not lost: what remains is one last tier, and the caller says so.
|
|
func TestACycleIsOneLastTierAndSaidSo(t *testing.T) {
|
|
edges := []inventory.Edge{{From: "a", To: "b", Kind: inventory.EdgeStandsOn}, {From: "b", To: "a", Kind: inventory.EdgeStandsOn}}
|
|
tiers := tiersOf([]string{"a", "b"}, edges)
|
|
if len(tiers) != 1 || len(tiers[0]) != 2 || !hasCycle(tiers, edges) {
|
|
t.Fatalf("a cycle should be one tier of two, said: %v", tiers)
|
|
}
|
|
}
|
|
|
|
// novox/hq 04-ISSUES/211: a merge moving the toolchain and a bundle compiled in it builds the
|
|
// bundle a tier after the toolchain, not beside it.
|
|
func TestABundleIsPlannedAfterTheToolchainItIsCompiledIn(t *testing.T) {
|
|
edges := []inventory.Edge{{From: "node-tools", To: "mesh-tools", Kind: inventory.EdgeStandsOn}}
|
|
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"},
|
|
[]string{"mesh-tools", "node-tools"}, edges)
|
|
if len(p.Tiers) != 2 || p.Tiers[0][0] != "mesh-tools" || p.Tiers[1][0] != "node-tools" {
|
|
t.Fatalf("the toolchain, then the bundle: %v", p.Tiers)
|
|
}
|
|
}
|
|
|
|
// novox/hq 04-ISSUES/214: a plan whose build outcome was recorded while no controller followed it —
|
|
// the controller rebuilding itself — settles from the build records instead of waiting for ever.
|
|
func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
|
|
asked := time.Date(2026, 10, 3, 19, 20, 0, 0, time.UTC)
|
|
p := inventory.Plan{ID: "plan-1", Tiers: [][]string{{"mesh-controller", "builder"}, {"route-proxy"}},
|
|
Modules: map[string]*inventory.PlanModule{
|
|
"mesh-controller": {State: "asked", AskedAt: &asked},
|
|
"builder": {State: "asked", AskedAt: &asked},
|
|
}}
|
|
records := map[string][]inventory.Build{
|
|
// Newest first, as Builds answers: the build after the ask is the outcome.
|
|
"mesh-controller": {
|
|
{ID: "build-2", Commit: "2ebbb799", At: asked.Add(4 * time.Minute)},
|
|
{ID: "build-1", Commit: "06ea2168", At: asked.Add(-10 * time.Minute)},
|
|
},
|
|
// Only a build from before the ask: not this ask's outcome.
|
|
"builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}},
|
|
}
|
|
if !settleFromRecords(&p, p.Tiers[0], records, nil) {
|
|
t.Fatal("nothing settled, though the controller's build is recorded after the ask")
|
|
}
|
|
if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil {
|
|
t.Errorf("the controller's ask is %+v, want built from 2ebbb799", s)
|
|
}
|
|
if s := p.Modules["builder"]; s.State != "asked" {
|
|
t.Errorf("an ask with no record after it was settled: %+v", s)
|
|
}
|
|
|
|
// novox/hq 04-ISSUES/219: a build recorded after the ask but asked before it — an earlier
|
|
// plan's late outcome — is not this ask's, built or failed.
|
|
r := inventory.Plan{ID: "plan-3", Tiers: [][]string{{"postgres"}},
|
|
Modules: map[string]*inventory.PlanModule{"postgres": {State: "asked", AskedAt: &asked}}}
|
|
late := map[string][]inventory.Build{"postgres": {
|
|
{ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)},
|
|
}}
|
|
if settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "asked" {
|
|
t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"])
|
|
}
|
|
// Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it.
|
|
late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73",
|
|
Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)})
|
|
if !settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "built" ||
|
|
r.Modules["postgres"].Commit != "4bcd5f73" {
|
|
t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"])
|
|
}
|
|
|
|
// A failure recorded after the ask fails the plan, as hearing it would have.
|
|
q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}},
|
|
Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}}
|
|
settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}, nil)
|
|
if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" {
|
|
t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"])
|
|
}
|
|
}
|
|
|
|
// The first machine's report, made between the send to it and the send to the rest, opens the gate for it:
|
|
// each machine is judged from its own send, not from the last one (novox/hq issue 256).
|
|
func TestTheGateJudgesEachMachineFromItsOwnSend(t *testing.T) {
|
|
built := time.Date(2026, 10, 5, 18, 22, 0, 0, time.UTC)
|
|
firstSent := built.Add(31 * time.Second)
|
|
firstReported := built.Add(43 * time.Second)
|
|
restSent := built.Add(58 * time.Second)
|
|
restReported := built.Add(74 * time.Second)
|
|
reports := []inventory.Reported{
|
|
{Node: "ace", At: &firstReported},
|
|
{Node: "g14", At: &restReported},
|
|
}
|
|
since := func(node string) time.Time {
|
|
if node == "ace" {
|
|
return firstSent
|
|
}
|
|
return restSent
|
|
}
|
|
if ok, waiting := appliedEach("build-agent", since, []string{"ace", "g14"}, reports); !ok {
|
|
t.Fatalf("the gate still waits on %v, though each reported after its own send", waiting)
|
|
}
|
|
// The old reading, every machine from the last send, is what held the plan.
|
|
if ok, _ := applied("build-agent", restSent, []string{"ace", "g14"}, reports); ok {
|
|
t.Fatal("the single-moment reading should hold the first machine back")
|
|
}
|
|
early := built.Add(10 * time.Second)
|
|
if ok, waiting := appliedEach("build-agent", since, []string{"ace"}, []inventory.Reported{{Node: "ace", At: &early}}); ok || waiting[0] != "ace" {
|
|
t.Fatal("a report from before the machine was sent opened the gate")
|
|
}
|
|
}
|
|
|
|
// novox/hq issue 278 and ADR 0236's open question: a change to the build agent — its manifest alone, as
|
|
// the catalogue's data sections were, or its program — rebuilds the build agent and nothing it builds.
|
|
// What it builds is ordered after it in a plan that holds both, never added to one for its sake. The
|
|
// merge that rebuilt 103 modules with the agent in tier 0 rebuilt them for a file read as shared code;
|
|
// the agent stood first only because everything else is built by it.
|
|
func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) {
|
|
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
|
agent := fromRepo("build-agent", repo, "modules/build-agent")
|
|
redis := fromRepo("redis", repo, "modules/redis")
|
|
postgres := fromRepo("postgres", repo, "modules/postgres")
|
|
entries := []inventory.Entry{agent, redis, postgres}
|
|
edges := []inventory.Edge{
|
|
{From: "redis", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
|
{From: "redis", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
|
{From: "postgres", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
|
{From: "mesh-tools", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
|
{From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
|
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
|
|
}
|
|
for _, paths := range [][]string{
|
|
{"modules/build-agent/module.json"}, // its manifest alone
|
|
{"modules/build-agent/cmd/agent/main.go", "modules/build-agent/module.json"}, // its program too
|
|
} {
|
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", Paths: paths,
|
|
ModuleDirs: []string{"modules/build-agent"}, ModuleDirsSaid: true}
|
|
touched := whatTheMergeTouched(entries, entries, m, nil)
|
|
if len(touched) != 1 || touched[0].Manifest.Module != "build-agent" {
|
|
t.Fatalf("%v touched %v", paths, touched)
|
|
}
|
|
p := planOfMerge(m, []string{"build-agent"}, edges)
|
|
if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "build-agent" || len(p.Modules) != 1 {
|
|
t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers)
|
|
}
|
|
}
|
|
// With what it builds moved beside it, both go in one tier (novox/hq ADR 0360 §3): built-by orders nothing.
|
|
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"},
|
|
[]string{"build-agent", "redis"}, edges)
|
|
if len(p.Tiers) != 1 || !slices.Equal(p.Tiers[0], []string{"build-agent", "redis"}) || len(p.Modules) != 2 {
|
|
t.Fatalf("the agent and what moved beside it, in one tier: %v", p.Tiers)
|
|
}
|
|
}
|