package main import ( "context" "encoding/json" "reflect" "slices" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" "github.com/novox/mesh-controller/internal/link" ) // What the review of issue 318's fix found (novox/hq ADR 0254): every module of a send leaves it with a // verdict, a pass is counted no faster than the gate's spacing, a carried build's verdict carries only its own // wait, and a provider waiting for a login says who waits on it. // withGateBounds sets the gate's bounds for one test. func withGateBounds(t *testing.T, settle, every, bound time.Duration) { t.Helper() wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound gateSettle, gateEvery, gateBound = settle, every, bound t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) } // factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says // of its modules' resources. func factsOn(t *testing.T, machine string, since time.Time, rs ...inventory.ResourceHealth) func() gateFacts { return func() gateFacts { now := time.Now() at := now return gateFacts{now: now, reports: map[string]inventory.Reported{machine: {Node: machine, Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, rolledBack: map[string][]lease.Rollback{}, served: map[string]served{machine: {runtime: true, tools: map[string]bool{}}}, health: map[string]inventory.NodeHealth{machine: {Node: machine, HeardAt: now, Resources: rs}}, } } } func healthyContainer(module string) inventory.ResourceHealth { return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, State: link.StateHealthy} } // **A fault decided before the settle time does not strand the module beside it** (review (a)): the node // tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass, // counted over the gate's spacing — before it says its own. The broken one alone is put back. func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) { open := aMesh(t) ctx := t.Context() withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second) since := time.Now().Add(-time.Second) base := factsOn(t, "laptop", since, healthyContainer("app")) wasGather := gatherGateFacts t.Cleanup(func() { gatherGateFacts = wasGather }) gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { f := base() f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack, At: since.Add(100 * time.Millisecond), Why: "the node tools did not answer"}} return f, nil } g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since} pairs := []judged{{module: broker_runtime, node: "laptop"}, {module: "app", node: "laptop"}} judgings := 0 for deadline := time.Now().Add(5 * time.Second); g.Verdict == "" && time.Now().Before(deadline); { if _, err := judgeMoves(ctx, open, g, pairs, time.Now()); err != nil { t.Fatal(err) } judgings++ time.Sleep(30 * time.Millisecond) } if g.Verdict != inventory.GateFailed || judgings < gatePasses { t.Fatalf("verdict %q after %d judging(s): %+v; want failed, after the module beside it was judged", g.Verdict, judgings, g) } if !reflect.DeepEqual(g.Passing, []string{"app"}) || !reflect.DeepEqual(g.Failing, []string{broker_runtime}) { t.Fatalf("passing %v, failing %v; want app kept and the node tools put back", g.Passing, g.Failing) } if !strings.Contains(g.Why, "witness") { t.Errorf("the verdict does not say what broke: %q", g.Why) } } // broker_runtime is the node tools' module name, a core component a witness judges. const broker_runtime = "node-tools" // **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick // while a module beside it is not yet healthy counts the healthy one once. func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) { open := aMesh(t) ctx := t.Context() withGateBounds(t, 0, time.Hour, time.Hour) since := time.Now().Add(-time.Minute) base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"}) wasGather := gatherGateFacts t.Cleanup(func() { gatherGateFacts = wasGather }) gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return base(), nil } g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since} pairs := []judged{{module: "app", node: "laptop"}, {module: "late", node: "laptop"}} now := time.Now() for i := 0; i < 3; i++ { if _, err := judgeMoves(ctx, open, g, pairs, now.Add(time.Duration(i)*time.Second)); err != nil { t.Fatal(err) } } if g.Healthy["app"] != 1 || g.Healthy["late"] != 0 { t.Fatalf("counted %v in three judgings within a second; want app once and late never", g.Healthy) } } // **A carried build's pass carries its own wait, never another's** (review (e)). func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) { b := aBacklog(t) ctx := t.Context() inv := b.open.inventory since := time.Now().Add(-time.Minute) g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since, Verdict: inventory.GatePassed, Why: "healthy 3 times over 2m0s" + waitsSaid(map[string]string{"late": "relogin needed on laptop: late waits"}), Waits: map[string]string{"late": "relogin needed on laptop: late waits"}, Carried: []inventory.CarriedMove{{Module: "app", Node: "laptop", From: "c1", To: "c2", Build: "build-app-c2"}, {Module: "late", Node: "laptop", From: "c1", To: "c2", Build: "build-late-c2"}}} passCarried(ctx, b.open, &inventory.Plan{ID: "release-test"}, g, "") app, _, _ := inv.GateOf(ctx, "build-app-c2") late, _, _ := inv.GateOf(ctx, "build-late-c2") if strings.Contains(app.Why, "waits for a person") || app.Verdict != inventory.GatePassed { t.Errorf("app's pass: %+v; want it without late's wait", app) } if !strings.Contains(late.Why, "relogin needed on laptop") { t.Errorf("late's pass: %+v; want its own wait", late) } } // **The tier path keeps the pass of the module the gate is kept on** (review (b)): a plan's first send // carried its own module, healthy, and a build waiting on that machine that never became healthy. At the // bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it. func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) { g := aGateMesh(t) ctx := t.Context() inv := g.open.inventory // `late` waits on anchor for a gate: c1 sent, c2 registered and built. for _, b := range []inventory.Build{ {ID: "build-late-1", Module: "late", Commit: "l1", Repository: "novox/mesh-catalog", Path: "modules/late", Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, {ID: "build-late-2", Module: "late", Commit: "l2", Repository: "novox/mesh-catalog", Path: "modules/late", Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, } { manifest, _ := json.Marshal(catalogue.Manifest{Module: "late", Version: b.Commit}) b.Manifest = manifest b.Made = []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + b.Commit}} if err := inv.RecordBuild(ctx, b); err != nil { t.Fatal(err) } if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "late", Version: b.Commit}, inventory.Source{ Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/late", BuiltFrom: b.Commit, Head: b.Commit, Asked: b.Asked}); err != nil { t.Fatal(err) } if b.Commit == "l1" { if _, err := inv.Assign(ctx, "anchor", "late"); err != nil { t.Fatal(err) } if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-late", map[string]string{"app": "c1", "late": "l1"}); err != nil { t.Fatal(err) } } } wasMoves := machineMoves t.Cleanup(func() { machineMoves = wasMoves }) machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { modules, err := open.inventory.Assigned(ctx, node) if err != nil { return nil, err } sent, known, err := open.inventory.SentBuilds(ctx, node) if err != nil { return nil, err } return f.moves(node, modules, sent, known, all), nil } gather := gatherGateFacts gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { f, err := gather(ctx, open, component) f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{ healthyContainer("app"), {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"}}}} return f, err } gateEvery, gateBound = 0, 400*time.Millisecond for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { advancePlans(ctx, g.open) if p := g.plan(t); p.State == inventory.PlanFailed || p.State == inventory.PlanDone { break } time.Sleep(30 * time.Millisecond) } p := g.plan(t) if p.State != inventory.PlanFailed { t.Fatalf("the plan is %s: %s; want it failed on late", p.State, p.Note) } if v, found, err := inv.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed || !strings.Contains(v.Why, "on its own") { t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err) } if failed, _ := inv.GateFailed(ctx, "build-late-2"); !failed { t.Fatal("late's build is not marked failed at its gate") } if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" || current["late"].Commit != "l1" { t.Fatalf("registered app %s, late %s; want app kept at c2 and late put back to l1", current["app"].Commit, current["late"].Commit) } } // **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that // needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent. func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory k := conditionsFrom register(t, open, catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}) register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}}) for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} { if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { t.Fatal(err) } } if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop", Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil { t.Fatal(err) } at := h0 say := func(machine string, rs ...link.ResourceHealth) { t.Helper() at = at.Add(time.Second) for i := range rs { rs[i].Since = at } if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil { t.Fatal(err) } } account := link.ResourceHealth{Module: "db", Resource: "db.operator", Kind: link.KindAccount, Target: "operator", Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group db"} unit := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: link.KindUnit, Target: "db.service", Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} for look := 0; look < 2; look++ { say("anchor", account, unit) say("laptop", shop) } list, err := k.Open(ctx) if err != nil { t.Fatal(err) } var keys []string var c conditions.Condition for _, x := range list { keys = append(keys, x.Key) if x.Key == "module.db.anchor.relogin-needed" { c = x } } if !reflect.DeepEqual(keys, []string{"module.db.anchor.relogin-needed"}) { t.Fatalf("open %v; want the provider's wait alone, its consumer held under it", keys) } if c.Severity != conditions.Urgent || !strings.Contains(c.Evidence[0].Said, "shop on laptop") { t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said) } } // **A broken module is put back at once** (issue 318 review): the laptop's witness put the node tools back // within a minute of a send that also moved `app`, and `app` reads not yet healthy because of it. The node // tools are marked and put back in the very judging that found them broken — their registered build is the // one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass. func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory withConditionsInMemory(t) was := doctorFrom doctorFrom = nil t.Cleanup(func() { doctorFrom = was }) old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute) build := func(module, commit string, asked time.Time) { manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"}) if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit, Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + commit}}}); err != nil { t.Fatal(err) } if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{ Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit, Asked: asked}); err != nil { t.Fatal(err) } } for _, m := range []string{"app", broker_runtime} { build(m, "c1", old) if _, err := inv.Assign(ctx, "laptop", m); err != nil { t.Fatal(err) } } if err := inv.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop", map[string]string{"app": "c1", broker_runtime: "c1"}); err != nil { t.Fatal(err) } build("app", "c2", recent) build(broker_runtime, "c2", recent) wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather }) machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { modules, _ := open.inventory.Assigned(ctx, node) sent, known, err := open.inventory.SentBuilds(ctx, node) if err != nil { return nil, err } return f.moves(node, modules, sent, known, all), nil } releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"laptop": true}, nil } var sends []string n := 0 sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { current, err := open.inventory.CurrentBuilds(ctx) if err != nil { return nil, err } for _, node := range names { n++ carried := map[string]string{"app": current["app"].Commit, broker_runtime: current[broker_runtime].Commit} sends = append(sends, node+":"+carried[broker_runtime]) digest := "d-" + node + "-" + time.Now().Format("150405.000000") + string(rune('a'+n)) if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { return nil, err } if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { return nil, err } } return names, nil } var seen []string // the node tools' registered build at each judging gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { now := time.Now() f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{}, rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} reports, _ := open.inventory.LastReports(ctx) for _, r := range reports { f.reports[r.Node] = r } current, _ := open.inventory.CurrentBuilds(ctx) seen = append(seen, current[broker_runtime].Commit) app := healthyContainer("app") if current[broker_runtime].Commit == "c2" { // The witness reverted the node tools; app cannot reach them yet. f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack, From: "c2", To: "c1", At: now, Why: "the node tools did not answer"}} app.State, app.Reason = link.StateUnhealthy, "down" } f.served["laptop"] = served{runtime: current[broker_runtime].Commit == "c1", tools: map[string]bool{}} f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}} return f, nil } withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second) release := func() inventory.Plan { t.Helper() plans, _ := inv.RecentPlans(ctx, 10) for _, p := range plans { if p.Release != nil { return p } } t.Fatal("no walk") return inventory.Plan{} } // Judged until the first judging has found the node tools broken, and one more. for i := 0; i < 20 && len(seen) < 2; i++ { advancePlans(ctx, open) } if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" { t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at the very next judging", seen) } if failed, _ := inv.GateFailed(ctx, "build-"+broker_runtime+"-c2"); !failed { t.Fatal("the node tools' build is not marked failed at once") } if p := release(); p.State != inventory.PlanRolling || p.Release.Gate == nil || p.Release.Gate.Verdict != "" { t.Fatalf("the walk is %s with gate %+v; want it still judging app", p.State, p.Release.Gate) } if !slices.Contains(sends, "laptop:c1") { t.Fatalf("sends %v; want the laptop sent the node tools' earlier build at once", sends) } for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { advancePlans(ctx, open) if release().State != inventory.PlanRolling { break } time.Sleep(20 * time.Millisecond) } p := release() if p.State != inventory.PlanFailed { t.Fatalf("the walk is %s: %s; want failed, for the node tools", p.State, p.Note) } if v, found, _ := inv.GateOf(ctx, "build-app-c2"); !found || v.Verdict != inventory.GatePassed || !strings.Contains(v.Why, "on its own") { t.Fatalf("app's verdict: %+v; want its own pass", v) } if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { t.Fatalf("app is registered at %s; want it kept at c2", current["app"].Commit) } } // **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous // build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group. func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory record := func(commit string, m catalogue.Manifest) { raw, _ := json.Marshal(m) at := time.Now().Add(-time.Hour) if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-lights-" + commit, Module: "lights", Commit: commit, Repository: "novox/mesh-catalog", Path: "modules/lights", Manifest: raw, Asked: at, At: at}); err != nil { t.Fatal(err) } } record("c1", catalogue.Manifest{Module: "lights"}) record("c2", catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user", "name": "operator", "groups": []any{"lights"}}}}) pairs := []judged{{module: "lights", node: "laptop"}} shelf := map[string]catalogue.Manifest{} for _, c := range []struct { from string want bool }{{"c1", true}, {"", false}, {"c0-never-built", false}} { g := &inventory.PlanGate{From: c.from, To: "c2"} if got := movesAddingGroups(ctx, inv, g, pairs, shelf)["lights"]; got != c.want { t.Errorf("a move from %q adds a group: %v; want %v", c.from, got, c.want) } } }