mesh-controller/cmd/mesh-controller/gate_test.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/lease"
	"github.com/novox/mesh-controller/internal/link"
)

// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8).

// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered,
// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is
// sent and reports it — with the gate's bounds shortened to judge in three steps.
type gateMesh struct {
	open   *stores
	sent   [][]string
	health map[string]health // per machine, what a judging finds; healthy when unsaid
	keeper *conditions.Keeper
	told   *conditions.Told
}

func aGateMesh(t *testing.T) *gateMesh {
	t.Helper()
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	g := &gateMesh{open: open, health: map[string]health{}}
	g.keeper, _ = withConditionsInMemory(t)
	g.told = &conditions.Told{}
	was := doctorFrom
	doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told}
	t.Cleanup(func() { doctorFrom = was })

	for _, b := range []inventory.Build{
		{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
			Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
		{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
			Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
	} {
		manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
		b.Manifest = manifest
		if err := inv.RecordBuild(ctx, b); err != nil {
			t.Fatal(err)
		}
	}
	register := func(commit string, asked time.Time) {
		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
			inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
				Head: commit, Asked: asked}); err != nil {
			t.Fatal(err)
		}
	}
	register("c1", time.Now().Add(-2*time.Hour))
	for _, n := range []string{"anchor", "laptop"} {
		if _, err := inv.Assign(ctx, n, "app"); err != nil {
			t.Fatal(err)
		}
		if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
			t.Fatal(err)
		}
	}
	register("c2", time.Now().Add(-time.Minute))

	// Every send: recorded with the build the module is at, applied and reported by the machine.
	n := 0
	wasSend := sendRollout
	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
		g.sent = append(g.sent, append([]string(nil), names...))
		current, err := open.inventory.CurrentBuilds(ctx)
		if err != nil {
			return nil, err
		}
		for _, node := range names {
			n++
			digest := fmt.Sprintf("d-%s-%d", node, n)
			if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
				return nil, err
			}
			if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
				Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
				return nil, err
			}
		}
		return names, nil
	}
	t.Cleanup(func() { sendRollout = wasSend })

	// What a judging reads: the store's reports, and a health the test says per machine.
	wasGather := gatherGateFacts
	gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
		f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
			rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
		reports, err := open.inventory.LastReports(ctx)
		if err != nil {
			return f, err
		}
		for _, r := range reports {
			if g.health[r.Node] == healthBroken {
				r.Outcome = inventory.OutcomeFailed
			}
			f.reports[r.Node] = r
		}
		for node, h := range g.health {
			if h == healthNotYet {
				// Not reported on the send: neither the send made last, nor the gate's own (issue 352).
				f.rolledBack[node] = nil
				r := f.reports[node]
				r.Current, r.Declared, r.ReportedSequence = false, "", 0
				f.reports[node] = r
			}
		}
		return f, nil
	}
	t.Cleanup(func() { gatherGateFacts = wasGather })

	wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
	// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
	gateSettle, gateEvery = 0, time.Hour
	t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })

	built := time.Now().UTC()
	plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
		Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
		Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
			Build: "build-2"}}}
	if err := inv.SavePlan(ctx, &plan); err != nil {
		t.Fatal(err)
	}
	return g
}

func (g *gateMesh) plan(t *testing.T) inventory.Plan {
	t.Helper()
	p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate")
	if err != nil {
		t.Fatal(err)
	}
	return p
}

// A module's build that fails its gate on the first machine is put back there — the previous build
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
// condition and an event, never registered or rolled back again.
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
	judgeLeavesAsTheCore(t)
	g := aGateMesh(t)
	ctx := t.Context()
	inv := g.open.inventory

	advancePlans(ctx, g.open) // the first machine is sent the new build
	if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) {
		t.Fatalf("sent %v, not the first machine alone", g.sent)
	}
	if p := g.plan(t); p.Modules["app"].Previous != "c1" {
		t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"])
	}

	g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging
	gateEvery = 0
	advancePlans(ctx, g.open)

	p := g.plan(t)
	gate := p.Modules["app"].Gate
	if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed ||
		gate.Rollback != inventory.RolledBack {
		t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
	}
	if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) {
		t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent)
	}
	if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
		t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
	}
	if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" {
		t.Fatalf("anchor was last sent %v, not the previous build", sent)
	}
	if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" {
		t.Fatalf("laptop was sent the failed build: %v", sent)
	}
	if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed {
		t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err)
	}

	// Said: a condition for the operator, and the event.
	open, err := g.keeper.Open(ctx)
	if err != nil {
		t.Fatal(err)
	}
	var key string
	for _, c := range open {
		if c.Kind == kindRolledBack {
			key = c.Key
		}
	}
	if key != "build.app.anchor.rolled-back" {
		t.Fatalf("no rolled-back condition for app on anchor: %+v", open)
	}
	saidIt := false
	for _, e := range g.told.Said() {
		saidIt = saidIt || e.Event == link.KeyRolledBack
	}
	if !saidIt {
		t.Fatalf("the rollback was not said as an event: %+v", g.told.Said())
	}

	// Never again: more passes send nothing, a second judging rolls nothing back, and the failed build
	// heard again is not registered.
	advancePlans(ctx, g.open)
	state := p.Modules["app"]
	gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again")
	if len(g.sent) != 2 {
		t.Fatalf("sent again after the rollback: %v", g.sent)
	}
	_, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app",
		Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"}),
		Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}})
	if err == nil || !strings.Contains(err.Error(), "failed its gate") {
		t.Fatalf("the failed build was registered again: %v", err)
	}
	if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
		t.Fatalf("the module moved to %s", current["app"].Commit)
	}
	if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") {
		t.Fatalf("a plan stopped at a failed gate was retried: %v", err)
	}

	// The probe keeps the condition while the newest verdict is the failure.
	obs, err := probeGates(ctx, doctorFrom)
	if err != nil || len(obs) != 1 || obs[0].Key() != key {
		t.Fatalf("the gate's probe found %+v %v", obs, err)
	}
}

// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then
// the rest are sent, the verdict kept, and the plan done.
func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
	g := aGateMesh(t)
	ctx := t.Context()
	gateEvery = 0
	for i := 0; i < 6; i++ {
		advancePlans(ctx, g.open)
	}
	p := g.plan(t)
	if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) {
		t.Fatalf("sent %v", g.sent)
	}
	gate := p.Modules["app"].Gate
	if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) {
		t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
	}
	if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
		t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err)
	}
	if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
		t.Fatalf("a passing build left a condition: %+v %v", obs, err)
	}
}

// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
	g := aGateMesh(t)
	ctx := t.Context()
	gateSettle = time.Hour    // a core module could not pass for an hour
	advancePlans(ctx, g.open) // the first machine is sent the new build
	advancePlans(ctx, g.open) // one healthy judging
	p := g.plan(t)
	gate := p.Modules["app"].Gate
	if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 {
		t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note)
	}
	if !strings.Contains(gate.Why, "healthy 1 times") {
		t.Errorf("the verdict says %q", gate.Why)
	}
}

// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
func TestAModuleIsJudgedByItsClass(t *testing.T) {
	for _, c := range []struct {
		module  string
		claims  []catalogue.Claim
		passes  int
		settles bool
	}{
		{"app", nil, 1, false},
		{"nzbget", nil, 1, false},
		{"mesh-controller", nil, 3, true},
		{"mesh-host", nil, 3, true},
		{"node-tools", nil, 3, true},
		{"nats", nil, 3, true},
		{catalogue.DeliverySeat, nil, 3, true},
		{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
	} {
		p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
		if p != c.passes || s != c.settles {
			t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
		}
	}
}

// A first machine that never becomes healthy fails its gate at the bound, and is put back.
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
	judgeLeavesAsTheCore(t)
	g := aGateMesh(t)
	ctx := t.Context()
	advancePlans(ctx, g.open)
	g.health["anchor"] = healthNotYet
	gateEvery = 0
	advancePlans(ctx, g.open)
	if p := g.plan(t); !p.Open() || len(g.sent) != 1 {
		t.Fatalf("judged before the bound: %s %v", p.State, g.sent)
	}
	gateBound = -time.Second
	advancePlans(ctx, g.open)
	p := g.plan(t)
	if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed ||
		!strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack {
		t.Fatalf("the plan is %s, its gate %+v", p.State, gate)
	}
}

// The health a judging finds, per core component and for a module.
func TestTheHealthDefinitions(t *testing.T) {
	now := time.Now()
	since := now.Add(-time.Minute)
	applied := func(node string) gateFacts {
		at := now
		return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node,
			Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
			served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}}
	}
	m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}}

	f := applied("anchor")
	f.served["anchor"] = served{runtime: true, tools: map[string]bool{}}
	if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") {
		t.Errorf("a module whose tools are not served is %v (%s)", h, why)
	}
	f.served["anchor"].tools["app"] = true
	if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
		t.Errorf("a module applied with its tools served is %v (%s)", h, why)
	}
	// A condition raised about it on that machine since it was sent: not yet healthy.
	f.judged = true
	f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
		Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}}
	if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet {
		t.Errorf("a module with a new condition about it is %v", h)
	}
	f.open[0].Raised = since.Add(-time.Hour)
	if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
		t.Errorf("a condition older than the send counted against it: %v", h)
	}
	// A witness that put it back: broken.
	f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb",
		Outcome: lease.OutcomeRolledBack, Why: "x", At: now}}
	if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
		t.Errorf("a component a witness put back is %v", h)
	}
	// The node tools answer, or not.
	f = applied("anchor")
	if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet {
		t.Errorf("node tools not answering are %v", h)
	}
	f.served["anchor"] = served{runtime: true}
	if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood {
		t.Errorf("node tools answering are %v", h)
	}
	// The controller: the lease held since the send, by a controller that says it is ready.
	f = applied("control")
	f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}}
	if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet {
		t.Errorf("a lease held by a controller older than the build is %v", h)
	}
	f.holder.Taken = now
	if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood {
		t.Errorf("a new controller holding the lease and ready is %v", h)
	}
	f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"}
	if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet ||
		!strings.Contains(why, "first run") {
		t.Errorf("a controller not ready is %v (%s)", h, why)
	}
	// What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running
	// resource of it on that machine is stated healthy since the send; starting and unhealthy wait.
	f = applied("anchor")
	stated := func(state, reason string, heard time.Time) {
		f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard,
			Resources: []inventory.ResourceHealth{
				{Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason},
				{Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}}
	}
	plain := catalogue.Manifest{Module: "app"}
	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
		t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why)
	}
	stated(link.StateStarting, "", now)
	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") {
		t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why)
	}
	stated(link.StateUnhealthy, "restarting", now)
	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") {
		t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why)
	}
	stated(link.StateHealthy, "", since.Add(-time.Minute))
	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
		t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why)
	}
	stated(link.StateHealthy, "", now)
	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
		t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why)
	}
	f.healthErr = errors.New("the store is away")
	if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
		t.Errorf("health that cannot be read is never read as healthy: %v", h)
	}
	// And the condition it raises after the send holds it, as every condition about it does.
	f.healthErr = nil
	f.judged = true
	f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{
		Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}}
	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") {
		t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why)
	}

	// A machine that refused what it was sent: broken.
	f = applied("anchor")
	r := f.reports["anchor"]
	r.Outcome = inventory.OutcomeRefused
	f.reports["anchor"] = r
	if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
		t.Errorf("a refusal is %v", h)
	}
}

// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused,
// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits.
func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}},
		Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}}
	if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
		Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil {
		t.Fatal(err)
	}
	u, err := inv.UpgradeOf(ctx, "nats")
	if err != nil || u.RollOut || u.From != catalogue.FromBus {
		t.Fatalf("the bus's policy is %+v %v", u, err)
	}
	if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) {
		t.Fatalf("a person rolled the bus out: %v", err)
	}
	var sent [][]string
	was := sendRollout
	sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
		sent = append(sent, names)
		return names, nil
	}
	t.Cleanup(func() { sendRollout = was })
	now := time.Now().UTC()
	plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now,
		State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}},
		Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}}
	if err := inv.SavePlan(ctx, &plan); err != nil {
		t.Fatal(err)
	}
	advancePlans(ctx, open)
	if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 {
		t.Fatalf("a plan sent the bus: %s %v", p.State, sent)
	}
	held, err := busHeld(ctx, inv, []string{"anchor", "laptop"})
	if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" {
		t.Fatalf("a push may send the bus's machine: %v %v", held, err)
	}
	// Nor may any other send — a plan's for another module on that machine carried the bus with it.
	if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) {
		t.Fatalf("a send to the bus's machine was not refused: %v", err)
	}
	// A rebuild that made the same artifacts is no move.
	for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} {
		b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}}
		b.Asked, b.At = time.Now(), time.Now()
		if err := inv.RecordBuild(ctx, b); err != nil {
			t.Fatal(err)
		}
	}
	if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
		t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err)
	}
	if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second),
		At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image",
			Reference: "registry/nats@sha256:new"}}}); err != nil {
		t.Fatal(err)
	}
	// The planned step refuses to start without its word on reversibility, and without a snapshot taken
	// first by the bus machine's backup holder.
	if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") {
		t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err)
	}
	wasSnapshot := takeBusSnapshot
	t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
	takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") }
	if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil ||
		!strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 {
		t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent)
	}
	takeBusSnapshot = func(_ context.Context, module, node string) (string, error) {
		return node + "'s restore point of " + module, nil
	}
	if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) {
		t.Fatalf("the step sent %v, not the bus's machine", sent)
	}
	s, found, err := inv.LatestBusStep(ctx)
	if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil ||
		!reflect.DeepEqual(s.Machines, []string{"anchor"}) {
		t.Fatalf("the step is %+v %v %v", s, found, err)
	}
}

// A merge that deletes a module's directory builds nothing for it: the module is forgotten where
// nothing holds it, and a plan whose build finds no manifest goes on instead of failing.
func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	asked := asksRecorded(t)
	for _, name := range []string{"gone", "kept"} {
		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{
			Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil {
			t.Fatal(err)
		}
	}
	m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef",
		Paths:   []string{"modules/gone/module.json", "modules/gone/index.ts"},
		Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}}
	if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
		t.Fatal(err)
	}
	if len(*asked) != 0 {
		t.Fatalf("a deleted module was asked to build: %v", *asked)
	}
	if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 {
		t.Fatalf("a merge that only deleted a module made a plan: %+v", plans)
	}
	shelf, err := inv.Catalogue(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if _, still := shelf["gone"]; still {
		t.Fatal("a deleted module nothing holds was not forgotten")
	}

	// Without the announcer saying what went: the build finds no manifest, and the plan goes on.
	asked0 := time.Now().UTC().Add(-time.Minute)
	plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0,
		State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}},
		Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}}
	if err := inv.SavePlan(ctx, &plan); err != nil {
		t.Fatal(err)
	}
	planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+
		"there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k")
	p, _ := inv.PlanByID(ctx, "plan-deleted")
	if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted {
		t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"])
	}
}

func mustJSON(t *testing.T, v any) []byte {
	t.Helper()
	b, err := json.Marshal(v)
	if err != nil {
		t.Fatal(err)
	}
	return b
}

func nodeID(t *testing.T, open *stores, name string) string {
	t.Helper()
	n, err := open.inventory.NodeByName(context.Background(), name)
	if err != nil {
		t.Fatal(err)
	}
	return n.ID
}

// What a machine's witness says in its reports is a condition while it says it, by the host's words:
// `core.<component>.<node>.<outcome>`, urgent for a rollback, a warning when it could not judge — and
// gone with the first report that no longer carries it.
func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
	open := aMesh(t)
	d := &doctor{open: open}
	at := time.Now().UTC()
	witnessed.heard("control", []lease.Rollback{
		{Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack,
			Why: "the new controller did not take the lease within 60s", At: at},
		{Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at},
	}, at)
	t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) })
	obs, err := probeGates(t.Context(), d)
	if err != nil {
		t.Fatal(err)
	}
	got := map[string]conditions.Severity{}
	for _, o := range obs {
		got[o.Key()] = o.Severity
	}
	want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent,
		"core.node-tools.control.unwitnessed": conditions.Warning}
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("the witness's verdicts are %v", got)
	}
	witnessed.heard("control", nil, time.Now())
	if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 {
		t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
	}
}

// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle
// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still
// open after a healthy reading.
func judgeLeavesAsTheCore(t *testing.T) {
	t.Helper()
	wasPasses, wasSettles := gateLeafPasses, gateLeafSettles
	gateLeafPasses, gateLeafSettles = gatePasses, true
	t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles })
}