mesh-controller/cmd/mesh-controller/gate_test.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/lease"
	"github.com/novox/mesh-controller/internal/link"
)

// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8).

// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered,
// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is
// sent and reports it — with the gate's bounds shortened to judge in three steps.
type gateMesh struct {
	open   *stores
	sent   [][]string
	health map[string]health // per machine, what a judging finds; healthy when unsaid
	keeper *conditions.Keeper
	told   *conditions.Told
}

func (g *gateMesh) plan(t *testing.T) inventory.Plan {
	t.Helper()
	p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate")
	if err != nil {
		t.Fatal(err)
	}
	return p
}

// A module's build that fails its gate on the first machine is put back there — the previous build
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
// condition and an event, never registered or rolled back again.

// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then
// the rest are sent, the verdict kept, and the plan done.

// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health
// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent.

// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).

// A first machine that never becomes healthy fails its gate at the bound, and is put back.

// The health a judging finds, per core component and for a module.
func TestTheHealthDefinitions(t *testing.T) {
	now := time.Now()
	since := now.Add(-time.Minute)
	applied := func(node string) gateFacts {
		at := now
		return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node,
			Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
			served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}}
	}
	m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}}

	f := applied("anchor")
	f.served["anchor"] = served{runtime: true, tools: map[string]bool{}}
	if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") {
		t.Errorf("a module whose tools are not served is %v (%s)", h, why)
	}
	f.served["anchor"].tools["app"] = true
	if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
		t.Errorf("a module applied with its tools served is %v (%s)", h, why)
	}
	// A condition raised about it on that machine since it was sent: not yet healthy.
	f.judged = true
	f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
		Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}}
	if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet {
		t.Errorf("a module with a new condition about it is %v", h)
	}
	f.open[0].Raised = since.Add(-time.Hour)
	if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
		t.Errorf("a condition older than the send counted against it: %v", h)
	}
	// A witness that put it back: broken.
	f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb",
		Outcome: lease.OutcomeRolledBack, Why: "x", At: now}}
	if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
		t.Errorf("a component a witness put back is %v", h)
	}
	// The node tools answer, or not.
	f = applied("anchor")
	if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet {
		t.Errorf("node tools not answering are %v", h)
	}
	f.served["anchor"] = served{runtime: true}
	if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood {
		t.Errorf("node tools answering are %v", h)
	}
	// The controller: the lease held since the send, by a controller that says it is ready.
	f = applied("control")
	f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}}
	if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet {
		t.Errorf("a lease held by a controller older than the build is %v", h)
	}
	f.holder.Taken = now
	if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood {
		t.Errorf("a new controller holding the lease and ready is %v", h)
	}
	f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"}
	if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet ||
		!strings.Contains(why, "first run") {
		t.Errorf("a controller not ready is %v (%s)", h, why)
	}
	// What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running
	// resource of it on that machine is stated healthy since the send; starting and unhealthy wait.
	f = applied("anchor")
	stated := func(state, reason string, heard time.Time) {
		f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard,
			Resources: []inventory.ResourceHealth{
				{Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason},
				{Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}}
	}
	plain := catalogue.Manifest{Module: "app"}
	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
		t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why)
	}
	stated(link.StateStarting, "", now)
	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") {
		t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why)
	}
	stated(link.StateUnhealthy, "restarting", now)
	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") {
		t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why)
	}
	stated(link.StateHealthy, "", since.Add(-time.Minute))
	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
		t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why)
	}
	stated(link.StateHealthy, "", now)
	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
		t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why)
	}
	f.healthErr = errors.New("the store is away")
	if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
		t.Errorf("health that cannot be read is never read as healthy: %v", h)
	}
	// And the condition it raises after the send holds it, as every condition about it does.
	f.healthErr = nil
	f.judged = true
	f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{
		Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}}
	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") {
		t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why)
	}

	// A machine that refused what it was sent: broken.
	f = applied("anchor")
	r := f.reports["anchor"]
	r.Outcome = inventory.OutcomeRefused
	f.reports["anchor"] = r
	if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
		t.Errorf("a refusal is %v", h)
	}
}

// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused,
// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits.

// A merge that deletes a module's directory builds nothing for it: the module is forgotten where
// nothing holds it, and a plan whose build finds no manifest goes on instead of failing.

func mustJSON(t *testing.T, v any) []byte {
	t.Helper()
	b, err := json.Marshal(v)
	if err != nil {
		t.Fatal(err)
	}
	return b
}

func nodeID(t *testing.T, open *stores, name string) string {
	t.Helper()
	n, err := open.inventory.NodeByName(context.Background(), name)
	if err != nil {
		t.Fatal(err)
	}
	return n.ID
}

// What a machine's witness says in its reports is a condition while it says it, by the host's words:
// `core.<component>.<node>.<outcome>`, urgent for a rollback, a warning when it could not judge — and
// gone with the first report that no longer carries it.

// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle
// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still
// open after a healthy reading.
func judgeLeavesAsTheCore(t *testing.T) {
	t.Helper()
	wasPasses, wasSettles := gateLeafPasses, gateLeafSettles
	gateLeafPasses, gateLeafSettles = gatePasses, true
	t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles })
}