package main import ( "context" "encoding/json" "errors" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" "github.com/novox/mesh-controller/internal/link" ) // The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8). // gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered, // a plan whose tier built c2, and every send recorded and answered — the machine applies what it is // sent and reports it — with the gate's bounds shortened to judge in three steps. type gateMesh struct { open *stores sent [][]string health map[string]health // per machine, what a judging finds; healthy when unsaid keeper *conditions.Keeper told *conditions.Told } func (g *gateMesh) plan(t *testing.T) inventory.Plan { t.Helper() p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate") if err != nil { t.Fatal(err) } return p } // A module's build that fails its gate on the first machine is put back there — the previous build // registered and sent to it again — and never reaches the second machine; it is marked, said as a // condition and an event, never registered or rolled back again. // A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then // the rest are sent, the verdict kept, and the plan done. // A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health // judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent. // A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the // core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524). // A first machine that never becomes healthy fails its gate at the bound, and is put back. // The health a judging finds, per core component and for a module. func TestTheHealthDefinitions(t *testing.T) { now := time.Now() since := now.Add(-time.Minute) applied := func(node string) gateFacts { at := now return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node, Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}} } m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}} f := applied("anchor") f.served["anchor"] = served{runtime: true, tools: map[string]bool{}} if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") { t.Errorf("a module whose tools are not served is %v (%s)", h, why) } f.served["anchor"].tools["app"] = true if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood { t.Errorf("a module applied with its tools served is %v (%s)", h, why) } // A condition raised about it on that machine since it was sent: not yet healthy. f.judged = true f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{ Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}} if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet { t.Errorf("a module with a new condition about it is %v", h) } f.open[0].Raised = since.Add(-time.Hour) if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood { t.Errorf("a condition older than the send counted against it: %v", h) } // A witness that put it back: broken. f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb", Outcome: lease.OutcomeRolledBack, Why: "x", At: now}} if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken { t.Errorf("a component a witness put back is %v", h) } // The node tools answer, or not. f = applied("anchor") if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet { t.Errorf("node tools not answering are %v", h) } f.served["anchor"] = served{runtime: true} if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood { t.Errorf("node tools answering are %v", h) } // The controller: the lease held since the send, by a controller that says it is ready. f = applied("control") f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}} if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet { t.Errorf("a lease held by a controller older than the build is %v", h) } f.holder.Taken = now if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood { t.Errorf("a new controller holding the lease and ready is %v", h) } f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"} if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet || !strings.Contains(why, "first run") { t.Errorf("a controller not ready is %v (%s)", h, why) } // What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running // resource of it on that machine is stated healthy since the send; starting and unhealthy wait. f = applied("anchor") stated := func(state, reason string, heard time.Time) { f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard, Resources: []inventory.ResourceHealth{ {Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason}, {Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}} } plain := catalogue.Manifest{Module: "app"} if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood { t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why) } stated(link.StateStarting, "", now) if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") { t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why) } stated(link.StateUnhealthy, "restarting", now) if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") { t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why) } stated(link.StateHealthy, "", since.Add(-time.Minute)) if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet { t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why) } stated(link.StateHealthy, "", now) if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood { t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why) } f.healthErr = errors.New("the store is away") if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet { t.Errorf("health that cannot be read is never read as healthy: %v", h) } // And the condition it raises after the send holds it, as every condition about it does. f.healthErr = nil f.judged = true f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{ Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}} if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") { t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why) } // A machine that refused what it was sent: broken. f = applied("anchor") r := f.reports["anchor"] r.Outcome = inventory.OutcomeRefused f.reports["anchor"] = r if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken { t.Errorf("a refusal is %v", h) } } // The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused, // a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits. // A merge that deletes a module's directory builds nothing for it: the module is forgotten where // nothing holds it, and a plan whose build finds no manifest goes on instead of failing. func mustJSON(t *testing.T, v any) []byte { t.Helper() b, err := json.Marshal(v) if err != nil { t.Fatal(err) } return b } func nodeID(t *testing.T, open *stores, name string) string { t.Helper() n, err := open.inventory.NodeByName(context.Background(), name) if err != nil { t.Fatal(err) } return n.ID } // What a machine's witness says in its reports is a condition while it says it, by the host's words: // `core...`, urgent for a rollback, a warning when it could not judge — and // gone with the first report that no longer carries it. // judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle // time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still // open after a healthy reading. func judgeLeavesAsTheCore(t *testing.T) { t.Helper() wasPasses, wasSettles := gateLeafPasses, gateLeafSettles gateLeafPasses, gateLeafSettles = gatePasses, true t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles }) }