206 lines
9.8 KiB
Go
206 lines
9.8 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/lease"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8).
|
|
|
|
// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered,
|
|
// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is
|
|
// sent and reports it — with the gate's bounds shortened to judge in three steps.
|
|
type gateMesh struct {
|
|
open *stores
|
|
sent [][]string
|
|
health map[string]health // per machine, what a judging finds; healthy when unsaid
|
|
keeper *conditions.Keeper
|
|
told *conditions.Told
|
|
}
|
|
|
|
func (g *gateMesh) plan(t *testing.T) inventory.Plan {
|
|
t.Helper()
|
|
p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
|
|
// A module's build that fails its gate on the first machine is put back there — the previous build
|
|
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
|
|
// condition and an event, never registered or rolled back again.
|
|
|
|
// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then
|
|
// the rest are sent, the verdict kept, and the plan done.
|
|
|
|
// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health
|
|
// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent.
|
|
|
|
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
|
|
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
|
|
|
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
|
|
|
// The health a judging finds, per core component and for a module.
|
|
func TestTheHealthDefinitions(t *testing.T) {
|
|
now := time.Now()
|
|
since := now.Add(-time.Minute)
|
|
applied := func(node string) gateFacts {
|
|
at := now
|
|
return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node,
|
|
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
|
|
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}}
|
|
}
|
|
m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}}
|
|
|
|
f := applied("anchor")
|
|
f.served["anchor"] = served{runtime: true, tools: map[string]bool{}}
|
|
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") {
|
|
t.Errorf("a module whose tools are not served is %v (%s)", h, why)
|
|
}
|
|
f.served["anchor"].tools["app"] = true
|
|
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
|
|
t.Errorf("a module applied with its tools served is %v (%s)", h, why)
|
|
}
|
|
// A condition raised about it on that machine since it was sent: not yet healthy.
|
|
f.judged = true
|
|
f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
|
|
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}}
|
|
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet {
|
|
t.Errorf("a module with a new condition about it is %v", h)
|
|
}
|
|
f.open[0].Raised = since.Add(-time.Hour)
|
|
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
|
|
t.Errorf("a condition older than the send counted against it: %v", h)
|
|
}
|
|
// A witness that put it back: broken.
|
|
f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb",
|
|
Outcome: lease.OutcomeRolledBack, Why: "x", At: now}}
|
|
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
|
|
t.Errorf("a component a witness put back is %v", h)
|
|
}
|
|
// The node tools answer, or not.
|
|
f = applied("anchor")
|
|
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet {
|
|
t.Errorf("node tools not answering are %v", h)
|
|
}
|
|
f.served["anchor"] = served{runtime: true}
|
|
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood {
|
|
t.Errorf("node tools answering are %v", h)
|
|
}
|
|
// The controller: the lease held since the send, by a controller that says it is ready.
|
|
f = applied("control")
|
|
f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}}
|
|
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet {
|
|
t.Errorf("a lease held by a controller older than the build is %v", h)
|
|
}
|
|
f.holder.Taken = now
|
|
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood {
|
|
t.Errorf("a new controller holding the lease and ready is %v", h)
|
|
}
|
|
f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"}
|
|
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet ||
|
|
!strings.Contains(why, "first run") {
|
|
t.Errorf("a controller not ready is %v (%s)", h, why)
|
|
}
|
|
// What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running
|
|
// resource of it on that machine is stated healthy since the send; starting and unhealthy wait.
|
|
f = applied("anchor")
|
|
stated := func(state, reason string, heard time.Time) {
|
|
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard,
|
|
Resources: []inventory.ResourceHealth{
|
|
{Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason},
|
|
{Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}}
|
|
}
|
|
plain := catalogue.Manifest{Module: "app"}
|
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
|
|
t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why)
|
|
}
|
|
stated(link.StateStarting, "", now)
|
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") {
|
|
t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why)
|
|
}
|
|
stated(link.StateUnhealthy, "restarting", now)
|
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") {
|
|
t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why)
|
|
}
|
|
stated(link.StateHealthy, "", since.Add(-time.Minute))
|
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
|
|
t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why)
|
|
}
|
|
stated(link.StateHealthy, "", now)
|
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
|
|
t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why)
|
|
}
|
|
f.healthErr = errors.New("the store is away")
|
|
if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
|
|
t.Errorf("health that cannot be read is never read as healthy: %v", h)
|
|
}
|
|
// And the condition it raises after the send holds it, as every condition about it does.
|
|
f.healthErr = nil
|
|
f.judged = true
|
|
f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{
|
|
Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}}
|
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") {
|
|
t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why)
|
|
}
|
|
|
|
// A machine that refused what it was sent: broken.
|
|
f = applied("anchor")
|
|
r := f.reports["anchor"]
|
|
r.Outcome = inventory.OutcomeRefused
|
|
f.reports["anchor"] = r
|
|
if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
|
|
t.Errorf("a refusal is %v", h)
|
|
}
|
|
}
|
|
|
|
// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused,
|
|
// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits.
|
|
|
|
// A merge that deletes a module's directory builds nothing for it: the module is forgotten where
|
|
// nothing holds it, and a plan whose build finds no manifest goes on instead of failing.
|
|
|
|
func mustJSON(t *testing.T, v any) []byte {
|
|
t.Helper()
|
|
b, err := json.Marshal(v)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return b
|
|
}
|
|
|
|
func nodeID(t *testing.T, open *stores, name string) string {
|
|
t.Helper()
|
|
n, err := open.inventory.NodeByName(context.Background(), name)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return n.ID
|
|
}
|
|
|
|
// What a machine's witness says in its reports is a condition while it says it, by the host's words:
|
|
// `core.<component>.<node>.<outcome>`, urgent for a rollback, a warning when it could not judge — and
|
|
// gone with the first report that no longer carries it.
|
|
|
|
// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle
|
|
// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still
|
|
// open after a healthy reading.
|
|
func judgeLeavesAsTheCore(t *testing.T) {
|
|
t.Helper()
|
|
wasPasses, wasSettles := gateLeafPasses, gateLeafSettles
|
|
gateLeafPasses, gateLeafSettles = gatePasses, true
|
|
t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles })
|
|
}
|