jochen
1cc6a2d759
The gate judged a module by what the mesh saw from outside, so a container that crash-looped after it applied passed it. Each machine's node-engine now states the health of every long-running resource it runs; the controller keeps the newest statement per machine, raises module.<module>.<machine>.unhealthy on the second statement in a row, clears it on the first that does not say it, and the gate passes a module only when every long-running resource of it is stated healthy since the send. An engine that states nothing is judged as before.
594 lines
26 KiB
Go
594 lines
26 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/lease"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8).
|
|
|
|
// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered,
|
|
// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is
|
|
// sent and reports it — with the gate's bounds shortened to judge in three steps.
|
|
type gateMesh struct {
|
|
open *stores
|
|
sent [][]string
|
|
health map[string]health // per machine, what a judging finds; healthy when unsaid
|
|
keeper *conditions.Keeper
|
|
told *conditions.Told
|
|
}
|
|
|
|
func aGateMesh(t *testing.T) *gateMesh {
|
|
t.Helper()
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
g := &gateMesh{open: open, health: map[string]health{}}
|
|
g.keeper, _ = withConditionsInMemory(t)
|
|
g.told = &conditions.Told{}
|
|
was := doctorFrom
|
|
doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told}
|
|
t.Cleanup(func() { doctorFrom = was })
|
|
|
|
for _, b := range []inventory.Build{
|
|
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
|
|
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
|
|
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
|
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
|
|
} {
|
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
|
|
b.Manifest = manifest
|
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
register := func(commit string, asked time.Time) {
|
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
|
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
|
|
Head: commit, Asked: asked}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
register("c1", time.Now().Add(-2*time.Hour))
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
register("c2", time.Now().Add(-time.Minute))
|
|
|
|
// Every send: recorded with the build the module is at, applied and reported by the machine.
|
|
n := 0
|
|
wasSend := sendRollout
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
g.sent = append(g.sent, append([]string(nil), names...))
|
|
current, err := open.inventory.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, node := range names {
|
|
n++
|
|
digest := fmt.Sprintf("d-%s-%d", node, n)
|
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
|
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return names, nil
|
|
}
|
|
t.Cleanup(func() { sendRollout = wasSend })
|
|
|
|
// What a judging reads: the store's reports, and a health the test says per machine.
|
|
wasGather := gatherGateFacts
|
|
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
|
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
|
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
|
reports, err := open.inventory.LastReports(ctx)
|
|
if err != nil {
|
|
return f, err
|
|
}
|
|
for _, r := range reports {
|
|
if g.health[r.Node] == healthBroken {
|
|
r.Outcome = inventory.OutcomeFailed
|
|
}
|
|
f.reports[r.Node] = r
|
|
}
|
|
for node, h := range g.health {
|
|
if h == healthNotYet {
|
|
f.rolledBack[node] = nil
|
|
r := f.reports[node]
|
|
r.Current = false
|
|
f.reports[node] = r
|
|
}
|
|
}
|
|
return f, nil
|
|
}
|
|
t.Cleanup(func() { gatherGateFacts = wasGather })
|
|
|
|
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
|
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
|
|
gateSettle, gateEvery = 0, time.Hour
|
|
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
|
|
|
|
built := time.Now().UTC()
|
|
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
|
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
|
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
|
|
Build: "build-2"}}}
|
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return g
|
|
}
|
|
|
|
func (g *gateMesh) plan(t *testing.T) inventory.Plan {
|
|
t.Helper()
|
|
p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
|
|
// A module's build that fails its gate on the first machine is put back there — the previous build
|
|
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
|
|
// condition and an event, never registered or rolled back again.
|
|
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
|
|
g := aGateMesh(t)
|
|
ctx := t.Context()
|
|
inv := g.open.inventory
|
|
|
|
advancePlans(ctx, g.open) // the first machine is sent the new build
|
|
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) {
|
|
t.Fatalf("sent %v, not the first machine alone", g.sent)
|
|
}
|
|
if p := g.plan(t); p.Modules["app"].Previous != "c1" {
|
|
t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"])
|
|
}
|
|
|
|
g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging
|
|
gateEvery = 0
|
|
advancePlans(ctx, g.open)
|
|
|
|
p := g.plan(t)
|
|
gate := p.Modules["app"].Gate
|
|
if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed ||
|
|
gate.Rollback != inventory.RolledBack {
|
|
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
|
}
|
|
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) {
|
|
t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent)
|
|
}
|
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
|
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
|
|
}
|
|
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" {
|
|
t.Fatalf("anchor was last sent %v, not the previous build", sent)
|
|
}
|
|
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" {
|
|
t.Fatalf("laptop was sent the failed build: %v", sent)
|
|
}
|
|
if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed {
|
|
t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err)
|
|
}
|
|
|
|
// Said: a condition for the operator, and the event.
|
|
open, err := g.keeper.Open(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var key string
|
|
for _, c := range open {
|
|
if c.Kind == kindRolledBack {
|
|
key = c.Key
|
|
}
|
|
}
|
|
if key != "build.app.anchor.rolled-back" {
|
|
t.Fatalf("no rolled-back condition for app on anchor: %+v", open)
|
|
}
|
|
saidIt := false
|
|
for _, e := range g.told.Said() {
|
|
saidIt = saidIt || e.Event == link.KeyRolledBack
|
|
}
|
|
if !saidIt {
|
|
t.Fatalf("the rollback was not said as an event: %+v", g.told.Said())
|
|
}
|
|
|
|
// Never again: more passes send nothing, a second judging rolls nothing back, and the failed build
|
|
// heard again is not registered.
|
|
advancePlans(ctx, g.open)
|
|
state := p.Modules["app"]
|
|
gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again")
|
|
if len(g.sent) != 2 {
|
|
t.Fatalf("sent again after the rollback: %v", g.sent)
|
|
}
|
|
_, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app",
|
|
Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"})})
|
|
if err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
|
t.Fatalf("the failed build was registered again: %v", err)
|
|
}
|
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
|
t.Fatalf("the module moved to %s", current["app"].Commit)
|
|
}
|
|
if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") {
|
|
t.Fatalf("a plan stopped at a failed gate was retried: %v", err)
|
|
}
|
|
|
|
// The probe keeps the condition while the newest verdict is the failure.
|
|
obs, err := probeGates(ctx, doctorFrom)
|
|
if err != nil || len(obs) != 1 || obs[0].Key() != key {
|
|
t.Fatalf("the gate's probe found %+v %v", obs, err)
|
|
}
|
|
}
|
|
|
|
// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then
|
|
// the rest are sent, the verdict kept, and the plan done.
|
|
func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
|
g := aGateMesh(t)
|
|
ctx := t.Context()
|
|
gateEvery = 0
|
|
for i := 0; i < 6; i++ {
|
|
advancePlans(ctx, g.open)
|
|
}
|
|
p := g.plan(t)
|
|
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
|
t.Fatalf("sent %v", g.sent)
|
|
}
|
|
gate := p.Modules["app"].Gate
|
|
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses {
|
|
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
|
|
}
|
|
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
|
t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err)
|
|
}
|
|
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
|
t.Fatalf("a passing build left a condition: %+v %v", obs, err)
|
|
}
|
|
}
|
|
|
|
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
|
|
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
|
|
g := aGateMesh(t)
|
|
ctx := t.Context()
|
|
advancePlans(ctx, g.open)
|
|
g.health["anchor"] = healthNotYet
|
|
gateEvery = 0
|
|
advancePlans(ctx, g.open)
|
|
if p := g.plan(t); !p.Open() || len(g.sent) != 1 {
|
|
t.Fatalf("judged before the bound: %s %v", p.State, g.sent)
|
|
}
|
|
gateBound = -time.Second
|
|
advancePlans(ctx, g.open)
|
|
p := g.plan(t)
|
|
if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed ||
|
|
!strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack {
|
|
t.Fatalf("the plan is %s, its gate %+v", p.State, gate)
|
|
}
|
|
}
|
|
|
|
// The health a judging finds, per core component and for a module.
|
|
func TestTheHealthDefinitions(t *testing.T) {
|
|
now := time.Now()
|
|
since := now.Add(-time.Minute)
|
|
applied := func(node string) gateFacts {
|
|
at := now
|
|
return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node,
|
|
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
|
|
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}}
|
|
}
|
|
m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}}
|
|
|
|
f := applied("anchor")
|
|
f.served["anchor"] = served{runtime: true, tools: map[string]bool{}}
|
|
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") {
|
|
t.Errorf("a module whose tools are not served is %v (%s)", h, why)
|
|
}
|
|
f.served["anchor"].tools["app"] = true
|
|
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
|
|
t.Errorf("a module applied with its tools served is %v (%s)", h, why)
|
|
}
|
|
// A condition raised about it on that machine since it was sent: not yet healthy.
|
|
f.judged = true
|
|
f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
|
|
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}}
|
|
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet {
|
|
t.Errorf("a module with a new condition about it is %v", h)
|
|
}
|
|
f.open[0].Raised = since.Add(-time.Hour)
|
|
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
|
|
t.Errorf("a condition older than the send counted against it: %v", h)
|
|
}
|
|
// A witness that put it back: broken.
|
|
f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb",
|
|
Outcome: lease.OutcomeRolledBack, Why: "x", At: now}}
|
|
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
|
|
t.Errorf("a component a witness put back is %v", h)
|
|
}
|
|
// The node tools answer, or not.
|
|
f = applied("anchor")
|
|
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet {
|
|
t.Errorf("node tools not answering are %v", h)
|
|
}
|
|
f.served["anchor"] = served{runtime: true}
|
|
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood {
|
|
t.Errorf("node tools answering are %v", h)
|
|
}
|
|
// The controller: the lease held since the send, by a controller that says it is ready.
|
|
f = applied("control")
|
|
f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}}
|
|
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet {
|
|
t.Errorf("a lease held by a controller older than the build is %v", h)
|
|
}
|
|
f.holder.Taken = now
|
|
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood {
|
|
t.Errorf("a new controller holding the lease and ready is %v", h)
|
|
}
|
|
f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"}
|
|
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet ||
|
|
!strings.Contains(why, "first run") {
|
|
t.Errorf("a controller not ready is %v (%s)", h, why)
|
|
}
|
|
// What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running
|
|
// resource of it on that machine is stated healthy since the send; starting and unhealthy wait.
|
|
f = applied("anchor")
|
|
stated := func(state, reason string, heard time.Time) {
|
|
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard,
|
|
Resources: []inventory.ResourceHealth{
|
|
{Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason},
|
|
{Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}}
|
|
}
|
|
plain := catalogue.Manifest{Module: "app"}
|
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
|
|
t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why)
|
|
}
|
|
stated(link.StateStarting, "", now)
|
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") {
|
|
t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why)
|
|
}
|
|
stated(link.StateUnhealthy, "restarting", now)
|
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") {
|
|
t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why)
|
|
}
|
|
stated(link.StateHealthy, "", since.Add(-time.Minute))
|
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
|
|
t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why)
|
|
}
|
|
stated(link.StateHealthy, "", now)
|
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
|
|
t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why)
|
|
}
|
|
f.healthErr = errors.New("the store is away")
|
|
if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
|
|
t.Errorf("health that cannot be read is never read as healthy: %v", h)
|
|
}
|
|
// And the condition it raises after the send holds it, as every condition about it does.
|
|
f.healthErr = nil
|
|
f.judged = true
|
|
f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{
|
|
Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}}
|
|
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") {
|
|
t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why)
|
|
}
|
|
|
|
// A machine that refused what it was sent: broken.
|
|
f = applied("anchor")
|
|
r := f.reports["anchor"]
|
|
r.Outcome = inventory.OutcomeRefused
|
|
f.reports["anchor"] = r
|
|
if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
|
|
t.Errorf("a refusal is %v", h)
|
|
}
|
|
}
|
|
|
|
// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused,
|
|
// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits.
|
|
func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}},
|
|
Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}}
|
|
if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
|
Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
u, err := inv.UpgradeOf(ctx, "nats")
|
|
if err != nil || u.RollOut || u.From != catalogue.FromBus {
|
|
t.Fatalf("the bus's policy is %+v %v", u, err)
|
|
}
|
|
if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) {
|
|
t.Fatalf("a person rolled the bus out: %v", err)
|
|
}
|
|
var sent [][]string
|
|
was := sendRollout
|
|
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
|
|
sent = append(sent, names)
|
|
return names, nil
|
|
}
|
|
t.Cleanup(func() { sendRollout = was })
|
|
now := time.Now().UTC()
|
|
plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now,
|
|
State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}},
|
|
Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}}
|
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
advancePlans(ctx, open)
|
|
if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 {
|
|
t.Fatalf("a plan sent the bus: %s %v", p.State, sent)
|
|
}
|
|
held, err := busHeld(ctx, inv, []string{"anchor", "laptop"})
|
|
if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" {
|
|
t.Fatalf("a push may send the bus's machine: %v %v", held, err)
|
|
}
|
|
// Nor may any other send — a plan's for another module on that machine carried the bus with it.
|
|
if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) {
|
|
t.Fatalf("a send to the bus's machine was not refused: %v", err)
|
|
}
|
|
// A rebuild that made the same artifacts is no move.
|
|
for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} {
|
|
b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}}
|
|
b.Asked, b.At = time.Now(), time.Now()
|
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
|
|
t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err)
|
|
}
|
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second),
|
|
At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
|
Reference: "registry/nats@sha256:new"}}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The planned step refuses to start without its word on reversibility, and without a snapshot taken
|
|
// first by the bus machine's backup holder.
|
|
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") {
|
|
t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err)
|
|
}
|
|
wasSnapshot := takeBusSnapshot
|
|
t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
|
|
takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") }
|
|
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil ||
|
|
!strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 {
|
|
t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent)
|
|
}
|
|
takeBusSnapshot = func(_ context.Context, module, node string) (string, error) {
|
|
return node + "'s restore point of " + module, nil
|
|
}
|
|
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) {
|
|
t.Fatalf("the step sent %v, not the bus's machine", sent)
|
|
}
|
|
s, found, err := inv.LatestBusStep(ctx)
|
|
if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil ||
|
|
!reflect.DeepEqual(s.Machines, []string{"anchor"}) {
|
|
t.Fatalf("the step is %+v %v %v", s, found, err)
|
|
}
|
|
}
|
|
|
|
// A merge that deletes a module's directory builds nothing for it: the module is forgotten where
|
|
// nothing holds it, and a plan whose build finds no manifest goes on instead of failing.
|
|
func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
asked := asksRecorded(t)
|
|
for _, name := range []string{"gone", "kept"} {
|
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{
|
|
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef",
|
|
Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"},
|
|
Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}}
|
|
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(*asked) != 0 {
|
|
t.Fatalf("a deleted module was asked to build: %v", *asked)
|
|
}
|
|
if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 {
|
|
t.Fatalf("a merge that only deleted a module made a plan: %+v", plans)
|
|
}
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, still := shelf["gone"]; still {
|
|
t.Fatal("a deleted module nothing holds was not forgotten")
|
|
}
|
|
|
|
// Without the announcer saying what went: the build finds no manifest, and the plan goes on.
|
|
asked0 := time.Now().UTC().Add(-time.Minute)
|
|
plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0,
|
|
State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}},
|
|
Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}}
|
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+
|
|
"there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k")
|
|
p, _ := inv.PlanByID(ctx, "plan-deleted")
|
|
if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted {
|
|
t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"])
|
|
}
|
|
}
|
|
|
|
func mustJSON(t *testing.T, v any) []byte {
|
|
t.Helper()
|
|
b, err := json.Marshal(v)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return b
|
|
}
|
|
|
|
func nodeID(t *testing.T, open *stores, name string) string {
|
|
t.Helper()
|
|
n, err := open.inventory.NodeByName(context.Background(), name)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return n.ID
|
|
}
|
|
|
|
// What a machine's witness says in its reports is a condition while it says it, by the host's words:
|
|
// `core.<component>.<node>.<outcome>`, urgent for a rollback, a warning when it could not judge — and
|
|
// gone with the first report that no longer carries it.
|
|
func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
|
|
open := aMesh(t)
|
|
d := &doctor{open: open}
|
|
at := time.Now().UTC()
|
|
witnessed.heard("control", []lease.Rollback{
|
|
{Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack,
|
|
Why: "the new controller did not take the lease within 60s", At: at},
|
|
{Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at},
|
|
}, at)
|
|
t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) })
|
|
obs, err := probeGates(t.Context(), d)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := map[string]conditions.Severity{}
|
|
for _, o := range obs {
|
|
got[o.Key()] = o.Severity
|
|
}
|
|
want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent,
|
|
"core.node-tools.control.unwitnessed": conditions.Warning}
|
|
if !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("the witness's verdicts are %v", got)
|
|
}
|
|
witnessed.heard("control", nil, time.Now())
|
|
if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 {
|
|
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
|
|
}
|
|
}
|