Archived
Hold a consumer's findings under its unhealthy provider, and say them once there (hq ADR 0240, to-be 48 Phase C)
With twelve consumers of the database provision, one provider down would be twelve conditions for one fault and twelve gates failed for something none of them did. A consumer's check names the provision it exercises; while the provider composed for it — its recorded binding, or the machine its credential comes from — is unhealthy on the record, what that check finds raises nothing of its own: the provider's condition lists it as waiting and is urgent, and the consumer's gate waits, past its bound too, rather than putting a build back. Liveness findings and checks naming no provision stay the consumer's own, and once the provider is healthy a consumer still failing is raised at once.
This commit is contained in:
@@ -0,0 +1,172 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, "how it is checked"): one
|
||||
// unhealthy database provider and three consumers failing their checks that need it — one condition, at the
|
||||
// provider, urgent, the consumers listed as waiting; their gates wait, not fail; once the provider is
|
||||
// healthy, a consumer still failing is its own. A consumer failing while its provider is healthy is raised
|
||||
// on its own from the start.
|
||||
func TestOneProviderDownAndThreeConsumersFailingAreOneCondition(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
k := conditionsFrom
|
||||
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
|
||||
consumers := []string{"shop", "wiki", "crm"}
|
||||
for _, c := range consumers {
|
||||
register(t, open, catalogue.Manifest{Module: c, Version: "1", Requires: []string{"postgres-database"}})
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "db"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
machineOf := map[string]string{"shop": "laptop", "wiki": "laptop", "crm": "anchor"}
|
||||
for _, c := range consumers {
|
||||
if _, err := inv.Assign(ctx, machineOf[c], c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, machineOf[c], []inventory.Binding{{Machine: machineOf[c], Consumer: c,
|
||||
Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
at := h0
|
||||
say := func(machine string, rs ...link.ResourceHealth) {
|
||||
t.Helper()
|
||||
at = at.Add(time.Second)
|
||||
for i := range rs {
|
||||
rs[i].Since = at
|
||||
}
|
||||
if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
dbDown := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateUnhealthy,
|
||||
Reason: "its command: the database refuses connections", Check: "exec"}
|
||||
dbUp := dbDown
|
||||
dbUp.State, dbUp.Reason = link.StateHealthy, ""
|
||||
failing := func(module string) link.ResourceHealth {
|
||||
return link.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
|
||||
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
||||
}
|
||||
fine := func(module string) link.ResourceHealth {
|
||||
r := failing(module)
|
||||
r.State, r.Reason = link.StateHealthy, ""
|
||||
return r
|
||||
}
|
||||
openKeys := func() []conditions.Condition {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// Two looks of the provider down and its consumers failing, in either order on each machine.
|
||||
for look := 0; look < 2; look++ {
|
||||
say("laptop", failing("shop"), failing("wiki"))
|
||||
say("anchor", dbDown, failing("crm"))
|
||||
}
|
||||
raised := openKeys()
|
||||
if len(raised) != 1 || raised[0].Key != "module.db.anchor.unhealthy" {
|
||||
var keys []string
|
||||
for _, c := range raised {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
t.Fatalf("one provider down and three consumers failing raised %v; want the provider's alone", keys)
|
||||
}
|
||||
c := raised[0]
|
||||
if c.Severity != conditions.Urgent {
|
||||
t.Errorf("consumers wait on the provider and its condition is %s", c.Severity)
|
||||
}
|
||||
said := c.Evidence[0].Said
|
||||
for _, w := range []string{"shop on laptop", "wiki on laptop", "crm on anchor"} {
|
||||
if !strings.Contains(said, w) {
|
||||
t.Errorf("the provider's condition does not list %s as waiting on it: %s", w, said)
|
||||
}
|
||||
}
|
||||
|
||||
// Their gates wait, not fail: the judging is neither a pass nor a fault, past the bound too.
|
||||
f, err := gatherGateFacts(ctx, open, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.open, f.openErr, f.judged = raised, nil, false
|
||||
for _, m := range consumers {
|
||||
h, why := moduleHealthWord(m, machineOf[m], h0, f)
|
||||
if h != healthWaiting || !strings.Contains(why, "waits on db on anchor") {
|
||||
t.Errorf("%s's gate: %v %q; want it waiting on its provider", m, h, why)
|
||||
}
|
||||
}
|
||||
// And a whole judging past the bound puts nothing back: it waits.
|
||||
long := h0.Add(-time.Hour)
|
||||
for _, m := range []string{"anchor", "laptop"} {
|
||||
f.reports[m] = inventory.Reported{Node: m, Outcome: inventory.OutcomeApplied, At: &f.now, Current: true}
|
||||
}
|
||||
gatherWas := gatherGateFacts
|
||||
defer func() { gatherGateFacts = gatherWas }()
|
||||
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return f, nil }
|
||||
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &long}
|
||||
verdict, err := judgeMoves(ctx, open, g, []judged{{module: "shop", node: "laptop"}}, time.Now())
|
||||
if err != nil || verdict != "" || !strings.Contains(g.Last, "waits on db on anchor") {
|
||||
t.Fatalf("a held consumer's gate past its bound: verdict %q (%v), last %q; want it waiting", verdict, err, g.Last)
|
||||
}
|
||||
|
||||
// The provider healthy again: a consumer still failing is now its own, at once (its streak stood).
|
||||
say("anchor", dbUp, fine("crm"))
|
||||
say("laptop", failing("shop"), fine("wiki"))
|
||||
var keys []string
|
||||
for _, c := range openKeys() {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
if !slices.Equal(keys, []string{"module.shop.laptop.unhealthy"}) {
|
||||
t.Fatalf("after the provider recovered: %v; want shop's own and nothing else", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A consumer failing while its provider is healthy is raised on its own.
|
||||
func TestAConsumerFailingBesideAHealthyProviderIsItsOwn(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
|
||||
for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
|
||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
|
||||
Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
healthy := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateHealthy, Check: "exec"}
|
||||
shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", State: link.StateUnhealthy,
|
||||
Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
|
||||
for i := 1; i <= 2; i++ {
|
||||
at := h0.Add(time.Duration(i) * time.Minute)
|
||||
_ = stateHealth(ctx, inv, conditionsFrom, "anchor", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{healthy}}, at)
|
||||
if err := stateHealth(ctx, inv, conditionsFrom, "laptop", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{shop}}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
list, _ := conditionsFrom.Open(ctx)
|
||||
if len(list) != 1 || list[0].Key != "module.shop.laptop.unhealthy" {
|
||||
t.Fatalf("a consumer failing beside a healthy provider raised %v", list)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user