diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 55f3098..1c91964 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -77,6 +77,10 @@ type health int const ( healthGood health = iota + // healthWaiting is not a pass and not a fault: what the module's checks find waits on an unhealthy + // provider (ADR 0240 rule 5), so the judging waits — past the bound too — rather than putting back a + // build for something it did not do. + healthWaiting healthNotYet healthBroken ) @@ -109,6 +113,8 @@ type gateFacts struct { // healthErr is why they could not be read. health map[string]inventory.NodeHealth healthErr error + // heldOn is, per "@", the provider its findings are held under (ADR 0240 rule 5). + heldOn map[string]string } // gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A @@ -147,6 +153,17 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string) } else { f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what") } + // Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail. + if f.healthErr == nil && f.openErr == nil { + if hold, err := readHolding(ctx, inv, f.open); err == nil { + f.heldOn = map[string]string{} + for machine := range f.health { + for module, p := range hold.heldModules(machine) { + f.heldOn[module+"@"+machine] = p.Module + " on " + p.Node + } + } + } + } if theLease != nil { h, found, err := theLease.holder(ctx) switch { @@ -435,6 +452,10 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs // What broke is put back; what was only not yet healthy beside it is too — they moved together. g.Failing = failing decide(g, inventory.GateFailed, why, now) + case worst == healthWaiting: + // Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either — + // the provider's own condition says what is wrong (ADR 0240 rule 5). + g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing case worst == healthNotYet: g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing if now.Sub(*g.Since) > gateBound { diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 94ea528..23ace29 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "time" + "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -93,13 +94,15 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke if k == nil { return nil } - return judgeModuleHealth(ctx, k, node, unhealthy, streaks, now) + return judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now) } // judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a // resource of it is unhealthy — or on the first while it is already open — and clears every one this -// statement no longer says. -func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string, +// statement no longer says. **A consumer whose findings wait on an unhealthy provider is held** (to-be 48 +// §6): raised as nothing of its own, listed at the provider's condition, which is urgent while anyone +// waits on it. +func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error { open, err := k.Open(ctx) if err != nil { @@ -111,6 +114,12 @@ func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string, standing[c.Key] = c } } + var hold *holding + if inv != nil { + if hold, err = readHolding(ctx, inv, open); err != nil { + return err + } + } var problems []string modules := make([]string, 0, len(unhealthy)) for m := range unhealthy { @@ -118,8 +127,23 @@ func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string, } sort.Strings(modules) seen := map[string]bool{} + heldOn := map[string]string{} + providers := map[catalogue.Chosen]bool{} for _, m := range modules { o := moduleUnhealthyObservation(m, node, unhealthy[m]) + if hold != nil { + if p, held := hold.heldUnder(node, m, unhealthy[m]); held { + // Held under the provider's condition: nothing of its own, and the provider's says it waits. + heldOn[o.Key()] = p.Module + " on " + p.Node + providers[p] = true + continue + } + if waiters := hold.waitersOn(catalogue.Chosen{Node: node, Module: m}); len(waiters) > 0 { + o.Severity = conditions.Urgent + o.Said += "; " + waitingWords(waiters) + o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters)) + } + } seen[o.Key()] = true c, isOpen := standing[o.Key()] if streaks[m] < moduleUnhealthyAfter && !isOpen { @@ -137,7 +161,18 @@ func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string, continue } module := strings.TrimSuffix(c.Subject.ID, "."+node) - if _, err := k.Clear(ctx, key, fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)); err != nil { + why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module) + if on, held := heldOn[key]; held { + why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on) + } + if _, err := k.Clear(ctx, key, why); err != nil { + problems = append(problems, err.Error()) + } + } + // And each provider a consumer here now waits on, when its own condition is open: said again with who + // waits on it, so the wait is listed at the provider whichever machine's statement arrived first. + for p := range providers { + if err := sayWaiters(ctx, k, hold, p, now); err != nil { problems = append(problems, err.Error()) } } @@ -147,6 +182,37 @@ func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string, return nil } +// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest +// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks. +func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error { + var raisedAt *conditions.Condition + for i, c := range hold.open { + if c.Key == moduleUnhealthyKey(p.Module, p.Node) { + raisedAt = &hold.open[i] + } + } + if raisedAt == nil { + return nil + } + var rs []inventory.ResourceHealth + for _, r := range hold.healths[p.Node].Resources { + if r.Module == p.Module && r.State == link.StateUnhealthy { + rs = append(rs, r) + } + } + if len(rs) == 0 { + return nil + } + o := moduleUnhealthyObservation(p.Module, p.Node, rs) + if waiters := hold.waitersOn(p); len(waiters) > 0 { + o.Severity = conditions.Urgent + o.Said += "; " + waitingWords(waiters) + o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters)) + } + _, err := k.Observe(ctx, o) + return err +} + // moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module, // the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence. func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation { @@ -211,6 +277,10 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea case link.StateStarting: return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine) case link.StateUnhealthy: + if on, held := f.heldOn[module+"@"+machine]; held { + return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind, + r.Resource, machine, on) + } return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r)) default: return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State, diff --git a/cmd/mesh-controller/provider_hold.go b/cmd/mesh-controller/provider_hold.go new file mode 100644 index 0000000..ea975bd --- /dev/null +++ b/cmd/mesh-controller/provider_hold.go @@ -0,0 +1,192 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, to-be 48 §6, Phase C). +// +// With twelve consumers of the database provision and thirty-six of a route, one provider down would be +// twelve conditions for one fault and twelve gates failed for something none of them did. So a consumer's +// check names, in `needs`, the provision it exercises; while **that provision's provider for this +// consumer** — the one the controller composed the consumer against: its recorded binding (ADR 0232), or +// the provider its credential for the provision is from — is unhealthy on the record, what the check finds +// is held under the provider's condition: listed there as waiting on it, raised as nothing of its own, and +// the consumer's gate waits rather than fails. The provider's condition is urgent while consumers wait. +// +// **Only what the check finds is held.** A consumer that is down or restarting is its own, whatever its +// provider does; so is anything a check that names no provision finds, and anything found while the +// provider is healthy. A machine-level fault is never pinned on a module (issue 281), and this does not +// change that. + +// holding is what one reading of the record needs to say who waits on whom: every machine's newest +// statement, the open conditions, and the catalogue — read once, asked many times. +type holding struct { + ctx context.Context + inv *inventory.Inventory + healths map[string]inventory.NodeHealth + open []conditions.Condition + shelf map[string]catalogue.Manifest + // providers memoises providerFor by machine, consumer and provision. + providers map[string]providerLookup +} + +type providerLookup struct { + chosen catalogue.Chosen + ok bool +} + +// readHolding reads what the hold is judged from. +func readHolding(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) { + healths, err := inv.Healths(ctx) + if err != nil { + return nil, err + } + return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil +} + +// heldFinding says a resource's state is a finding of its declared check that names a provision: what +// may be held. Down and restarting are liveness, the resource's own. +func heldFinding(r inventory.ResourceHealth) bool { + return r.State == link.StateUnhealthy && r.Check != "" && r.Needs != "" && + r.Reason != "down" && r.Reason != "restarting" +} + +// providerFor is the provider composed for a consumer's provision: its recorded binding, else the +// machine its credential for the provision comes from and the module there that provides it. +func (h *holding) providerFor(machine, consumer, provision string) (catalogue.Chosen, bool) { + key := machine + "\x00" + consumer + "\x00" + provision + if p, known := h.providers[key]; known { + return p.chosen, p.ok + } + chosen, ok := h.lookUpProvider(machine, consumer, provision) + h.providers[key] = providerLookup{chosen, ok} + return chosen, ok +} + +func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue.Chosen, bool) { + if bound, err := h.inv.BindingsFor(h.ctx, machine); err == nil { + if c, ok := bound[consumer][provision]; ok && c.Node != "" && c.Module != "" { + return c, true + } + } + secrets, err := h.inv.SecretsOf(h.ctx, machine, consumer) + if err != nil { + return catalogue.Chosen{}, false + } + for _, s := range secrets { + if s.Name != provision || s.Provider == "" { + continue + } + if h.shelf == nil { + if h.shelf, err = h.inv.Catalogue(h.ctx); err != nil { + return catalogue.Chosen{}, false + } + } + assigned, err := h.inv.Assigned(h.ctx, s.Provider) + if err != nil { + return catalogue.Chosen{}, false + } + for _, module := range assigned { + for _, offer := range h.shelf[module].Offers() { + if offer == provision { + return catalogue.Chosen{Node: s.Provider, Module: module}, true + } + } + } + } + return catalogue.Chosen{}, false +} + +// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest +// statement says a resource of it is unhealthy. +func (h *holding) unhealthy(p catalogue.Chosen) bool { + key := moduleUnhealthyKey(p.Module, p.Node) + for _, c := range h.open { + if c.Key == key { + return true + } + } + for _, r := range h.healths[p.Node].Resources { + if r.Module == p.Module && r.State == link.StateUnhealthy { + return true + } + } + return false +} + +// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding +// of a check naming a provision whose provider for this consumer is unhealthy on the record. False when any +// is the consumer's own. +func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) { + var on catalogue.Chosen + for _, r := range rs { + if r.State != link.StateUnhealthy { + continue + } + if !heldFinding(r) { + return catalogue.Chosen{}, false + } + p, ok := h.providerFor(machine, module, r.Needs) + if !ok || (p.Node == machine && p.Module == module) || !h.unhealthy(p) { + return catalogue.Chosen{}, false + } + on = p + } + return on, on.Module != "" +} + +// waitersOn is every consumer held under a provider, as " on ", sorted. +func (h *holding) waitersOn(p catalogue.Chosen) []string { + var out []string + for machine, nh := range h.healths { + byModule := map[string][]inventory.ResourceHealth{} + for _, r := range nh.Resources { + if r.Module != "" && r.State == link.StateUnhealthy { + byModule[r.Module] = append(byModule[r.Module], r) + } + } + for module, rs := range byModule { + if on, held := h.heldUnder(machine, module, rs); held && on == p { + out = append(out, module+" on "+machine) + } + } + } + sort.Strings(out) + return out +} + +// heldModules is, for one machine's statement, each module whose finding is held, with the provider. +func (h *holding) heldModules(machine string) map[string]catalogue.Chosen { + out := map[string]catalogue.Chosen{} + byModule := map[string][]inventory.ResourceHealth{} + for _, r := range h.healths[machine].Resources { + if r.Module != "" && r.State == link.StateUnhealthy { + byModule[r.Module] = append(byModule[r.Module], r) + } + } + for module, rs := range byModule { + if on, held := h.heldUnder(machine, module, rs); held { + out[module] = on + } + } + return out +} + +// moduleUnhealthyKey is a module's health condition's key on a machine. +func moduleUnhealthyKey(module, machine string) string { + return conditions.ScopeModule + "." + module + "." + machine + ".unhealthy" +} + +// waitingWords is the provider's evidence that consumers wait on it. +func waitingWords(waiters []string) string { + return fmt.Sprintf("waiting on it — %s", strings.Join(waiters, ", ")) +} diff --git a/cmd/mesh-controller/provider_hold_test.go b/cmd/mesh-controller/provider_hold_test.go new file mode 100644 index 0000000..d07e9c7 --- /dev/null +++ b/cmd/mesh-controller/provider_hold_test.go @@ -0,0 +1,172 @@ +package main + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, "how it is checked"): one +// unhealthy database provider and three consumers failing their checks that need it — one condition, at the +// provider, urgent, the consumers listed as waiting; their gates wait, not fail; once the provider is +// healthy, a consumer still failing is its own. A consumer failing while its provider is healthy is raised +// on its own from the start. +func TestOneProviderDownAndThreeConsumersFailingAreOneCondition(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + k := conditionsFrom + register(t, open, catalogue.Manifest{Module: "db", Version: "1", + Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}) + consumers := []string{"shop", "wiki", "crm"} + for _, c := range consumers { + register(t, open, catalogue.Manifest{Module: c, Version: "1", Requires: []string{"postgres-database"}}) + } + if _, err := inv.Assign(ctx, "anchor", "db"); err != nil { + t.Fatal(err) + } + machineOf := map[string]string{"shop": "laptop", "wiki": "laptop", "crm": "anchor"} + for _, c := range consumers { + if _, err := inv.Assign(ctx, machineOf[c], c); err != nil { + t.Fatal(err) + } + if err := inv.RecordBindings(ctx, machineOf[c], []inventory.Binding{{Machine: machineOf[c], Consumer: c, + Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil { + t.Fatal(err) + } + } + + at := h0 + say := func(machine string, rs ...link.ResourceHealth) { + t.Helper() + at = at.Add(time.Second) + for i := range rs { + rs[i].Since = at + } + if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil { + t.Fatal(err) + } + } + dbDown := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateUnhealthy, + Reason: "its command: the database refuses connections", Check: "exec"} + dbUp := dbDown + dbUp.State, dbUp.Reason = link.StateHealthy, "" + failing := func(module string) link.ResourceHealth { + return link.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, + State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} + } + fine := func(module string) link.ResourceHealth { + r := failing(module) + r.State, r.Reason = link.StateHealthy, "" + return r + } + openKeys := func() []conditions.Condition { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + return list + } + + // Two looks of the provider down and its consumers failing, in either order on each machine. + for look := 0; look < 2; look++ { + say("laptop", failing("shop"), failing("wiki")) + say("anchor", dbDown, failing("crm")) + } + raised := openKeys() + if len(raised) != 1 || raised[0].Key != "module.db.anchor.unhealthy" { + var keys []string + for _, c := range raised { + keys = append(keys, c.Key) + } + t.Fatalf("one provider down and three consumers failing raised %v; want the provider's alone", keys) + } + c := raised[0] + if c.Severity != conditions.Urgent { + t.Errorf("consumers wait on the provider and its condition is %s", c.Severity) + } + said := c.Evidence[0].Said + for _, w := range []string{"shop on laptop", "wiki on laptop", "crm on anchor"} { + if !strings.Contains(said, w) { + t.Errorf("the provider's condition does not list %s as waiting on it: %s", w, said) + } + } + + // Their gates wait, not fail: the judging is neither a pass nor a fault, past the bound too. + f, err := gatherGateFacts(ctx, open, "") + if err != nil { + t.Fatal(err) + } + f.open, f.openErr, f.judged = raised, nil, false + for _, m := range consumers { + h, why := moduleHealthWord(m, machineOf[m], h0, f) + if h != healthWaiting || !strings.Contains(why, "waits on db on anchor") { + t.Errorf("%s's gate: %v %q; want it waiting on its provider", m, h, why) + } + } + // And a whole judging past the bound puts nothing back: it waits. + long := h0.Add(-time.Hour) + for _, m := range []string{"anchor", "laptop"} { + f.reports[m] = inventory.Reported{Node: m, Outcome: inventory.OutcomeApplied, At: &f.now, Current: true} + } + gatherWas := gatherGateFacts + defer func() { gatherGateFacts = gatherWas }() + gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return f, nil } + g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &long} + verdict, err := judgeMoves(ctx, open, g, []judged{{module: "shop", node: "laptop"}}, time.Now()) + if err != nil || verdict != "" || !strings.Contains(g.Last, "waits on db on anchor") { + t.Fatalf("a held consumer's gate past its bound: verdict %q (%v), last %q; want it waiting", verdict, err, g.Last) + } + + // The provider healthy again: a consumer still failing is now its own, at once (its streak stood). + say("anchor", dbUp, fine("crm")) + say("laptop", failing("shop"), fine("wiki")) + var keys []string + for _, c := range openKeys() { + keys = append(keys, c.Key) + } + if !slices.Equal(keys, []string{"module.shop.laptop.unhealthy"}) { + t.Fatalf("after the provider recovered: %v; want shop's own and nothing else", keys) + } +} + +// A consumer failing while its provider is healthy is raised on its own. +func TestAConsumerFailingBesideAHealthyProviderIsItsOwn(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + register(t, open, catalogue.Manifest{Module: "db", Version: "1", + Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}) + register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}}) + for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} { + if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { + t.Fatal(err) + } + } + if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop", + Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil { + t.Fatal(err) + } + healthy := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateHealthy, Check: "exec"} + shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", State: link.StateUnhealthy, + Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} + for i := 1; i <= 2; i++ { + at := h0.Add(time.Duration(i) * time.Minute) + _ = stateHealth(ctx, inv, conditionsFrom, "anchor", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{healthy}}, at) + if err := stateHealth(ctx, inv, conditionsFrom, "laptop", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{shop}}, at); err != nil { + t.Fatal(err) + } + } + list, _ := conditionsFrom.Open(ctx) + if len(list) != 1 || list[0].Key != "module.shop.laptop.unhealthy" { + t.Fatalf("a consumer failing beside a healthy provider raised %v", list) + } +}