package main import ( "context" "fmt" "sort" "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, to-be 48 §6, Phase C). // // With twelve consumers of the database provision and thirty-six of a route, one provider down would be // twelve conditions for one fault and twelve gates failed for something none of them did. So a consumer's // check names, in `needs`, the provision it exercises; while **that provision's provider for this // consumer** — the one the controller composed the consumer against: its recorded binding (ADR 0232), or // the provider its credential for the provision is from — is unhealthy on the record, what the check finds // is held under the provider's condition: listed there as waiting on it, raised as nothing of its own, and // the consumer's gate waits rather than fails. The provider's condition is urgent while consumers wait. // // **Only what the check finds is held.** A consumer that is down or restarting is its own, whatever its // provider does; so is anything a check that names no provision finds, and anything found while the // provider is healthy. A machine-level fault is never pinned on a module (issue 281), and this does not // change that. // holding is what one reading of the record needs to say who waits on whom: every machine's newest // statement, the open conditions, and the catalogue — read once, asked many times. type holding struct { ctx context.Context inv *inventory.Inventory healths map[string]inventory.NodeHealth open []conditions.Condition shelf map[string]catalogue.Manifest // providers memoises providerFor by machine, consumer and provision. providers map[string]providerLookup // waits is what the waits of a statement are checked against, read as they are asked for (novox/hq issue 450). waits operatorWaitFacts } // checked is a machine's newest statement as the controller judges it: each wait checked (ADR 0283 decision 3), so // a wait that does not check out reads as unhealthy, as it did when the statement was judged (novox/hq issue 450). // What is stored is what the machine said, the waits unchecked; read as stored, a provider whose wait failed its // check seems to wait for the operator while its unhealthy condition is open. func (h *holding) checked(machine string) []inventory.ResourceHealth { rs := h.healths[machine].Resources mods := waitingModules(rs) if len(mods) == 0 { return rs } if h.waits.manifests == nil { h.waits.manifests = map[string]catalogue.Manifest{} } for _, module := range mods { if _, has := h.waits.manifests[module]; !has { if m, ok := h.manifestOf(module); ok { h.waits.manifests[module] = m } } } readWaitFacts(h.ctx, h.inv, machine, mods, nil, &h.waits) return checkWaiting(machine, rs, h.waits) } type providerLookup struct { chosen catalogue.Chosen ok bool } // readHolding reads what the hold is judged from. func readHolding(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) { healths, err := inv.Healths(ctx) if err != nil { return nil, err } return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil } // readHoldingFor is how a judging reads its holding: nothing without a store. A variable so a test can hand a // judging the record it holds under (novox/hq issue 405). var readHoldingFor = func(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) { if inv == nil { return nil, nil } return readHolding(ctx, inv, open) } // heldFinding says a resource's state is a finding of its declared check that names a provision: what // may be held. Down and restarting are liveness, the resource's own. func heldFinding(r inventory.ResourceHealth) bool { return r.State == link.StateUnhealthy && r.Check != "" && r.Needs != "" && r.Reason != "down" && r.Reason != "restarting" } // providerFor is the provider composed for a consumer's provision: its recorded binding, else the // machine its credential for the provision comes from and the module there that provides it. func (h *holding) providerFor(machine, consumer, provision string) (catalogue.Chosen, bool) { key := machine + "\x00" + consumer + "\x00" + provision if p, known := h.providers[key]; known { return p.chosen, p.ok } chosen, ok := h.lookUpProvider(machine, consumer, provision) h.providers[key] = providerLookup{chosen, ok} return chosen, ok } func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue.Chosen, bool) { if bound, err := h.inv.BindingsFor(h.ctx, machine); err == nil { if c, ok := bound[consumer][provision]; ok && c.Node != "" && c.Module != "" { return c, true } } secrets, err := h.inv.SecretsOf(h.ctx, machine, consumer) if err != nil { return catalogue.Chosen{}, false } for _, s := range secrets { if s.Name != provision || s.Provider == "" { continue } if h.shelf == nil { if h.shelf, err = h.inv.Catalogue(h.ctx); err != nil { return catalogue.Chosen{}, false } } assigned, err := h.inv.Assigned(h.ctx, s.Provider) if err != nil { return catalogue.Chosen{}, false } for _, module := range assigned { for _, offer := range h.shelf[module].Offers() { if offer == provision { return catalogue.Chosen{Node: s.Provider, Module: module}, true } } } } return catalogue.Chosen{}, false } // providerState is how a provider stands on the record: unhealthy when its unhealthy condition is open or its // machine's newest statement says a resource of it is unhealthy; else waiting for the operator when that statement // says a resource of it waits, with the waits it names, or its needs-operator condition is open (waits then // unknown); else healthy. func (h *holding) providerState(p catalogue.Chosen) (unhealthy, waiting bool, waits []inventory.Wait) { for _, c := range h.open { if c.Key == moduleUnhealthyKey(p.Module, p.Node) { return true, false, nil } } for _, r := range h.checked(p.Node) { if r.Module != p.Module { continue } switch r.State { case link.StateUnhealthy: return true, false, nil case link.StateWaiting: waiting = true waits = append(waits, r.Waits...) } } if !waiting { for _, c := range h.open { waiting = waiting || c.Key == needsOperatorKey(p.Module, p.Node) } } return false, waiting, waits } // manifestOf is a module's manifest from the catalogue, read once; false when it cannot be read. func (h *holding) manifestOf(module string) (catalogue.Manifest, bool) { if h.shelf == nil && h.inv != nil { shelf, err := h.inv.Catalogue(h.ctx) if err != nil { return catalogue.Manifest{}, false } h.shelf = shelf } m, ok := h.shelf[module] return m, ok } // covering is the waits of a provider that cover a provision it gives (novox/hq issue 405): a module that waits // says nothing else of it is wrong and names each part that waits (ADR 0283 decision 1), so only a consumer of // the waiting part waits on it. A wait covers a provision when its part is that provision, or the secret it waits // for is the provision's shared credential (ADR 0158). Nothing when no wait can be matched: a consumer failing // then is not held, since holding it would hide a fault that may be its own. func (h *holding) covering(p catalogue.Chosen, provision string, waits []inventory.Wait) []inventory.Wait { credential := "" if m, ok := h.manifestOf(p.Module); ok { credential, _ = m.SharedCredentialOf(provision) } var out []inventory.Wait for _, w := range waits { if w.Part == provision || (w.Secret != "" && w.Secret == credential) { out = append(out, w) } } return out } // heldUnderProvider is the provider a consumer's findings are held under, and — when that provider only waits for the // operator, for the part the consumer needs — the waits that hold it. type heldUnderProvider struct { provider catalogue.Chosen // waits is set when every finding held waits on a provider that only waits for the operator: the consumer's // gate then reads as ADR 0254's waits for a person, a pass with the wait carried (ADR 0283 decision 4). waits []inventory.Wait } // heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding // of a check naming a provision whose provider for this consumer is unhealthy on the record, or waits for the // operator for the part that gives it (novox/hq issue 405). False when any is the consumer's own. func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) { by, held := h.heldWith(machine, module, rs) return by.provider, held } func (h *holding) heldWith(machine, module string, rs []inventory.ResourceHealth) (heldUnderProvider, bool) { var on heldUnderProvider onlyWaits := true for _, r := range rs { if r.State != link.StateUnhealthy { continue } if !heldFinding(r) { return heldUnderProvider{}, false } p, ok := h.providerFor(machine, module, r.Needs) if !ok || (p.Node == machine && p.Module == module) { return heldUnderProvider{}, false } unhealthy, waiting, waits := h.providerState(p) switch { case unhealthy: onlyWaits = false case waiting: covered := h.covering(p, r.Needs, waits) if len(covered) == 0 { return heldUnderProvider{}, false } on.waits = append(on.waits, covered...) default: return heldUnderProvider{}, false } on.provider = p } if !onlyWaits { on.waits = nil } return on, on.provider.Module != "" } // waitingUncovered is a provider of a consumer's failing finding that waits for the operator for a part the // finding cannot be matched to (novox/hq issue 405): the consumer is raised on its own, and its condition says // the provider waits, so neither is hidden. func (h *holding) waitingUncovered(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) { for _, r := range rs { if r.State != link.StateUnhealthy || !heldFinding(r) { continue } p, ok := h.providerFor(machine, module, r.Needs) if !ok || (p.Node == machine && p.Module == module) { continue } if unhealthy, waiting, waits := h.providerState(p); !unhealthy && waiting && len(h.covering(p, r.Needs, waits)) == 0 { return p, true } } return catalogue.Chosen{}, false } // waitersOn is every consumer held under a provider, as " on ", sorted. func (h *holding) waitersOn(p catalogue.Chosen) []string { var out []string for machine, nh := range h.healths { byModule := map[string][]inventory.ResourceHealth{} for _, r := range nh.Resources { if r.Module != "" && r.State == link.StateUnhealthy { byModule[r.Module] = append(byModule[r.Module], r) } } for module, rs := range byModule { if on, held := h.heldUnder(machine, module, rs); held && on == p { out = append(out, module+" on "+machine) } } } sort.Strings(out) return out } // heldModules is, for one machine's statement, each module whose finding is held, with the provider. func (h *holding) heldModules(machine string) map[string]heldUnderProvider { out := map[string]heldUnderProvider{} byModule := map[string][]inventory.ResourceHealth{} for _, r := range h.healths[machine].Resources { if r.Module != "" && r.State == link.StateUnhealthy { byModule[r.Module] = append(byModule[r.Module], r) } } for module, rs := range byModule { if on, held := h.heldWith(machine, module, rs); held { out[module] = on } } return out } // moduleUnhealthyKey is a module's health condition's key on a machine. func moduleUnhealthyKey(module, machine string) string { return conditions.ScopeModule + "." + module + "." + machine + ".unhealthy" } // waitingWords is the provider's evidence that consumers wait on it. func waitingWords(waiters []string) string { return fmt.Sprintf("waiting on it — %s", strings.Join(waiters, ", ")) }