mesh-controller/cmd/mesh-controller/provider_hold.go

package main

import (
	"context"
	"fmt"
	"sort"
	"strings"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, to-be 48 §6, Phase C).
//
// With twelve consumers of the database provision and thirty-six of a route, one provider down would be
// twelve conditions for one fault and twelve gates failed for something none of them did. So a consumer's
// check names, in `needs`, the provision it exercises; while **that provision's provider for this
// consumer** — the one the controller composed the consumer against: its recorded binding (ADR 0232), or
// the provider its credential for the provision is from — is unhealthy on the record, what the check finds
// is held under the provider's condition: listed there as waiting on it, raised as nothing of its own, and
// the consumer's gate waits rather than fails. The provider's condition is urgent while consumers wait.
//
// **Only what the check finds is held.** A consumer that is down or restarting is its own, whatever its
// provider does; so is anything a check that names no provision finds, and anything found while the
// provider is healthy. A machine-level fault is never pinned on a module (issue 281), and this does not
// change that.

// holding is what one reading of the record needs to say who waits on whom: every machine's newest
// statement, the open conditions, and the catalogue — read once, asked many times.
type holding struct {
	ctx     context.Context
	inv     *inventory.Inventory
	healths map[string]inventory.NodeHealth
	open    []conditions.Condition
	shelf   map[string]catalogue.Manifest
	// providers memoises providerFor by machine, consumer and provision.
	providers map[string]providerLookup
}

type providerLookup struct {
	chosen catalogue.Chosen
	ok     bool
}

// readHolding reads what the hold is judged from.
func readHolding(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
	healths, err := inv.Healths(ctx)
	if err != nil {
		return nil, err
	}
	return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
}

// heldFinding says a resource's state is a finding of its declared check that names a provision: what
// may be held. Down and restarting are liveness, the resource's own.
func heldFinding(r inventory.ResourceHealth) bool {
	return r.State == link.StateUnhealthy && r.Check != "" && r.Needs != "" &&
		r.Reason != "down" && r.Reason != "restarting"
}

// providerFor is the provider composed for a consumer's provision: its recorded binding, else the
// machine its credential for the provision comes from and the module there that provides it.
func (h *holding) providerFor(machine, consumer, provision string) (catalogue.Chosen, bool) {
	key := machine + "\x00" + consumer + "\x00" + provision
	if p, known := h.providers[key]; known {
		return p.chosen, p.ok
	}
	chosen, ok := h.lookUpProvider(machine, consumer, provision)
	h.providers[key] = providerLookup{chosen, ok}
	return chosen, ok
}

func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue.Chosen, bool) {
	if bound, err := h.inv.BindingsFor(h.ctx, machine); err == nil {
		if c, ok := bound[consumer][provision]; ok && c.Node != "" && c.Module != "" {
			return c, true
		}
	}
	secrets, err := h.inv.SecretsOf(h.ctx, machine, consumer)
	if err != nil {
		return catalogue.Chosen{}, false
	}
	for _, s := range secrets {
		if s.Name != provision || s.Provider == "" {
			continue
		}
		if h.shelf == nil {
			if h.shelf, err = h.inv.Catalogue(h.ctx); err != nil {
				return catalogue.Chosen{}, false
			}
		}
		assigned, err := h.inv.Assigned(h.ctx, s.Provider)
		if err != nil {
			return catalogue.Chosen{}, false
		}
		for _, module := range assigned {
			for _, offer := range h.shelf[module].Offers() {
				if offer == provision {
					return catalogue.Chosen{Node: s.Provider, Module: module}, true
				}
			}
		}
	}
	return catalogue.Chosen{}, false
}

// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest
// statement says a resource of it is unhealthy.
func (h *holding) unhealthy(p catalogue.Chosen) bool {
	key := moduleUnhealthyKey(p.Module, p.Node)
	for _, c := range h.open {
		if c.Key == key {
			return true
		}
	}
	for _, r := range h.healths[p.Node].Resources {
		if r.Module == p.Module && r.State == link.StateUnhealthy {
			return true
		}
	}
	return false
}

// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
// of a check naming a provision whose provider for this consumer is unhealthy on the record. False when any
// is the consumer's own.
func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
	var on catalogue.Chosen
	for _, r := range rs {
		if r.State != link.StateUnhealthy {
			continue
		}
		if !heldFinding(r) {
			return catalogue.Chosen{}, false
		}
		p, ok := h.providerFor(machine, module, r.Needs)
		if !ok || (p.Node == machine && p.Module == module) || !h.unhealthy(p) {
			return catalogue.Chosen{}, false
		}
		on = p
	}
	return on, on.Module != ""
}

// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
func (h *holding) waitersOn(p catalogue.Chosen) []string {
	var out []string
	for machine, nh := range h.healths {
		byModule := map[string][]inventory.ResourceHealth{}
		for _, r := range nh.Resources {
			if r.Module != "" && r.State == link.StateUnhealthy {
				byModule[r.Module] = append(byModule[r.Module], r)
			}
		}
		for module, rs := range byModule {
			if on, held := h.heldUnder(machine, module, rs); held && on == p {
				out = append(out, module+" on "+machine)
			}
		}
	}
	sort.Strings(out)
	return out
}

// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
func (h *holding) heldModules(machine string) map[string]catalogue.Chosen {
	out := map[string]catalogue.Chosen{}
	byModule := map[string][]inventory.ResourceHealth{}
	for _, r := range h.healths[machine].Resources {
		if r.Module != "" && r.State == link.StateUnhealthy {
			byModule[r.Module] = append(byModule[r.Module], r)
		}
	}
	for module, rs := range byModule {
		if on, held := h.heldUnder(machine, module, rs); held {
			out[module] = on
		}
	}
	return out
}

// moduleUnhealthyKey is a module's health condition's key on a machine.
func moduleUnhealthyKey(module, machine string) string {
	return conditions.ScopeModule + "." + module + "." + machine + ".unhealthy"
}

// waitingWords is the provider's evidence that consumers wait on it.
func waitingWords(waiters []string) string {
	return fmt.Sprintf("waiting on it — %s", strings.Join(waiters, ", "))
}