mesh-delivery/modules/mesh-delivery-assigner/internal/assign/wants.go

package assign

import (
	"fmt"
	"slices"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
type Module struct {
	Name     string
	FP       string
	Built    bool
	Held     bool
	Granted  bool
	Manifest catalogue.Manifest
	Nodes    []string
	// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
	// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
	Composed func(node, fp string) (delivery.ComposedDigest, bool)
	// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
	// with this digest (ADR 0539 §14). Nil: none did.
	Allowed func(node, digest string) bool
	// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
	Released bool
	// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
	// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
	Restored func(node string) bool
	// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
	Kept bool
	// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
	Now time.Time
}

func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
	if m.Composed == nil {
		return delivery.ComposedDigest{}, false
	}
	return m.Composed(node, fp)
}

// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
type Nodes struct {
	Online map[string]bool
	Haves  map[string]map[string]delivery.Have
	// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
	Providers map[string]map[string][]string
}

// healthy says whether a have runs fp healthy.
func healthy(h delivery.Have, fp string) bool {
	return h.FP == fp && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
}

// Canary is the module's canary: the first of its nodes, by name, that is online (ADR 0435 §3; one ring, ordered by
// name, ADR 0528 clause 3). Empty when none is online.
func Canary(nodes []string, online map[string]bool) string {
	sorted := slices.Sorted(slices.Values(nodes))
	for _, n := range sorted {
		if online[n] {
			return n
		}
	}
	return ""
}

// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
// limit (ADR 0528 clause 3).
func Holds(m Module, node string, n Nodes) (bool, int, string) {
	switch {
	case !m.Built:
		return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
	case m.Held:
		return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
	}
	canary := Canary(m.Nodes, n.Online)
	switch {
	case canary == "":
		return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
	case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
		return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
	}
	for _, provision := range m.Manifest.Requires {
		if ok, why := providedHealthy(provision, node, n); !ok {
			return false, delivery.RuleProviders, why
		}
	}
	if !m.Granted {
		return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
	}
	if _, ok := m.composition(node, m.FP); !ok {
		return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
	}
	return true, 0, ""
}

// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
// offers it) and holds.
func providedHealthy(provision, node string, n Nodes) (bool, string) {
	providers := n.Providers[provision]
	if len(providers) == 0 {
		return true, ""
	}
	names := make([]string, 0, len(providers))
	for p := range providers {
		names = append(names, p)
	}
	sort.Strings(names)
	for _, p := range names {
		nodes := providers[p]
		at := node
		if !slices.Contains(nodes, node) {
			at = Canary(nodes, n.Online)
		}
		if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
			return true, ""
		}
	}
	return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
}

// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
// valuable or rebuildable; cache and none do not).
func keepsData(m catalogue.Manifest) bool {
	for _, it := range m.DataItems() {
		if it.Class == catalogue.ClassIrreplaceable || it.Class == catalogue.ClassValuable || it.Class == catalogue.ClassRebuildable {
			return true
		}
	}
	return false
}

// Outcome is what one module's pass decides: each node's want to write (only those that change), and a hold to write.
type Outcome struct {
	Wants map[string]delivery.Want
	// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
	Held *delivery.Held
	// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
	// record and its ask are taken away, and the wants move by the rules again in this pass.
	Lifted bool
	// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
	Record []RecordNeed
	// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
	// back: the operator is asked whether to restore each (ADR 0539 §19).
	Restore []string
	// Rules is, by node, the rule holding the node back from m.FP and why.
	Rules map[string]Holding
}

// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
// controller's judgement of the move and its words.
type RecordNeed struct {
	Node      string
	Digest    string
	Judgement delivery.Judgement
	Words     string
}

// Holding is a rule holding a node back, and why.
type Holding struct {
	Rule  int
	Words string
}

// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
// module holding data is not set back but asked about (Restore). A
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
	out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
	held := m.Held
	if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
		out.Lifted, held = true, false
	}
	if !held && !out.Lifted && !m.Released && m.FP != "" {
		for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
			if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
				out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
				held = true
				break
			}
		}
	}
	mm := m
	mm.Held = held
	for _, node := range m.Nodes {
		if m.FP == "" {
			break
		}
		before, had := wants[node]
		var next delivery.Want
		// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
		// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
		// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
		if w, ok := m.userListRevision(node, before, had, n); ok {
			if !had || !delivery.SameWant(normal(before), normal(w)) {
				out.Wants[node] = w
			}
			continue
		}
		switch ok, rule, words := Holds(mm, node, n); {
		case ok:
			d, _ := m.composition(node, m.FP)
			next = delivery.Want{FP: m.FP, Composed: d.Digest}
			if had && !before.Removed && before.FP == m.FP && before.Composed == d.Digest {
				next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
			}
			running, ran := n.Haves[node][m.Name]
			if need, wait := m.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
				if need != nil {
					out.Record = append(out.Record, *need)
				}
				continue
			}
		case held && had && before.FP == m.FP:
			out.Rules[node] = Holding{Rule: rule, Words: words}
			last := lastHealthy[node]
			switch {
			case keepsData(m.Manifest) && (m.Restored == nil || !m.Restored(node)):
				next = before
				next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked whether to restore it", m.Name, short(m.FP))
				out.Restore = append(out.Restore, node)
			case last == "":
				next = before
				next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node)
			default:
				d, there := m.composition(node, last)
				if !there {
					next = before
					next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
					break
				}
				next = delivery.Want{FP: last, Composed: d.Digest, Newest: m.FP, Rule: rule, Words: words}
			}
		default:
			out.Rules[node] = Holding{Rule: rule, Words: words}
			if !had || before.Removed {
				continue
			}
			next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
			if before.FP == m.FP {
				next = delivery.Want{FP: m.FP, Composed: before.Composed}
			}
		}
		if !had || !delivery.SameWant(normal(before), normal(next)) {
			out.Wants[node] = next
		}
	}
	for node, before := range wants {
		if !slices.Contains(m.Nodes, node) && !before.Removed {
			out.Wants[node] = delivery.Want{Removed: true, Words: fmt.Sprintf("%s is no longer assigned to %s", m.Name, node)}
		}
	}
	return out
}

// allRunHealthy says whether every node of m that runs the module (has a have of it) runs m.FP healthy, and at least one does:
// a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven modules stayed
// held on every machine after one transient failed check). A node set back to another fingerprint does not run m.FP, so its
// hold stays.
func allRunHealthy(m Module, n Nodes) bool {
	ran := 0
	for _, node := range m.Nodes {
		h, ok := n.Haves[node][m.Name]
		if !ok {
			continue
		}
		if !healthy(h, m.FP) {
			return false
		}
		ran++
	}
	return ran > 0
}

// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
// it, and the node's own words. The node is the held record's own field.
func HeldWords(h delivery.Have, at time.Time) string {
	words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
	if !at.IsZero() {
		words += ", seen at " + at.UTC().Format(time.RFC3339)
	}
	if d := strings.TrimSpace(h.Detail); d != "" {
		return words + ": " + d
	}
	return words + ": the node gave no words"
}

// LiftedWords say why a hold was lifted by itself.
func LiftedWords(module, fp string) string {
	return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
}

// normal is a want without its version, as a decoded and a composed one are compared.
func normal(w delivery.Want) delivery.Want {
	w.V = 0
	return w
}

func short(fp string) string {
	part, err := delivery.FingerprintKeyPart(fp)
	if err != nil || len(part) < 12 {
		return fp
	}
	return part[:12]
}

// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus,
// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of
// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the
// same fingerprint judged neither (a user list only, §7).
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
	if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
		return nil, false
	}
	// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
	// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
	// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
	// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
	if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
		return nil, false
	}
	// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
	prev := before.FP
	if !had {
		prev = running
	}
	if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
		return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
	}
	if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
		return nil, false
	}
	if m.Allowed != nil && m.Allowed(node, d.Digest) {
		return nil, false
	}
	return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
}

// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d delivery.ComposedDigest) bool {
	if d.Policy != "" {
		return d.Policy == delivery.PolicyRecord
	}
	policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
	return policy == catalogue.PolicyRecord
}

// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
func (m Module) userListRevision(node string, before delivery.Want, had bool, n Nodes) (delivery.Want, bool) {
	if !catalogue.ProvidesBus(m.Manifest) {
		return delivery.Want{}, false
	}
	running, ran := n.Haves[node][m.Name]
	if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) {
		return delivery.Want{}, false
	}
	d, ok := m.composition(node, running.FP)
	if !ok || !d.UserListOnly {
		return delivery.Want{}, false
	}
	w := delivery.Want{FP: running.FP, Composed: d.Digest}
	if had && before.Composed == d.Digest {
		w.Backup, w.Words = before.Backup, before.Words
	}
	return w, true
}