mesh-delivery/modules/mesh-delivery-assigner/internal/assign/wants.go

package assign

import (
	"fmt"
	"slices"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
type Module struct {
	Name    string
	FP      string
	Built   bool
	Held    bool
	Granted bool
	// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
	// that runs an older one names it (issue 1945). Nil: none.
	GrantedAt func(fp string) bool
	// GrantedOn says whether the controller's grant of FP names the module's accounts on a node, when the grant names any:
	// a grant written for the module's other nodes says nothing of a node whose account is not issued and sealed (issue
	// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
	GrantedOn func(node string) bool
	Manifest  catalogue.Manifest
	Nodes     []string
	// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
	// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
	Composed func(node, fp string) (delivery.ComposedDigest, bool)
	// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
	// with this digest (ADR 0539 §14). Nil: none did.
	Allowed func(node, digest string) bool
	// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
	Released bool
	// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
	// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
	Restored func(node string) bool
	// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
	Kept bool
	// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
	Now time.Time
	// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
	Puts map[string]delivery.Put
}

func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
	if m.Composed == nil {
		return delivery.ComposedDigest{}, false
	}
	return m.Composed(node, fp)
}

// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
type Nodes struct {
	Online map[string]bool
	Haves  map[string]map[string]delivery.Have
	// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
	Providers map[string]map[string][]string
}

// healthy says whether a have runs fp healthy.
func healthy(h delivery.Have, fp string) bool {
	return h.FP == fp && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
}

// Canary is the module's canary: the first of its nodes, by name, that is online (ADR 0435 §3; one ring, ordered by
// name, ADR 0528 clause 3). Empty when none is online.
func Canary(nodes []string, online map[string]bool) string {
	sorted := slices.Sorted(slices.Values(nodes))
	for _, n := range sorted {
		if online[n] {
			return n
		}
	}
	return ""
}

// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
// limit (ADR 0528 clause 3).
func Holds(m Module, node string, n Nodes) (bool, int, string) {
	return holds(m, node, n, true)
}

// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
// composition.
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
	switch {
	case !m.Built:
		return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
	case m.Held:
		return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
	}
	if ordering {
		canary := Canary(m.Nodes, n.Online)
		switch {
		case canary == "":
			return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
		case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
			return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
		}
		for _, provision := range m.Manifest.Requires {
			if ok, why := providedHealthy(provision, node, n); !ok {
				return false, delivery.RuleProviders, why
			}
		}
	}
	if !m.Granted {
		return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
	}
	if m.GrantedOn != nil && !m.GrantedOn(node) {
		return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
			short(m.FP))
	}
	if _, ok := m.composition(node, m.FP); !ok {
		return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
	}
	return true, 0, ""
}

// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
// offers it) and holds.
func providedHealthy(provision, node string, n Nodes) (bool, string) {
	providers := n.Providers[provision]
	if len(providers) == 0 {
		return true, ""
	}
	names := make([]string, 0, len(providers))
	for p := range providers {
		names = append(names, p)
	}
	sort.Strings(names)
	for _, p := range names {
		nodes := providers[p]
		at := node
		if !slices.Contains(nodes, node) {
			at = Canary(nodes, n.Online)
		}
		if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
			return true, ""
		}
	}
	return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
}

// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
// valuable or rebuildable; cache and none do not).
func keepsData(m catalogue.Manifest) bool {
	for _, it := range m.DataItems() {
		if it.Class == catalogue.ClassIrreplaceable || it.Class == catalogue.ClassValuable || it.Class == catalogue.ClassRebuildable {
			return true
		}
	}
	return false
}

// Outcome is what one module's pass decides: each node's want to write (only those that change), and a hold to write.
type Outcome struct {
	Wants map[string]delivery.Want
	// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
	Held *delivery.Held
	// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
	// record and its ask are taken away, and the wants move by the rules again in this pass.
	Lifted bool
	// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
	Record []RecordNeed
	// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
	// back: the operator is asked whether to restore each (ADR 0539 §19).
	Restore []string
	// Rules is, by node, the rule holding the node back from m.FP and why.
	Rules map[string]Holding
	// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
	// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
	// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
	// own composition and is never here.
	Neither map[string]Holding
}

// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
// controller's judgement of the move and its words.
type RecordNeed struct {
	Node      string
	Digest    string
	Judgement delivery.Judgement
	Words     string
}

// Holding is a rule holding a node back, and why.
type Holding struct {
	Rule  int
	Words string
}

// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
// module holding data is not set back but asked about (Restore). A
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
	out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
	held := m.Held
	if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
		out.Lifted, held = true, false
	}
	if !held && !out.Lifted && !m.Released && m.FP != "" {
		for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
			if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
				out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
				held = true
				break
			}
		}
	}
	mm := m
	mm.Held = held
	for _, node := range m.Nodes {
		if m.FP == "" {
			break
		}
		before, had := wants[node]
		var next delivery.Want
		p, put := m.Puts[node]
		if !put && had && before.Put && before.Hold {
			// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
			// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
			// without hold is not kept this way: its put takes it further once heard.
			p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
				Step: delivery.PutHolding}, true
		}
		if put {
			if want, owns := PutOwns(p); owns {
				if want != nil {
					w := *want
					// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
					if ul, ok := m.userListRevision(node, before, had, n); ok && ul.FP == w.FP {
						w.Composed = ul.Composed
					}
					if !had || !delivery.SameWant(normal(before), normal(w)) {
						out.Wants[node] = w
					}
				}
				continue
			}
		}
		// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
		// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
		// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
		if w, ok := m.userListRevision(node, before, had, n); ok {
			if !had || !delivery.SameWant(normal(before), normal(w)) {
				out.Wants[node] = w
			}
			continue
		}
		ok, rule, words := Holds(mm, node, n)
		target := m.FP
		if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
			// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
			// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
			// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
			// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
			// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
			// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
			// order the move from it as from any want.
			switch _, composed := m.composition(node, have.FP); {
			case !ran:
			case have.FP == m.FP:
				ok, rule, words = holds(mm, node, n, false)
			case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
				// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
				// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
				// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
				// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
			case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
				ok, target = true, have.FP
			}
		}
		switch {
		case ok:
			d, _ := m.composition(node, target)
			next = delivery.Want{FP: target, Composed: d.Digest}
			at := m
			if target != m.FP {
				// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
				next.Newest, next.Rule, next.Words = m.FP, rule, words
				at.FP = target
			}
			if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
				next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
			}
			running, ran := n.Haves[node][m.Name]
			// A removed want is no want: a first want after it goes through the record ask as a first want does.
			asked, hadLive := before, had
			if before.Removed {
				asked, hadLive = delivery.Want{}, false
			}
			if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
				if need != nil {
					out.Record = append(out.Record, *need)
				}
				continue
			}
		case held && had && before.FP == m.FP:
			out.Rules[node] = Holding{Rule: rule, Words: words}
			last := lastHealthy[node]
			switch {
			case keepsData(m.Manifest) && (m.Restored == nil || !m.Restored(node)):
				next = before
				next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked whether to restore it", m.Name, short(m.FP))
				out.Restore = append(out.Restore, node)
			case last == "":
				next = before
				next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node)
			default:
				d, there := m.composition(node, last)
				if !there {
					next = before
					next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
					break
				}
				next = delivery.Want{FP: last, Composed: d.Digest, Newest: m.FP, Rule: rule, Words: words}
			}
		default:
			out.Rules[node] = Holding{Rule: rule, Words: words}
			if !had || before.Removed {
				// The node runs the module, or its have names it, and gets no want for it: the module is on neither list (issue 1945), and
				// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
				// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
				// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
				h, there := n.Haves[node][m.Name]
				switch {
				case there && h.Phase != delivery.PhaseRemoved && !held:
					out.Neither[node] = Holding{Rule: rule, Words: words}
				case had && (!there || h.Phase == delivery.PhaseRemoved):
					// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
					// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
					// would have its node agent remove the module and stop holding its composition's resources back.
					next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
					if !delivery.SameWant(normal(before), normal(next)) {
						out.Wants[node] = next
					}
				}
				continue
			}
			next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
			if before.FP == m.FP {
				// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
				next = delivery.Want{FP: m.FP, Composed: before.Composed}
			} else if d, there := m.composition(node, before.FP); there && d.Digest != "" && d.Digest != before.Composed {
				// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
				// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
				// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
				at := m
				at.FP = before.FP
				running, ran := n.Haves[node][m.Name]
				need, wait := at.mustAsk(node, before, true, ran, running.FP, running.Composed == "", d)
				if need != nil {
					out.Record = append(out.Record, *need)
				}
				if !wait {
					next.Composed = d.Digest
				}
			}
		}
		if !had || !delivery.SameWant(normal(before), normal(next)) {
			out.Wants[node] = next
		}
	}
	for node, before := range wants {
		if !slices.Contains(m.Nodes, node) && !before.Removed {
			out.Wants[node] = delivery.Want{Removed: true, Words: fmt.Sprintf("%s is no longer assigned to %s", m.Name, node)}
		}
	}
	return out
}

// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
func NoWantWords(rule int, words string) string {
	return fmt.Sprintf("no want: rule %d: %s", rule, words)
}

// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
// moving). A failed health while it
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
// the world's lastHealthy) already asks for phase running.
func failed(h delivery.Have) bool {
	return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
}

// runs says whether a have is of a module the node runs: running, whatever its health.
func runs(h delivery.Have) bool {
	return h.Phase == delivery.PhaseRunning
}

// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
// at least one does: a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven
// modules stayed held on every machine after one transient failed check). A node set back to another fingerprint does not run
// m.FP, and a failed one does not run it healthy, so the hold stays; a have that is removed, held for its backup or still moving
// (backing up, applying, checking) is not counted, or it would keep the hold up for ever.
func allRunHealthy(m Module, n Nodes) bool {
	ran := 0
	for _, node := range m.Nodes {
		h, ok := n.Haves[node][m.Name]
		if !ok || (h.Phase != delivery.PhaseRunning && h.Phase != delivery.PhaseFailed) {
			continue
		}
		if !healthy(h, m.FP) {
			return false
		}
		ran++
	}
	return ran > 0
}

// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
// it, and the node's own words. The node is the held record's own field.
func HeldWords(h delivery.Have, at time.Time) string {
	words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
	if !at.IsZero() {
		words += ", seen at " + at.UTC().Format(time.RFC3339)
	}
	if d := strings.TrimSpace(h.Detail); d != "" {
		return words + ": " + d
	}
	return words + ": the node gave no words"
}

// LiftedWords say why a hold was lifted by itself.
func LiftedWords(module, fp string) string {
	return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
}

// normal is a want without its version, as a decoded and a composed one are compared.
func normal(w delivery.Want) delivery.Want {
	w.V = 0
	return w
}

func short(fp string) string {
	part, err := delivery.FingerprintKeyPart(fp)
	if err != nil || len(part) < 12 {
		return fp
	}
	return part[:12]
}

// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus,
// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of
// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the
// same fingerprint judged neither (a user list only, §7).
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
	if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
		return nil, false
	}
	// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
	// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
	// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
	// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
	if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
		return nil, false
	}
	// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
	prev := before.FP
	if !had {
		prev = running
	}
	if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
		return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
	}
	if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
		return nil, false
	}
	if m.Allowed != nil && m.Allowed(node, d.Digest) {
		return nil, false
	}
	return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
}

// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d delivery.ComposedDigest) bool {
	if d.Policy != "" {
		return d.Policy == delivery.PolicyRecord
	}
	policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
	return policy == catalogue.PolicyRecord
}

// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
func (m Module) userListRevision(node string, before delivery.Want, had bool, n Nodes) (delivery.Want, bool) {
	if !catalogue.ProvidesBus(m.Manifest) {
		return delivery.Want{}, false
	}
	running, ran := n.Haves[node][m.Name]
	if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) {
		return delivery.Want{}, false
	}
	d, ok := m.composition(node, running.FP)
	if !ok || !d.UserListOnly {
		return delivery.Want{}, false
	}
	w := delivery.Want{FP: running.FP, Composed: d.Digest}
	if had && before.Composed == d.Digest {
		w.Backup, w.Words = before.Backup, before.Words
	}
	return w, true
}

// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's ControllerGrant.NamesNode, which a
// put reads by too).
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
	return g.NamesNode(node, module)
}