bus-state is answered with no credential of a consumer's own, so ADR 0603 §5 counts the provider's own have: the assigner reads, per pass, what the running healthy builds on online nodes offer (their manifests through the builder's builds) and holds a consumer with a bus-state entry under the provisions' rule until one offers it, handed or not and with no provisioner key. The words name bus state, and the delivery view names the edge from them. Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
785 lines
38 KiB
Go
785 lines
38 KiB
Go
package assign
|
|
|
|
import (
|
|
"fmt"
|
|
"maps"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/provisions"
|
|
|
|
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
|
)
|
|
|
|
// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
|
|
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
|
|
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
|
|
type Module struct {
|
|
Name string
|
|
FP string
|
|
Built bool
|
|
Held bool
|
|
Granted bool
|
|
// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
|
|
// that runs an older one names it (issue 1945). Nil: none.
|
|
GrantedAt func(fp string) bool
|
|
// GrantedOn says whether the controller's grant of FP names the module's accounts on a node, when the grant names any:
|
|
// a grant written for the module's other nodes says nothing of a node whose account is not issued and sealed (issue
|
|
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
|
|
GrantedOn func(node string) bool
|
|
Manifest catalogue.Manifest
|
|
// Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want
|
|
// waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it.
|
|
Provisions func(node string) map[string]provisions.Provision
|
|
// Answering is the manifests of the modules placed somewhere that provide a provision: what catalogue.Awaited
|
|
// judges a want's provisions by (issue 774). Nil: none known, and no key is waited for that is not there.
|
|
Answering func(provision string) []catalogue.Manifest
|
|
// Offered says whether any module offers a provision by name or by its own name (catalogue.Manifest.Offers) among every
|
|
// manifest the trunk holds and the modules the mesh provides itself, placed or not. Nil: not known, and no requirement
|
|
// is waited on for want of an offer.
|
|
Offered func(provision string) bool
|
|
// Invokes are the seat verbs and module tools the module graph says the module invokes and keeps as edges to the
|
|
// modules serving them, those it leaves out on a cycle not among them (ADR 0614 §2, §3): the want waits until each is
|
|
// served (Served). Nil: no graph node is read, and no want waits on a verb.
|
|
Invokes []string
|
|
// Served says whether a build serving a seat verb or module tool runs healthy, counted from its holder's have (ADR 0614
|
|
// §2, as ADR 0603 §5 counts a provider's). Nil: nothing is read of what runs, and no want waits on a verb.
|
|
Served func(invoke string) bool
|
|
// Offering says whether a build whose manifest offers a provision runs healthy on a node present, counted from the
|
|
// provider's own have (ADR 0603 §5): what a want asking bus state waits for (ADR 0614 §5). Nil: nothing is read of what
|
|
// runs, and no want waits on bus state.
|
|
Offering func(provision string) bool
|
|
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
|
|
Chosen *catalogue.PersonPolicy
|
|
Nodes []string
|
|
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
|
|
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
|
|
Composed func(node, fp string) (delivery.ComposedDigest, bool)
|
|
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
|
|
// with this digest (ADR 0539 §14). Nil: none did.
|
|
Allowed func(node, digest string) bool
|
|
// Release names the release whose Approve carries the build fp, as mesh-delivery's gate holds it for the release's heads,
|
|
// read by the build's source revision (ADR 0601 §1): that Approve is the yes to a move of the build alone. False when no
|
|
// approved release is known to carry it. Nil: none is read.
|
|
Release func(fp string) (string, bool)
|
|
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
|
|
Released bool
|
|
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
|
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
|
|
Restored func(node string) bool
|
|
// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
|
|
Kept bool
|
|
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
|
|
Now time.Time
|
|
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
|
|
Puts map[string]delivery.Put
|
|
// Needs are the reader capabilities the manifest at FP needs (the indexer's modules value, catalogue.ManifestNeeds, ADR
|
|
// 0614 §4): a node's want moves only while that node's engine states each of them. Nil: none.
|
|
Needs []string
|
|
// Self says whether node composes the module itself: the catalogue's placement names the node (ADR 0547 §2). It is the one
|
|
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
|
|
// the node was placed does not outlive the placement (issue 1126). Nil: no node does.
|
|
Self func(node string) bool
|
|
}
|
|
|
|
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
|
|
|
|
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
|
if m.Composed == nil {
|
|
return delivery.ComposedDigest{}, false
|
|
}
|
|
return m.Composed(node, fp)
|
|
}
|
|
|
|
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
|
|
type Nodes struct {
|
|
Online map[string]bool
|
|
Haves map[string]map[string]delivery.Have
|
|
// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
|
|
Providers map[string]map[string][]string
|
|
// Reads is, by node, the reader capabilities its node-engine states in its node record (delivery.NodeRecord.ManifestReads,
|
|
// ADR 0614 §4). A node with none, or none read, is read as stating the baseline (catalogue.StatedManifestReads).
|
|
Reads map[string][]string
|
|
}
|
|
|
|
// healthy says whether a have runs fp healthy.
|
|
func healthy(h delivery.Have, fp string) bool {
|
|
return h.FP == fp && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
|
|
}
|
|
|
|
// Canary is the module's canary: the first of its nodes, by name, that is online (ADR 0435 §3; one ring, ordered by
|
|
// name, ADR 0528 clause 3). Empty when none is online.
|
|
func Canary(nodes []string, online map[string]bool) string {
|
|
sorted := slices.Sorted(slices.Values(nodes))
|
|
for _, n := range sorted {
|
|
if online[n] {
|
|
return n
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
|
|
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
|
|
// limit (ADR 0528 clause 3). Rule 5, every provider healthy, holds no want (issue 1960): a module whose provider is down
|
|
// is wanted all the same, and said as not reached (Unreached), since a site runs while the proxy in front of it is
|
|
// down and one module never blocks another.
|
|
func Holds(m Module, node string, n Nodes) (bool, int, string) {
|
|
return holds(m, node, n, true)
|
|
}
|
|
|
|
// holds is Holds; with ordering false it leaves out the canary (3), which orders a move among nodes, and keeps those
|
|
// without which there is nothing to want: the build done, not held, the grant and the composition.
|
|
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
|
switch {
|
|
case !m.Built:
|
|
return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
|
|
case m.Held:
|
|
return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
|
|
}
|
|
if ordering {
|
|
canary := Canary(m.Nodes, n.Online)
|
|
switch {
|
|
case canary == "":
|
|
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
|
|
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
|
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
|
}
|
|
}
|
|
if !m.Granted {
|
|
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
|
|
}
|
|
if m.GrantedOn != nil && !m.GrantedOn(node) {
|
|
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
|
|
short(m.FP))
|
|
}
|
|
if words, lacking := m.readerLacking(node, n); lacking {
|
|
return false, delivery.RuleComposed, words
|
|
}
|
|
if _, ok := m.composition(node, m.FP); !ok {
|
|
return false, delivery.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
|
|
}
|
|
if words, waiting := m.provisionWaiting(node); waiting {
|
|
return false, delivery.RuleProvisions, words
|
|
}
|
|
if words, waiting := m.verbWaiting(node); waiting {
|
|
return false, delivery.RuleProvisions, words
|
|
}
|
|
if words, waiting := m.busStateWaiting(node); waiting {
|
|
return false, delivery.RuleProvisions, words
|
|
}
|
|
return true, 0, ""
|
|
}
|
|
|
|
// readerLacking says why the module's want on node is held for the node's engine (ADR 0614 §4): a reader capability its
|
|
// manifest needs that the engine does not state, read as the baseline when it states none. It is held under rule 7 (the
|
|
// composition), since no rule of its own is numbered in this release, with words naming the capability; it moves once the
|
|
// engine states it. The node-engine's own new build is held only by a word of its own manifest no engine states.
|
|
func (m Module) readerLacking(node string, n Nodes) (string, bool) {
|
|
missing := catalogue.MissingManifestReads(m.Needs, n.Reads[node])
|
|
if len(missing) == 0 {
|
|
return "", false
|
|
}
|
|
return graph.ReaderWaitWords(node, missing, m.Name, short(m.FP)), true // the words the delivery view names the edge by
|
|
}
|
|
|
|
// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has
|
|
// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none.
|
|
func (m Module) provisionWaiting(node string) (string, bool) {
|
|
if m.Provisions == nil {
|
|
return "", false
|
|
}
|
|
if missing := m.MissingProvisions(node); len(missing) > 0 {
|
|
return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true
|
|
}
|
|
keyed := m.Provisions(node)
|
|
// Only a provision the newest manifest still requires is waited on (ADR 0603 §9): a key of an older revision's edge,
|
|
// asked or retiring for the operator's answer (ADR 0570 §5), holds nothing. A requirement catalogue.Awaited leaves out is
|
|
// not waited on when something offers it (a module's own name, a module the mesh provides itself, a provision the
|
|
// composition does not key: optional, a toolchain on the module's own machine, a login the controller makes). One that
|
|
// nothing offers is waited on (ADR 0603 §5: an unknown provider has provided nothing), so a consumer never moves ahead of
|
|
// a provider whose module the indexer has not yet taken in; it holds loudly, since the watchman raises the hold past its
|
|
// bound (ADR 0603 §8).
|
|
needed := m.Manifest.Requires
|
|
if m.Answering != nil {
|
|
needed = catalogue.Awaited(m.Manifest, m.Answering)
|
|
have := map[string]bool{}
|
|
for _, p := range keyed {
|
|
have[p.Provision] = true
|
|
}
|
|
for _, r := range slices.Sorted(slices.Values(m.Manifest.Requires)) {
|
|
if have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(needed, r) {
|
|
needed = append(needed, r) // the provisioner keys it from the manifests, whoever is placed yet
|
|
continue
|
|
}
|
|
if m.Offered == nil || m.Manifest.OptionalRequires[r] || len(m.Answering(r)) > 0 || m.Offered(r) || slices.Contains(needed, r) {
|
|
continue
|
|
}
|
|
needed = append(needed, r)
|
|
if !have[r] {
|
|
return fmt.Sprintf("no module offers %s yet for %s on %s: an unknown provider has provided nothing, so the want waits for one, and the watchman raises it past its bound", r, m.Name, node), true
|
|
}
|
|
}
|
|
}
|
|
for _, key := range slices.Sorted(maps.Keys(keyed)) {
|
|
if p := keyed[key]; p.State != provisions.Provided && slices.Contains(needed, p.Provision) {
|
|
words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
|
|
if p.Why != "" {
|
|
words += ": " + p.Why
|
|
}
|
|
return words, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
// MissingProvisions are the provisions a want of the module waits for (catalogue.Awaited, the one rule the plan
|
|
// composition and the seat mesh-provisioner key by, issue 774) that have no key on node yet, sorted; none while the
|
|
// seat's state is not read. A requirement the composition keys nothing for (a toolchain on the module's own machine, a
|
|
// login the controller still makes for a module not handed) is not waited for: on 2026-10-30 every new build of
|
|
// grafana, dnsmasq, mesh-knowledge, mesh-mail and the builders waited here for a key nobody would make.
|
|
func (m Module) MissingProvisions(node string) []string {
|
|
if m.Provisions == nil || m.Answering == nil {
|
|
return nil
|
|
}
|
|
have := map[string]bool{}
|
|
for _, p := range m.Provisions(node) {
|
|
have[p.Provision] = true
|
|
}
|
|
var out []string
|
|
for _, r := range catalogue.Awaited(m.Manifest, m.Answering) {
|
|
if !have[r] {
|
|
out = append(out, r)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
|
|
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
|
|
// offers it) and holds.
|
|
func providedHealthy(provision, node string, n Nodes) (bool, string) {
|
|
providers := n.Providers[provision]
|
|
if len(providers) == 0 {
|
|
return true, ""
|
|
}
|
|
names := make([]string, 0, len(providers))
|
|
for p := range providers {
|
|
names = append(names, p)
|
|
}
|
|
sort.Strings(names)
|
|
for _, p := range names {
|
|
nodes := providers[p]
|
|
at := node
|
|
if !slices.Contains(nodes, node) {
|
|
at = Canary(nodes, n.Online)
|
|
}
|
|
if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
|
|
return true, ""
|
|
}
|
|
}
|
|
return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
|
|
}
|
|
|
|
// unreached is why m on node may be wanted and not reached: the first provision it requires that no provider runs
|
|
// healthy on node or at its canary (rule 5, said and never holding a want; issue 1960); empty when all do.
|
|
func unreached(m Module, node string, n Nodes) string {
|
|
for _, provision := range m.Manifest.Requires {
|
|
if ok, why := providedHealthy(provision, node, n); !ok {
|
|
return why
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
|
|
// valuable or rebuildable; cache and none do not).
|
|
func keepsData(m catalogue.Manifest) bool {
|
|
for _, it := range m.DataItems() {
|
|
if it.Class == catalogue.ClassIrreplaceable || it.Class == catalogue.ClassValuable || it.Class == catalogue.ClassRebuildable {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Outcome is what one module's pass decides: each node's want to write (only those that change), and a hold to write.
|
|
type Outcome struct {
|
|
Wants map[string]delivery.Want
|
|
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
|
|
Held *delivery.Held
|
|
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
|
|
// record and its ask are taken away, and the wants move by the rules again in this pass.
|
|
Lifted bool
|
|
// NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is
|
|
// that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once.
|
|
NodeFailed map[string]NodeFailure
|
|
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
|
Record []RecordNeed
|
|
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
|
// back: the operator is asked whether to restore each (ADR 0539 §19).
|
|
Restore []string
|
|
// Rules is, by node, the rule holding the node back from m.FP and why.
|
|
Rules map[string]Holding
|
|
// Unreached is, by node, why the module may be wanted there and not reached: no provider of a provision it requires runs
|
|
// healthy (rule 5, issue 1960). It holds no want back; the pass says it.
|
|
Unreached map[string]string
|
|
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
|
|
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
|
|
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
|
|
// own composition and is never here.
|
|
Neither map[string]Holding
|
|
}
|
|
|
|
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
|
|
// controller's judgement of the move and its words.
|
|
type RecordNeed struct {
|
|
Node string
|
|
Digest string
|
|
Judgement delivery.Judgement
|
|
Words string
|
|
}
|
|
|
|
// Holding is a rule holding a node back, and why. The bus module is held back by the rules alone, as every module (ADR 0601 §1,
|
|
// §2): no bar of its own.
|
|
type Holding struct {
|
|
Rule int
|
|
Words string
|
|
}
|
|
|
|
// Reason is the holding in words: the rule and its words.
|
|
func (h Holding) Reason() string {
|
|
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
|
}
|
|
|
|
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
|
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
|
|
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
|
|
// module holding data is not set back but asked about (Restore). A
|
|
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
|
|
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
|
|
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
|
|
held := m.Held
|
|
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
|
|
out.Lifted, held = true, false
|
|
}
|
|
if !held && !out.Lifted && !m.Released && m.FP != "" {
|
|
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
|
h, ok := n.Haves[node][m.Name]
|
|
if !ok || h.FP != m.FP || !failed(h) {
|
|
continue
|
|
}
|
|
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
|
|
// F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back,
|
|
// and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its
|
|
// first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's
|
|
// condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557
|
|
// §2, ADR 0429 §6).
|
|
if out.NodeFailed == nil {
|
|
out.NodeFailed = map[string]NodeFailure{}
|
|
}
|
|
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
|
|
continue
|
|
}
|
|
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
|
held = true
|
|
break
|
|
}
|
|
}
|
|
mm := m
|
|
mm.Held = held
|
|
for _, node := range m.Nodes {
|
|
if m.FP == "" {
|
|
break
|
|
}
|
|
before, had := wants[node]
|
|
var next delivery.Want
|
|
p, put := m.Puts[node]
|
|
if !put && had && before.Put && before.Hold {
|
|
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
|
|
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
|
|
// without hold is not kept this way: its put takes it further once heard.
|
|
p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
|
|
Step: delivery.PutHolding}, true
|
|
}
|
|
if put {
|
|
if want, owns := PutOwns(p); owns {
|
|
if want != nil {
|
|
w := *want
|
|
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
|
|
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.FP == w.FP {
|
|
w.Composed = ul.Composed
|
|
}
|
|
if !had || !delivery.SameWant(normal(before), normal(w)) {
|
|
out.Wants[node] = w
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
|
|
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
|
|
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
|
|
if w, ok := m.userListRevision(node, before, had, n); ok {
|
|
if !had || !delivery.SameWant(normal(before), normal(w)) {
|
|
out.Wants[node] = w
|
|
}
|
|
continue
|
|
}
|
|
ok, rule, words := Holds(mm, node, n)
|
|
target := m.FP
|
|
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == delivery.RuleCanary {
|
|
// The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel,
|
|
// link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form,
|
|
// composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want.
|
|
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
|
|
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
|
|
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
|
|
older := ran && runs(have) && have.FP != m.FP && strings.HasPrefix(have.FP, delivery.SourcePrefix2)
|
|
if older {
|
|
_, composed := m.composition(node, have.FP)
|
|
older = composed && m.GrantedAt != nil && m.GrantedAt(have.FP)
|
|
}
|
|
oldForm := ran && runs(have) && have.FP != m.FP && !strings.HasPrefix(have.FP, delivery.SourcePrefix2)
|
|
switch {
|
|
case older:
|
|
ok, target = true, have.FP
|
|
case oldForm:
|
|
// held back by rule 3 as written: on neither list until the canary runs F healthy
|
|
default:
|
|
ok, rule, words = holds(mm, node, n, false)
|
|
}
|
|
}
|
|
if why := unreached(m, node, n); why != "" {
|
|
if out.Unreached == nil {
|
|
out.Unreached = map[string]string{}
|
|
}
|
|
out.Unreached[node] = why
|
|
}
|
|
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
|
|
n.Haves[node][m.Name].FP != m.FP {
|
|
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
|
|
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
|
|
// it would be on neither list and its node-engine would give the module up (issue 1945).
|
|
ok, rule, words = false, delivery.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
|
|
}
|
|
switch {
|
|
case ok:
|
|
d, _ := m.composition(node, target)
|
|
next = delivery.Want{FP: target, Composed: d.Digest}
|
|
at := m
|
|
if target != m.FP {
|
|
// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
|
|
next.Newest, next.Rule, next.Words = m.FP, rule, words
|
|
at.FP = target
|
|
}
|
|
// A placed node's want names the build alone: it is the same target while the build is.
|
|
if had && !before.Removed && !before.Put && before.FP == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
|
|
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
|
}
|
|
running, ran := n.Haves[node][m.Name]
|
|
// A removed want is no want: a first want after it goes through the record ask as a first want does.
|
|
asked, hadLive := before, had
|
|
if before.Removed {
|
|
asked, hadLive = delivery.Want{}, false
|
|
}
|
|
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
|
|
if need != nil {
|
|
out.Record = append(out.Record, *need)
|
|
}
|
|
continue
|
|
}
|
|
case held && had && before.FP == m.FP:
|
|
out.Rules[node] = Holding{Rule: rule, Words: words}
|
|
last := lastHealthy[node]
|
|
switch {
|
|
case keepsData(m.Manifest) && (m.Restored == nil || !m.Restored(node)):
|
|
next = before
|
|
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked whether to restore it", m.Name, short(m.FP))
|
|
out.Restore = append(out.Restore, node)
|
|
case last == "":
|
|
next = before
|
|
next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node)
|
|
default:
|
|
d, there := m.composition(node, last)
|
|
if !there {
|
|
next = before
|
|
next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
|
|
break
|
|
}
|
|
next = delivery.Want{FP: last, Composed: d.Digest, Newest: m.FP, Rule: rule, Words: words}
|
|
}
|
|
default:
|
|
out.Rules[node] = Holding{Rule: rule, Words: words}
|
|
if !had || before.Removed {
|
|
// The node runs the module, or its have names it, and gets no want for it: the module is on neither list (issue 1945), and
|
|
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
|
|
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
|
|
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
|
|
h, there := n.Haves[node][m.Name]
|
|
switch {
|
|
case there && h.Phase != delivery.PhaseRemoved && !held:
|
|
out.Neither[node] = Holding{Rule: rule, Words: words}
|
|
case had && (!there || h.Phase == delivery.PhaseRemoved):
|
|
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
|
|
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
|
|
// would have its node agent remove the module and stop holding its composition's resources back.
|
|
next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
|
|
if !delivery.SameWant(normal(before), normal(next)) {
|
|
out.Wants[node] = next
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
|
|
if before.FP == m.FP {
|
|
// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
|
|
next = delivery.Want{FP: m.FP, Composed: before.Composed}
|
|
} else if d, there := m.composition(node, before.FP); there && d.Digest != "" && d.Digest != before.Composed {
|
|
// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
|
|
// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
|
|
// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
|
|
at := m
|
|
at.FP = before.FP
|
|
running, ran := n.Haves[node][m.Name]
|
|
need, wait := at.mustAsk(node, before, true, ran, running.FP, running.Composed == "", d)
|
|
if need != nil {
|
|
out.Record = append(out.Record, *need)
|
|
}
|
|
if !wait {
|
|
next.Composed = d.Digest
|
|
}
|
|
}
|
|
}
|
|
next, write := m.selfDerived(node, before, had, next, n, &out)
|
|
if !write {
|
|
continue
|
|
}
|
|
if !had || !delivery.SameWant(normal(before), normal(next)) {
|
|
out.Wants[node] = next
|
|
}
|
|
}
|
|
for node, before := range wants {
|
|
if !slices.Contains(m.Nodes, node) && !before.Removed {
|
|
out.Wants[node] = delivery.Want{Removed: true, Words: fmt.Sprintf("%s is no longer assigned to %s", m.Name, node)}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// selfDerived is next with its Self derived from the placement now (Module.Self), whichever rule made it, and whether it is
|
|
// written. A placed node's want names the build alone (ADR 0547 §2). An unplaced node's want that a rule kept from the want
|
|
// written while the node was placed (held at F, held back by rule 3 or 5, a data module asked about) names the controller's
|
|
// composition of the same fingerprint, asked as any move of a record module is (ADR 0539 §14): on 2026-10-27, after the
|
|
// placement seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
|
|
// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or
|
|
// while its ask waits, the want written stays and the node's holding says why.
|
|
func (m Module) selfDerived(node string, before delivery.Want, had bool, next delivery.Want, n Nodes, out *Outcome) (delivery.Want, bool) {
|
|
switch {
|
|
case next.Removed:
|
|
return next, true
|
|
case m.selfOn(node):
|
|
next.Self, next.Composed = true, ""
|
|
return next, true
|
|
case !next.Self && next.Composed != "":
|
|
return next, true
|
|
}
|
|
next.Self, next.Yes = false, ""
|
|
d, there := m.composition(node, next.FP)
|
|
if !there || d.Digest == "" {
|
|
out.Rules[node] = Holding{Rule: delivery.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
|
|
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.FP))}
|
|
return next, false
|
|
}
|
|
at := m
|
|
at.FP = next.FP
|
|
running, ran := n.Haves[node][m.Name]
|
|
if need, wait := at.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
|
|
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
|
|
out.Record = append(out.Record, *need)
|
|
}
|
|
return next, false
|
|
}
|
|
next.Composed = d.Digest
|
|
return next, true
|
|
}
|
|
|
|
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
|
|
func NoWantWords(rule int, words string) string {
|
|
return fmt.Sprintf("no want: rule %d: %s", rule, words)
|
|
}
|
|
|
|
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
|
|
// moving). A failed health while it
|
|
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
|
|
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
|
|
// the world's lastHealthy) already asks for phase running.
|
|
func failed(h delivery.Have) bool {
|
|
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
|
|
}
|
|
|
|
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
|
|
func healthyElsewhere(m Module, n Nodes, node string) []string {
|
|
var out []string
|
|
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
|
|
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
|
|
// healthy now.
|
|
if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) {
|
|
out = append(out, other)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and
|
|
// the hold and the set-back stand at the second failure (ADR 0557 §2).
|
|
func failedElsewhere(m Module, n Nodes, node string) bool {
|
|
for _, other := range m.Nodes {
|
|
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.FP == m.FP && failed(h) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone.
|
|
func FailedAloneWords(module, fp string, failing []string) string {
|
|
return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp),
|
|
strings.Join(failing, ", "))
|
|
}
|
|
|
|
// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's
|
|
// health, phase and words (HeldWords without a time), and the healthy witnesses by name.
|
|
type NodeFailure struct {
|
|
Others []string
|
|
Words string
|
|
}
|
|
|
|
// runs says whether a have is of a module the node runs: running, whatever its health.
|
|
func runs(h delivery.Have) bool {
|
|
return h.Phase == delivery.PhaseRunning
|
|
}
|
|
|
|
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
|
|
// at least one does: a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven
|
|
// modules stayed held on every machine after one transient failed check). A node set back to another fingerprint does not run
|
|
// m.FP, and a failed one does not run it healthy, so the hold stays; a have that is removed, held for its backup or still moving
|
|
// (backing up, applying, checking) is not counted, or it would keep the hold up for ever.
|
|
func allRunHealthy(m Module, n Nodes) bool {
|
|
ran := 0
|
|
for _, node := range m.Nodes {
|
|
h, ok := n.Haves[node][m.Name]
|
|
if !ok || (h.Phase != delivery.PhaseRunning && h.Phase != delivery.PhaseFailed) {
|
|
continue
|
|
}
|
|
if !healthy(h, m.FP) {
|
|
return false
|
|
}
|
|
ran++
|
|
}
|
|
return ran > 0
|
|
}
|
|
|
|
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
|
|
// it, and the node's own words. The node is the held record's own field.
|
|
func HeldWords(h delivery.Have, at time.Time) string {
|
|
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
|
|
if !at.IsZero() {
|
|
words += ", seen at " + at.UTC().Format(time.RFC3339)
|
|
}
|
|
if d := strings.TrimSpace(h.Detail); d != "" {
|
|
return words + ": " + d
|
|
}
|
|
return words + ": the node gave no words"
|
|
}
|
|
|
|
// LiftedWords say why a hold was lifted by itself.
|
|
func LiftedWords(module, fp string) string {
|
|
return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
|
|
}
|
|
|
|
// normal is a want without its version, as a decoded and a composed one are compared.
|
|
func normal(w delivery.Want) delivery.Want {
|
|
w.V = 0
|
|
return w
|
|
}
|
|
|
|
func short(fp string) string {
|
|
part, err := delivery.FingerprintKeyPart(fp)
|
|
if err != nil || len(part) < 12 {
|
|
return fp
|
|
}
|
|
return part[:12]
|
|
}
|
|
|
|
// mustAsk says whether the move to the composition d on node waits for the operator's yes (need names the ask, ADR 0539 §14). A
|
|
// module the node never ran (it has no want and no have), a move judged neither, a module that is not of policy record and a want
|
|
// that stays are written as any want. The bus module is no exception: it moves by the rules (ADR 0601 §1), its node backing up its
|
|
// streams before it applies (§2); the assigner asks no snapshot for it.
|
|
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
|
|
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
|
|
return nil, false
|
|
}
|
|
// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
|
|
// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
|
|
// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
|
|
// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
|
|
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
|
|
return nil, false
|
|
}
|
|
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
|
|
return nil, false
|
|
}
|
|
if m.Allowed != nil && m.Allowed(node, d.Digest) {
|
|
return nil, false
|
|
}
|
|
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
|
}
|
|
|
|
// recorded says whether the module is of policy record: a person's choice at the assigner over the manifest's word (ADR
|
|
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
|
|
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
|
func (m Module) recorded(d delivery.ComposedDigest) bool {
|
|
if m.Chosen != nil {
|
|
policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen)
|
|
return policy == catalogue.PolicyRecord
|
|
}
|
|
if d.Policy != "" {
|
|
return d.Policy == delivery.PolicyRecord
|
|
}
|
|
policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
|
|
return policy == catalogue.PolicyRecord
|
|
}
|
|
|
|
// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
|
|
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
|
|
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
|
|
func (m Module) userListRevision(node string, before delivery.Want, had bool, n Nodes) (delivery.Want, bool) {
|
|
if !catalogue.ProvidesBus(m.Manifest) {
|
|
return delivery.Want{}, false
|
|
}
|
|
running, ran := n.Haves[node][m.Name]
|
|
if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) {
|
|
return delivery.Want{}, false
|
|
}
|
|
d, ok := m.composition(node, running.FP)
|
|
if !ok || !d.UserListOnly {
|
|
return delivery.Want{}, false
|
|
}
|
|
w := delivery.Want{FP: running.FP, Composed: d.Digest}
|
|
if had && before.Composed == d.Digest {
|
|
w.Backup, w.Words = before.Backup, before.Words
|
|
}
|
|
return w, true
|
|
}
|
|
|
|
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's Grant.NamesNode, which a
|
|
// put reads by too).
|
|
func GrantNamesNode(g delivery.Grant, node, module string) bool {
|
|
return g.NamesNode(node, module)
|
|
}
|