jochen
0d1bb035cb
Under a module's own record (its manifest's, or its irreplaceable data's), every move waited for a person's push, even one that only changed a tools bundle: the database server stayed behind on ace on 2026-10-15 although its walk said it recreated none of its containers. ADR 0377 lets such a move roll out. A move is judged per machine against the build that machine was last sent: it goes when no container, mounted file, data path, data declaration, provider or (for a module with data) long-running process changes. A person's record, the bus and any move that cannot be judged still wait. The walk keeps per machine which way each recorded module went and why, says it in its note and log, and status names a machine behind with `push <node>` only for a move that waits.
281 lines
10 KiB
Go
281 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
|
|
// issue 259, ADR 0221).
|
|
//
|
|
// A named push ends by sending every other machine whose declaration differs from what it was last
|
|
// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
|
|
// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
|
|
// out makes every machine running it differ from the merge on, and so did a module whose plan was
|
|
// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
|
|
// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
|
|
// everywhere at once.
|
|
//
|
|
// What tells the two apart is which build of each module the machine was last sent, kept with every
|
|
// send. A machine any of whose modules would move to a build its policy or an open plan holds back
|
|
// is not sent by a push that did not name it; the push says which, and why, and how to send it.
|
|
|
|
// heldBack is why a push that did not name a machine must not send it, empty when it may.
|
|
//
|
|
// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
|
|
// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
|
|
// machine was last sent — including a module the machine was never sent at all. A move is held when
|
|
// the module's policy records rather than rolls out, or when an open plan has not yet sent this
|
|
// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
|
|
// is not known, so a held upgrade cannot be told from anything else.
|
|
func heldBack(node string, modules []string, sent map[string]string, known bool,
|
|
current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
|
|
if !known {
|
|
return []string{"which builds it was last sent is not known — it was last sent before the " +
|
|
"mesh kept them, or sent a declaration by hand"}
|
|
}
|
|
var why []string
|
|
for _, m := range modules {
|
|
now := current[m]
|
|
was, carried := sent[m]
|
|
if carried && was == now.Commit {
|
|
continue
|
|
}
|
|
move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
|
|
if !now.RollOut {
|
|
why = append(why, move+", which its upgrade policy records rather than rolls out")
|
|
continue
|
|
}
|
|
if id := planStillToSend(plans, m, node); id != "" {
|
|
why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
|
|
}
|
|
}
|
|
sort.Strings(why)
|
|
return why
|
|
}
|
|
|
|
// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
|
|
// one holding the module that has neither finished sending it nor sent it here first, and has not
|
|
// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
|
|
// per machine.
|
|
func planStillToSend(plans []inventory.Plan, module, node string) string {
|
|
for _, p := range plans {
|
|
s, holds := p.Modules[module]
|
|
if !p.Open() || !holds {
|
|
continue
|
|
}
|
|
if s == nil {
|
|
return p.ID
|
|
}
|
|
if s.SentAt != nil || s.State == "failed" {
|
|
continue
|
|
}
|
|
first := false
|
|
for _, n := range s.First {
|
|
if n == node {
|
|
first = true
|
|
}
|
|
}
|
|
if !first {
|
|
return p.ID
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func fromBuild(was string, carried bool) string {
|
|
if !carried {
|
|
return "(never sent it) "
|
|
}
|
|
return "from " + buildName(was) + " "
|
|
}
|
|
|
|
func buildName(commit string) string {
|
|
if commit == "" {
|
|
return "a build with no source"
|
|
}
|
|
return shortCommit(commit)
|
|
}
|
|
|
|
// heldMachines reads, for each machine named, why a push that did not name it must not send it
|
|
// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
|
|
// to the send, which says why.
|
|
func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
|
|
out := map[string][]string{}
|
|
if len(names) == 0 {
|
|
return out, nil
|
|
}
|
|
inv := open.inventory
|
|
current, err := inv.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
plans, err := inv.OpenPlans(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
f, err := readMoveFacts(ctx, inv)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, node := range names {
|
|
plan, _, err := planFor(ctx, open, node)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
modules := make([]string, 0, len(plan.Modules))
|
|
for _, m := range plan.Modules {
|
|
modules = append(modules, m.Module)
|
|
}
|
|
sent, known, err := inv.SentBuilds(ctx, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// A rebuild that put the same thing on the machine is no move (ADR 0236): read as the build it
|
|
// runs.
|
|
same := map[string]string{}
|
|
for m, was := range sent {
|
|
same[m] = was
|
|
if f.identical(m, was, current[m].Commit) {
|
|
same[m] = current[m].Commit
|
|
}
|
|
}
|
|
// A move under a module's own `record` that recreates nothing and moves no data goes as a module that rolls
|
|
// out goes (novox/hq ADR 0377): a walk's to send, not a person's.
|
|
here, copied := current, false
|
|
for m, was := range same {
|
|
if now, held := current[m]; held && now.OwnRecord() && !f.identical(m, was, now.Commit) && f.rolls(m, was) {
|
|
if !copied {
|
|
copied = true
|
|
here = make(map[string]inventory.CurrentBuild, len(current))
|
|
for k, v := range current {
|
|
here[k] = v
|
|
}
|
|
}
|
|
now.RollOut = true
|
|
here[m] = now
|
|
}
|
|
}
|
|
why := heldBack(node, modules, same, known, here, plans)
|
|
// And a build no gate has seen is not carried by a send that does not judge it (ADR 0236).
|
|
for _, mv := range f.moves(node, modules, same, known, false) {
|
|
if planStillToSend(plans, mv.Module, node) == "" {
|
|
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
|
|
"plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
|
}
|
|
}
|
|
if len(why) > 0 {
|
|
sort.Strings(why)
|
|
out[node] = why
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
|
|
// was not sent, that whatever else it is owed waits with it, and the command that sends it.
|
|
func sayHeld(w io.Writer, node string, why []string) {
|
|
fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
|
|
"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
|
|
"grant from this push among it — waits with it. `push %s` sends it\n",
|
|
node, strings.Join(why, "; "), node)
|
|
}
|
|
|
|
// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
|
|
// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
|
|
// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
|
|
//
|
|
// `handled` is every machine already sent or already said; it is not considered again. Answers the
|
|
// machines that could not be composed, as refusals.
|
|
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
|
|
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
|
|
keep keeping, w io.Writer) ([]string, error) {
|
|
inv := open.inventory
|
|
var refusals []string
|
|
// Bounded by the node count: a node is marked handled the round it is considered and is never
|
|
// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
|
|
// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
|
|
// cascade legitimately still converging, so it is said rather than passed over in silence.
|
|
rounds := 0
|
|
for {
|
|
would, err := wouldSend(ctx, open, nodes)
|
|
if err != nil {
|
|
return refusals, err
|
|
}
|
|
behind, err := inv.Waiting(ctx, would)
|
|
if err != nil {
|
|
return refusals, err
|
|
}
|
|
var also []string
|
|
for _, m := range behind {
|
|
if !handled[m.Node] {
|
|
also = append(also, m.Node)
|
|
}
|
|
}
|
|
if len(also) == 0 {
|
|
return refusals, nil
|
|
}
|
|
if rounds++; rounds > len(nodes) {
|
|
fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
|
|
"should not happen; run `push --behind` to finish\n",
|
|
rounds-1, strings.Join(also, ", "))
|
|
return refusals, nil
|
|
}
|
|
sort.Strings(also)
|
|
held, err := heldMachines(ctx, open, also)
|
|
if err != nil {
|
|
return refusals, err
|
|
}
|
|
var sending []string
|
|
for _, name := range also {
|
|
// Every candidate this round is marked handled — the sent ones so they are not
|
|
// re-listed, the held ones because they stay held, and the refused ones so a machine
|
|
// that cannot be composed does not make the loop spin on it for ever.
|
|
handled[name] = true
|
|
if why, isHeld := held[name]; isHeld {
|
|
sayHeld(w, name, why)
|
|
continue
|
|
}
|
|
sending = append(sending, name)
|
|
}
|
|
if len(sending) == 0 {
|
|
continue
|
|
}
|
|
fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
|
|
"declaration since changed; sending it too\n", strings.Join(sending, ", "))
|
|
// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
|
|
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
|
|
// Held for this round only, and after the last round's were given back, so two pushes
|
|
// cascading into each other's machines never each wait on the other.
|
|
refused, err := sendRound(ctx, open, sending, compose, d, holder, keep)
|
|
refusals = append(refusals, refused...)
|
|
if err != nil {
|
|
return refusals, err
|
|
}
|
|
}
|
|
}
|
|
|
|
// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
|
|
// is left out of it said, and its declaration allocated.
|
|
func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) {
|
|
return func(held context.Context, node string) (sendable, error) {
|
|
plan, settings, err := planFor(held, open, node)
|
|
if err != nil {
|
|
return sendable{}, err
|
|
}
|
|
reportUnhostable(node, plan)
|
|
reportKept(held, plan)
|
|
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
|
|
if err == nil {
|
|
reportLeftOut(node, declared)
|
|
}
|
|
return declared, err
|
|
}
|
|
}
|