jochen
2421b82ad2
A named push flushed every other machine whose declaration differed from what it was last sent (hq ADR 0083). Under an upgrade policy of `record`, or a plan still waiting on its first machine (ADR 0218), every machine running the module differs, so `push <one>` sent the held build to all of them (hq issue 259). Each send now records which build of each module it carried (node.sent_builds, migration 0061). The cascade, and the bus holder added to a named push, skip a machine any of whose modules would move to a build its policy records or an open plan has not sent it, and say which module, which build, why, and that `push <node>` sends it. A machine whose last send was not recorded is held until it is named. The named machine itself, a whole-mesh push and `push --behind` are unchanged.
242 lines
8.8 KiB
Go
242 lines
8.8 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
|
|
// issue 259, ADR 0221).
|
|
//
|
|
// A named push ends by sending every other machine whose declaration differs from what it was last
|
|
// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
|
|
// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
|
|
// out makes every machine running it differ from the merge on, and so did a module whose plan was
|
|
// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
|
|
// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
|
|
// everywhere at once.
|
|
//
|
|
// What tells the two apart is which build of each module the machine was last sent, kept with every
|
|
// send. A machine any of whose modules would move to a build its policy or an open plan holds back
|
|
// is not sent by a push that did not name it; the push says which, and why, and how to send it.
|
|
|
|
// heldBack is why a push that did not name a machine must not send it, empty when it may.
|
|
//
|
|
// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
|
|
// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
|
|
// machine was last sent — including a module the machine was never sent at all. A move is held when
|
|
// the module's policy records rather than rolls out, or when an open plan has not yet sent this
|
|
// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
|
|
// is not known, so a held upgrade cannot be told from anything else.
|
|
func heldBack(node string, modules []string, sent map[string]string, known bool,
|
|
current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
|
|
if !known {
|
|
return []string{"which builds it was last sent is not known — it was last sent before the " +
|
|
"mesh kept them, or sent a declaration by hand"}
|
|
}
|
|
var why []string
|
|
for _, m := range modules {
|
|
now := current[m]
|
|
was, carried := sent[m]
|
|
if carried && was == now.Commit {
|
|
continue
|
|
}
|
|
move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
|
|
if !now.RollOut {
|
|
why = append(why, move+", which its upgrade policy records rather than rolls out")
|
|
continue
|
|
}
|
|
if id := planStillToSend(plans, m, node); id != "" {
|
|
why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
|
|
}
|
|
}
|
|
sort.Strings(why)
|
|
return why
|
|
}
|
|
|
|
// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
|
|
// one holding the module that has neither finished sending it nor sent it here first, and has not
|
|
// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
|
|
// per machine.
|
|
func planStillToSend(plans []inventory.Plan, module, node string) string {
|
|
for _, p := range plans {
|
|
s, holds := p.Modules[module]
|
|
if !p.Open() || !holds {
|
|
continue
|
|
}
|
|
if s == nil {
|
|
return p.ID
|
|
}
|
|
if s.SentAt != nil || s.State == "failed" {
|
|
continue
|
|
}
|
|
first := false
|
|
for _, n := range s.First {
|
|
if n == node {
|
|
first = true
|
|
}
|
|
}
|
|
if !first {
|
|
return p.ID
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func fromBuild(was string, carried bool) string {
|
|
if !carried {
|
|
return "(never sent it) "
|
|
}
|
|
return "from " + buildName(was) + " "
|
|
}
|
|
|
|
func buildName(commit string) string {
|
|
if commit == "" {
|
|
return "a build with no source"
|
|
}
|
|
return shortCommit(commit)
|
|
}
|
|
|
|
// heldMachines reads, for each machine named, why a push that did not name it must not send it
|
|
// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
|
|
// to the send, which says why.
|
|
func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
|
|
out := map[string][]string{}
|
|
if len(names) == 0 {
|
|
return out, nil
|
|
}
|
|
inv := open.inventory
|
|
current, err := inv.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
plans, err := inv.OpenPlans(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, node := range names {
|
|
plan, _, err := planFor(ctx, open, node)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
modules := make([]string, 0, len(plan.Modules))
|
|
for _, m := range plan.Modules {
|
|
modules = append(modules, m.Module)
|
|
}
|
|
sent, known, err := inv.SentBuilds(ctx, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 {
|
|
out[node] = why
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
|
|
// was not sent, that whatever else it is owed waits with it, and the command that sends it.
|
|
func sayHeld(w io.Writer, node string, why []string) {
|
|
fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
|
|
"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
|
|
"grant from this push among it — waits with it. `push %s` sends it\n",
|
|
node, strings.Join(why, "; "), node)
|
|
}
|
|
|
|
// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
|
|
// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
|
|
// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
|
|
//
|
|
// `handled` is every machine already sent or already said; it is not considered again. Answers the
|
|
// machines that could not be composed, as refusals.
|
|
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
|
|
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
|
|
w io.Writer) ([]string, error) {
|
|
inv := open.inventory
|
|
var refusals []string
|
|
// Bounded by the node count: a node is marked handled the round it is considered and is never
|
|
// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
|
|
// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
|
|
// cascade legitimately still converging, so it is said rather than passed over in silence.
|
|
rounds := 0
|
|
for {
|
|
would, err := wouldSend(ctx, open, nodes)
|
|
if err != nil {
|
|
return refusals, err
|
|
}
|
|
behind, err := inv.Waiting(ctx, would)
|
|
if err != nil {
|
|
return refusals, err
|
|
}
|
|
var also []string
|
|
for _, m := range behind {
|
|
if !handled[m.Node] {
|
|
also = append(also, m.Node)
|
|
}
|
|
}
|
|
if len(also) == 0 {
|
|
return refusals, nil
|
|
}
|
|
if rounds++; rounds > len(nodes) {
|
|
fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
|
|
"should not happen; run `push --behind` to finish\n",
|
|
rounds-1, strings.Join(also, ", "))
|
|
return refusals, nil
|
|
}
|
|
sort.Strings(also)
|
|
held, err := heldMachines(ctx, open, also)
|
|
if err != nil {
|
|
return refusals, err
|
|
}
|
|
var sending []string
|
|
for _, name := range also {
|
|
// Every candidate this round is marked handled — the sent ones so they are not
|
|
// re-listed, the held ones because they stay held, and the refused ones so a machine
|
|
// that cannot be composed does not make the loop spin on it for ever.
|
|
handled[name] = true
|
|
if why, isHeld := held[name]; isHeld {
|
|
sayHeld(w, name, why)
|
|
continue
|
|
}
|
|
sending = append(sending, name)
|
|
}
|
|
if len(sending) == 0 {
|
|
continue
|
|
}
|
|
fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
|
|
"declaration since changed; sending it too\n", strings.Join(sending, ", "))
|
|
// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
|
|
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
|
|
// Held for this round only, and after the last round's were given back, so two pushes
|
|
// cascading into each other's machines never each wait on the other.
|
|
refused, err := sendRound(ctx, open, sending, compose, d, holder)
|
|
refusals = append(refusals, refused...)
|
|
if err != nil {
|
|
return refusals, err
|
|
}
|
|
}
|
|
}
|
|
|
|
// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
|
|
// is left out of it said, and its declaration allocated.
|
|
func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) {
|
|
return func(held context.Context, node string) (sendable, error) {
|
|
plan, settings, err := planFor(held, open, node)
|
|
if err != nil {
|
|
return sendable{}, err
|
|
}
|
|
reportUnhostable(node, plan)
|
|
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
|
|
if err == nil {
|
|
reportLeftOut(node, declared)
|
|
}
|
|
return declared, err
|
|
}
|
|
}
|