Keep a named push from sending builds a policy or a plan holds back
A named push flushed every other machine whose declaration differed from what it was last sent (hq ADR 0083). Under an upgrade policy of `record`, or a plan still waiting on its first machine (ADR 0218), every machine running the module differs, so `push <one>` sent the held build to all of them (hq issue 259). Each send now records which build of each module it carried (node.sent_builds, migration 0061). The cascade, and the bus holder added to a named push, skip a machine any of whose modules would move to a build its policy records or an open plan has not sent it, and say which module, which build, why, and that `push <node>` sends it. A machine whose last send was not recorded is held until it is named. The named machine itself, a whole-mesh push and `push --behind` are unchanged.
This commit is contained in:
jochen
@@ -0,0 +1,241 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
|
||||
// issue 259, ADR 0221).
|
||||
//
|
||||
// A named push ends by sending every other machine whose declaration differs from what it was last
|
||||
// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
|
||||
// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
|
||||
// out makes every machine running it differ from the merge on, and so did a module whose plan was
|
||||
// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
|
||||
// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
|
||||
// everywhere at once.
|
||||
//
|
||||
// What tells the two apart is which build of each module the machine was last sent, kept with every
|
||||
// send. A machine any of whose modules would move to a build its policy or an open plan holds back
|
||||
// is not sent by a push that did not name it; the push says which, and why, and how to send it.
|
||||
|
||||
// heldBack is why a push that did not name a machine must not send it, empty when it may.
|
||||
//
|
||||
// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
|
||||
// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
|
||||
// machine was last sent — including a module the machine was never sent at all. A move is held when
|
||||
// the module's policy records rather than rolls out, or when an open plan has not yet sent this
|
||||
// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
|
||||
// is not known, so a held upgrade cannot be told from anything else.
|
||||
func heldBack(node string, modules []string, sent map[string]string, known bool,
|
||||
current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
|
||||
if !known {
|
||||
return []string{"which builds it was last sent is not known — it was last sent before the " +
|
||||
"mesh kept them, or sent a declaration by hand"}
|
||||
}
|
||||
var why []string
|
||||
for _, m := range modules {
|
||||
now := current[m]
|
||||
was, carried := sent[m]
|
||||
if carried && was == now.Commit {
|
||||
continue
|
||||
}
|
||||
move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
|
||||
if !now.RollOut {
|
||||
why = append(why, move+", which its upgrade policy records rather than rolls out")
|
||||
continue
|
||||
}
|
||||
if id := planStillToSend(plans, m, node); id != "" {
|
||||
why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
|
||||
}
|
||||
}
|
||||
sort.Strings(why)
|
||||
return why
|
||||
}
|
||||
|
||||
// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
|
||||
// one holding the module that has neither finished sending it nor sent it here first, and has not
|
||||
// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
|
||||
// per machine.
|
||||
func planStillToSend(plans []inventory.Plan, module, node string) string {
|
||||
for _, p := range plans {
|
||||
s, holds := p.Modules[module]
|
||||
if !p.Open() || !holds {
|
||||
continue
|
||||
}
|
||||
if s == nil {
|
||||
return p.ID
|
||||
}
|
||||
if s.SentAt != nil || s.State == "failed" {
|
||||
continue
|
||||
}
|
||||
first := false
|
||||
for _, n := range s.First {
|
||||
if n == node {
|
||||
first = true
|
||||
}
|
||||
}
|
||||
if !first {
|
||||
return p.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func fromBuild(was string, carried bool) string {
|
||||
if !carried {
|
||||
return "(never sent it) "
|
||||
}
|
||||
return "from " + buildName(was) + " "
|
||||
}
|
||||
|
||||
func buildName(commit string) string {
|
||||
if commit == "" {
|
||||
return "a build with no source"
|
||||
}
|
||||
return shortCommit(commit)
|
||||
}
|
||||
|
||||
// heldMachines reads, for each machine named, why a push that did not name it must not send it
|
||||
// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
|
||||
// to the send, which says why.
|
||||
func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
|
||||
out := map[string][]string{}
|
||||
if len(names) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
inv := open.inventory
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
modules := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
modules = append(modules, m.Module)
|
||||
}
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 {
|
||||
out[node] = why
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
|
||||
// was not sent, that whatever else it is owed waits with it, and the command that sends it.
|
||||
func sayHeld(w io.Writer, node string, why []string) {
|
||||
fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
|
||||
"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
|
||||
"grant from this push among it — waits with it. `push %s` sends it\n",
|
||||
node, strings.Join(why, "; "), node)
|
||||
}
|
||||
|
||||
// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
|
||||
// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
|
||||
// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
|
||||
//
|
||||
// `handled` is every machine already sent or already said; it is not considered again. Answers the
|
||||
// machines that could not be composed, as refusals.
|
||||
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
|
||||
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
|
||||
w io.Writer) ([]string, error) {
|
||||
inv := open.inventory
|
||||
var refusals []string
|
||||
// Bounded by the node count: a node is marked handled the round it is considered and is never
|
||||
// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
|
||||
// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
|
||||
// cascade legitimately still converging, so it is said rather than passed over in silence.
|
||||
rounds := 0
|
||||
for {
|
||||
would, err := wouldSend(ctx, open, nodes)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
behind, err := inv.Waiting(ctx, would)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
var also []string
|
||||
for _, m := range behind {
|
||||
if !handled[m.Node] {
|
||||
also = append(also, m.Node)
|
||||
}
|
||||
}
|
||||
if len(also) == 0 {
|
||||
return refusals, nil
|
||||
}
|
||||
if rounds++; rounds > len(nodes) {
|
||||
fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
|
||||
"should not happen; run `push --behind` to finish\n",
|
||||
rounds-1, strings.Join(also, ", "))
|
||||
return refusals, nil
|
||||
}
|
||||
sort.Strings(also)
|
||||
held, err := heldMachines(ctx, open, also)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
var sending []string
|
||||
for _, name := range also {
|
||||
// Every candidate this round is marked handled — the sent ones so they are not
|
||||
// re-listed, the held ones because they stay held, and the refused ones so a machine
|
||||
// that cannot be composed does not make the loop spin on it for ever.
|
||||
handled[name] = true
|
||||
if why, isHeld := held[name]; isHeld {
|
||||
sayHeld(w, name, why)
|
||||
continue
|
||||
}
|
||||
sending = append(sending, name)
|
||||
}
|
||||
if len(sending) == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
|
||||
"declaration since changed; sending it too\n", strings.Join(sending, ", "))
|
||||
// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
|
||||
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
|
||||
// Held for this round only, and after the last round's were given back, so two pushes
|
||||
// cascading into each other's machines never each wait on the other.
|
||||
refused, err := sendRound(ctx, open, sending, compose, d, holder)
|
||||
refusals = append(refusals, refused...)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
|
||||
// is left out of it said, and its declaration allocated.
|
||||
func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) {
|
||||
return func(held context.Context, node string) (sendable, error) {
|
||||
plan, settings, err := planFor(held, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
reportUnhostable(node, plan)
|
||||
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||
if err == nil {
|
||||
reportLeftOut(node, declared)
|
||||
}
|
||||
return declared, err
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user