package main import ( "context" "fmt" "io" "sort" "strings" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" ) // A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq // issue 259, ADR 0221). // // A named push ends by sending every other machine whose declaration differs from what it was last // sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A // digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls // out makes every machine running it differ from the merge on, and so did a module whose plan was // still waiting on its first machine (ADR 0218): `push ` sent all four machines the build // that was meant to be walked through the mesh one machine at a time, and a fault in it was met // everywhere at once. // // What tells the two apart is which build of each module the machine was last sent, kept with every // send. A machine any of whose modules would move to a build its policy or an open plan holds back // is not sent by a push that did not name it; the push says which, and why, and how to send it. // heldBack is why a push that did not name a machine must not send it, empty when it may. // // `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as // Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the // machine was last sent — including a module the machine was never sent at all. A move is held when // the module's policy records rather than rolls out, or when an open plan has not yet sent this // machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried // is not known, so a held upgrade cannot be told from anything else. func heldBack(node string, modules []string, sent map[string]string, known bool, current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string { if !known { return []string{"which builds it was last sent is not known — it was last sent before the " + "mesh kept them, or sent a declaration by hand"} } var why []string for _, m := range modules { now := current[m] was, carried := sent[m] if carried && was == now.Commit { continue } move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit)) if !now.RollOut { why = append(why, move+", which its upgrade policy records rather than rolls out") continue } if id := planStillToSend(plans, m, node); id != "" { why = append(why, move+", which "+id+" has not sent it yet (one machine first)") } } sort.Strings(why) return why } // planStillToSend is the open plan that has a module's new build still to send this machine, or empty: // one holding the module that has neither finished sending it nor sent it here first, and has not // failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made // per machine. func planStillToSend(plans []inventory.Plan, module, node string) string { for _, p := range plans { s, holds := p.Modules[module] if !p.Open() || !holds { continue } if s == nil { return p.ID } if s.SentAt != nil || s.State == "failed" { continue } first := false for _, n := range s.First { if n == node { first = true } } if !first { return p.ID } } return "" } func fromBuild(was string, carried bool) string { if !carried { return "(never sent it) " } return "from " + buildName(was) + " " } func buildName(commit string) string { if commit == "" { return "a build with no source" } return shortCommit(commit) } // heldMachines reads, for each machine named, why a push that did not name it must not send it // (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left // to the send, which says why. func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) { out := map[string][]string{} if len(names) == 0 { return out, nil } inv := open.inventory current, err := inv.CurrentBuilds(ctx) if err != nil { return nil, err } plans, err := inv.OpenPlans(ctx) if err != nil { return nil, err } for _, node := range names { plan, _, err := planFor(ctx, open, node) if err != nil { continue } modules := make([]string, 0, len(plan.Modules)) for _, m := range plan.Modules { modules = append(modules, m.Module) } sent, known, err := inv.SentBuilds(ctx, node) if err != nil { return nil, err } if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 { out[node] = why } } return out, nil } // sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it // was not sent, that whatever else it is owed waits with it, and the command that sends it. func sayHeld(w io.Writer, node string, why []string) { fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+ "holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+ "grant from this push among it — waits with it. `push %s` sends it\n", node, strings.Join(why, "; "), node) } // flushBehind is the end of a named push: every other machine now behind is sent too, by name, over // as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose // modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221). // // `handled` is every machine already sent or already said; it is not considered again. Answers the // machines that could not be composed, as refusals. func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool, compose func(held context.Context, node string) (sendable, error), d delivery, holder string, w io.Writer) ([]string, error) { inv := open.inventory var refusals []string // Bounded by the node count: a node is marked handled the round it is considered and is never // considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard // against a logic error, not a real limit — if it were ever hit, that is a bug rather than a // cascade legitimately still converging, so it is said rather than passed over in silence. rounds := 0 for { would, err := wouldSend(ctx, open, nodes) if err != nil { return refusals, err } behind, err := inv.Waiting(ctx, would) if err != nil { return refusals, err } var also []string for _, m := range behind { if !handled[m.Node] { also = append(also, m.Node) } } if len(also) == 0 { return refusals, nil } if rounds++; rounds > len(nodes) { fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+ "should not happen; run `push --behind` to finish\n", rounds-1, strings.Join(also, ", ")) return refusals, nil } sort.Strings(also) held, err := heldMachines(ctx, open, also) if err != nil { return refusals, err } var sending []string for _, name := range also { // Every candidate this round is marked handled — the sent ones so they are not // re-listed, the held ones because they stay held, and the refused ones so a machine // that cannot be composed does not make the loop spin on it for ever. handled[name] = true if why, isHeld := held[name]; isHeld { sayHeld(w, name, why) continue } sending = append(sending, name) } if len(sending) == 0 { continue } fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+ "declaration since changed; sending it too\n", strings.Join(sending, ", ")) // Tolerantly, exactly as the named send: a machine that cannot be composed is collected as // a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066). // Held for this round only, and after the last round's were given back, so two pushes // cascading into each other's machines never each wait on the other. refused, err := sendRound(ctx, open, sending, compose, d, holder) refusals = append(refusals, refused...) if err != nil { return refusals, err } } } // composeForPush is how a push composes one machine: its set resolved, what it cannot host and what // is left out of it said, and its declaration allocated. func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) { return func(held context.Context, node string) (sendable, error) { plan, settings, err := planFor(held, open, node) if err != nil { return sendable{}, err } reportUnhostable(node, plan) declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating) if err == nil { reportLeftOut(node, declared) } return declared, err } }