package main import ( "context" "fmt" "os" "slices" "sort" "time" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" ) // What a walk touches, its bound and its outcome (novox/hq ADR 0308 decision 1, decision 5): said in // `plan-moved` on the save that starts the walk and the save that ends it, beside the walk whole, for the holder // of the operator's channel to hold what the walk touches while it runs and tell its result once. // // **What a walk touches** is every module it moves on every machine it sends it to, and every module a moved // tool runner (node-tools) or node-engine (mesh-host) carries on that machine: what restarts there. At the start // that is what the walk will send — each module of its tiers whose policy rolls it out, on every machine running // it; at the end it is what it reached — the machines each module was sent to, first and rest, and what a gated // send carried with it. // // **Its bound** is twice the largest delivery budget among the walk's deliveries (deliveryBudgets: a core // module's ten minutes, a leaf module's budget in force), and fifteen minutes, not doubled, for a walk none of // whose deliveries has a budget: a release plan (builds waiting for a person's gate), a walk whose every module's // policy records rather than rolls out (held for a person, ADR 0282 decision 1), or a walk kept without a class. // walkNoBudgetBound is the bound of a walk with no budget (ADR 0308 decision 5): not doubled. const walkNoBudgetBound = 15 * time.Minute // budgetNone is the budget class of a walk none of whose deliveries has a budget. const budgetNone = "none" // walkSaid is what `plan-moved` adds on a walk's start and end, beside the walk whole. type walkSaid struct { Walk string `json:"walk"` Phase string `json:"phase"` // Touches is, per machine, the modules the walk moves or reaches there and what a moved tool runner or // node-engine carries there; never null. Touches []walkTouch `json:"touches"` // Outcome is on the end only: delivered, failed or stopped. Outcome string `json:"outcome,omitempty"` // BoundSeconds and BudgetClass are said on both, read from the walk's class at the start. BoundSeconds int `json:"bound_seconds"` BudgetClass string `json:"budget_class"` // StartedAt is when the walk was said started, on both; EndedAt when it was said ended, on the end: the // bound runs from StartedAt, whenever the event is heard. StartedAt *time.Time `json:"started_at,omitempty"` EndedAt *time.Time `json:"ended_at,omitempty"` } // walkTouch is what a walk touches on one machine. type walkTouch struct { Machine string `json:"machine"` Modules []string `json:"modules"` // Carried names those of Modules that are there only because a moved tool runner or node-engine carries // them: what restarts with it, not what the walk builds. Carried []string `json:"carried,omitempty"` } // planMovedBody is `plan-moved`'s body: the walk whole, as before, and on its start and end what it touches. type planMovedBody struct { inventory.Plan *walkSaid } // walkFacts is what the touches are read from: the controller's inventory. type walkFacts interface { Running(ctx context.Context, module string) ([]string, error) UpgradeOf(ctx context.Context, module string) (inventory.Upgrade, error) Assigned(ctx context.Context, node string) ([]string, error) SentBuilds(ctx context.Context, node string) (map[string]string, bool, error) Catalogue(ctx context.Context) (map[string]catalogue.Manifest, error) } // sayWalk is what `plan-moved` adds to the save that started or ended the walk; nil on every other save. A fact // that cannot be read is said on the controller's log and left out, never in the way of the event: a walk said // with fewer touches is held less, never longer. func sayWalk(ctx context.Context, facts walkFacts, p inventory.Plan) *walkSaid { if p.WalkSaid == "" { return nil } w := &walkSaid{Walk: p.ID, Phase: p.WalkSaid} if p.Times != nil { w.StartedAt = p.Times.Started if p.WalkSaid == inventory.WalkEnded { w.EndedAt, w.Outcome = p.Times.Ended, p.Outcome() } } read := readWalkFacts(ctx, facts, p) w.BudgetClass = walkBudgetClass(p, read.rollsOut) w.BoundSeconds = int(walkBound(w.BudgetClass) / time.Second) var moved map[string][]string if p.WalkSaid == inventory.WalkStarted { moved = walkSends(p, read.running, read.rollsOut) } else { moved = walkReached(p, read.running) } var shelf map[string]catalogue.Manifest if len(moved[broker.RuntimeModule]) > 0 { // Which modules the tool runner serves is the catalogue's: read only when the walk moves it. shelf = read.shelf() } w.Touches = touchesOf(moved, read.onMachine(moved), shelf) return w } // walkBudgetClass is the largest budget class among the walk's deliveries: its class read at the cut when one of // its modules rolls out, and none when it has no budget. func walkBudgetClass(p inventory.Plan, rollsOut func(string) bool) string { if p.Release != nil || p.Times == nil { return budgetNone } if _, budgeted := deliveryBudgets[p.Times.Class]; !budgeted { return budgetNone } for _, tier := range p.Tiers { for _, m := range tier { if rollsOut(m) { return p.Times.Class } } } return budgetNone } // walkBound is a walk's bound by its budget class: twice its budget, or fifteen minutes with none. func walkBound(class string) time.Duration { if b, ok := deliveryBudgets[class]; ok { return 2 * b } return walkNoBudgetBound } // walkSends is what a walk will send at its start, per module the machines it goes to: each module of its tiers // whose policy rolls it out, on every machine running it (for a release plan, those of its order). func walkSends(p inventory.Plan, running func(string) []string, rollsOut func(string) bool) map[string][]string { out := map[string][]string{} for _, tier := range p.Tiers { for _, m := range tier { if s := p.Modules[m]; s != nil && s.State == planDeleted { continue } if p.Release == nil && !rollsOut(m) { continue } for _, n := range running(m) { if p.Release != nil && !slices.Contains(p.Release.Order, n) { continue } out[m] = appendNew(out[m], n) } } } return out } // walkReached is what a walk sent by its end, per module the machines it reached: its first machines, its rest, // and every module a gated send carried with it. A release plan reached the machines it walked: those done and // the one being judged. func walkReached(p inventory.Plan, running func(string) []string) map[string][]string { out := map[string][]string{} carried := func(g *inventory.PlanGate) { if g == nil { return } for _, c := range g.Carried { out[c.Module] = appendNew(out[c.Module], c.Node) } } if r := p.Release; r != nil { reached := append([]string{}, r.Done...) if r.Gate != nil { reached = appendNew(reached, r.Gate.Machines...) carried(r.Gate) } for _, g := range r.Gates { if g != nil && !g.Pending { reached = appendNew(reached, g.Machines...) carried(g) } } for _, tier := range p.Tiers { for _, m := range tier { for _, n := range running(m) { if slices.Contains(reached, n) { out[m] = appendNew(out[m], n) } } } } return out } for m, s := range p.Modules { if s == nil { continue } out[m] = appendNew(out[m], s.First...) for n := range s.Rest { out[m] = appendNew(out[m], n) } carried(s.Gate) if len(out[m]) == 0 { delete(out, m) } } return out } // touchesOf turns what a walk moves, per module, into what it touches, per machine: the modules it moves there, // and, where it moves the tool runner, every module there whose tools it serves, and where it moves the // node-engine, every module there. Sorted by machine and module. func touchesOf(moved map[string][]string, on map[string][]string, shelf map[string]catalogue.Manifest) []walkTouch { byMachine := map[string][]string{} for m, machines := range moved { for _, n := range machines { byMachine[n] = appendNew(byMachine[n], m) } } out := make([]walkTouch, 0, len(byMachine)) for n, modules := range byMachine { t := walkTouch{Machine: n, Modules: modules} engine, runner := slices.Contains(modules, hostModule), slices.Contains(modules, broker.RuntimeModule) if engine || runner { for _, m := range on[n] { if slices.Contains(t.Modules, m) || (!engine && !servesTools(shelf[m])) { continue } t.Modules = append(t.Modules, m) t.Carried = append(t.Carried, m) } } sort.Strings(t.Modules) sort.Strings(t.Carried) out = append(out, t) } sort.Slice(out, func(i, j int) bool { return out[i].Machine < out[j].Machine }) return out } // servesTools says the tool runner serves something of this module: a bundle it loads entrypoints from, or the // module's own tools. func servesTools(m catalogue.Manifest) bool { if len(m.Tools) > 0 { return true } for _, b := range m.Bundles { if len(b.Loads) > 0 { return true } } return false } // walkRead is the facts a walk's touches are worked out from, read once per event. type walkRead struct { running func(string) []string rollsOut func(string) bool onMachine func(moved map[string][]string) map[string][]string shelf func() map[string]catalogue.Manifest } // readWalkFacts reads what the walk's touches need. Without facts (a test, a command with no store), nothing runs // anywhere and every module rolls out: the touches are then what the walk itself recorded. func readWalkFacts(ctx context.Context, facts walkFacts, p inventory.Plan) walkRead { cantRead := func(what string, err error) { fmt.Fprintf(os.Stderr, "%s: %s cannot be read for what the walk touches: %v\n", p.ID, what, err) } r := walkRead{ running: func(string) []string { return nil }, rollsOut: func(string) bool { return true }, onMachine: func(map[string][]string) map[string][]string { return nil }, shelf: func() map[string]catalogue.Manifest { return nil }, } if facts == nil { return r } running := map[string][]string{} r.running = func(m string) []string { if on, read := running[m]; read { return on } on, err := facts.Running(ctx, m) if err != nil { cantRead("where "+m+" runs", err) } running[m] = on return on } rolls := map[string]bool{} r.rollsOut = func(m string) bool { if yes, read := rolls[m]; read { return yes } u, err := facts.UpgradeOf(ctx, m) if err != nil { cantRead(m+"'s policy", err) } // A policy that cannot be read is taken to roll out: the walk is then held more, never less. rolls[m] = err != nil || u.RollOut return rolls[m] } r.onMachine = func(moved map[string][]string) map[string][]string { out := map[string][]string{} for _, core := range []string{hostModule, broker.RuntimeModule} { for _, n := range moved[core] { if _, read := out[n]; read { continue } var there []string if sent, known, err := facts.SentBuilds(ctx, n); err != nil { cantRead("what "+n+" was sent", err) } else if known { for m := range sent { there = append(there, m) } } if assigned, err := facts.Assigned(ctx, n); err != nil { cantRead("what "+n+" is assigned", err) } else { there = appendNew(there, assigned...) } sort.Strings(there) out[n] = there } } return out } r.shelf = func() map[string]catalogue.Manifest { shelf, err := facts.Catalogue(ctx) if err != nil { cantRead("the catalogue", err) } return shelf } return r }