jochen
facc551ff2
The router holds what an open walk touches and tells its result once; it can do so only if plan-moved names the walk's start and end, the modules and machines it moves, what a moved tool runner or node-engine carries there, and how long to hold.
335 lines
11 KiB
Go
335 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"slices"
|
|
"sort"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// What a walk touches, its bound and its outcome (novox/hq ADR 0308 decision 1, decision 5): said in
|
|
// `plan-moved` on the save that starts the walk and the save that ends it, beside the walk whole, for the holder
|
|
// of the operator's channel to hold what the walk touches while it runs and tell its result once.
|
|
//
|
|
// **What a walk touches** is every module it moves on every machine it sends it to, and every module a moved
|
|
// tool runner (node-tools) or node-engine (mesh-host) carries on that machine: what restarts there. At the start
|
|
// that is what the walk will send — each module of its tiers whose policy rolls it out, on every machine running
|
|
// it; at the end it is what it reached — the machines each module was sent to, first and rest, and what a gated
|
|
// send carried with it.
|
|
//
|
|
// **Its bound** is twice the largest delivery budget among the walk's deliveries (deliveryBudgets: a core
|
|
// module's ten minutes, a leaf module's budget in force), and fifteen minutes, not doubled, for a walk none of
|
|
// whose deliveries has a budget: a release plan (builds waiting for a person's gate), a walk whose every module's
|
|
// policy records rather than rolls out (held for a person, ADR 0282 decision 1), or a walk kept without a class.
|
|
|
|
// walkNoBudgetBound is the bound of a walk with no budget (ADR 0308 decision 5): not doubled.
|
|
const walkNoBudgetBound = 15 * time.Minute
|
|
|
|
// budgetNone is the budget class of a walk none of whose deliveries has a budget.
|
|
const budgetNone = "none"
|
|
|
|
// walkSaid is what `plan-moved` adds on a walk's start and end, beside the walk whole.
|
|
type walkSaid struct {
|
|
Walk string `json:"walk"`
|
|
Phase string `json:"phase"`
|
|
// Touches is, per machine, the modules the walk moves or reaches there and what a moved tool runner or
|
|
// node-engine carries there; never null.
|
|
Touches []walkTouch `json:"touches"`
|
|
// Outcome is on the end only: delivered, failed or stopped.
|
|
Outcome string `json:"outcome,omitempty"`
|
|
// BoundSeconds and BudgetClass are said on both, read from the walk's class at the start.
|
|
BoundSeconds int `json:"bound_seconds"`
|
|
BudgetClass string `json:"budget_class"`
|
|
// StartedAt is when the walk was said started, on both; EndedAt when it was said ended, on the end: the
|
|
// bound runs from StartedAt, whenever the event is heard.
|
|
StartedAt *time.Time `json:"started_at,omitempty"`
|
|
EndedAt *time.Time `json:"ended_at,omitempty"`
|
|
}
|
|
|
|
// walkTouch is what a walk touches on one machine.
|
|
type walkTouch struct {
|
|
Machine string `json:"machine"`
|
|
Modules []string `json:"modules"`
|
|
// Carried names those of Modules that are there only because a moved tool runner or node-engine carries
|
|
// them: what restarts with it, not what the walk builds.
|
|
Carried []string `json:"carried,omitempty"`
|
|
}
|
|
|
|
// planMovedBody is `plan-moved`'s body: the walk whole, as before, and on its start and end what it touches.
|
|
type planMovedBody struct {
|
|
inventory.Plan
|
|
*walkSaid
|
|
}
|
|
|
|
// walkFacts is what the touches are read from: the controller's inventory.
|
|
type walkFacts interface {
|
|
Running(ctx context.Context, module string) ([]string, error)
|
|
UpgradeOf(ctx context.Context, module string) (inventory.Upgrade, error)
|
|
Assigned(ctx context.Context, node string) ([]string, error)
|
|
SentBuilds(ctx context.Context, node string) (map[string]string, bool, error)
|
|
Catalogue(ctx context.Context) (map[string]catalogue.Manifest, error)
|
|
}
|
|
|
|
// sayWalk is what `plan-moved` adds to the save that started or ended the walk; nil on every other save. A fact
|
|
// that cannot be read is said on the controller's log and left out, never in the way of the event: a walk said
|
|
// with fewer touches is held less, never longer.
|
|
func sayWalk(ctx context.Context, facts walkFacts, p inventory.Plan) *walkSaid {
|
|
if p.WalkSaid == "" {
|
|
return nil
|
|
}
|
|
w := &walkSaid{Walk: p.ID, Phase: p.WalkSaid}
|
|
if p.Times != nil {
|
|
w.StartedAt = p.Times.Started
|
|
if p.WalkSaid == inventory.WalkEnded {
|
|
w.EndedAt, w.Outcome = p.Times.Ended, p.Outcome()
|
|
}
|
|
}
|
|
read := readWalkFacts(ctx, facts, p)
|
|
w.BudgetClass = walkBudgetClass(p, read.rollsOut)
|
|
w.BoundSeconds = int(walkBound(w.BudgetClass) / time.Second)
|
|
var moved map[string][]string
|
|
if p.WalkSaid == inventory.WalkStarted {
|
|
moved = walkSends(p, read.running, read.rollsOut)
|
|
} else {
|
|
moved = walkReached(p, read.running)
|
|
}
|
|
var shelf map[string]catalogue.Manifest
|
|
if len(moved[broker.RuntimeModule]) > 0 {
|
|
// Which modules the tool runner serves is the catalogue's: read only when the walk moves it.
|
|
shelf = read.shelf()
|
|
}
|
|
w.Touches = touchesOf(moved, read.onMachine(moved), shelf)
|
|
return w
|
|
}
|
|
|
|
// walkBudgetClass is the largest budget class among the walk's deliveries: its class read at the cut when one of
|
|
// its modules rolls out, and none when it has no budget.
|
|
func walkBudgetClass(p inventory.Plan, rollsOut func(string) bool) string {
|
|
if p.Release != nil || p.Times == nil {
|
|
return budgetNone
|
|
}
|
|
if _, budgeted := deliveryBudgets[p.Times.Class]; !budgeted {
|
|
return budgetNone
|
|
}
|
|
for _, tier := range p.Tiers {
|
|
for _, m := range tier {
|
|
if rollsOut(m) {
|
|
return p.Times.Class
|
|
}
|
|
}
|
|
}
|
|
return budgetNone
|
|
}
|
|
|
|
// walkBound is a walk's bound by its budget class: twice its budget, or fifteen minutes with none.
|
|
func walkBound(class string) time.Duration {
|
|
if b, ok := deliveryBudgets[class]; ok {
|
|
return 2 * b
|
|
}
|
|
return walkNoBudgetBound
|
|
}
|
|
|
|
// walkSends is what a walk will send at its start, per module the machines it goes to: each module of its tiers
|
|
// whose policy rolls it out, on every machine running it (for a release plan, those of its order).
|
|
func walkSends(p inventory.Plan, running func(string) []string, rollsOut func(string) bool) map[string][]string {
|
|
out := map[string][]string{}
|
|
for _, tier := range p.Tiers {
|
|
for _, m := range tier {
|
|
if s := p.Modules[m]; s != nil && s.State == planDeleted {
|
|
continue
|
|
}
|
|
if p.Release == nil && !rollsOut(m) {
|
|
continue
|
|
}
|
|
for _, n := range running(m) {
|
|
if p.Release != nil && !slices.Contains(p.Release.Order, n) {
|
|
continue
|
|
}
|
|
out[m] = appendNew(out[m], n)
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// walkReached is what a walk sent by its end, per module the machines it reached: its first machines, its rest,
|
|
// and every module a gated send carried with it. A release plan reached the machines it walked: those done and
|
|
// the one being judged.
|
|
func walkReached(p inventory.Plan, running func(string) []string) map[string][]string {
|
|
out := map[string][]string{}
|
|
carried := func(g *inventory.PlanGate) {
|
|
if g == nil {
|
|
return
|
|
}
|
|
for _, c := range g.Carried {
|
|
out[c.Module] = appendNew(out[c.Module], c.Node)
|
|
}
|
|
}
|
|
if r := p.Release; r != nil {
|
|
reached := append([]string{}, r.Done...)
|
|
if r.Gate != nil {
|
|
reached = appendNew(reached, r.Gate.Machines...)
|
|
carried(r.Gate)
|
|
}
|
|
for _, tier := range p.Tiers {
|
|
for _, m := range tier {
|
|
for _, n := range running(m) {
|
|
if slices.Contains(reached, n) {
|
|
out[m] = appendNew(out[m], n)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
for m, s := range p.Modules {
|
|
if s == nil {
|
|
continue
|
|
}
|
|
out[m] = appendNew(out[m], s.First...)
|
|
for n := range s.Rest {
|
|
out[m] = appendNew(out[m], n)
|
|
}
|
|
carried(s.Gate)
|
|
if len(out[m]) == 0 {
|
|
delete(out, m)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// touchesOf turns what a walk moves, per module, into what it touches, per machine: the modules it moves there,
|
|
// and, where it moves the tool runner, every module there whose tools it serves, and where it moves the
|
|
// node-engine, every module there. Sorted by machine and module.
|
|
func touchesOf(moved map[string][]string, on map[string][]string, shelf map[string]catalogue.Manifest) []walkTouch {
|
|
byMachine := map[string][]string{}
|
|
for m, machines := range moved {
|
|
for _, n := range machines {
|
|
byMachine[n] = appendNew(byMachine[n], m)
|
|
}
|
|
}
|
|
out := make([]walkTouch, 0, len(byMachine))
|
|
for n, modules := range byMachine {
|
|
t := walkTouch{Machine: n, Modules: modules}
|
|
engine, runner := slices.Contains(modules, hostModule), slices.Contains(modules, broker.RuntimeModule)
|
|
if engine || runner {
|
|
for _, m := range on[n] {
|
|
if slices.Contains(t.Modules, m) || (!engine && !servesTools(shelf[m])) {
|
|
continue
|
|
}
|
|
t.Modules = append(t.Modules, m)
|
|
t.Carried = append(t.Carried, m)
|
|
}
|
|
}
|
|
sort.Strings(t.Modules)
|
|
sort.Strings(t.Carried)
|
|
out = append(out, t)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool { return out[i].Machine < out[j].Machine })
|
|
return out
|
|
}
|
|
|
|
// servesTools says the tool runner serves something of this module: a bundle it loads entrypoints from, or the
|
|
// module's own tools.
|
|
func servesTools(m catalogue.Manifest) bool {
|
|
if len(m.Tools) > 0 {
|
|
return true
|
|
}
|
|
for _, b := range m.Bundles {
|
|
if len(b.Loads) > 0 {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// walkRead is the facts a walk's touches are worked out from, read once per event.
|
|
type walkRead struct {
|
|
running func(string) []string
|
|
rollsOut func(string) bool
|
|
onMachine func(moved map[string][]string) map[string][]string
|
|
shelf func() map[string]catalogue.Manifest
|
|
}
|
|
|
|
// readWalkFacts reads what the walk's touches need. Without facts (a test, a command with no store), nothing runs
|
|
// anywhere and every module rolls out: the touches are then what the walk itself recorded.
|
|
func readWalkFacts(ctx context.Context, facts walkFacts, p inventory.Plan) walkRead {
|
|
cantRead := func(what string, err error) {
|
|
fmt.Fprintf(os.Stderr, "%s: %s cannot be read for what the walk touches: %v\n", p.ID, what, err)
|
|
}
|
|
r := walkRead{
|
|
running: func(string) []string { return nil },
|
|
rollsOut: func(string) bool { return true },
|
|
onMachine: func(map[string][]string) map[string][]string { return nil },
|
|
shelf: func() map[string]catalogue.Manifest { return nil },
|
|
}
|
|
if facts == nil {
|
|
return r
|
|
}
|
|
running := map[string][]string{}
|
|
r.running = func(m string) []string {
|
|
if on, read := running[m]; read {
|
|
return on
|
|
}
|
|
on, err := facts.Running(ctx, m)
|
|
if err != nil {
|
|
cantRead("where "+m+" runs", err)
|
|
}
|
|
running[m] = on
|
|
return on
|
|
}
|
|
rolls := map[string]bool{}
|
|
r.rollsOut = func(m string) bool {
|
|
if yes, read := rolls[m]; read {
|
|
return yes
|
|
}
|
|
u, err := facts.UpgradeOf(ctx, m)
|
|
if err != nil {
|
|
cantRead(m+"'s policy", err)
|
|
}
|
|
// A policy that cannot be read is taken to roll out: the walk is then held more, never less.
|
|
rolls[m] = err != nil || u.RollOut
|
|
return rolls[m]
|
|
}
|
|
r.onMachine = func(moved map[string][]string) map[string][]string {
|
|
out := map[string][]string{}
|
|
for _, core := range []string{hostModule, broker.RuntimeModule} {
|
|
for _, n := range moved[core] {
|
|
if _, read := out[n]; read {
|
|
continue
|
|
}
|
|
var there []string
|
|
if sent, known, err := facts.SentBuilds(ctx, n); err != nil {
|
|
cantRead("what "+n+" was sent", err)
|
|
} else if known {
|
|
for m := range sent {
|
|
there = append(there, m)
|
|
}
|
|
}
|
|
if assigned, err := facts.Assigned(ctx, n); err != nil {
|
|
cantRead("what "+n+" is assigned", err)
|
|
} else {
|
|
there = appendNew(there, assigned...)
|
|
}
|
|
sort.Strings(there)
|
|
out[n] = there
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
r.shelf = func() map[string]catalogue.Manifest {
|
|
shelf, err := facts.Catalogue(ctx)
|
|
if err != nil {
|
|
cantRead("the catalogue", err)
|
|
}
|
|
return shelf
|
|
}
|
|
return r
|
|
}
|