package main import ( "context" "encoding/json" "errors" "flag" "fmt" "slices" "sort" "strings" "sync" "time" "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/builder" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/chain" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // A merge produces a tiered plan the mesh keeps (novox/hq ADR 0162). // // The handler that hears the merge computes the plan from the catalogue's one dependency relation, // writes it to the store, asks the first tier and returns — the receive loop is never held by a // build. Every outcome taken in advances the plan it belongs to; a look at the plans, asked for by each event that can // move one (plan_looks.go, ADR 0406 §5), advances what outcomes alone cannot (a tier waiting for machines to report); // a controller replaced mid-plan finds the plan where it left it and looks first. // gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since // novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on. var gateFirstMachine = false // planWaitBound is how long a plan may wait on one thing before `status` names it red. const planWaitBound = 30 * time.Minute // tiersOf sorts a set of modules into tiers along the ordering edges among them: tier 0 depends // on nothing else in the set, tier 1 only on tier 0, and so on. An edge to a module outside the set says // nothing about the order inside it. A cycle — which the catalogue should never produce — puts // what remains in one last tier rather than losing it, and is said by the caller. func tiersOf(set []string, edges []inventory.Edge) [][]string { in := map[string]bool{} for _, m := range set { in[m] = true } deps := map[string]map[string]bool{} for _, m := range set { deps[m] = map[string]bool{} } for _, e := range edges { // A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's // build needs nothing of A's first. Stands-on and declared order after the base is built, and the // build seat's holder follows the controller that defines its worker (EdgeWorkerOf, issue 206). if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages { continue } // **Being built by the builder orders nothing** (novox/hq ADR 0360 §3): whichever builder runs builds it. if e.Kind == inventory.EdgeBuiltBy { continue } deps[e.From][e.To] = true } placed := map[string]bool{} var tiers [][]string for len(placed) < len(set) { var tier []string for _, m := range set { if placed[m] { continue } free := true for d := range deps[m] { if !placed[d] { free = false break } } if free { tier = append(tier, m) } } if len(tier) == 0 { // A cycle: everything left, together, and the caller says so. for _, m := range set { if !placed[m] { tier = append(tier, m) } } } sort.Strings(tier) for _, m := range tier { placed[m] = true } tiers = append(tiers, tier) } return tiers } // reachableFrom is the moved modules plus everything that depends on them, through every layer: // what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine // is not changed by a new build machine, so a built-by edge orders and gates a plan and never // widens it — the first plan of 2026-10-01 took the whole catalogue along for a controller change. func reachableFrom(moved []string, edges []inventory.Edge) []string { in := map[string]bool{} for _, m := range moved { in[m] = true } for grew := true; grew; { grew = false for _, e := range edges { // Built-by and worker-of order a plan; neither widens it. A new build machine changes // nothing it builds, and a new controller changes nothing about the holder it orders. A // packages edge, read from a record made before novox/hq ADR 0267, widens nothing either: // a shared file moves each module whose build source holds it, directly. if e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf || e.Kind == inventory.EdgePackages || e.Kind == edgeChangeOrder { continue } if in[e.To] && !in[e.From] { in[e.From] = true grew = true } } } out := make([]string, 0, len(in)) for m := range in { out = append(out, m) } sort.Strings(out) return out } // hasCycle says whether the tiers' last tier holds modules that still depend on each other. A packages // or built-by edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is // no cycle, and saying one was is a false report in every such plan's log. func hasCycle(tiers [][]string, edges []inventory.Edge) bool { if len(tiers) == 0 { return false } last := map[string]bool{} for _, m := range tiers[len(tiers)-1] { last[m] = true } for _, e := range edges { if e.Kind == inventory.EdgePackages || e.Kind == inventory.EdgeBuiltBy { continue } if last[e.From] && last[e.To] { return true } } return false } // planOfMerge is the plan one merge produces: the moved modules and everything reachable from them, // tiered, with the merge it answers. func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inventory.Plan { p := planOfMoves(moved, edges) merged, _ := time.Parse(time.RFC3339Nano, m.MergedAt) p.Repository, p.Branch, p.Commit, p.Merged = m.Owner+"/"+m.Repo, m.Base, m.Commit, merged.UTC() return p } // planOfMoves is the walk of a set of moved modules (novox/hq ADR 0162, 0276): they and everything reachable // from them, tiered along the graph, with no merge named yet. func planOfMoves(moved []string, edges []inventory.Edge) inventory.Plan { set := reachableFrom(moved, edges) tiers := tiersOf(set, edges) modules := map[string]*inventory.PlanModule{} for _, name := range set { modules[name] = &inventory.PlanModule{} } return inventory.Plan{ ID: fmt.Sprintf("plan-%d", time.Now().UnixNano()), Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: tiers, Modules: modules, } } // gates is what the next tier needs running from this one: a module of the tier that a later // tier is built by — the runtime dependency — and whose policy rolls it out, must be applied by // the machines running it before the next tier is asked. A base an image stands on need only be // built; a source another module packages need not even be that. func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) []string { if p.Tier >= len(p.Tiers) { return nil } inTier := map[string]bool{} for _, m := range p.Tiers[p.Tier] { inTier[m] = true } later := map[string]bool{} for _, tier := range p.Tiers[p.Tier+1:] { for _, m := range tier { later[m] = true } } seen := map[string]bool{} var out []string // The plan's own order (ADR 0419) is not in the catalogue's graph: it is kept on the plan. if p.Delivery != nil { edges = append(append([]inventory.Edge{}, edges...), p.Delivery.Order...) } for _, e := range edges { // A delivery group's order inside a walk (novox/hq ADR 0276 decision 5) gates as built-by does: the // member before is running on its machines before the member after is asked. // A built-by edge waits for nothing since novox/hq ADR 0360 §3. ordered := e.Kind == edgeChangeOrder if later[e.From] && inTier[e.To] && ordered && !seen[e.To] && rollsOut(e.To) { seen[e.To] = true out = append(out, e.To) } } sort.Strings(out) return out } // noMoveWhy is what a plan says of a module of its tier whose build was made from the source every machine // running it runs (novox/hq issue 280): nothing is sent for it. // noMovePrefix opens it, and is what noMove reads: one prefix for the writer and the reader. const ( noMovePrefix = "no move" noMoveWhy = noMovePrefix + ": made from the source every machine running it runs" ) // noMove is whether the plan marked the module no move and sent it nothing. Read from what the plan says, // so a plan kept before the mark had a name of its own is read the same (novox/hq issue 648). func noMove(s *inventory.PlanModule) bool { return s != nil && s.FirstAt == nil && s.Gate == nil && strings.HasPrefix(s.Why, noMovePrefix) } // applied says whether every machine running the module has reported since the module was built. // appliedEach is applied with a moment of its own for each machine: the reports that count are the ones // after that machine was sent the build (novox/hq issue 256). func appliedEach(module string, since func(node string) time.Time, running []string, reports []inventory.Reported) (bool, []string) { at := map[string]*time.Time{} for _, r := range reports { at[r.Node] = r.At } var waiting []string for _, n := range running { if t := at[n]; t == nil || t.Before(since(n)) { waiting = append(waiting, n) } } return len(waiting) == 0, waiting } // gateNeed is a module the next tier waits for: the machines running it, and from when each must have reported. type gateNeed struct { module string running []string since func(node string) time.Time } // silentNodes is the machines a walk sent and heard nothing from past ApplySilentAfter, on whichever tier it sent // them (novox/hq ADR 0360 §5, issue 1160): the one rule both the gates between tiers and the walk's end read, through // walkReports. A machine that reports on its send is not silent from then on. func silentNodes(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[string]bool { _, _, silent := walkReports(p, reports, now, prov) out := map[string]bool{} for _, n := range silent { out[n] = true } return out } // withoutSilent is the machines of running that are not silent: a later tier is not sent to a machine the walk heard // nothing from, which keeps its want of the plan's builds for the send it gets when it is heard again (ADR 0360 §5). func withoutSilent(running []string, silent map[string]bool) []string { return slices.DeleteFunc(slices.Clone(running), func(n string) bool { return silent[n] }) } // settleTierGate is the wait between tiers on the reports of the machines running what the next tier needs: what is // still waited for ("module on a, b"), the silent machines left out of the wait, sorted, and whether every machine the // gate waits on is silent, so that nothing it needs was delivered anywhere (issue 1160). func settleTierGate(needs []gateNeed, reports []inventory.Reported, silent map[string]bool) (waiting, quiet []string, lost bool) { all, silentAll := 0, 0 seenQuiet := map[string]bool{} seenNode := map[string]bool{} for _, need := range needs { for _, n := range need.running { if !seenNode[n] { seenNode[n] = true all++ if silent[n] { silentAll++ } } } _, on := appliedEach(need.module, need.since, need.running, reports) var still []string for _, n := range on { if silent[n] { if !seenQuiet[n] { seenQuiet[n] = true quiet = append(quiet, n) } continue } still = append(still, n) } if len(still) > 0 { waiting = append(waiting, fmt.Sprintf("%s on %s", need.module, strings.Join(still, ", "))) } } sort.Strings(quiet) return waiting, quiet, all > 0 && silentAll == all } func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) { at := map[string]*time.Time{} for _, r := range reports { at[r.Node] = r.At } var waiting []string for _, n := range running { if t := at[n]; t == nil || t.Before(builtAt) { waiting = append(waiting, n) } } return len(waiting) == 0, waiting } // askTier asks the build machine for the named modules of the plan's tier — all of it, or what of it is still // unasked (novox/hq issue 754) — and marks each asked. A module // the catalogue no longer holds, or whose ask could not be made, is a failure of the plan: a tier // half asked is a tier that will never complete. func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, names []string) error { entries, err := inv.Catalogued(ctx) if err != nil { return err } byName := map[string]inventory.Entry{} for _, e := range entries { byName[e.Manifest.Module] = e } for _, name := range names { askModule(ctx, p, name, byName) } return nil } // unaskedOf are the modules of a plan's current tier it has still to ask: those it never asked, and one it holds // built that it never asked for (novox/hq issue 754) — a build taken in by a controller from before a walk took // only its own asks, somebody else's, which the walk would otherwise send as its own (ADR 0219). A tier partly asked is asked the rest, not // waited on for ever: a walk that waits for every build of a tier it never asked whole never moves again. func unaskedOf(p inventory.Plan) []string { if p.Tier >= len(p.Tiers) { return nil } var out []string for _, m := range p.Tiers[p.Tier] { s := p.Modules[m] switch { case s == nil || s.State == "": out = append(out, m) case s.State == "built" && s.Build == "" && s.AskedAt == nil: out = append(out, m) } } return out } // askedInTier says the plan asked a module of its current tier itself: under an id, or at a time it kept. func askedInTier(p inventory.Plan) bool { if p.Tier >= len(p.Tiers) { return false } for _, m := range p.Tiers[p.Tier] { if s := p.Modules[m]; s != nil && (s.Build != "" || s.AskedAt != nil) { return true } } return false } // takesOutcome says whether a build's outcome is the plan's for a module of its current tier (novox/hq ADR 0219, // issues 219 and 754). **A walk that waits for its delivery's word asks nothing, and so is answered by nothing** // (ADR 0239): a rebuild that lands in the wait is somebody else's build. **Nor is a module the plan never asked // answered**: taken, the plan held a module built that it never asked, and waited for the rest of the tier for // ever. Asked under an id, the plan takes that id's outcome and no other; asked before ids were kept, a build // asked at or after the plan's ask. func takesOutcome(p inventory.Plan, state *inventory.PlanModule, asked time.Time, id string) bool { if p.Waiting() || state == nil || (state.Build == "" && state.AskedAt == nil) { return false } if state.Build != "" { return state.Build == id } return !askedBefore(asked, state.AskedAt) } // askABuild is how a plan asks for one build, not waited for, and learns the id it asked under. A // variable so a test of what a plan does around an ask needs no build machine. // // Set in init, not where it is declared: an assignment pending on a build asks for one too (novox/hq issue // 325), and a build's outcome makes pending assignments, so the two refer to each other. var askABuild func(ctx context.Context, source buildSource, path, ref string) (string, error) func init() { askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) { // Kept by each asker with its own name once the ask is made (novox/hq issue 325). return buildOneAsked(ctx, source, path, ref, 0, false, "") } } // reuseKey carries a walk's leave to take a check's build from askModule to the ask it makes (novox/hq ADR 0350 §2). type reuseKey struct{} // withReuse is ctx carrying a walk's leave to take a check's build. func withReuse(ctx context.Context, r *link.ReuseAsk) context.Context { return context.WithValue(ctx, reuseKey{}, r) } // reuseIn is the leave ctx carries; nil for every ask but a walk's. func reuseIn(ctx context.Context) *link.ReuseAsk { r, _ := ctx.Value(reuseKey{}).(*link.ReuseAsk) return r } // reuseAsked is the leave an ask carries: a walk's, and never a dry run's — a dry run registers nothing, so it // takes nothing either (novox/hq ADR 0350 §2, §6). func reuseAsked(ctx context.Context, dryRun bool) *link.ReuseAsk { if dryRun { return nil } return reuseIn(ctx) } // reuseAskOf is a walk's leave to take a check's build: the pull requests of the merges it answers. func reuseAskOf(p *inventory.Plan) *link.ReuseAsk { r := &link.ReuseAsk{} if p.Delivery == nil { return r } seen := map[link.PullRef]bool{} for _, m := range p.Delivery.Merges { pull, ok := pullOf(m) if !ok { continue } if !seen[pull] { seen[pull] = true r.Pulls = append(r.Pulls, pull) } } return r } // checkedBuildsOf are the builds the build seat's checks of those pull requests said they made for their merges, // within the week check builds are kept (novox/hq ADR 0350 §2): read from the seat's own outcomes on the bus. A // variable so a test needs no bus. var checkedBuildsOf = func(ctx context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) { if len(pulls) == 0 { return nil, nil } js, err := aBus() if err != nil { return nil, err } defer js.Close() return link.ReadCheckBuilds(ctx, js, buildSeatHeld(ctx), pulls, time.Now().Add(-builder.CheckBuildsKept)) } // mergedHeadsOf are the head commits the walk's merges merged, by pull request (novox/hq issue 703): a pull request // merged twice in one walk, or whose head the announcement did not say, has none, and takes no check's build. func mergedHeadsOf(p *inventory.Plan) map[link.PullRef]string { heads := map[link.PullRef]string{} if p.Delivery == nil { return heads } seen := map[link.PullRef]bool{} for _, m := range p.Delivery.Merges { pull, ok := pullOf(m) if !ok { continue } if seen[pull] && heads[pull] != m.Head { heads[pull] = "" continue } seen[pull] = true heads[pull] = m.Head } return heads } // walkVerdicts are the checks' builds a walk read, by the walk's plan: read once per walk (novox/hq issue 649), not // once per module it asks for. Kept a day; a walk lasts minutes. var walkVerdicts = struct { sync.Mutex read map[string]verdictsRead }{read: map[string]verdictsRead{}} type verdictsRead struct { at time.Time builds []link.CheckedBuild } // reuseWithVerdicts is a walk's leave with the builds its pull requests' verdicts name: the only builds the builder // may take (ADR 0350 §2), and **only those of a check of the head each merge merged** (novox/hq issue 703): a build // a check made at an earlier head of the same pull request is never registered. Read once per walk. When they // cannot be read the leave names none, and the walk builds, saying why. func reuseWithVerdicts(ctx context.Context, walk string, r *link.ReuseAsk, heads map[link.PullRef]string) *link.ReuseAsk { walkVerdicts.Lock() read, known := walkVerdicts.read[walk] walkVerdicts.Unlock() if !known || walk == "" { builds, err := checkedBuildsOf(ctx, r.Pulls) if err != nil { fmt.Printf(" the checks' builds of %d pull request(s) could not be read, so the walk builds: %v\n", len(r.Pulls), err) return r } read = verdictsRead{at: time.Now(), builds: builds} if walk != "" { walkVerdicts.Lock() for id, w := range walkVerdicts.read { if time.Since(w.at) > 24*time.Hour { delete(walkVerdicts.read, id) } } walkVerdicts.read[walk] = read walkVerdicts.Unlock() } } r.Builds = nil for _, b := range read.builds { if link.SameHead(b.Head, heads[b.Pull]) { r.Builds = append(r.Builds, b) } } return r } // pullOf is a merge's pull request: owner and repository, the last two parts of however the repository is spelled // (owner/repo, a path on the git seat, a URL), and its number. False when the merge names none. func pullOf(m inventory.PlanMerge) (link.PullRef, bool) { parts := strings.Split(strings.Trim(strings.TrimSuffix(m.Repository, ".git"), "/"), "/") if len(parts) < 2 || m.Number <= 0 || parts[len(parts)-2] == "" || parts[len(parts)-1] == "" { return link.PullRef{}, false } return link.PullRef{Owner: parts[len(parts)-2], Repo: parts[len(parts)-1], Number: m.Number}, true } // askModule asks the build machine for one module of a plan and marks it asked, with the id it was // asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked. func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) { now := time.Now().UTC() state := p.Modules[name] if state == nil { state = &inventory.PlanModule{} p.Modules[name] = state } e, known := byName[name] if !known { state.State = "failed" state.Why = "no longer in the catalogue" p.State = inventory.PlanFailed p.Note = name + " is no longer in the catalogue" return } source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat} fmt.Printf(" tier %d: ", p.Tier) // The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215): the branch contains every // commit a batch's walk carries — but for a merge walked alone after a failed walk (novox/hq ADR 0276 // decision 3), built on its own commit to find which merge brought the failure. ref := followedBranch(e.Source.Ref) carried := p.CommitOn(e.Source.Repository, ref) if p.Delivery != nil && p.Delivery.Alone && carried != "" { ref = carried } // **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull // requests it answers, whose checks name the part of a differing key. id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source, e.Source.Path, ref) if err != nil { state.State = "failed" state.Why = err.Error() p.State = inventory.PlanFailed p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err) return } // The commit of the module's own repository the walk carries (novox/hq ADR 0276): a batch's walk carries // one per repository. commit := carried if commit == "" { commit = p.Commit } recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: e.Source.Repository, Seat: e.Source.Seat, Path: e.Source.Path, Ref: followedBranch(e.Source.Ref), Commit: commit, For: "plan"}) state.State = "asked" state.AskedAt = &now state.Build = id state.Why, state.Commit, state.BuiltAt, state.Made = "", "", nil, "" } // planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and // advances what that completes. Called from the daemon's take-in of every outcome. // // **Only a build asked at or after the plan's ask is its outcome** (novox/hq 04-ISSUES/219). Two // plans a few minutes apart both ask for a module; the earlier plan's build, finishing late, is not // the later plan's answer — it stood on the bases from before the later plan's merge, and taking it // would send machines, and the next tier, what the later merge replaced. asked is zero when the // build's request time is not known, and such an outcome is taken as before. func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) { planBuiltSaying(ctx, open, module, commit, failed, asked, id, "") } // planBuiltSaying is planBuilt keeping what the build did — "reused from " or "built: " (novox/hq // ADR 0350 §4) — on the module of the walk it answers. func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id, made string) { inv := open.inventory // One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather // than write over what the holder is about to save. Not taken, it is still in the build records, // which the holder settles the plan from (issue 214). release, err := inv.HoldPlans(ctx, true) if err != nil { fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err) return } defer release() plans, err := inv.OpenPlans(ctx) if err != nil { fmt.Printf("plans: cannot read them: %v\n", err) return } now := time.Now().UTC() for i := range plans { p := &plans[i] if p.Tier >= len(p.Tiers) { continue } inTier := false for _, m := range p.Tiers[p.Tier] { if m == module { inTier = true } } if !inTier { continue } // **The plan's own ask is its outcome, by id** (novox/hq ADR 0219); another build of the module // is, as before, when it was asked at or after the plan's ask (issue 219). // **A module asked under an id is answered by that id's outcome and no other** (novox/hq ADR // 0219): a replay, a rebuild beside the plan, or an older ask finishing late is somebody else's // build, made from other source, and settling the plan with it would send that. A plan from // before ids were kept is matched as it was: by when the build was asked (issue 219). A walk // waiting for its word, or a module it never asked, takes no outcome at all (issue 754). state := p.Modules[module] if !takesOutcome(*p, state, asked, id) { continue } if failed != "" && deletedAtSource(failed) { // Deleted at its source by the merge, not broken (novox/hq ADR 0236): the plan goes on. state.State, state.Why = planDeleted, "deleted at its source: "+firstLine(failed) forgetDeleted(ctx, inv, module, p) } else if failed != "" { state.State = "failed" state.Why = failed p.State = inventory.PlanFailed p.Note = fmt.Sprintf("%s failed to build in tier %d", module, p.Tier) sayUnsent(p, func(m string) bool { u, err := inv.UpgradeOf(ctx, m) return err == nil && u.RollOut }) } else { state.State = "built" state.BuiltAt = &now state.Commit = commit state.Made = made } if err := inv.SavePlan(ctx, p); err != nil { fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err) continue } if p.State == inventory.PlanFailed { fmt.Printf("%s: %s; the tiers after it are not asked\n", p.ID, p.Note) } } advanceHeld(ctx, open) } // startedBeside is the id of a started walk open beside this one that it is related to — a merge's walk past // its wait, not the backlog's — with why, or empty: a walk waits only for a started walk it shares something with // (novox/hq ADR 0276 decision 4, ADR 0407 decision 4, issue 973). func startedBeside(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (behind, why string, err error) { plans, err := inv.OpenPlans(ctx) if err != nil { return "", "", err } started := startedWalks(plans, p.ID, p.Created) if len(started) == 0 { return "", "", nil } edges, err := inv.Dependencies(ctx) if err != nil { return "", "", err } onPath, err := ownPathNow(ctx, inv) if err != nil { return "", "", err } open, err := openWalksOf(ctx, inv, started, onPath) if err != nil { return "", "", err } mine, err := openWalkOf(ctx, inv, p, onPath) if err != nil { return "", "", err } behind, why = relatedOpen(mine.fp, mine.own, open, edges) return behind, why, nil } // deferredNote begins the note of a walk deferred behind another. const deferredNote = "let go; starts once " // deferred says a walk has its word and has not started: let go while another walk was started, it waits for // that one to end (startedBeside), and its note says so. Read as a wait, not as a tier running late: `plans` // and `status` say its note, and S3 leaves it out. A let-go walk whose first ask failed carries that error as // its note instead, and is watched as before. func deferred(p inventory.Plan) bool { return p.Open() && p.Release == nil && p.Tier == 0 && !askedAny(p) && p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go != nil && strings.HasPrefix(p.Note, deferredNote) } // askedAny says a plan asked any module. func askedAny(p inventory.Plan) bool { for _, s := range p.Modules { if s != nil && s.State != "" { return true } } return false } // keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold // on the plans so it never lengthens it: measured, never acted on, and an error only said. func keepWalkPhases(ctx context.Context, open *stores) { if err := recordWalkPhases(ctx, open.inventory, time.Now()); err != nil { fmt.Printf("plans: the phases of the walks ended lately could not be kept: %v\n", err) } } // advanceHeld moves every open plan as far as the facts allow, for a caller holding the plans: a tier whose modules // are all built and whose gates are applied gives way to the next; the last tier done is the plan done. It says // whether a step failed, which the look at the plans (lookAtPlans) retries. // // **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two controllers — the // old and the new while a machine hands its controller over — would each ask a tier the other had just asked. func advanceHeld(ctx context.Context, open *stores) (failed bool) { inv := open.inventory // A walk that ended lets the next batch be cut at once, not at a window's next expiry (novox/hq ADR 0276, ADR 0407). defer func() { if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil { fmt.Printf("batches: %v\n", err) } }() plans, err := inv.OpenPlans(ctx) if err != nil { fmt.Printf("plans: cannot read them: %v\n", err) return true } if len(plans) == 0 { return false } edges, err := inv.Dependencies(ctx) if err != nil { fmt.Printf("plans: cannot read the dependencies: %v\n", err) return true } rollsOut := func(module string) bool { u, err := inv.UpgradeOf(ctx, module) return err == nil && u.RollOut } for i := range plans { p := &plans[i] for p.Open() { // **Saved only when the step changed it** (novox/hq issue 296). Every step was saved, and // the steps run on every tick and after every build outcome on the mesh: a plan whose tier // stood still for half an hour was written every few seconds, its revision moved, `plan-moved` // said on the bus, and its last save read as when it began building. before := planSnapshot(*p) sends := &sendsMade{} moved, err := advanceOnce(withSendsMade(ctx, sends), open, p, edges, rollsOut) if err != nil { failed = true // Kept in the plan, so `plans` says why it has not moved rather than the log alone; // the state is left as it was and the step is tried again by the next look. Said and // kept when it is new: the same refusal on every tick is one fact, not one per tick. p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again" // A refusal for the bus's planned step is a person's to end: S17 says it from its first moment. if refusedForTheBus(err) { busRefusedFirst.mark(p.ID, time.Now()) } if planSnapshot(*p) != before { fmt.Printf("%s: %v\n", p.ID, err) if err := inv.SavePlan(ctx, p); err != nil { fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err) } } break } if p.State == inventory.PlanFailed { sayUnsent(p, rollsOut) } if planSnapshot(*p) != before { if err := inv.SavePlan(ctx, p); err != nil { fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err) failed = true break } // Said once the send is kept (ADR 0406 §5, issue 974): what an outsider hears of a tier sent. saySends(ctx, *p, sends) } if !moved { break } } } return failed } // planSnapshot is a plan as a save would write it, to compare before and after a step: what the store // stamps itself (when it was saved, its revision and epoch, when its tier was entered) is left out. func planSnapshot(p inventory.Plan) string { p.Updated, p.Revision, p.Epoch, p.TierEntered = time.Time{}, 0, 0, time.Time{} b, err := json.Marshal(p) if err != nil { // Unreadable is never the same: the plan is saved, as every step was before. return fmt.Sprintf("unmarshallable %p %v", &p, err) } return string(b) } // advanceOnce takes one step of one plan and says whether anything changed. func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) (bool, error) { inv := open.inventory if p.Release != nil { return advanceRelease(ctx, open, p) } // **A walk that waits for its delivery's word asks nothing** (novox/hq ADR 0239): nothing of it is // registered before its turn, so no other send carries it to a machine. if p.Waiting() { note := waitingNote(*p) changed := p.Note != note p.Note = note return changed, nil } if p.Tier >= len(p.Tiers) { p.State = inventory.PlanDone fmt.Printf("%s: done — %s at %s, %d tier(s)\n", p.ID, p.Repository, short(p.Commit), len(p.Tiers)) return true, nil } tier := p.Tiers[p.Tier] // Not yet asked: ask — the whole tier, or what of it is still unasked (novox/hq issue 754). if unasked := unaskedOf(*p); len(unasked) > 0 { // **One walk at a time beside a related one** (novox/hq ADR 0276, ADR 0407 decision 4, issue 973): a walk // about to ask its first tier while a started walk it is related to is open waits for that one to end, // let go or not, and says so and why. A started walk it shares nothing with does not hold it. if p.Tier == 0 && !askedInTier(*p) { if behind, why, err := startedBeside(ctx, inv, *p); err != nil { return false, err } else if behind != "" { note := fmt.Sprintf("%s%s ended — %s", deferredNote, behind, why) changed := p.Note != note p.Note = note return changed, nil } } // **A backup before the send** (novox/hq ADR 0404, issue 1065): a walk that starts asks the backup holders of // the machines running a moved module that keeps data, and asks its first tier only when every one is done. if p.Tier == 0 && !askedInTier(*p) { if proceed, err := backupsFirst(ctx, inv, p); err != nil { return false, err } else if !proceed { return false, nil } } if len(unasked) < len(tier) { fmt.Printf("%s: tier %d was asked in part; asking the rest: %s\n", p.ID, p.Tier, strings.Join(unasked, ", ")) } if err := askTier(ctx, inv, p, unasked); err != nil { return false, err } return true, nil } // **Asked: settle from the build records first** (novox/hq 04-ISSUES/214). An outcome is taken // in by whichever controller hears it, and a merge to the controller's own repository replaces // the controller in its first tier: the build that produced the new one is recorded, and the // plan never hears it. The record is the fact; a build recorded after the ask is that tier's // outcome, whoever was listening. recorded, byID, err := recordsOfAsked(ctx, inv, p, tier) if err != nil { return false, err } if settleFromRecords(p, tier, recorded, byID) { return true, nil } // Asked: wait for every build. var latest time.Time for _, m := range tier { s := p.Modules[m] if s == nil || (s.State != "built" && s.State != planDeleted) { return false, nil } if s.BuiltAt != nil && s.BuiltAt.After(latest) { latest = *s.BuiltAt } } // Built: send every module of the tier whose policy rolls out, once, to the machines running // it — whether or not its source commit moved. A dependent rebuilt because its base moved, or // a module that packages another repository's source, keeps its commit; the catalogue announces // no move for it and its machines would keep the old image until somebody pushed (novox/hq // issue 189). A module whose policy records is built and left, as its policy says. // // **One machine first, unless the module's policy says together** (novox/hq issue 249, ADR // 0218). The plan sent every machine running the module at once, and the operator's policy — // one at a time, stopping at the first that fails, which an announced upgrade honours — was not // read here at all: a module whose new declarations broke it broke everywhere in the same // minute. Now the first machine is sent, the plan records it and waits for that machine's report // after the send to say it applied what it was sent; only then are the rest sent. A first machine // that fails or refuses stops the module's rollout and the plan with it, the rest untouched. // // **One send per machine for the whole tier** (novox/hq issue 281). A send carries the machine's // whole declaration (ADR 0221), and the plan sent the first machine once per module: on 2026-10-06 a // tier of seventy modules sent one laptop a new declaration every few seconds, the node-engine set // aside one for the next, the node tools never settled, and the gate blamed a module for the machine // being behind. Now every module of the tier whose first machine is the same is sent there in one // send, judged there by one gate — kept on the first of them, the others gated by it — and the rest // of the tier's machines are sent once, together, when the gates have passed. // **A machine silent past the bound is left as it was, on every tier** (novox/hq ADR 0360 §5, issue 1160): the // walk's end names it, and the gates between tiers and the sends of the tiers after do not wait for it or go to it. allReports, err := inv.LastReports(ctx) if err != nil { return false, err } silent := silentNodes(*p, allReports, time.Now().UTC(), provLookup(ctx)) // **A machine away when a tier is sent is not sent to and not waited for** (ADR 0435 §1, issue 1161): read once, // for a tier that has something left to send. var away map[string]bool for _, m := range tier { if s := p.Modules[m]; s != nil && s.SentAt == nil && s.FirstAt == nil && s.State != planDeleted { away = judgedAway(ctx, open, time.Now().UTC()) break } } var pending []string firstOn := map[string][]string{} runningOf := map[string][]string{} var rest []string restTo := map[string][]string{} together := map[string]bool{} for _, m := range tier { state := p.Modules[m] if state == nil || state.SentAt != nil || state.State == planDeleted { continue } // **A recorded module goes where its move recreates nothing and moves no data** (novox/hq ADR 0377): judged // per machine against the build it was last sent, said per machine, and sent only there. var goes map[string]bool if !rollsOut(m) { out, err := judgeRecordedInWalk(ctx, inv, p, m, state) if err != nil { return false, err } if len(out) == 0 { continue } goes = map[string]bool{} for _, n := range out { goes[n] = true } } running, err := inv.Running(ctx, m) if err != nil { return false, err } if goes != nil { running = slices.DeleteFunc(running, func(n string) bool { return !goes[n] }) } if state.FirstAt == nil { running = withoutSilent(running, silent) passed := withoutSilent(running, away) if len(passed) < len(running) { for _, n := range running { if away[n] && !slices.Contains(p.Away, n) { fmt.Printf("%s: %s is away; %s is not sent to it and not waited for\n", p.ID, n, m) } } rememberAway(p, running, away) running = passed } } runningOf[m] = running // **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source // of the build every machine running it was sent, it is registered with that build's artifacts — // nothing to send, and nothing for a gate to judge. if state.FirstAt == nil { if same, err := unchangedOnEvery(ctx, inv, m, state.Build, running); err != nil { return false, err } else if same { now := time.Now().UTC() state.SentAt = &now state.Why = noMoveWhy fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m) continue } } policy, err := inv.UpgradeOf(ctx, m) if err != nil { return false, err } // **Every machine at once** (novox/hq ADR 0360 §1, §2): a module that rolls out is sent to every machine // running it in one step, with no first machine and no gate, whatever module it is. A module a controller // before ADR 0360 already sent to its first machine finishes under its gate. if !gateFirstMachine && state.FirstAt == nil { policy.Together = true } var reports []inventory.Reported if !policy.Together { // Read for the choice of the first machine as well as for its report. if reports, err = inv.LastReports(ctx); err != nil { return false, err } } now := time.Now().UTC() // **Sent with others, judged with them** (issue 281): the gate of the send that carried it is its // verdict on its first machine. A failure there stopped the plan already. if state.GatedBy != "" { lead := p.Modules[state.GatedBy] if lead == nil || lead.Gate == nil || lead.Gate.Verdict == "" { pending = append(pending, fmt.Sprintf("%s judged with %s on %s", m, state.GatedBy, strings.Join(state.First, ", "))) continue } if lead.Gate.Verdict != inventory.GatePassed { continue } passedWith(m, state, state.GatedBy, lead.Gate) } // Read after its pass is taken over from the send that carried it, so a passed gate is never judged // again from the first machine's later reports (novox/hq issue 335). step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound) switch { case step.failed != "": // The first machine refused or failed what it was sent, or never said: the gate failed, and // what it carried is put back there (novox/hq ADR 0236); the rest are left as they were. failFirstSend(ctx, open, p, m, state, firstRunning(state.First, running), step.failed, step.rest) return true, nil case step.waiting != "": at := "" if state.FirstAt != nil { at = state.FirstAt.Local().Format("15:04") } pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, at)) continue } // **The gate** (novox/hq ADR 0236, to-be 45 §8): the first machine reported the build applied; // it is judged by its health before anything else is sent — the rest, or, where it is the only // machine, the plan's next step. if state.GatedBy == "" && !policy.Together && state.FirstAt != nil && len(state.First) > 0 { verdict, err := judgeGate(ctx, open, p, m, state, running, now) if err != nil { return false, err } switch verdict { case "": if putBackBroken(ctx, open, p, state.Gate, m, state) { return true, nil } pending = append(pending, fmt.Sprintf("%s judged on %s: %s", m, strings.Join(state.Gate.Machines, ", "), gateLine(state.Gate))) continue case inventory.GateFailed: failFirstSend(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why, step.rest) return true, nil case inventory.GateSuperseded: // Another send moved this module on its first machine (novox/hq issue 352): the build is not // judged, not marked, not put back; the plan ends here, said, and a newer plan carries on. state.Why = "superseded: " + state.Gate.Why p.State = inventory.PlanSuperseded p.Note = fmt.Sprintf("%s's judging on %s was superseded: %s", m, strings.Join(state.Gate.Machines, ", "), state.Gate.Why) fmt.Printf("%s: %s\n", p.ID, p.Note) return true, nil case inventory.GatePassed: if !state.Gate.Kept { gatePassed(ctx, open, p, m, state) state.Gate.Kept = true } } } switch { case len(step.send) == 0: // No machine runs it: nothing to send, and nothing to wait for. state.SentAt = &now continue case step.first: firstOn[step.send[0]] = append(firstOn[step.send[0]], m) continue } rest = append(rest, m) restTo[m] = step.send together[m] = policy.Together } // The first machines: one send each for everything of the tier that goes there first, one machine // per step — the plan is kept between them, so the next machine's send sees what this one walks. if len(firstOn) > 0 { machines := make([]string, 0, len(firstOn)) for n := range firstOn { machines = append(machines, n) } sort.Strings(machines) for _, node := range machines { err := firstSend(ctx, open, p, node, firstOn[node], runningOf) if errors.Is(err, errWalkedElsewhere) { // A build this machine would carry is being judged elsewhere: it goes once that gate passed. pending = append(pending, fmt.Sprintf("%s for %s: %v", node, strings.Join(firstOn[node], ", "), err)) continue } if err != nil { // Not marked sent, so the next step tries again (issue 249): a grant that could not be // issued is a send that did not happen. return false, fmt.Errorf("sending %s to %s first after tier %d: %w", strings.Join(firstOn[node], ", "), node, p.Tier, err) } return true, nil } } // The rest, after their gates passed, and what rolls out together: every machine once. if len(rest) > 0 { var machines []string scope := sendScope{modules: map[string]bool{}} for _, m := range rest { for _, n := range restTo[m] { if !slices.Contains(machines, n) { machines = append(machines, n) } } if together[m] { // What its policy rolls out together may move wherever it goes; anything else no gate has // seen still stops the send. scope.modules[m] = true } } sort.Strings(machines) was := map[string]map[string]chain.Ref{} for _, n := range machines { was[n] = ranBefore(ctx, inv, n) } sent, err := sendRollout(withScope(sendingAs(ctx, p), scope), open, machines) if err != nil { return false, fmt.Errorf("sending %s to %s after tier %d: %w", strings.Join(rest, ", "), strings.Join(machines, ", "), p.Tier, err) } now := time.Now().UTC() // What each machine of the rest was sent, kept for when it reports it applied (novox/hq ADR 0282 decision // 6): measured, never acted on. sentWhat := sentNow(ctx, open.inventory, sent) for _, m := range rest { p.Modules[m].SentAt = &now p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat) } // One `sent` for each machine, with what it was sent (ADR 0406 §5, issue 974). for _, n := range sent { var carried []string for _, m := range rest { if slices.Contains(restTo[m], n) { carried = append(carried, m) } } noteSends(ctx, p.Tier, []string{n}, carried, now) noteChanges(p, n, carried, was[n], now) } fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "), strings.Join(sent, ", ")) return true, nil } if len(pending) > 0 { note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(pending, "; ") + " to report it applied before the rest are sent" changed := p.State != inventory.PlanRolling || p.Note != note p.State = inventory.PlanRolling p.Note = note return changed, nil } // And wait for what the next tier needs running. leftAsWas := "" needed := gates(*p, edges, rollsOut) if len(needed) > 0 { reports, err := inv.LastReports(ctx) if err != nil { return false, err } var needs []gateNeed for _, m := range needed { running, err := inv.Running(ctx, m) if err != nil { return false, err } state := p.Modules[m] if state == nil { state = &inventory.PlanModule{} p.Modules[m] = state } if state.State == planDeleted { continue } // A machine passed as away was sent nothing to wait on (ADR 0435 §1, issue 1161). running = runningHeard(*p, running, state) // The plan sends what it waits for. A rebuild from the same source commit is not a // move the catalogue announces — the build machine rebuilt for a controller change // is one — so the roll-out that opens this gate is the plan's to make, once, and // the reports that open it are the ones after the send. since := latest if state.BuiltAt != nil { since = *state.BuiltAt } // **A build that is no move is waited on by the send that last carried it** (novox/hq issue 648). // Nothing was sent for it, so no report follows its marking or its build: a machine that does not // happen to report again held the walk for ever. Each machine runs it once it answered its last // send — the one whose builds made it no move — whenever that was. if noMove(state) { byNode := map[string]inventory.Reported{} for _, r := range reports { byNode[r.Node] = r } needs = append(needs, gateNeed{m, running, func(node string) time.Time { if s := byNode[node].Sent; s != nil { return *s } return time.Time{} }}) continue } // **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218) // a module is sent twice — the first machine, then the rest — and SentAt is the second. // Asked of every machine, the first machine's report, made between the two sends, read as // older than the build, and the gate waited for a report it already had, for ever. sinceFor := func(node string) time.Time { at := since sent := state.SentAt for _, n := range state.First { if n == node { sent = state.FirstAt } } if sent != nil && sent.After(at) { at = *sent } return at } needs = append(needs, gateNeed{m, running, sinceFor}) } waiting, quiet, lost := settleTierGate(needs, reports, silent) if lost { // Every machine the next tier waits on is silent: nothing it needs was delivered anywhere, so the plan does // not go on as if it had been (issue 1160). p.State = inventory.PlanFailed p.Note = fmt.Sprintf("tier %d built; every machine it waits on said nothing on its send within %s: %s", p.Tier, applySilentAfter, strings.Join(quiet, ", ")) fmt.Printf("%s: %s\n", p.ID, p.Note) return true, nil } if len(quiet) > 0 { fmt.Printf("%s: tier %d: %s said nothing on its send within %s and is left as it was; the plan goes on without it\n", p.ID, p.Tier, strings.Join(quiet, ", "), applySilentAfter) leftAsWas = fmt.Sprintf("%s said nothing on its send within %s and is left as it was", strings.Join(quiet, ", "), applySilentAfter) } if len(waiting) > 0 { note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(waiting, "; ") + " to be applied" changed := p.State != inventory.PlanRolling || p.Note != note p.State = inventory.PlanRolling p.Note = note return changed, nil } } // **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §5): after its last tier, every machine it sent // has reported on that send, or is silent past ApplySilentAfter; never a health reading. end := "" if p.Tier == len(p.Tiers)-1 { reports, err := inv.LastReports(ctx) if err != nil { return false, err } waiting, failed, silent := walkReports(*p, reports, time.Now().UTC(), provLookup(ctx)) if len(waiting) > 0 { note := "sent; waiting for " + strings.Join(waiting, ", ") + " to report on their send" changed := p.State != inventory.PlanRolling || p.Note != note p.State = inventory.PlanRolling p.Note = note return changed, nil } var said []string if len(failed) > 0 { said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", ")) } if len(silent) > 0 { said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", applySilentAfter, strings.Join(silent, ", "))) } said = append(said, recordedSaid(*p)...) end = strings.Join(said, "; ") // **The new controller's user list follows its walk** (ADR 0236 §34, read without the gate since ADR 0360): // once a walk that moved the controller ends, the bus's machine is sent the list it composes, if behind. if s := p.Modules[catalogue.ControllerSeatName]; s != nil && s.SentAt != nil && s.Gate == nil && !noMove(s) { carryUserList(ctx, open, p) } } if end == "" { end = leftAsWas } // A machine passed as away is said where the others passed are said (ADR 0435 §1, issue 1161). if said := awaySaying(*p); said != "" { end = strings.TrimPrefix(end+"; "+said, "; ") } p.Tier++ p.State = inventory.PlanBuilding p.Note = end if p.Tier < len(p.Tiers) { fmt.Printf("%s: tier %d done; asking tier %d: %s\n", p.ID, p.Tier-1, p.Tier, strings.Join(p.Tiers[p.Tier], ", ")) } return true, nil } // walkReports is, for every machine a walk sent its rest, whether it reported on that send (novox/hq ADR 0360 §4): // those still waiting, those that reported a failed or refused apply, and those silent past ApplySilentAfter, // each sorted. Pure. func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) (waiting, failed, silent []string) { byNode := map[string]inventory.Reported{} for _, r := range reports { byNode[r.Node] = r } // The modules of this delivery sent to each machine: a machine is judged on those, not on every module its // declaration hands to the provisioner (ADR 0362 §1-2, issue 840). carried := map[string][]string{} for m, s := range p.Modules { if s == nil { continue } for node := range s.Rest { carried[node] = append(carried[node], m) } } seen := map[string]bool{} for _, s := range p.Modules { if s == nil { continue } for node, sent := range s.Rest { sent = sent.Only(carried[node]...) if seen[node] { continue } r := byNode[node] at := sent.At if at.IsZero() && s.SentAt != nil { at = *s.SentAt } // A module handed to the provisioner is applied only once its client said so too (ADR 0362 §1-5). outcome, reported := provisionedOutcome(node, sent, r, r.At != nil && sent.ReportsOn(r), prov) switch { case reported: seen[node] = true switch { case outcome == inventory.OutcomeFailed || outcome == inventory.OutcomeRefused: failed = append(failed, node) case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed): failed = append(failed, node+" ("+outcome+")") } case !at.IsZero() && now.Sub(at) > applySilentAfter: seen[node] = true silent = append(silent, node) default: if !slices.Contains(waiting, node) { waiting = append(waiting, node) } } } } // A machine one module's send was reported on and another's not yet is the same send: reported. waiting = slices.DeleteFunc(waiting, func(n string) bool { return seen[n] }) sort.Strings(waiting) sort.Strings(failed) sort.Strings(silent) return waiting, failed, silent } // restOf is, for one module, every machine of its rest that the send reached and what it carried there. func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration { out := map[string]inventory.SentDeclaration{} for _, n := range to { if slices.Contains(sent, n) { out[n] = what[n] } } if len(out) == 0 { return nil } return out } // firstSend sends one machine, in one send, every module of the plan's tier whose first machine it is // (novox/hq issue 281), and records the send on each: what that machine ran of it before — read once, // before the send, so a module the same send carries is never read as already moved — the machines it // went to, and the gate. The gate is kept on the first of them and judges everything the send moved; // the others are gated by it. func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string, runningOf map[string][]string) error { inv := open.inventory before, beforeKnown, _ := inv.SentBuilds(ctx, node) // **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan // before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from // the readings of a gate already judging it there. // Only a send the machine reported it applied is one it has; one it never reported on is sent, as before // (issue 280: what it ran before is then that build, and a failure leaves it as it was). if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 && appliedLastSend(ctx, inv, node) { if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 { adoptSend(ctx, open, p, node, modules, before, runningOf) return nil } } // **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced // between the send and keeping it reads it here, and a failure still puts the machine back. kept := false for _, m := range modules { s := p.Modules[m] if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] { s.Previous, kept = before[m], true } } if kept { if err := inv.SavePlan(ctx, p); err != nil { return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err) } } owns := make([]inventory.CarriedMove, 0, len(modules)) for _, m := range modules { s := p.Modules[m] owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build}) } // What the assignments name before the send, for the rollout to say what it changed; and the send is made as this // rollout, so what is recorded of it names it (novox/hq ADR 0408, issue 1095). was := ranBefore(ctx, inv, node) ctx = sendingAs(ctx, p) // A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges // all of it there. carried, sent, grants, err := gatedSend(ctx, open, node, owns) if err != nil { return err } now := time.Now().UTC() lead := modules[0] // What each machine was just sent, kept on the gate (novox/hq issue 352): its report is held against // this send, whatever it is sent after. sentWhat := sentNow(ctx, inv, sent) for _, m := range modules { s := p.Modules[m] // What the first machine ran before (ADR 0236), kept above before the send. s.First, s.FirstAt = sent, &now s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]), From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat} s.GatedBy = "" if m == lead { s.Gate.Carried = carried s.Gate.Grants = grants } else { s.GatedBy = lead } } noteSends(ctx, p.Tier, sent, modules, now) for _, n := range sent { noteChanges(p, n, modules, was, now) } p.State = inventory.PlanRolling p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "), strings.Join(sent, ", ")) fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n", p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", ")) // The grants step taken before it (novox/hq issue 490), said with it. if said := grantsSaid(grants); said != "" { p.Note += "; " + said } // What the send recreates, said with it (novox/hq ADR 0245). if said := recreationsSaid(carried); said != "" { p.Note += "; " + said fmt.Printf("%s: %s\n", p.ID, said) } return nil } // adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts // from the machine's last send, and takes over the readings of an open gate of another walk judging the same build // there — whose carried move also says what the machine ran before, for a rollback. func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string, before map[string]string, runningOf map[string][]string) { inv := open.inventory since := time.Now().UTC() last, found, err := inv.SentTo(ctx, node) if err == nil && found && !last.At.IsZero() { since = last.At } lead := modules[0] other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build) for _, m := range modules { s := p.Modules[m] // What it ran before: kept by this walk before its send was made (§6); else what the other walk's send // moved it from; else the build itself, so a failure leaves it as it was (issue 280). switch { case s.Previous != "": case m == lead && from != "": s.Previous = from default: s.Previous = before[m] } at := since s.First, s.FirstAt = []string{node}, &at s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]), From: s.Previous, To: s.Commit, Since: &at} if found { s.Gate.Sent = map[string]inventory.SentDeclaration{node: last} } s.GatedBy = "" if m == lead { s.Gate.TakeReadings(other, modules, id) } else { s.GatedBy = lead } } p.State = inventory.PlanRolling p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send", p.Tier, node, strings.Join(modules, ", ")) if id != "" { p.Note += ", going on from the readings of " + id } fmt.Printf("%s: %s\n", p.ID, p.Note) } // appliedLastSend says a machine reported it applied the declaration it was sent last. func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool { reports, err := inv.LastReports(ctx) if err != nil { return false } for _, r := range reports { if r.Node == node { return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied } } return false } // judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the // walk's id, and what the machine ran before; nil when none does. func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) { plans, err := inv.OpenPlans(ctx) if err != nil { return nil, "", "" } // The same build, by its id where both name one, else by its commit. same := func(b, c string) bool { if b != "" && build != "" { return b == build } return sameCommit(c, commit) } usable := func(g *inventory.PlanGate) bool { return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) && slices.Contains(g.Machines, node) } for _, q := range plans { if q.ID == self { continue } if r := q.Release; r != nil { gates := []*inventory.PlanGate{r.Gate} for _, g := range r.Gates { gates = append(gates, g) } for _, g := range gates { if !usable(g) { continue } for _, c := range g.Carried { if c.Module == module && c.Node == node && same(c.Build, c.To) { return g, q.ID, c.From } } } continue } if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) { return s.Gate, q.ID, s.Previous } } return nil, "", "" } // passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate // on the first of them passed, and its pass was kept for every build it carried (passCarried). func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) { if s.Gate == nil { s.Gate = &inventory.PlanGate{Machines: g.Machines, From: s.Previous, To: s.Commit, Since: g.Since} } if s.Gate.Verdict != "" { return } s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+whyFor(g, module), g.JudgedAt, g.Took s.Gate.Passes, s.Gate.Kept = g.Passes, true } // failFirstSend stops the plan at a first send whose gate failed, or whose machine refused or failed // what it was sent: what the gate found wanting is put back there, each once — the module the gate is // kept on only when it is among them (issue 281: the module whose gate it is was blamed for whatever // the send carried) — and the machines after it are left as they were. func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule, machines []string, why string, rest []string) { batched, back := batchingRollbacks(ctx) defer func() { sendRollbacks(ctx, open, p, back) p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(rest, ", "))) fmt.Printf("%s: %s\n", p.ID, p.Note) }() g := state.Gate if g != nil && g.Verdict == inventory.GatePassed { // **A guard: a build that passed its gate is never put back for what came after** (novox/hq issue 335). // Not reached while nextRollout answers a passed gate with the rest to send, and advanceOnce reads it // after a carried module takes over its lead's pass; it is here so that a path added later cannot // overturn a verdict. If it is reached, the plan stops and says why, and the build is not marked failed. // The module is left a stopped rollout (its Why said, sent first and not to the rest), which // `plans retry` takes: it sends the first machine again, and the passed gate then sends the rest. state.Why = "passed its gate; its walk then stopped: " + why p.State = inventory.PlanFailed p.Note = fmt.Sprintf("%s passed its gate on %s in tier %d (%s) and is kept; its walk stopped after: %s", module, strings.Join(g.Machines, ", "), p.Tier, g.Why, why) return } if g != nil && slices.Contains(g.Returned, module) { // Put back at once when it broke: its rollback was made then, and is not made again. state.Why = "put back when it broke; its send failed: " + g.Why p.State = inventory.PlanFailed p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was put back when it broke", strings.Join(g.Machines, ", "), p.Tier, g.Why, module) } else if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) { gateFailed(batched, open, p, module, state, machines, why) } else if slices.Contains(g.Passing, module) && state.Build != "" { // **The module the gate is kept on keeps its own pass too** (issue 318 review): healthy for the passes // asked while another of its send failed, its build has a verdict, and other walks do not wait on it. if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module, Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines, Verdict: inventory.GatePassed, Why: passedAloneWhy(g, module), Component: g.Component, JudgingFrom: g.Since}); err != nil { fmt.Printf("%s: %s passed its gate on its own, and the verdict could not be kept: %v\n", p.ID, module, err) } state.Why = "passed on its own; stopped with the send that carried it: " + g.Why p.State = inventory.PlanFailed p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s passed on its own and is kept", strings.Join(g.Machines, ", "), p.Tier, g.Why, module) } else { state.Why = "not found wanting; stopped with the send that carried it: " + g.Why p.State = inventory.PlanFailed p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was not found wanting and is left as it is", strings.Join(g.Machines, ", "), p.Tier, g.Why, module) } if state.Gate != nil && len(state.Gate.Carried) > 0 { failCarried(batched, open, p, state.Gate, module) } } // rolloutStep is what a plan does next with one built module's machines (novox/hq issue 249). type rolloutStep struct { // send is the machines to send now; first, whether they are the first machine's send. send []string first bool // waiting names the first machines whose report the rest wait for. waiting string // failed says how a first machine did not take it; rest is what is then left alone. failed string rest []string } // nextRollout is the next step of one module's rollout in a plan (novox/hq issue 249, ADR 0218). // // Together, every machine running it at once, as the policy says. Otherwise one machine first — the // first by name among those that have reported within the bound, so a laptop that is away is not // the one the rest wait on; the first by name when none has; the same choice on every controller and // every resume — and the rest once each machine the first send reached reports, about the // declaration it was last sent, that it applied it. Compared by the store's own record of what was // sent (Reported.Current), never by this controller's clock against the machine's. // // **A first machine that fails, refuses, or does not report within the bound stops the rollout // there** (ADR 0218 §2), naming the machine; the rest are not sent. A wait with no end is not a // rollout: it held the plan open for ever, read as work in progress (issue 254). func nextRollout(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported, now time.Time, bound time.Duration) rolloutStep { if len(running) == 0 { return rolloutStep{} } if together { return rolloutStep{send: running} } byNode := map[string]inventory.Reported{} for _, r := range reports { byNode[r.Node] = r } if s.FirstAt == nil { sorted := append([]string{}, running...) sort.Strings(sorted) for _, n := range sorted { if r, said := byNode[n]; said && r.At != nil && now.Sub(*r.At) <= bound { return rolloutStep{send: []string{n}, first: true} } } return rolloutStep{send: sorted[:1], first: true} } sentFirst := map[string]bool{} for _, n := range s.First { sentFirst[n] = true } var rest []string for _, n := range running { if !sentFirst[n] { rest = append(rest, n) } } // **A passed gate is the first machine's verdict, and its later reports are not** (novox/hq issue 335). // Once the build passed there, what that machine reports next is about whatever it was sent after — // another walk's send, a push — and says nothing of this build. On 2026-10-08 a build passed on the // laptop, its send to the rest waited on another walk, that walk sent the laptop a new declaration it did // not report for half an hour, and the plan read the silence as the first machine never applying the // passed build: it marked it failed at its gate and put it back. The rest are sent, as the pass said. if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed { return rolloutStep{send: rest} } var waiting, failed []string for _, n := range s.First { r, said := byNode[n] // Only a report about what this plan sent it — or what it was sent after that — says anything about // this build (novox/hq issue 352); a plan from before sends were kept on the gate reads the report // against the send made last, as before. reported := r.Current if s.Gate != nil && s.Gate.Sent != nil { sent, kept := s.Gate.Sent[n] reported = kept && sent.ReportsOn(r) } if !said || r.At == nil || !reported { waiting = append(waiting, n) continue } switch r.Outcome { case inventory.OutcomeApplied: case inventory.OutcomeFailed, inventory.OutcomeRefused: failed = append(failed, n+" "+r.Outcome+" what it was sent") default: waiting = append(waiting, n) } } if len(failed) > 0 { return rolloutStep{failed: strings.Join(failed, "; "), rest: rest} } if len(waiting) > 0 { if now.Sub(*s.FirstAt) > bound { return rolloutStep{failed: fmt.Sprintf("%s did not report it applied within %s", strings.Join(waiting, ", "), bound), rest: rest} } return rolloutStep{waiting: strings.Join(waiting, ", ")} } return rolloutStep{send: rest} } // sayUnsent adds to an ended plan's note the modules it built and never sent (novox/hq issue 249). // An announced move of a module a plan held was left to that plan; a plan that ends without // sending it — failed elsewhere, or closed by hand — would leave its machines behind with nothing // saying so. A module whose rollout stopped at its first machine is not among them: that stop was // the point. Said once. func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) { var unsent []string for name, s := range p.Modules { if s != nil && s.State == "built" && s.SentAt == nil && s.FirstAt == nil && rollsOut(name) { unsent = append(unsent, name) } } if len(unsent) == 0 || strings.Contains(p.Note, "built and never sent") { return } sort.Strings(unsent) p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `push --behind` sends them" } // planLine is one plan as `status` says it, given the least bound of a tier. func planLine(p inventory.Plan, now time.Time) string { return planLineWith(p, now, pauseView{}, tierAtLeast) } // inTierSince is when a plan began waiting on what it waits on now, which `plans`, `status` and the // watchdog of a plan's progress (S3) all count from (novox/hq issue 296): a walk waiting for its // delivery's word from when it was opened, as S16 counts; any other plan from when it entered its tier, // or, when the delivery let it go after that, from the go. Never from the plan's last save: a plan is // saved for many reasons while its tier stands still, and a clock reset by each save said a build queued // for minutes had been building for a few seconds. A plan read without the stamp (one from before it // was kept) counts from its last save, the only time it has. func inTierSince(p inventory.Plan) time.Time { if p.Waiting() { return p.Created } since := p.TierEntered if since.IsZero() { since = p.Updated } if d := p.Delivery; d != nil && d.Go != nil && d.Go.After(since) { since = *d.Go } return since } // planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan // waiting on builds nobody will take until a person resumes the seat says so, and is not late. bound is // the plan's tier bound, the one its stalled condition is raised at (tierBounds): LATE is that condition // said on the line (novox/hq issue 296). The machines running what it moves are not named: planLineOn. func planLineWith(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration) string { return planLineOn(p, now, pause, bound, nil) } // planLineOn is planLineWith naming the plan by what it moves and where (planHeadline), given what runs each // module; nil names no machine. func planLineOn(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration, running func(string) []string) string { return planLineSaid(p, now, pause, bound, running, nil) } // planLineSaid is planLineOn saying, of a walk waiting for its delivery's word, what the delivery's owner says // holds it (novox/hq issue 529); nil waits says it as before. func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration, running func(string) []string, waits walkWaits) string { name := planHeadline(p, running) where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers)) switch p.State { case inventory.PlanAssembling, inventory.PlanQueued: // A batch not yet a walk (novox/hq ADR 0276): what it holds and how long is left. return batchWords(p, now) case inventory.PlanDone: return fmt.Sprintf("%s · done, %d tier(s)", name, len(p.Tiers)) case inventory.PlanFailed: return fmt.Sprintf("%s · FAILED at %s: %s", name, where, p.Note) case inventory.PlanSuperseded: return fmt.Sprintf("%s · %s", name, p.Note) } since := now.Sub(inTierSince(p)).Round(time.Second) if p.Waiting() { // Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239). return fmt.Sprintf("%s · %s, %s, for %s", name, where, waitingSaid(p, waits[p.ID]), since) } if deferred(p) { // Nor is waiting for the walk before it to end (a related walk, novox/hq ADR 0276, ADR 0407). return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since) } if waitingOnBackups(p) { return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since) } if waiting, paused := pausedWaiting(p, pause, now); paused { return fmt.Sprintf("%s · %s, %s", name, where, waiting) } late := "" if since > bound { late = " — LATE" } what := "building" if p.State == inventory.PlanRolling { what = p.Note } return fmt.Sprintf("%s · %s, %s for %s%s", name, where, what, since, late) } // planHeadline names a plan as a person knows it (asked by the operator, 2026-10-10: a plan called by its // repository alone said nothing of what it delivers): each repository as its pull request and title, what the // plan moves, and the machines running that — "mesh-catalog #175 a tap shows its outcome · messenger, // telegram → novox". A record naming no pull request is named by its commit, as before; one naming no moves // says none; running nil names no machine. func planHeadline(p inventory.Plan, running func(string) []string) string { var repos []string moves := map[string]bool{} for _, c := range p.Carried() { seg := repoName(c.Repository) + " " + short(c.Commit) if p.Delivery != nil { for _, m := range p.Delivery.Merges { if !strings.EqualFold(m.Repository, c.Repository) { continue } for _, n := range m.Moves { moves[n] = true } if m.Commit == c.Commit && m.Number > 0 { seg = repoName(c.Repository) + " " + pullWords(m) } } } repos = append(repos, seg) } out := strings.Join(repos, " + ") if len(moves) == 0 { return out } names := sortedKeysOf(boolsToStrings(moves)) out += " · " + strings.Join(names, ", ") if running == nil { return out } nodes := map[string]bool{} for _, n := range names { for _, node := range running(n) { nodes[node] = true } } if len(nodes) > 0 { out += " → " + strings.Join(sortedKeysOf(boolsToStrings(nodes)), ", ") } return out } // planFailedBuild marks the module a failed build was for when the result names no module: by the // repository and path the plan's modules were asked at. // // **By the id first** (novox/hq ADR 0219): a plan keeps the id it asked each module under, so an // outcome that never learnt its module's name — cancelled, killed, failed at the clone — is matched // to the module it was asked for exactly. Repository and path remain for a plan from before ids // were kept. func planFailedBuild(ctx context.Context, open *stores, result link.BuildResult) { asked, _ := link.BuildAskedAt(result.ID) if plans, err := open.inventory.OpenPlans(ctx); err == nil { if module := moduleAskedAs(plans, result.ID); module != "" { planBuilt(ctx, open, module, result.Commit, result.Failed, asked, result.ID) return } } entries, err := open.inventory.Catalogued(ctx) if err != nil { return } for _, e := range entries { if repositoryMatches(e.Source.Repository, result.Repository) && e.Source.Path == result.Path { planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed, asked, result.ID) return } } } // moduleAskedAs is the module an open plan's current tier asked for under this id, or nothing. func moduleAskedAs(plans []inventory.Plan, id string) string { if id == "" { return "" } for _, p := range plans { if p.Tier >= len(p.Tiers) { continue } for _, m := range p.Tiers[p.Tier] { if s := p.Modules[m]; s != nil && s.Build == id { return m } } } return "" } func repositoryMatches(a, b string) bool { trim := func(s string) string { return strings.ToLower(strings.TrimSuffix(s, ".git")) } return trim(a) == trim(b) || strings.HasSuffix(trim(a), "/"+trim(b)) || strings.HasSuffix(trim(b), "/"+trim(a)) } // planStatus is one plan as `status --json` says it. type planStatus struct { ID string `json:"id"` Repository string `json:"repository"` Commit string `json:"commit"` State string `json:"state"` Tier int `json:"tier"` Tiers int `json:"tiers"` Waiting string `json:"waiting,omitempty"` Since time.Time `json:"since"` Late bool `json:"late"` } func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView, bounds tierBounds) []planStatus { out := make([]planStatus, 0, len(plans)) for _, p := range plans { ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State, Tier: p.Tier, Tiers: len(p.Tiers), Since: p.Updated} if p.Open() { ps.Since = inTierSince(p) ps.Waiting = p.Note if ps.Waiting == "" { ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier) } // A walk waiting for its delivery's word is no tier late (ADR 0239): S16 is its watchdog. ps.Late = !p.Waiting() && now.Sub(ps.Since) > bounds.of(p.Repository) // Paused is a person's decision, not lateness (novox/hq ADR 0219). if waiting, paused := pausedWaiting(p, pause, now); paused { ps.Waiting, ps.Late = waiting, false } } out = append(out, ps) } return out } // openPlans is the open plans among the recent ones, and how many have waited past the bound. func openPlans(plans []inventory.Plan, now time.Time, pause pauseView, bounds tierBounds) ([]inventory.Plan, int) { var open []inventory.Plan late := 0 for _, p := range plans { if p.Open() { open = append(open, p) if _, paused := pausedWaiting(p, pause, now); paused || p.Waiting() { continue } if now.Sub(inTierSince(p)) > bounds.of(p.Repository) { late++ } } } return open, late } // plansCommand says what the last merges produced and where each stands; given an id, one plan // tier by tier with every module's state. func plansCommand(ctx context.Context, args []string) error { set := flag.NewFlagSet("plans", flag.ContinueOnError) limit := set.Int("n", 10, "how many to show") whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules") paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root") modules := set.String("modules", "", "or the modules it would change, comma-separated") moduleDirs := set.String("module-dirs", "", "with --paths: the directories holding a module.json at the commit, "+ "comma-separated, as the forge's announcer says them (novox/hq issue 278)") // Following a plan (novox/hq issue 229): `plans wait `. until := set.String("until", "", "with wait: what to wait for — moved (any change), tier (its next tier) or done (its end)") bound := set.String("bound", "", "with wait: how long to wait at most, as 15s or seconds (default and at most the "+ "longest a call is answered within)") // Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7). why := addHandActFlags(set) positionals, err := parseAround(set, args) if err != nil { return err } if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close" || positionals[0] == "go") { // Refused before anything is opened: a repair by hand says why. if err := why.require("plans " + positionals[0]); err != nil { return err } } open, err := openStores(ctx) if err != nil { return err } defer open.Close() inv := open.inventory now := time.Now() if len(positionals) == 2 && positionals[0] == "wait" { return planWaitCommand(ctx, inv, positionals[1], *until, *bound) } if len(positionals) == 1 { p, err := inv.PlanByID(ctx, positionals[0]) if err != nil { return err } bounds := readTierBounds(ctx, inv, now) fmt.Printf("%s — %s\n", p.ID, planLineSaid(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}), bounds.of(p.Repository), func(module string) []string { on, _ := inv.Running(ctx, module) return on }, waitsOf(ctx, inv, []inventory.Plan{p}))) if r := p.Release; r != nil { // A release plan's walk (ADR 0236): machines done, the one judged, those to come. fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "), orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", "))) if len(r.Failed)+len(r.Silent) > 0 { fmt.Printf(" failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", "))) } gates := []*inventory.PlanGate{r.Gate} for _, n := range r.Order { gates = append(gates, r.Gates[n]) } for _, g := range gates { if g == nil { continue } fmt.Printf(" %s\n", gateLine(g)) for _, c := range g.Carried { fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To)) if c.Recreates != "" { fmt.Printf(" %-22s %s\n", "", c.Recreates) } } } return nil } for i, tier := range p.Tiers { marker := " " if i == p.Tier && p.Open() { marker = ">" } fmt.Printf("%s tier %d\n", marker, i) for _, m := range tier { s := p.Modules[m] state := "not yet asked" if s != nil && s.State != "" { state = s.State if s.Commit != "" { state += " from " + short(s.Commit) } if s.Build != "" && s.State != "built" { state += " (" + s.Build + ")" } if s.Why != "" { state += ": " + s.Why } } fmt.Printf(" %-22s %s\n", m, state) // The rollout's record at its gate (novox/hq to-be 45 §8): first machine, from and to, // verdict, time to it, rolled back or not. switch { case s != nil && s.GatedBy != "" && (s.Gate == nil || s.Gate.Verdict == ""): // Sent with others, judged by the gate of that send (issue 281). fmt.Printf(" %-22s judged with %s, in the send to %s\n", "", s.GatedBy, strings.Join(s.First, ", ")) case s != nil && s.Gate != nil: fmt.Printf(" %-22s %s\n", "", gateLine(s.Gate)) } // Which way a recorded module went on each machine (novox/hq ADR 0377 §6). if s != nil { for _, n := range sortedKeysOf(s.Recorded) { fmt.Printf(" %-22s on %s %s\n", "", n, s.Recorded[n]) } } // What the send to its first machine did to its containers (ADR 0245). if s != nil && s.Gate != nil { for _, c := range s.Gate.Carried { if c.Recreates != "" { fmt.Printf(" %-22s on %s %s\n", "", c.Node, c.Recreates) } } } } } // The assignments it changed, each with the build and commit before and after (novox/hq ADR 0408, issue 1095). for _, line := range changedLines(p) { fmt.Println(line) } return nil } if *whatIf != "" { return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules), moduleDirsOf(*moduleDirs)) } // `go` (novox/hq ADR 0239): a person's word in place of the delivery's owner, for a walk that waits for it // — mesh-delivery down, or a person who will not wait. Recorded in the hand-act log with why. if len(positionals) == 2 && positionals[0] == "go" { why.record(ctx, "plans go", positionals[1:]) var p inventory.Plan if err := sayingOnTheBus(ctx, inv, func() (err error) { p, err = letGo(ctx, inv, positionals[1], "a person ("+whoAsked()+")", strings.TrimSpace(*why.why)) return err }); err != nil { return err } fmt.Printf("%s (%s at %s) is let go by hand; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository, short(p.Commit)) return nil } // `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on. if len(positionals) == 2 && positionals[0] == "retry" { // Said on the bus (novox/hq ADR 0308 decision 1): a retried walk starts again, and its start is said // by the verb that starts it. var said string if err := sayingOnTheBus(ctx, inv, func() (err error) { said, err = retryPlan(ctx, open, positionals[1]) return err }); err != nil { return err } fmt.Println(said) return nil } // `stop`, or `close` (novox/hq issue 254): a person ending a plan that will not move again — one // waiting on a report that cannot come — so it stops reading as work in progress. Marked failed // with who ended it; what it asked still builds and registers. if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") { how := "stopped" if positionals[0] == "close" { how = "closed" } // Said on the bus (novox/hq ADR 0308 decision 1): the walk's end, outcome stopped, is said by the verb // that ends it, so what it touched is not held until its bound. var p inventory.Plan if err := sayingOnTheBus(ctx, inv, func() error { release, err := inv.HoldPlans(ctx, true) if err != nil { return err } defer release() if p, err = inv.PlanByID(ctx, positionals[1]); err != nil { return err } if !p.Open() { return fmt.Errorf("%s is already %s", p.ID, p.State) } why.record(ctx, "plans "+positionals[0], positionals[1:]) stopByHand(&p, how, "a person ("+whoAsked()+")", strings.TrimSpace(*why.why)) sayUnsent(&p, func(m string) bool { u, err := inv.UpgradeOf(ctx, m) return err == nil && u.RollOut }) return inv.SavePlan(ctx, &p) }); err != nil { return err } fmt.Printf("%s %s at tier %d of %d; what was asked still builds and registers, nothing further is asked\n", p.ID, how, p.Tier, len(p.Tiers)) return nil } plans, err := inv.RecentPlans(ctx, *limit) if err != nil { return err } // **The batch being assembled first** (novox/hq ADR 0276 decision 7): what it holds, how long is left, // and that its plan is not yet calculated. batches, err := inv.Batches(ctx) if err != nil { return err } if len(plans) == 0 && len(batches) == 0 { fmt.Println("no merge has produced a plan yet") return nil } for _, b := range batches { fmt.Printf("%-28s %s\n", b.ID, batchWords(b, now)) // One line per repository: its pull request, what it answers, what it moves. for _, line := range batchLines(b) { fmt.Printf("%-28s %s\n", "", line) } } pause := buildSeatPause(ctx, inv, plans) bounds := readTierBounds(ctx, inv, now) running := func(module string) []string { on, _ := inv.Running(ctx, module) return on } waits := waitsOf(ctx, inv, plans) for _, p := range plans { if p.Batch() { continue } fmt.Printf("%-28s %s\n", p.ID, planLineSaid(p, now, pause, bounds.of(p.Repository), running, waits)) } return nil } // waitsOf asks the delivery's owner what holds the walks among these that wait for its word (novox/hq issue 529), // once, and only when one waits; nil when none waits, no holder is on record, or the bus or the owner does not // answer within a few seconds — the lines then read as before. func waitsOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) walkWaits { waiting := false for _, p := range plans { waiting = waiting || p.Open() && p.Waiting() } if !waiting { return nil } entries, err := inv.Catalogued(ctx) if err != nil { return nil } ctx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() var waits walkWaits _ = onTheBus(func(conn *nats.Conn) error { waits = walksWaiting(ctx, conn, deliverySeatHeld(entries)) return nil }) return waits } // planWhatIf is the plan a merge would produce, computed the way the merge handler computes one // and saved nowhere: the modules the repository's changed files touch (or the modules named), what // packages their source, everything reachable from them, in tiers. For reading before merging. func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string, paths, modules []string, moduleDirs *[]string) error { owner, repo, found := strings.Cut(repository, "/") if !found { return fmt.Errorf("--what-if takes owner/repository, not %q", repository) } m := link.SourceMoved{Owner: owner, Repo: repo, Base: "main", Commit: "what-if", Paths: paths} if moduleDirs != nil { m.ModuleDirs, m.ModuleDirsSaid = *moduleDirs, true } entries, err := inv.Catalogued(ctx) if err != nil { return err } read, err := readForPlanning(ctx, inv) if err != nil { return err } var from, packaging []inventory.Entry deleted := map[string]bool{} named := map[string]bool{} for _, name := range modules { named[name] = true } if len(named) == 0 { // The changed files: the planner's own answer, as the merge handler, the merge gate and a pull // request's check ask it (novox/hq ADR 0238). r := reachOfMerge(m, entries, read, nil) from, packaging = append(append([]inventory.Entry{}, r.Touched...), r.Deleted...), r.Packaging for _, e := range r.Deleted { deleted[e.Manifest.Module] = true } } else { for _, e := range entries { switch { case named[e.Manifest.Module]: from = append(from, e) case readsFrom(read[e.Manifest.Module], m): packaging = append(packaging, e) } } } moved := append(append([]inventory.Entry{}, from...), packaging...) if len(moved) == 0 { fmt.Printf("a merge of %s changing %s would build nothing the mesh holds\n", repository, orNone(strings.Join(append(paths, modules...), ", "))) return nil } edges, err := inv.Dependencies(ctx) if err != nil { return err } var names []string for _, e := range moved { names = append(names, e.Manifest.Module) } p := planOfMerge(m, names, edges) fmt.Printf("a merge of %s would build %d module(s) in %d tier(s):\n", repository, len(p.Modules), len(p.Tiers)) why := map[string]string{} for _, e := range packaging { why[e.Manifest.Module] = whyMoved(e, read[e.Manifest.Module], m) } if len(named) == 0 { for _, e := range from { why[e.Manifest.Module] = whyMoved(e, read[e.Manifest.Module], m) if deleted[e.Manifest.Module] { why[e.Manifest.Module] = "its manifest is removed: deleted at its source, forgotten where nothing holds it, not built" } } } rolls := map[string]string{} for i, tier := range p.Tiers { fmt.Printf(" tier %d\n", i) for _, name := range tier { how := "built; its policy records, so nothing is sent" if u, err := inv.UpgradeOf(ctx, name); err == nil && u.RollOut { running, _ := inv.Running(ctx, name) reports, _ := inv.LastReports(ctx) how = "built, then sent to " + orNone(strings.Join(running, ", ")) // One machine first unless the policy says together (novox/hq issue 249). if first := nextRollout(inventory.PlanModule{}, running, u.Together, reports, time.Now(), planWaitBound); first.first && len(running) > 1 { how = fmt.Sprintf("built, then sent to %s first and to the rest once it has applied it", first.send[0]) } rolls[name] = how } fmt.Printf(" %-22s %s\n", name, how) reason := why[name] switch { case reason == "" && len(named) > 0 && named[name]: reason = "named" case reason == "": reason = "it stands on a module the merge moves" } fmt.Printf(" %-22s why: %s\n", "", reason) } } if hasCycle(p.Tiers, edges) { fmt.Println(" the last tier depends on itself and would be built together, in no order") } return nil } // moduleDirsOf is --module-dirs as an announcer would say it: nothing when not given, so the what-if // reads as an announcement from an announcer that does not say. func moduleDirsOf(s string) *[]string { if strings.TrimSpace(s) == "" { return nil } dirs := splitList(s) return &dirs } func splitList(s string) []string { var out []string for _, part := range strings.Split(s, ",") { if part = strings.TrimSpace(part); part != "" { out = append(out, part) } } return out } // recordsOfAsked is what settleFromRecords reads: for every module of the tier still `asked`, its last // builds and the record of its own ask, by id. func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, tier []string) ( map[string][]inventory.Build, map[string]inventory.Build, error) { recorded := map[string][]inventory.Build{} byID := map[string]inventory.Build{} for _, m := range tier { if s := p.Modules[m]; s != nil && s.State == "asked" { builds, err := inv.Builds(ctx, m, 5) if err != nil { return nil, nil, err } recorded[m] = builds // Its own ask's record, by id — found even when the outcome named no module (ADR 0219). if s.Build != "" { b, found, err := inv.BuildByID(ctx, s.Build) if err != nil { return nil, nil, err } if found { byID[s.Build] = b } } } } return recorded, byID, nil } // settleFromRecords marks every module of the tier still `asked` built — or failed — from a build // recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214). // Newest first, as Builds answers: the first record after the ask is the outcome of that ask. // // The record of the plan's own ask, by its id, is that outcome before anything else (novox/hq ADR // 0219): the plan asked under it, and a failure recorded without a module — cancelled, killed — is // found by nothing else. func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build, byID map[string]inventory.Build) bool { changed := false for _, m := range tier { s := p.Modules[m] if s == nil || s.State != "asked" || s.AskedAt == nil { continue } var outcome *inventory.Build for i := range recorded[m] { // Asked under an id, only that id's record answers (ADR 0219), and it is looked up below. if s.Build != "" { break } b := recorded[m][i] if b.At.Before(*s.AskedAt) { break } // Recorded after the ask and asked before it: an earlier ask's late outcome, not this // one's (novox/hq 04-ISSUES/219). if askedBefore(b.Asked, s.AskedAt) { continue } outcome = &b } if own, found := byID[s.Build]; s.Build != "" && found { outcome = &own } if outcome == nil { continue } at := outcome.At if !outcome.Worked() && deletedAtSource(outcome.Failed) { s.State, s.Why = planDeleted, "deleted at its source: "+firstLine(outcome.Failed) fmt.Printf("%s: %s was deleted at its source (%s): not built, and the plan goes on\n", p.ID, m, outcome.ID) changed = true continue } if outcome.Worked() { s.State = "built" s.BuiltAt = &at s.Commit = outcome.Commit } else { s.State = "failed" s.Why = outcome.Failed p.State = inventory.PlanFailed p.Note = fmt.Sprintf("%s failed to build in tier %d", m, p.Tier) } fmt.Printf("%s: %s settled from the build records as %s (%s)\n", p.ID, m, s.State, outcome.ID) changed = true } return changed } // askedBefore is whether a build asked at asked was asked before a plan asked for its module — and // so is not that plan's outcome (novox/hq 04-ISSUES/219). False when either time is not known. func askedBefore(asked time.Time, planAsked *time.Time) bool { return !asked.IsZero() && planAsked != nil && asked.Before(*planAsked) } // firstRunning is the machines sent first that run the module — not the machine holding the bus, sent // with them only for its user list. func firstRunning(first, running []string) []string { var out []string for _, n := range first { if slices.Contains(running, n) { out = append(out, n) } } if len(out) == 0 { return first } return out } // planDeleted is a plan's module that the merge deleted at its source: not built, not sent, and no // failure of the plan (novox/hq ADR 0236). const planDeleted = "deleted" // forgetDeleted forgets a module a plan found deleted at its source, where nothing holds it, and says // it otherwise. func forgetDeleted(ctx context.Context, inv *inventory.Inventory, module string, p *inventory.Plan) { switch err := inv.ForgetModule(ctx, module); { case err == nil: fmt.Printf("%s: %s was deleted at its source: forgotten, and the plan goes on\n", p.ID, module) default: fmt.Printf("%s: %s was deleted at its source and is not built; it is kept until a person unassigns it and "+ "`module forget %s`: %v\n", p.ID, module, module, firstLine(err.Error())) } } // stopByHand ends a walk a person stopped or closed (`plans stop|close`): failed, with who stopped it and why // kept as the delivery's owner keeps its own stop, so the walk ends with outcome stopped, not failed (novox/hq // ADR 0308 decision 1). func stopByHand(p *inventory.Plan, how, by, why string) { p.State = inventory.PlanFailed p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + why if p.Delivery == nil { p.Delivery = &inventory.PlanDelivery{} } p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why }