package main import ( "context" "encoding/json" "errors" "fmt" "os" "slices" "sort" "strconv" "strings" "time" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // **Merges are assembled in a rolling window, and each batch is delivered by one walk** (novox/hq ADR 0276, // issue 362). // // Every merge opened a walk of its own, and the next merge of the branch superseded it. On 2026-10-10 two // catalogue merges eighteen seconds apart left one walk that no delivery held, and the operator started it by // hand 58 minutes later. A merge now joins the open **batch**. Each merge restarts the **merge window**; it // closes when none came for `merge-window` (90 seconds) or at the latest `merge-window-at-most` (10 minutes) // after the batch's first merge. A closed batch becomes one walk when no walk is open: one plan for every // commit in it, with at most one commit per repository, the latest of its branch, which contains the earlier // ones. The walk names every merge it answers. // // - One walk at a time beside a related one (ADR 0407 decision 4, issue 973). Merges that come while a walk // runs join the next batch; a closed batch that shares nothing with an open walk is cut beside it. A walk // that started is never superseded; one still waiting for its delivery's word is folded into the walk of a // batch related to it, which answers every merge it answered — the only supersession left. // - A merge heard after a later merge of its branch was put in a walk is answered by that walk: a batch // still assembling, an open walk that builds everything it moves, or a walk done that did, which then // names it at once. Otherwise it joins the next batch. // - A failed walk marks nothing delivered. Each earlier merge it carried in a later commit is walked on its // own commit, the newest first, one walk each, until one is delivered; the older ones are then answered // by that one. // // **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, ADR 0306 decisions // 2 and 3, issue 525): it joins a batch it is related to, and one that is related to none and whose delivery // group has no member still to merge is a batch whose window is closed at once (batchToJoin). Such a batch is cut // at once, beside any open walk it shares nothing with, and queued behind the first one it is related to // (cutUnrelated). // // A merge on the controller's own path (a module whose walk waits for nobody's word) never shares a batch with // one that waits for mesh-delivery's: each kind has a batch of its own, an own-path batch is cut first and folds // no waiting catalogue walk, and a catalogue walk let go while another walk runs starts once it ended — so no // catalogue batch's walk starts without the word (decided during the build, 2026-10-10). // // The batch, its merges and their times are in the store (batched_merge, and the batch's own plan record in // the state `assembling` or `queued`), so a restarted controller resumes the window where it stood. // The merge window's defaults. The controller's settings `merge-window` and `merge-window-at-most` reach it in // the file MESH_MERGE_WINDOW_FILE names (module.json), read at every look, so a change takes effect without a // restart. var ( mergeWindowDefault = 90 * time.Second mergeWindowAtMostDefault = 10 * time.Minute ) // mergeWindowFileVar names the file the controller's window settings are written to. const mergeWindowFileVar = "MESH_MERGE_WINDOW_FILE" // batchLateAfter is how long past its maximum a batch may still be assembling, with no walk open, before // the controller is said to have failed to cut it (ADR 0276 decision 9). const batchLateAfter = time.Minute // mergeGrace is how long before a batch's first merge a merge may have been made and still be told apart from one // made before the controller began keeping merges (owedLate). const mergeGrace = 10 * time.Minute // edgeChangeOrder is the order the rules of a plan draw among its modules (novox/hq ADR 0419 clauses 3 and 4, delivery_edges.go) // made an edge of its tiers: the module after depends on the module before, and is gated on it. const edgeChangeOrder = "change-order" // mergeWindowOf is the window and its maximum, from the controller's settings; a value unreadable is its // default, and said. func mergeWindowOf() (window, atMost time.Duration) { window, atMost = mergeWindowDefault, mergeWindowAtMostDefault path := os.Getenv(mergeWindowFileVar) if path == "" { return window, atMost } raw, err := os.ReadFile(path) if err != nil { fmt.Fprintf(os.Stderr, "the merge window's settings cannot be read (%v); %s and %s are used\n", err, window, atMost) return window, atMost } var values map[string]any if err := json.Unmarshal(raw, &values); err != nil { fmt.Fprintf(os.Stderr, "the merge window's settings are not JSON (%v); %s and %s are used\n", err, window, atMost) return window, atMost } for key, to := range map[string]*time.Duration{"merge-window": &window, "merge-window-at-most": &atMost} { v, has := values[key] if !has || v == "" { continue } if d, ok := durationSetting(v); ok { *to = d } else { fmt.Fprintf(os.Stderr, "the setting %s=%v is no duration; %s is used\n", key, v, *to) } } return window, atMost } // durationSetting reads a duration as a setting says it: "90s", "10m", or a number of seconds. func durationSetting(v any) (time.Duration, bool) { switch t := v.(type) { case string: if d, err := time.ParseDuration(strings.TrimSpace(t)); err == nil && d >= 0 { return d, true } if n, err := strconv.ParseFloat(strings.TrimSpace(t), 64); err == nil && n >= 0 { return time.Duration(n * float64(time.Second)), true } case float64: if t >= 0 { return time.Duration(t * float64(time.Second)), true } } return 0, false } // mergeFacts is what hearMerge judges a merge by, kept apart so the catch-up (ADR 0457) judges a merge the way hearing it will. type mergeFacts struct { merged time.Time touches []inventory.Entry later []inventory.BatchedMerge owed bool from, packaging []inventory.Entry already int addsModuleToBuilds bool // a first merge into a repository the mesh builds nothing from: firstBuilds acts on it } // factsOfMerge reads what hearing a merge turns on: the modules it touches whatever the history, the later merges of its // branch, whether it is owed late, and the modules it moves. func factsOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, entries []inventory.Entry, read map[string][]inventory.ReadRepository) (mergeFacts, error) { repository := m.Owner + "/" + m.Repo f := mergeFacts{} var err error f.merged, err = time.Parse(time.RFC3339Nano, m.MergedAt) if err != nil { f.merged = now } // **What it touches, history or not** (review of this change): a merge heard after a later merge of its // branch was cut reads as history for every module that cut marked seen, and was dropped from the record // here, never named by the walk that carried it. raw := m raw.MergedAt = "" f.touches = wouldMove(raw, entries, read) if len(f.touches) > 0 { if f.later, err = inv.LaterMergesOf(ctx, repository, m.Base, f.merged.UTC()); err != nil { return f, err } } if f.owed, err = owedLate(ctx, inv, m, f.merged, f.touches); err != nil { return f, err } f.from, f.packaging, f.already = mergeCandidates(m, entries, read) f.addsModuleToBuilds = f.already == 0 && len(addedDirs(m, entries, read)) > 0 return f, nil } // plans says hearing the merge puts it in a batch or a walk. func (f mergeFacts) plans() bool { return len(f.from) > 0 || len(f.packaging) > 0 || len(f.later) > 0 || f.owed } // heldMerge marks a failure to act on a merge, whatever it was, as "not now": the announcement is held on the bus and // handed over again, so the merge joins the next batch, never taken (issue 1583, after 921). The bus acknowledges it // only when hearMerge answers without ErrTryAgain, and a merge heard once is not announced again: the bus answering // "jetstream not enabled", a bucket missing for a moment or a store read cut short each lost the merge when only an // outage of a known kind (notNow) was held. The delay between hands-over is bounded (link.MergeRedeliverAfter). func heldMerge(err error) error { if err == nil { return nil } if errors.Is(err, link.ErrTryAgain) { return err } return fmt.Errorf("%w: the merge is not kept yet: %w", link.ErrTryAgain, err) } // hearMerge puts a merge the forge announced into the batch that answers it, at the moment it was heard, // and cuts what is due. func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.Time) error { // One merge acted on at a time, whoever hands it over (novox/hq issue 266). actingOnMerges.Lock() defer actingOnMerges.Unlock() inv := f.open.inventory repository := m.Owner + "/" + m.Repo if kept, known, err := inv.MergeOf(ctx, repository, m.Commit); err != nil { return heldMerge(err) } else if known { fmt.Printf("%s merged into %s (%.8s) was heard before; %s answers it\n", repository, m.Base, m.Commit, orWords(kept.Plan, "a walk of its own commit, once the walks before it end")) return nil } entries, err := inv.Catalogued(ctx) if err != nil { return heldMerge(err) } read, err := readForPlanning(ctx, inv) if err != nil { return heldMerge(err) } facts, err := factsOfMerge(ctx, inv, m, now, entries, read) if err != nil { return heldMerge(err) } merged, touches := facts.merged, facts.touches if !facts.plans() { already := facts.already // "Already built from it" and "nothing reads it" are different facts, and reading the first // as the second sends somebody looking for a broken trigger when the mesh is up to date. if already > 0 { fmt.Printf("%s merged into %s (%.8s); %d module(s) the mesh holds are already built from it\n", repository, m.Base, m.Commit, already) return nil } if already == 0 { if done, err := f.firstBuilds(ctx, m, entries, read, now); err != nil || done { return heldMerge(err) } } fmt.Print(nothingReadsIt(repository, m)) return nil } // A module the merge adds is registered from the builder's first done key of it, which the builder's write wakes // (novox/hq ADR 0501 §6): nothing is asked here. // **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk. own := ownPath(touches) waits, holding, err := groupWaitHolding(ctx, inv, m, deliverySeatHeld(entries)) if err != nil { return heldMerge(err) } release, err := inv.HoldPlans(ctx, true) if err != nil { return heldMerge(err) } defer release() // Again under the hold: a controller handing over may have kept it since. if _, known, err := inv.MergeOf(ctx, repository, m.Commit); err != nil || known { return heldMerge(err) } var moves []string for _, e := range touches { moves = append(moves, e.Manifest.Module) } sort.Strings(moves) event, err := json.Marshal(keptMerge{SourceMoved: m, Moves: moves}) if err != nil { return err } heard := inventory.BatchedMerge{Repository: repository, Branch: m.Base, Commit: m.Commit, Merged: merged.UTC(), Heard: now, Event: event} // **A merge heard after a later merge of its branch** is answered by the walk holding that one. // **A merge on the controller's own path never shares a batch with one that waits for the delivery's // word** (decided during the build, 2026-10-10): batched together, the catalogue's deliveries would start // with the controller's and skip their turn. Each kind has a batch of its own. if p, ok, err := answeredByALaterMerge(ctx, inv, heard, touches, own, deliverySeatHeld(entries)); err != nil { return heldMerge(err) } else if ok { heard.Plan = p.ID if _, err := inv.AddMerge(ctx, heard); err != nil { return heldMerge(err) } if p.Batch() { if err := keepBatch(ctx, inv, &p, now, ""); err != nil { return unsaved(err) } } else if err := nameMerges(ctx, inv, &p); err != nil { return unsaved(err) } fmt.Printf("%s merged into %s (%.8s) after a later merge of its branch was put in %s (%s): that %s "+ "answers it\n", repository, m.Base, m.Commit, p.ID, p.State, walkOrBatch(p)) return cutBatchesHeld(ctx, f.open, now) } batch, why, err := batchToJoin(ctx, inv, heard, entries, read, now, own, waits) if err != nil { return heldMerge(err) } heard.Plan = batch.ID if added, err := inv.AddMerge(ctx, heard); err != nil || !added { return heldMerge(err) } holdFor(&batch, m.Head, waits, holding, own, now) if err := keepBatch(ctx, inv, &batch, now, ""); err != nil { return unsaved(err) } fmt.Printf("%s merged into %s (%.8s); %s, it joins %s — %s\n", repository, m.Base, m.Commit, why, batch.ID, batchWords(batch, now)) return cutBatchesHeld(ctx, f.open, now) } // holdFor keeps in a batch that a merge of a group joined whether that group still has a member to merge (ADR 0418 // clause 3): held while it has, the window running again from the moment the last hold is cleared. A group with a // merge of the controller's own path makes its batch an own-path one: such a plan waits for nobody's word, and is // started by the controller when the group's batch closes (clause 6). func holdFor(batch *inventory.Plan, head, why string, holding, own bool, now time.Time) { if batch.Delivery == nil { batch.Delivery = &inventory.PlanDelivery{} } if batch.Delivery.Batch == nil { batch.Delivery.Batch = &inventory.PlanBatch{Own: own} } w := batch.Delivery.Batch w.Own = w.Own || own if head == "" { return } if holding { if w.Holds == nil { w.Holds = map[string]string{} } w.Holds[head] = why return } if _, held := w.Holds[head]; held { holds := map[string]string{} for h, y := range w.Holds { if h != head { holds[h] = y } } w.Holds = holds if len(holds) == 0 { w.Holds, w.HeldAt = nil, now } } } // owedLate says a merge that reads as history is still the record's to answer (review of this change): a cut // marks every module it moved seen at that moment, so a merge of that branch // handed over late — made before the cut, heard after it — reads as history and was dropped. One that touches // a module the mesh holds, of a branch whose merges the controller keeps, made since it began keeping them, // is kept; one made before it began is old news, as before. func owedLate(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, merged time.Time, touches []inventory.Entry) (bool, error) { if len(touches) == 0 { return false, nil } kept, first, err := inv.KeptOnBranch(ctx, m.Owner+"/"+m.Repo, m.Base) if err != nil || !kept { return false, err } return !merged.Before(first.Add(-mergeGrace)), nil } // answeredByALaterMerge is the plan that answers a merge heard after a later merge of its branch was put in // one (ADR 0276 decision 4): a batch not yet cut, whose walk plans every file of the repository's merges; an // open walk, or a walk done, that builds every module this merge would move — built from the branch after the // later merge, which contains it. A failed or stopped walk answers nothing more, and a walk folded into // another is followed to that one. False when none does: the merge joins the next batch. func answeredByALaterMerge(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, moves []inventory.Entry, own, held bool) (inventory.Plan, bool, error) { later, err := inv.LaterMergesOf(ctx, heard.Repository, heard.Branch, heard.Merged) if err != nil || len(later) == 0 { return inventory.Plan{}, false, err } for _, l := range later { p, err := followTakeOver(ctx, inv, l.Plan) if err != nil { return inventory.Plan{}, false, err } switch { case p.Batch(): // A batch of the other kind is not this merge's: it joins the batch of its own kind. if p.OwnPath() == own { return p, true, nil } case p.Open() || p.State == inventory.PlanDone: // A walk that waited for nobody's word is not a catalogue merge's to be answered by while the delivery // seat has a holder: its delivery would skip its turn (decided during the build, 2026-10-10). With no // holder on record nothing waits, and any walk that built it answers. waited := p.Delivery != nil && p.Delivery.Awaits != "" if buildsEvery(p, moves) && (own || waited || !held) { return p, true, nil } } } return inventory.Plan{}, false, nil } // followTakeOver is a plan, or the walk that took it over when it was folded before it started. func followTakeOver(ctx context.Context, inv *inventory.Inventory, id string) (inventory.Plan, error) { p, err := inv.PlanByID(ctx, id) for hops := 0; err == nil && hops < 16 && p.State == inventory.PlanSuperseded && p.Delivery != nil && p.Delivery.TakenOverBy != ""; hops++ { p, err = inv.PlanByID(ctx, p.Delivery.TakenOverBy) } return p, err } // buildsEvery says a walk builds every one of the modules, and that there is one: a merge that moves nothing // is no walk's to name. func buildsEvery(p inventory.Plan, modules []inventory.Entry) bool { if len(modules) == 0 { return false } for _, e := range modules { if _, in := p.Modules[e.Manifest.Module]; !in { return false } } return true } // ownPath says a merge touches a module on the controller's own path, whose walk waits for nobody's word. func ownPath(touches []inventory.Entry) bool { for _, e := range touches { if onTheOwnPath(e.Manifest.Module, e.Manifest) { return true } } return false } // **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, as ADR 0306 // decisions 2 and 3 read it). batchToJoin is the batch a merge heard now joins, and why in words: // // - an assembling or queued batch of its kind it is related to (related.go): it joins it and waits for the // window, which its coming restarts. Related to two or more, the newer are folded into the oldest, which // answers all their merges: a merge never waits in two windows, and what it shares with each walks in one. // - none, and its delivery group has a member still to merge, or mesh-delivery could not say (groupWait, waits // names why): a new batch, which waits for the window, beside the ones it shares nothing with; its words say // why. // - none, and no member of its group is still to merge: a new batch whose window is closed at once. Cut at // once means only that: with a walk open that it is related to the batch is queued behind it as any closed // batch is (ADR 0276 decision 4, ADR 0407 decision 4); beside one it shares nothing with it is cut. func batchToJoin(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, entries []inventory.Entry, read map[string][]inventory.ReadRepository, now time.Time, own bool, waits string) (inventory.Plan, string, error) { edges, err := inv.Dependencies(ctx) if err != nil { return inventory.Plan{}, "", err } // **A group is one batch, whatever kinds its members are** (novox/hq ADR 0418 clause 6): a merge joins the batch // of the other kind that holds a merge from its head. if group, ok, err := batchOfGroup(ctx, inv, announcedOf(heard).Head, heard.Repository); err != nil { return inventory.Plan{}, "", err } else if ok && group.OwnPath() != own && group.Delivery != nil && group.Delivery.Batch != nil { group.Delivery.Batch.CutAtOnce = false return group, "its group " + announcedOf(heard).Head + " has a merge in " + group.ID + ", a batch of the other kind: " + "a group is one batch", nil } joins, whys, _, err := relatedBatches(ctx, inv, []inventory.BatchedMerge{heard}, own, "", entries, read, edges) if err != nil { return inventory.Plan{}, "", err } if len(joins) > 0 { into := joins[0] why := "related to " + into.ID + " (" + whys[0] + ")" if len(joins) > 1 { plans, moves, err := foldInto(ctx, inv, &into, joins[1:]) if err != nil { return inventory.Plan{}, "", err } if err := inv.SavePlansMoving(ctx, plans, moves); err != nil { return inventory.Plan{}, "", err } var folded []string for i, b := range joins[1:] { folded = append(folded, b.ID+" ("+whys[i+1]+")") } why += ", and to " + strings.Join(folded, ", ") + ", folded into it" } into.Delivery.Batch.CutAtOnce = false return into, why, nil } if waits != "" { // Its own window, beside the batches it shares nothing with: waiting is for its group's members, which // are related to it by their head and join it (ADR 0306 decision 2). b, err := newBatch(ctx, inv, heard, now, own, false, waits) return b, waits + ", it waits for the window in a batch of its own", err } b, err := newBatch(ctx, inv, heard, now, own, true, "") return b, "it shares nothing with a batch assembling and no member of a group is still to merge: it is cut at once", err } // relatedBatches are the batches not yet cut of a kind, oldest first, parted into those related to merges (with // the first reason of each) and the others. Each batch is judged by what its cut would move (ADR 0306 decision // 3): its merges' moves as they were kept and as they read now — the catalogue may have moved since a merge was // heard — widened to their dependents, and, for a catalogue batch, every walk waiting for its word, which the // cut folds in (cutBatch's carry). A merge kept with no moves leaves its batch unknown, related to everything. // except is a walk not to count as folded: the one being stopped. func relatedBatches(ctx context.Context, inv *inventory.Inventory, merges []inventory.BatchedMerge, own bool, except string, entries []inventory.Entry, read map[string][]inventory.ReadRepository, edges []inventory.Edge) (joins []inventory.Plan, whys []string, others []inventory.Plan, err error) { batches, err := inv.Batches(ctx) if err != nil { return nil, nil, nil, err } var kind []inventory.Plan for _, b := range batches { if b.OwnPath() == own { kind = append(kind, b) } } if len(kind) == 0 { return nil, nil, nil, nil } carry := newFootprint() if !own { // An own-path batch folds no waiting walk (cutBatchesHeld); a catalogue batch folds those it is related to // (cutUnrelated). Every one is counted here, which only makes a merge join a batch more readily: it waits // for the window, never walks beside what it shares something with (issue 973). plans, err := inv.OpenPlans(ctx) if err != nil { return nil, nil, nil, err } for _, p := range plans { if p.Release != nil || !p.Waiting() || p.ID == except { continue } kept, err := inv.MergesOf(ctx, p.ID) if err != nil { return nil, nil, nil, err } carry = unionFootprint(carry, footprintOfWalk(p, kept)) } } mine := footprintAsCut(merges, entries, read, edges) for _, b := range kind { held, err := inv.MergesOf(ctx, b.ID) if err != nil { return nil, nil, nil, err } if b.Delivery == nil { b.Delivery = &inventory.PlanDelivery{} } if b.Delivery.Batch == nil { b.Delivery.Batch = &inventory.PlanBatch{Own: own} } if ok, why := related(mine, unionFootprint(footprintAsCut(held, entries, read, edges), carry), edges); ok { joins, whys = append(joins, b), append(whys, why) } else { others = append(others, b) } } return joins, whys, others, nil } // footprintAsCut is a batch's footprint as a cut now would read it: footprintOfBatch, widened by the modules each // merge moves as the catalogue reads now (planBatch judges each merge again at the cut, so what it moves may // have changed since it was heard). Both are kept: when in doubt, related. func footprintAsCut(merges []inventory.BatchedMerge, entries []inventory.Entry, read map[string][]inventory.ReadRepository, edges []inventory.Edge) footprint { f := footprintOfBatch(merges, edges) var moved []string for _, m := range merges { s := announcedOf(m).SourceMoved if s.Owner == "" { continue } s.MergedAt = "" // news, whatever a cut marked seen since (as planBatch reads it) for _, e := range wouldMove(s, entries, read) { moved = append(moved, e.Manifest.Module) } } for _, name := range reachableFrom(moved, edges) { f.modules[name] = true } return f } // unionFootprint is what two changes reach together. func unionFootprint(a, b footprint) footprint { f := newFootprint() for _, from := range []footprint{a, b} { for k := range from.repos { f.repos[k] = true } for k := range from.modules { f.modules[k] = true } for k := range from.heads { f.heads[k] = true } } f.unknown = a.unknown if f.unknown == "" { f.unknown = b.unknown } return f } // foldInto folds batches into the oldest one related to a merge: the plans to write (into first, so no merge // names a plan the store does not hold) and the merges to move, for one SavePlansMoving. The folded batches name // the batch that took them over; a stop's closed window and its withheld modules (ADR 0299) go with their merges. func foldInto(ctx context.Context, inv *inventory.Inventory, into *inventory.Plan, from []inventory.Plan) ([]*inventory.Plan, []inventory.MergeMove, error) { if into.Delivery == nil { into.Delivery = &inventory.PlanDelivery{} } if into.Delivery.Batch == nil { into.Delivery.Batch = &inventory.PlanBatch{Own: into.OwnPath()} } plans := []*inventory.Plan{into} var moves []inventory.MergeMove for i := range from { f := from[i] merges, err := inv.MergesOf(ctx, f.ID) if err != nil { return nil, nil, err } for _, m := range merges { moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: into.ID, Alone: m.Alone}) } if f.Delivery == nil { f.Delivery = &inventory.PlanDelivery{} } if w := f.Delivery.Batch; w != nil { into.Delivery.Batch.Closed = into.Delivery.Batch.Closed || w.Closed into.Delivery.Batch.Withheld = union(into.Delivery.Batch.Withheld, w.Withheld) sort.Strings(into.Delivery.Batch.Withheld) } f.State = inventory.PlanSuperseded f.Delivery.Batch = nil f.Delivery.TakenOverBy = into.ID f.Note = fmt.Sprintf("folded into %s before it was cut: a merge related to both joined it, and %s answers "+ "its merges", into.ID, into.ID) plans = append(plans, &f) } return plans, moves, nil } // newBatch is a new batch of a kind, kept before its first merge names it: a merge never names a plan the store // does not hold. cutAtOnce closes its window at once; waits says why it waits for its group, when it does. func newBatch(ctx context.Context, inv *inventory.Inventory, first inventory.BatchedMerge, now time.Time, own, cutAtOnce bool, waits string) (inventory.Plan, error) { b := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch, Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}, Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own, CutAtOnce: cutAtOnce, Waits: waits}}} if err := inv.SavePlan(ctx, &b); err != nil { return inventory.Plan{}, err } return b, nil } // keepBatch writes a batch as its merges and the clock say it now: the commits it holds, the merges they // answer, its window, and assembling or queued behind a walk. Saved only when that changed. func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind string) error { return keepBatchBecause(ctx, inv, b, now, behind, "") } // keepBatchBecause is keepBatch for a batch queued behind a walk it is related to, which its words say (issue 973). func keepBatchBecause(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind, why string) error { merges, err := inv.MergesOf(ctx, b.ID) if err != nil { return err } window, atMost := mergeWindowOf() // The note's seconds left are no change of the batch's: it is saved when what it holds or its state moved. unNoted := func(p inventory.Plan) string { p.Note = "" return planSnapshot(p) } before := unNoted(*b) commits, named := carriedOf(merges) b.Commits = commits if len(commits) > 0 { newest := newestCommit(commits) b.Repository, b.Branch, b.Commit, b.Merged = newest.Repository, newest.Branch, newest.Commit, newest.Merged } if b.Delivery == nil { b.Delivery = &inventory.PlanDelivery{} } anchor := b.Created if was := b.Delivery.Batch; was != nil && was.HeldAt.After(anchor) { anchor = was.HeldAt // the maximum counts from the last hold's end (ADR 0418 clause 3) } closes, latest := windowOf(merges, anchor, window, atMost) b.Delivery.Merges = named var closed, cutAtOnce bool var withheld []string var waits string var holds map[string]string var heldAt time.Time if was := b.Delivery.Batch; was != nil { holds, heldAt = was.Holds, was.HeldAt closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299) cutAtOnce = was.CutAtOnce // the merge's, until another joins it (ADR 0304 decision 1) waits = was.Waits // its group's, as said when it was made (ADR 0306 decision 2) } b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, BehindWhy: why, Own: b.OwnPath(), Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits, Holds: holds, HeldAt: heldAt} b.State = inventory.PlanAssembling if behind != "" && windowClosed(*b, now) { b.State = inventory.PlanQueued } else { b.Delivery.Batch.Behind, b.Delivery.Batch.BehindWhy = "", "" } b.Note = batchWords(*b, now) if unNoted(*b) == before { return nil } if err := inv.SavePlan(ctx, b); err != nil { return err } // The window's clock follows the record, only when the record moved: a merge made the quiet key again, a // batch that opened made its most (ADR 0407 decision 1). armWindow(ctx, *b, now) return nil } // windowOf is when a batch's window closes, unless another merge comes, and when at the latest: the last // merge heard plus the window, and the batch's first merge plus the maximum. func windowOf(merges []inventory.BatchedMerge, created time.Time, window, atMost time.Duration) (closes, latest time.Time) { var last time.Time for _, m := range merges { if m.Heard.After(last) { last = m.Heard } } if last.IsZero() { last = created } return last.Add(window), created.Add(atMost) } // windowClosed says a batch's window is closed: closed by a stop, closed at once for a merge that shares // nothing (ADR 0304 decision 1), no merge for the window's length, or its maximum reached. **A batch holding a group // with a member still to merge is closed by none of the last three** (novox/hq ADR 0418 clause 3). func windowClosed(b inventory.Plan, now time.Time) bool { if b.Delivery == nil || b.Delivery.Batch == nil { return true } w := b.Delivery.Batch if w.Closed { return true } if len(w.Holds) > 0 { return false // a group's last member is still to merge: neither the window nor its maximum closes the batch } return w.CutAtOnce || !now.Before(w.ClosesAt) || !now.Before(w.AtMost) } // carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and // every merge named with the commit that carries it. func carriedOf(merges []inventory.BatchedMerge) ([]inventory.PlanCommit, []inventory.PlanMerge) { latest := map[string]inventory.BatchedMerge{} spelled := map[string]string{} for _, m := range merges { k := strings.ToLower(m.Repository) + "\x00" + m.Branch if l, ok := latest[k]; !ok || laterMerge(m, l) { latest[k] = m } var e link.SourceMoved if json.Unmarshal(m.Event, &e) == nil && e.Owner != "" { spelled[k] = e.Owner + "/" + e.Repo } } var commits []inventory.PlanCommit for k, l := range latest { repo := spelled[k] if repo == "" { repo = l.Repository } commits = append(commits, inventory.PlanCommit{Repository: repo, Branch: l.Branch, Commit: l.Commit, Merged: l.Merged}) } sort.Slice(commits, func(i, j int) bool { if !strings.EqualFold(commits[i].Repository, commits[j].Repository) { return strings.ToLower(commits[i].Repository) < strings.ToLower(commits[j].Repository) } return commits[i].Branch < commits[j].Branch }) var named []inventory.PlanMerge for _, m := range merges { k := strings.ToLower(m.Repository) + "\x00" + m.Branch repo := spelled[k] if repo == "" { repo = m.Repository } n := namedOf(m, repo) if l := latest[k]; l.Commit != m.Commit { n.Carried = l.Commit } named = append(named, n) } sort.SliceStable(named, func(i, j int) bool { return strings.ToLower(named[i].Repository) < strings.ToLower(named[j].Repository) }) return commits, named } // keptMerge is a merge as the record keeps it: the forge's announcement, and the modules it moved when it was // heard. The announcement reads back as a SourceMoved alone, which ignores the rest. type keptMerge struct { link.SourceMoved Moves []string `json:"moves,omitempty"` // FirstBuild says the merge is a first merge into a repository the mesh builds nothing from, kept to ask the first // build of its modules once mesh-delivery's word on it is kept (novox/hq ADR 0432, issue 1146). FirstBuild bool `json:"first_build,omitempty"` // Held says which first builds of the merge were refused for a word the running build-agent does not read: the merge // waits for a build-agent that reads them, and is never failed (novox/hq ADR 0482 §5, issue 1173). Held []heldFirstBuild `json:"held,omitempty"` } // announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none. func announcedOf(m inventory.BatchedMerge) keptMerge { var k keptMerge _ = json.Unmarshal(m.Event, &k) return k } // spelledOf is a kept merge's repository as the forge spelled it; empty when its announcement does not say. func spelledOf(m inventory.BatchedMerge) string { if e := announcedOf(m); e.Owner != "" { return e.Owner + "/" + e.Repo } return "" } // namedOf is one merge as a walk or batch names it: its commit, and its pull request and moves as announced. func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge { k := announcedOf(m) return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves, Head: k.HeadSHA, Merged: m.Merged, Heard: m.Heard} } // laterMerge says a was merged after b: by the forge's merge time, then by when each was heard. func laterMerge(a, b inventory.BatchedMerge) bool { if !a.Merged.Equal(b.Merged) { return a.Merged.After(b.Merged) } return a.Heard.After(b.Heard) } // newestCommit is the commit merged last among those a walk carries. func newestCommit(commits []inventory.PlanCommit) inventory.PlanCommit { newest := commits[0] for _, c := range commits[1:] { if c.Merged.After(newest.Merged) { newest = c } } return newest } // nameMerges writes the merges a walk answers onto its record, after a merge heard late was given to it. func nameMerges(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) error { merges, err := inv.MergesOf(ctx, p.ID) if err != nil { return err } // The walk's own commits stand: a merge heard late is carried by the one of its repository's branch. var named []inventory.PlanMerge for _, m := range merges { repo := m.Repository if e := spelledOf(m); e != "" { repo = e } n := namedOf(m, repo) if c := p.CommitOn(m.Repository, m.Branch); c != "" && c != m.Commit { n.Carried = c } named = append(named, n) } sort.SliceStable(named, func(i, j int) bool { return strings.ToLower(named[i].Repository) < strings.ToLower(named[j].Repository) }) if p.Delivery == nil { p.Delivery = &inventory.PlanDelivery{} } p.Delivery.Merges = named return inv.SavePlan(ctx, p) } // cutBatchesHeld takes the step that is due, for a caller holding the plans: after a failed walk, the search // for the merge that brought the failure; with no walk open, the next merge of that search walked alone; else // each closed batch cut into a walk with the waiting walks it is related to folded in, unless it is related to a // walk open, behind which it is said queued (cutUnrelated). While the search runs everything waits. func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error { inv := open.inventory // A merge found in two records is the half-done move of a crash: put right before anything is decided from it. if repaired, err := inv.RepairMerges(ctx); err != nil { return err } else if repaired > 0 { fmt.Printf("%d merge(s) found in two plans, the half of a move a restart interrupted, are kept by the plan they were moved to\n", repaired) } if err := searchAfterFailures(ctx, inv); err != nil { return err } alone, err := inv.AloneMerges(ctx) if err != nil { return err } if alone, err = coverAlone(ctx, inv, alone); err != nil { return err } plans, err := inv.OpenPlans(ctx) if err != nil { return err } var started, waiting []inventory.Plan for i := range plans { p := plans[i] if p.Release != nil { continue // the backlog's walk through the gates: no merge's } if p.Waiting() { waiting = append(waiting, p) } else { started = append(started, p) } } batches, err := inv.Batches(ctx) if err != nil { return err } // **A hold is looked at again at every cut** (ADR 0418 clause 3): a group whose last member merged or closed // since is released, and its window runs again from now. if batches, err = releaseHolds(ctx, inv, batches, now); err != nil { return err } keepAll := func(behind string) error { for i := range batches { if err := keepBatch(ctx, inv, &batches[i], now, behind); err != nil { return err } } return nil } if len(alone) > 0 { // A failed walk's search walks its merges alone, one at a time: everything waits for it (kept as it was, // issue 973). if len(started) > 0 { return keepAll(started[0].ID) } if len(waiting) > 0 { // The waiting walk is the one open; the search walks after it, and the batches wait behind both. return keepAll(waiting[0].ID) } return walkAlone(ctx, open, alone[0], now) } return cutUnrelated(ctx, open, batches, started, waiting, now) } // releaseHolds asks mesh-delivery again about each group a batch holds, and clears the holds of those with no member // left to merge. A group mesh-delivery cannot be asked about stays held: the safe error is to wait. It answers the // batches as kept. func releaseHolds(ctx context.Context, inv *inventory.Inventory, batches []inventory.Plan, now time.Time) ([]inventory.Plan, error) { var entries []inventory.Entry for i := range batches { b := &batches[i] if b.Delivery == nil || b.Delivery.Batch == nil || len(b.Delivery.Batch.Holds) == 0 { continue } if entries == nil { var err error if entries, err = inv.Catalogued(ctx); err != nil { return nil, err } } changed := false for head := range b.Delivery.Batch.Holds { open, isGroup, _, err := openMembersOf(ctx, inv, deliverySeatHeld(entries), head, "") if err != nil || (isGroup && len(open) > 0) { continue } holdFor(b, head, "", false, false, now) changed = true fmt.Printf("%s: group %s has no member left to merge; its window runs again from now\n", b.ID, head) } if changed { if err := keepBatch(ctx, inv, b, now, ""); err != nil { return nil, err } } } return batches, nil } // cutUnrelated cuts the closed batches that share nothing with an open walk, each into a walk beside the open // ones, and says the others queued behind the first walk they are related to (novox/hq ADR 0407 decision 4, // issue 973). A batch cut in this look counts as open for the batches after it. // // - An own-path batch goes first and folds no waiting walk; a catalogue batch folds the waiting walks it is // related to, and those it shares nothing with keep waiting. // - **The controller's own path is related to everything** (decided for issue 973): a walk or batch that // replaces the controller, the node-engine, the tool runner, the bus or mesh-delivery never runs beside // another, and nothing runs beside it. // - A batch whose window is still open is assembling; one related to an open walk is queued behind it, and // ADR 0276 decision 4 stands for it: nothing supersedes a walk that started. func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting []inventory.Plan, now time.Time) error { inv := open.inventory sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() }) var entries []inventory.Entry var read map[string][]inventory.ReadRepository var edges []inventory.Edge judged := false onPath, err := ownPathNow(ctx, inv) if err != nil { return err } openWalks, err := openWalksOf(ctx, inv, started, onPath) if err != nil { return err } for i := range batches { b := &batches[i] if !windowClosed(*b, now) { if err := keepBatch(ctx, inv, b, now, ""); err != nil { return err } continue } if !judged { var err error if entries, err = inv.Catalogued(ctx); err != nil { return err } if read, err = readForPlanning(ctx, inv); err != nil { return err } if edges, err = inv.Dependencies(ctx); err != nil { return err } judged = true } held, err := inv.MergesOf(ctx, b.ID) if err != nil { return err } fp := footprintAsCut(held, entries, read, edges) var fold, keep []inventory.Plan if !b.OwnPath() { for _, w := range waiting { ww, err := openWalkOf(ctx, inv, w, onPath) if err != nil { return err } if ok, _ := related(fp, ww.fp, edges); ok { fold = append(fold, w) fp = unionFootprint(fp, ww.fp) } else { keep = append(keep, w) } } } if behind, why := relatedOpen(fp, b.OwnPath(), openWalks, edges); behind != "" { if err := keepBatchBecause(ctx, inv, b, now, behind, why); err != nil { return err } continue } if err := keepBatch(ctx, inv, b, now, ""); err != nil { return err } if err := cutBatch(ctx, open, b, fold, now); err != nil { return err } waiting = keep if c := windowClockOf(); c != nil { c.Forget(ctx, b.ID) // a batch cut has no window left to close } cut, err := inv.PlanByID(ctx, b.ID) if err != nil { return err } if !cut.Open() { continue } w, err := openWalkOf(ctx, inv, cut, onPath) if err != nil { return err } if cut.Waiting() { waiting = append(waiting, cut) } else { openWalks = append(openWalks, w) } } return nil } // cutBatch makes a closed batch one walk, folding in the walks that wait for their word: it keeps its id, and // the folded walks name it as the walk that took them over. func cutBatch(ctx context.Context, open *stores, batch *inventory.Plan, waiting []inventory.Plan, now time.Time) error { inv := open.inventory var carry []string for _, w := range waiting { merges, err := inv.MergesOf(ctx, w.ID) if err != nil { return err } for _, m := range merges { if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, batch.ID, m.Alone); err != nil { return err } } // What the folded walk was to build is built by the batch's: its merges were acted on when it was cut, // so they read as history now. for name := range w.Modules { carry = append(carry, name) } } if err := planBatch(ctx, open, batch, carry, now); err != nil { return err } // Closed after the batch's walk is kept, never before: a controller replaced between the two leaves both // open, which the next cut settles, rather than neither. for i := range waiting { w := waiting[i] w.State = inventory.PlanSuperseded if w.Delivery == nil { w.Delivery = &inventory.PlanDelivery{} } w.Delivery.TakenOverBy = batch.ID w.Note = fmt.Sprintf("superseded at tier %d by %s (%s) before it started: that walk answers its merges", w.Tier, batch.ID, batch.Named()) if err := inv.SavePlan(ctx, &w); err != nil { return err } fmt.Printf(" %s is %s\n", w.ID, w.Note) } return nil } // walkAlone walks one merge on its own commit (ADR 0276 decision 3): a plan of its own, cut at once. func walkAlone(ctx context.Context, open *stores, m inventory.BatchedMerge, now time.Time) error { inv := open.inventory p := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: m.Repository, Branch: m.Branch, Commit: m.Commit, Merged: m.Merged, Created: now, State: inventory.PlanQueued, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}, Note: "walked alone after a failed walk carried it", Delivery: &inventory.PlanDelivery{Alone: true}} if err := inv.SavePlan(ctx, &p); err != nil { return err } if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, p.ID, true); err != nil { return err } fmt.Printf("%s %.8s, carried by a walk that failed, is walked on its own commit as %s\n", m.Repository, m.Commit, p.ID) return planBatch(ctx, open, &p, nil, now) } // planBatch plans a batch's merges as one walk, under the batch's id: per repository the latest merge of its // branch with every file the batch's merges of it changed, the moved modules and their dependents in tiers // along the graph and the order of any delivery group among them. func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry []string, now time.Time) error { inv := open.inventory merges, err := inv.MergesOf(ctx, batch.ID) if err != nil { return err } if len(merges) == 0 { batch.State = inventory.PlanSuperseded batch.Note = "a batch that holds no merge: nothing to walk" batch.Delivery.Batch = nil return inv.SavePlan(ctx, batch) } entries, err := inv.Catalogued(ctx) if err != nil { return err } read, err := readForPlanning(ctx, inv) if err != nil { return err } edges, err := inv.Dependencies(ctx) if err != nil { return err } commits, named := carriedOf(merges) var moved []string var members []batchMember unchanged := map[string]string{} for _, m := range combinedMerges(merges) { // **News, whatever was seen since** (review of this change): each merge was judged when it was heard, and // a cut, a fold or a failed walk since marks its modules seen — so a merge walked alone after a failed // walk, or folded, would read as history and move nothing. When it was made is not asked again here. m.MergedAt = "" // **Unchanged, said** (novox/hq issue 611): read before the merge is acted on, from the same records. for n, why := range unchangedOfMerge(m, entries, read) { unchanged[n] = why } names, err := movesOfMerge(ctx, inv, m, entries, read) if err != nil { return err } members = append(members, batchMember{merge: m, moved: names}) for _, n := range names { if !slices.Contains(moved, n) { moved = append(moved, n) } } } // **Whether it waits for its delivery's word** is read from what its merges moved (novox/hq ADR 0239), never // from what a folded walk carried along, which holds dependents too. awaits := awaitsFor(entries, moved) held := map[string]bool{} for _, e := range entries { held[e.Manifest.Module] = true } var also []string for _, n := range carry { // One the catalogue no longer holds would fail the walk's ask; it is nobody's to build. if held[n] && !slices.Contains(moved, n) { moved = append(moved, n) also = append(also, n) } } sort.Strings(also) ordered, err := changeOrderEdges(ctx, merges, members, entries, moved, edges) if err != nil { return err } var withheld []string plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...)) placedBy(&plan, ordered) // **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the // merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it // would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change, // and the stop named it. if batch.Delivery != nil && batch.Delivery.Batch != nil { if out := withhold(&plan, batch.Delivery.Batch.Withheld, moved); len(out) > 0 { withheld = out fmt.Printf(" withheld, a stopped merge's modules built only as dependents: %s\n", strings.Join(out, ", ")) } } plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch plan.Commits = commits newest := newestCommit(commits) plan.Repository, plan.Branch, plan.Commit, plan.Merged = newest.Repository, newest.Branch, newest.Commit, newest.Merged plan.Delivery = awaits if plan.Delivery == nil { plan.Delivery = &inventory.PlanDelivery{} } plan.Delivery.Merges = named plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone plan.Delivery.Withheld = withheld plan.Delivery.Order = ordered // **Left where it is, a module whose program did not change** (novox/hq issue 721): its check found it the same as // the build the mesh holds. for n, why := range leaveSameAsRunning(ctx, inv, &plan, moved, append(append([]inventory.Edge{}, edges...), ordered...)) { unchanged[n] = why } plan.Delivery.Unchanged = unchangedLeft(unchanged, plan.Modules) // **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on. plan.Times = walkTimesAtCut(*batch, plan, entries, now) if len(moved) == 0 || len(plan.Modules) == 0 { plan.State = inventory.PlanDone plan.Tiers = [][]string{} plan.Note = "moved nothing the mesh holds: nothing to walk" if len(plan.Delivery.Unchanged) > 0 { plan.Note = "moved nothing the mesh holds: nothing to walk; " + unchangedSaid(plan.Delivery.Unchanged) } if err := inv.SavePlan(ctx, &plan); err != nil { return err } *batch = plan fmt.Printf("%s (%s): its merges move nothing the mesh holds\n", plan.ID, plan.Named()) return nil } if hasCycle(plan.Tiers, edges) { fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n", strings.Join(plan.Tiers[len(plan.Tiers)-1], ", ")) } if len(also) > 0 { fmt.Printf(" %s, left unbuilt by a walk folded into it, are planned here again\n", strings.Join(also, ", ")) } var tiers []string for i, t := range plan.Tiers { var names []string for _, n := range t { if m := plan.Modules[n]; m != nil && m.Placed != "" { n += " " + m.Placed } names = append(names, n) } tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(names, ", "))) } var answers []string for _, n := range named { answers = append(answers, n.Repository+"@"+short(n.Commit)) } fmt.Printf("%s cut: %s, answering %s; %d module(s) in %d tier(s)\n %s\n", plan.ID, plan.Named(), strings.Join(answers, ", "), len(plan.Modules), len(plan.Tiers), strings.Join(tiers, "\n ")) if plan.Waiting() { plan.Note = waitingNote(plan) if err := inv.SavePlan(ctx, &plan); err != nil { return err } *batch = plan fmt.Printf(" %s waits for %s's word before its first tier is asked\n", plan.ID, plan.Delivery.Awaits) return nil } // Written and its first tier asked as one act on the plans (novox/hq issue 213), under the caller's hold. if err := askFirstTierAtCut(&plan, func(p *inventory.Plan) (bool, error) { return backupsFirst(ctx, inv, p) }, func() error { return askTier(ctx, inv, &plan, plan.Tiers[plan.Tier]) }); err != nil { return err } if err := inv.SavePlan(ctx, &plan); err != nil { return err } *batch = plan return nil } // askFirstTierAtCut asks a freshly cut plan's first tier, after the backups of the data it moves (novox/hq ADR 0404, // issue 1086): the cut asks the tier itself, so advanceOnce, which asks the backups before a first tier, finds it // asked and never takes them. A plan whose backups wait, failed or could not be worked out asks nothing; it is saved // unasked and the walk's own step goes on from its record. func askFirstTierAtCut(p *inventory.Plan, backups func(*inventory.Plan) (bool, error), ask func() error) error { proceed, err := backups(p) if err != nil { p.Note = backupsNote + "not asked, the backups of the plan could not be worked out: " + err.Error() + "; nothing is asked or sent, and the plan tries again" return nil } if !proceed { return nil } return ask() } // placedBy writes on each module of a plan the edges of its order that put it after another module of the plan, with the // rule that drew each (novox/hq ADR 0419 clause 5). func placedBy(plan *inventory.Plan, ordered []inventory.Edge) { for _, e := range ordered { m, in := plan.Modules[e.From] if _, before := plan.Modules[e.To]; !in || !before { continue } said := "after " + e.To + " (" + e.Why + ")" if m.Placed == "" { m.Placed = said } else if !strings.Contains(m.Placed, said) { m.Placed += "; " + said } } } // unchangedSaid names the modules a walk says matched what runs, for its note. func unchangedSaid(unchanged map[string]string) string { names := make([]string, 0, len(unchanged)) for n := range unchanged { names = append(names, n) } sort.Strings(names) return "unchanged, the build the mesh holds is theirs: " + strings.Join(names, ", ") } // unchangedLeft is what a walk says matched what runs: every module a merge left unchanged that the walk does not // build anyway, as another merge's module or a dependent (novox/hq issue 611). Nil when none is. func unchangedLeft(unchanged map[string]string, builds map[string]*inventory.PlanModule) map[string]string { out := map[string]string{} for n, why := range unchanged { if _, built := builds[n]; !built { out[n] = why } } if len(out) == 0 { return nil } return out } // withhold takes from a plan every module named that its merges do not move, and the tiers left empty: the // modules taken. func withhold(plan *inventory.Plan, names, moved []string) []string { var out []string for _, n := range names { if _, in := plan.Modules[n]; in && !slices.Contains(moved, n) { delete(plan.Modules, n) out = append(out, n) } } if len(out) == 0 { return nil } var tiers [][]string for _, t := range plan.Tiers { var kept []string for _, n := range t { if !slices.Contains(out, n) { kept = append(kept, n) } } if len(kept) > 0 { tiers = append(tiers, kept) } } if tiers == nil { tiers = [][]string{} } plan.Tiers = tiers sort.Strings(out) return out } // walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window // closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its // class. A merge walked alone had no window. func walkTimesAtCut(batch, walk inventory.Plan, entries []inventory.Entry, now time.Time) *inventory.PlanTimes { cut := now t := &inventory.PlanTimes{Cut: &cut, Class: classOf(walk, entries)} if batch.Delivery != nil && batch.Delivery.Batch != nil { w := batch.Delivery.Batch closed := w.ClosesAt if !w.AtMost.IsZero() && w.AtMost.Before(closed) { closed = w.AtMost } if w.CutAtOnce && !batch.Created.IsZero() { closed = batch.Created // closed when it was made (ADR 0304 decision 1) } if !closed.IsZero() { if closed.After(now) { closed = now } t.WindowClosed = &closed } } return t } // resolverSeat is the seat of the mesh's resolver: a module claiming it is a core module (ADR 0282 decision 1). const resolverSeat = "mesh-dns-resolver" // classOf is a walk's class (novox/hq ADR 0282 decision 1): core when it walks a module on the controller's own // path or one holding the mesh's resolver, leaf otherwise. func classOf(walk inventory.Plan, entries []inventory.Entry) string { modules := make([]string, 0, len(walk.Modules)) for m := range walk.Modules { modules = append(modules, m) } return classOfModules(modules, entries) } // classOfModules is the class of modules moved together: core when one is on the controller's own path or holds // the mesh's resolver, leaf otherwise (ADR 0282 decision 1). func classOfModules(modules []string, entries []inventory.Entry) string { resolvers := map[string]bool{} for _, e := range entries { for _, c := range e.Manifest.Claims { if c.Name == resolverSeat { resolvers[e.Manifest.Module] = true } } } onPath := ownPathOf(entries) for _, m := range modules { if _, own := onPath[m]; own || resolvers[m] { return inventory.ClassCore } } return inventory.ClassLeaf } // combinedMerges is a batch's merges as one merge per repository's branch: the latest, with every file the // merges of it changed and said to be a module's — on a linear trunk the latest contains the others. A file is // removed when the last merge of the batch that changed it removed it. merges are in the order they were made. func combinedMerges(merges []inventory.BatchedMerge) []link.SourceMoved { type combined struct { latest inventory.BatchedMerge m link.SourceMoved said bool removed map[string]bool paths []string dirs []string cut bool } by := map[string]*combined{} var keys []string for _, b := range merges { var e link.SourceMoved if err := json.Unmarshal(b.Event, &e); err != nil { continue } k := strings.ToLower(b.Repository) + "\x00" + b.Branch c, ok := by[k] if !ok { c = &combined{latest: b, m: e, said: true, removed: map[string]bool{}} by[k] = c keys = append(keys, k) } else if laterMerge(b, c.latest) { c.latest, c.m = b, e } c.said = c.said && e.ModuleDirsSaid c.cut = c.cut || e.PathsTruncated c.paths, c.dirs = union(c.paths, e.Paths), union(c.dirs, e.ModuleDirs) for _, path := range e.Paths { c.removed[path] = slices.Contains(e.Removed, path) } for _, path := range e.Removed { c.removed[path] = true } } sort.Strings(keys) var out []link.SourceMoved for _, k := range keys { c := by[k] m := c.m m.Paths, m.ModuleDirs, m.PathsTruncated, m.ModuleDirsSaid = c.paths, c.dirs, c.cut, c.said m.Removed = nil for _, path := range c.paths { if c.removed[path] { m.Removed = append(m.Removed, path) } } out = append(out, m) } return out } // union is a and b without repeats, in order. func union(a, b []string) []string { out := append([]string{}, a...) for _, s := range b { if !slices.Contains(out, s) { out = append(out, s) } } return out } // batchMember is one repository's merge in a batch and what it moves. type batchMember struct { merge link.SourceMoved moved []string } // changeOrderEdges is the order of a batch's moved modules as edges of its plan's tiers (novox/hq ADR 0419): drawn // per module from the module graph and from what the merges say of the manifests they touch (delivery_edges.go), // whatever repository the merges are in and whether or not they form a delivery group. No `after:` line is read. // A cycle is said, never lost: the plan puts the modules left in its last tier, and the composed check fails the // group before it merges. func changeOrderEdges(ctx context.Context, merges []inventory.BatchedMerge, members []batchMember, entries []inventory.Entry, moved []string, edges []inventory.Edge) ([]inventory.Edge, error) { // What each repository's branch held before the batch: the first parent of its earliest merge in it. earliest := map[string]inventory.BatchedMerge{} for _, k := range merges { key := strings.ToLower(k.Repository) + "\x00" + k.Branch if e, ok := earliest[key]; !ok || laterMerge(e, k) { earliest[key] = k } } var sources []changeSource for _, m := range members { s := changeSource{Owner: m.merge.Owner, Repo: m.merge.Repo, New: m.merge.Commit, Paths: m.merge.Paths, Truncated: m.merge.PathsTruncated, Removed: m.merge.Removed} if e, ok := earliest[strings.ToLower(m.merge.Owner+"/"+m.merge.Repo)+"\x00"+m.merge.Base]; ok { s.Since = e.Commit } if s.Truncated { s.Paths, s.Truncated = truncatedPaths(entries, s.Owner, s.Repo, moved), false } sources = append(sources, s) } changes, err := changesOf(ctx, sources) if err != nil { return nil, fmt.Errorf("what the merges say of their manifests cannot be read, so the order of their modules "+ "cannot be drawn: %w", err) } order := orderEdges(moved, changes, edges) for _, e := range order.Edges { fmt.Printf(" order: %s\n", edgeSaid(e)) } if said := order.CycleSaid(); said != "" { fmt.Printf(" %s\n", said) } return order.Edges, nil } // searchAfterFailures starts the search a failed walk leaves (ADR 0276 decision 3): every merge it answered // in a later commit of its repository waits to be walked on its own commit. A stopped walk starts none: a // person ended it. func searchAfterFailures(ctx context.Context, inv *inventory.Inventory) error { plans, err := inv.RecentPlans(ctx, 20) if err != nil { return err } for _, p := range plans { if p.State != inventory.PlanFailed || p.Delivery == nil || p.Delivery.Stopped != "" || strings.HasPrefix(p.Note, "stopped by hand") || strings.HasPrefix(p.Note, "closed by hand") { continue } merges, err := inv.MergesOf(ctx, p.ID) if err != nil { return err } for _, m := range merges { if m.Alone || p.CommitOf(m.Repository) == m.Commit { continue // its own commit was walked } if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, "", true); err != nil { return err } fmt.Printf("%s failed; %s %.8s, which it carried in %.8s, is walked on its own commit\n", p.ID, m.Repository, m.Commit, p.CommitOf(m.Repository)) } } return nil } // coverAlone gives every merge waiting to be walked alone to a walk done that holds a later merge of its // branch — one walked alone, or the failed walk retried by a person: the search stops at the first merge // delivered, and that walk answers the older ones. What is left waits, the newest first. func coverAlone(ctx context.Context, inv *inventory.Inventory, alone []inventory.BatchedMerge) ([]inventory.BatchedMerge, error) { var left []inventory.BatchedMerge for _, m := range alone { later, err := inv.LaterMergesOf(ctx, m.Repository, m.Branch, m.Merged) if err != nil { return nil, err } covered := false for _, l := range later { p, err := inv.PlanByID(ctx, l.Plan) if err != nil { return nil, err } if p.State != inventory.PlanDone { continue } if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, p.ID, true); err != nil { return nil, err } if err := nameMerges(ctx, inv, &p); err != nil { return nil, err } fmt.Printf("%s %.8s is answered by %s, which delivered a later merge of its branch on its own commit\n", m.Repository, m.Commit, p.ID) covered = true break } if !covered { left = append(left, m) } } return left, nil } // batchWords is a batch in the operator's words (ADR 0276 decision 7): "assembling: 13 s left (at the latest // 00:31:09); grouped: novox/mesh-catalog@48bda475 (answers 553b7191), …; plan not yet calculated". func batchWords(b inventory.Plan, now time.Time) string { grouped := groupedWords(b) if b.Delivery == nil || b.Delivery.Batch == nil { return "grouped: " + grouped + "; plan not yet calculated" } w := b.Delivery.Batch if w.Own { grouped += " (on the controller's own path)" } if b.State == inventory.PlanQueued { because := "" if w.BehindWhy != "" { because = " (" + w.BehindWhy + ")" } return fmt.Sprintf("queued behind %s%s; grouped: %s; plan not yet calculated", w.Behind, because, grouped) } if w.CutAtOnce { return fmt.Sprintf("cut at once, sharing nothing with a batch assembling; grouped: %s; plan not yet calculated", grouped) } left := w.ClosesAt.Sub(now) if w.AtMost.Before(w.ClosesAt) { left = w.AtMost.Sub(now) } if left < 0 { left = 0 } waits := "" if w.Waits != "" { waits = " — " + w.Waits } return fmt.Sprintf("assembling: %s left (at the latest %s)%s; grouped: %s; plan not yet calculated", secondsWords(left), w.AtMost.Local().Format("15:04:05"), waits, grouped) } // groupedWords is a batch's commits, each with the earlier merges it answers. func groupedWords(b inventory.Plan) string { var out []string for _, c := range b.Carried() { s := c.Repository + "@" + short(c.Commit) var answers []string if b.Delivery != nil { for _, m := range b.Delivery.Merges { if strings.EqualFold(m.Repository, c.Repository) && m.Carried == c.Commit { answers = append(answers, short(m.Commit)) } } } if len(answers) > 0 { s += " (answers " + strings.Join(answers, ", ") + ")" } out = append(out, s) } if len(out) == 0 { return "nothing yet" } return strings.Join(out, ", ") } // batchLines is a batch's grouped list as `plans` prints it under its line (asked by the operator, 2026-10-10): // one line per repository, its pull request and title, the earlier merges it answers, and what its merges move. func batchLines(b inventory.Plan) []string { var out []string for _, c := range b.Carried() { var head *inventory.PlanMerge var answers []string moves := map[string]bool{} if b.Delivery != nil { for i := range b.Delivery.Merges { m := &b.Delivery.Merges[i] if !strings.EqualFold(m.Repository, c.Repository) { continue } for _, n := range m.Moves { moves[n] = true } switch { case m.Commit == c.Commit: head = m case m.Carried == c.Commit: answers = append(answers, pullWords(*m)) } } } line := repoName(c.Repository) + " " + short(c.Commit) if head != nil && head.Number > 0 { line = repoName(c.Repository) + " " + pullWords(*head) } if len(answers) > 0 { line += " (answers " + strings.Join(answers, ", ") + ")" } if len(moves) > 0 { line += " · " + strings.Join(sortedKeysOf(boolsToStrings(moves)), ", ") } out = append(out, line) } return out } // pullWords is a merge as its pull request: "#175 a tap shows its outcome", the title cut at fifty runes; the // commit where the announcement named no pull request. func pullWords(m inventory.PlanMerge) string { if m.Number == 0 { return short(m.Commit) } s := fmt.Sprintf("#%d", m.Number) if t := cutTitle(m.Title, 50); t != "" { s += " " + t } return s } // cutTitle is a title cut at n runes, with an ellipsis where it was cut. func cutTitle(title string, n int) string { r := []rune(strings.TrimSpace(title)) if len(r) <= n { return string(r) } return strings.TrimSpace(string(r[:n-1])) + "…" } // boolsToStrings is a set's members, for sortedKeysOf. func boolsToStrings(set map[string]bool) map[string]string { out := make(map[string]string, len(set)) for k := range set { out[k] = "" } return out } // secondsWords is a short wait as a person reads it. func secondsWords(d time.Duration) string { if d < 2*time.Minute { return fmt.Sprintf("%d s", int(d.Round(time.Second)/time.Second)) } return d.Round(time.Second).String() } // walkOrBatch is what a plan record is, in a word. func walkOrBatch(p inventory.Plan) string { if p.Batch() { return "batch" } return "walk" } // orWords is s, or the words for when it is empty. func orWords(s, none string) string { if s == "" { return none } return s }