jschoubben
9f072c110c
mesh/merge-gate head 9f072c110c14 merged into trunk 60cc9ec363df — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head 9f072c110c14 merged into trunk 60cc9ec363df — pass: its merge-check.sh passed
mesh/delivery delivered
A rebuild that landed while a walk waited for mesh-delivery's word was taken as the walk's build of that module; once let go, the walk asked a tier only when every module of it was unasked, so it never asked the rest and stood still. A walk now takes only the outcomes of builds it asked, and none while it waits; a rebuild does not join a waiting walk and says it runs beside it; and a tier asked in part is asked the rest, including a module held built that the walk never asked for, which is somebody else's build and not sent as the walk's.
1538 lines
56 KiB
Go
1538 lines
56 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"regexp"
|
|
"slices"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// **Merges are assembled in a rolling window, and each batch is delivered by one walk** (novox/hq ADR 0276,
|
|
// issue 362).
|
|
//
|
|
// Every merge opened a walk of its own, and the next merge of the branch superseded it. On 2026-10-10 two
|
|
// catalogue merges eighteen seconds apart left one walk that no delivery held, and the operator started it by
|
|
// hand 58 minutes later. A merge now joins the open **batch**. Each merge restarts the **merge window**; it
|
|
// closes when none came for `merge-window` (90 seconds) or at the latest `merge-window-at-most` (10 minutes)
|
|
// after the batch's first merge. A closed batch becomes one walk when no walk is open: one plan for every
|
|
// commit in it, with at most one commit per repository, the latest of its branch, which contains the earlier
|
|
// ones. The walk names every merge it answers.
|
|
//
|
|
// - One walk at a time. Merges that come while a walk runs join the next batch. A walk that started is never
|
|
// superseded; one still waiting for its delivery's word is folded into the next batch's walk, which
|
|
// answers every merge it answered — the only supersession left.
|
|
// - A merge heard after a later merge of its branch was put in a walk is answered by that walk: a batch
|
|
// still assembling, an open walk that builds everything it moves, or a walk done that did, which then
|
|
// names it at once. Otherwise it joins the next batch.
|
|
// - A failed walk marks nothing delivered. Each earlier merge it carried in a later commit is walked on its
|
|
// own commit, the newest first, one walk each, until one is delivered; the older ones are then answered
|
|
// by that one.
|
|
//
|
|
// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, ADR 0306 decisions
|
|
// 2 and 3, issue 525): it joins a batch it is related to, and one that is related to none and whose delivery
|
|
// group has no member still to merge is a batch whose window is closed at once (batchToJoin). Walks stay one at
|
|
// a time: such a batch is cut when no walk is open, and queued behind the one that is otherwise.
|
|
//
|
|
// A merge on the controller's own path (a module whose walk waits for nobody's word) never shares a batch with
|
|
// one that waits for mesh-delivery's: each kind has a batch of its own, an own-path batch is cut first and folds
|
|
// no waiting catalogue walk, and a catalogue walk let go while another walk runs starts once it ended — so no
|
|
// catalogue batch's walk starts without the word (decided during the build, 2026-10-10).
|
|
//
|
|
// The batch, its merges and their times are in the store (batched_merge, and the batch's own plan record in
|
|
// the state `assembling` or `queued`), so a restarted controller resumes the window where it stood.
|
|
|
|
// The merge window's defaults. The controller's settings `merge-window` and `merge-window-at-most` reach it in
|
|
// the file MESH_MERGE_WINDOW_FILE names (module.json), read at every look, so a change takes effect without a
|
|
// restart.
|
|
var (
|
|
mergeWindowDefault = 90 * time.Second
|
|
mergeWindowAtMostDefault = 10 * time.Minute
|
|
)
|
|
|
|
// mergeWindowFileVar names the file the controller's window settings are written to.
|
|
const mergeWindowFileVar = "MESH_MERGE_WINDOW_FILE"
|
|
|
|
// batchCutEvery is how often the controller looks whether a batch's window closed.
|
|
const batchCutEvery = 5 * time.Second
|
|
|
|
// batchLateAfter is how long past its maximum a batch may still be assembling, with no walk open, before
|
|
// the controller is said to have failed to cut it (ADR 0276 decision 9).
|
|
const batchLateAfter = time.Minute
|
|
|
|
// edgeGroupOrder is a delivery group's order made an edge of a walk's tiers (ADR 0276 decision 5): the
|
|
// modules of the member after depend on the modules of the member before, and are gated on them.
|
|
const edgeGroupOrder = "group-order"
|
|
|
|
// mergeWindowOf is the window and its maximum, from the controller's settings; a value unreadable is its
|
|
// default, and said.
|
|
func mergeWindowOf() (window, atMost time.Duration) {
|
|
window, atMost = mergeWindowDefault, mergeWindowAtMostDefault
|
|
path := os.Getenv(mergeWindowFileVar)
|
|
if path == "" {
|
|
return window, atMost
|
|
}
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "the merge window's settings cannot be read (%v); %s and %s are used\n", err, window, atMost)
|
|
return window, atMost
|
|
}
|
|
var values map[string]any
|
|
if err := json.Unmarshal(raw, &values); err != nil {
|
|
fmt.Fprintf(os.Stderr, "the merge window's settings are not JSON (%v); %s and %s are used\n", err, window, atMost)
|
|
return window, atMost
|
|
}
|
|
for key, to := range map[string]*time.Duration{"merge-window": &window, "merge-window-at-most": &atMost} {
|
|
v, has := values[key]
|
|
if !has || v == "" {
|
|
continue
|
|
}
|
|
if d, ok := durationSetting(v); ok {
|
|
*to = d
|
|
} else {
|
|
fmt.Fprintf(os.Stderr, "the setting %s=%v is no duration; %s is used\n", key, v, *to)
|
|
}
|
|
}
|
|
return window, atMost
|
|
}
|
|
|
|
// durationSetting reads a duration as a setting says it: "90s", "10m", or a number of seconds.
|
|
func durationSetting(v any) (time.Duration, bool) {
|
|
switch t := v.(type) {
|
|
case string:
|
|
if d, err := time.ParseDuration(strings.TrimSpace(t)); err == nil && d >= 0 {
|
|
return d, true
|
|
}
|
|
if n, err := strconv.ParseFloat(strings.TrimSpace(t), 64); err == nil && n >= 0 {
|
|
return time.Duration(n * float64(time.Second)), true
|
|
}
|
|
case float64:
|
|
if t >= 0 {
|
|
return time.Duration(t * float64(time.Second)), true
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
// hearMerge puts a merge the forge announced into the batch that answers it, at the moment it was heard,
|
|
// and cuts what is due.
|
|
func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.Time) error {
|
|
// One merge acted on at a time, whoever hands it over: the bus, or the catch-up that reads back
|
|
// what the bus did not hand over (novox/hq issue 266).
|
|
actingOnMerges.Lock()
|
|
defer actingOnMerges.Unlock()
|
|
|
|
inv := f.open.inventory
|
|
repository := m.Owner + "/" + m.Repo
|
|
if kept, known, err := inv.MergeOf(ctx, repository, m.Commit); err != nil {
|
|
return notNow(err)
|
|
} else if known {
|
|
fmt.Printf("%s merged into %s (%.8s) was heard before; %s answers it\n", repository, m.Base, m.Commit,
|
|
orWords(kept.Plan, "a walk of its own commit, once the walks before it end"))
|
|
return nil
|
|
}
|
|
entries, err := inv.Catalogued(ctx)
|
|
if err != nil {
|
|
return notNow(err)
|
|
}
|
|
read, err := readForPlanning(ctx, inv)
|
|
if err != nil {
|
|
return notNow(err)
|
|
}
|
|
merged, err := time.Parse(time.RFC3339Nano, m.MergedAt)
|
|
if err != nil {
|
|
merged = now
|
|
}
|
|
// **What it touches, history or not** (review of this change): a merge heard after a later merge of its
|
|
// branch was cut reads as history for every module that cut marked seen, and was dropped from the record
|
|
// here, never named by the walk that carried it.
|
|
raw := m
|
|
raw.MergedAt = ""
|
|
touches := wouldMove(raw, entries, read)
|
|
var later []inventory.BatchedMerge
|
|
if len(touches) > 0 {
|
|
if later, err = inv.LaterMergesOf(ctx, repository, m.Base, merged.UTC()); err != nil {
|
|
return notNow(err)
|
|
}
|
|
}
|
|
owed, err := owedLate(ctx, inv, m, merged, touches)
|
|
if err != nil {
|
|
return notNow(err)
|
|
}
|
|
if from, packaging, already := mergeCandidates(m, entries, read); len(from) == 0 && len(packaging) == 0 && len(later) == 0 && !owed {
|
|
// "Already built from it" and "nothing reads it" are different facts, and reading the first
|
|
// as the second sends somebody looking for a broken trigger when the mesh is up to date.
|
|
if already > 0 {
|
|
fmt.Printf("%s merged into %s (%.8s); %d module(s) the mesh holds are already built from it\n",
|
|
repository, m.Base, m.Commit, already)
|
|
return nil
|
|
}
|
|
fmt.Printf("%s merged into %s (%.8s); nothing the mesh holds reads it\n", repository, m.Base, m.Commit)
|
|
return nil
|
|
}
|
|
// **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk.
|
|
own := ownPath(touches)
|
|
waits, err := groupWait(ctx, inv, m, deliverySeatHeld(entries), own)
|
|
if err != nil {
|
|
return notNow(err)
|
|
}
|
|
release, err := inv.HoldPlans(ctx, true)
|
|
if err != nil {
|
|
return notNow(err)
|
|
}
|
|
defer release()
|
|
// Again under the hold: a controller handing over may have kept it since.
|
|
if _, known, err := inv.MergeOf(ctx, repository, m.Commit); err != nil || known {
|
|
return notNow(err)
|
|
}
|
|
var moves []string
|
|
for _, e := range touches {
|
|
moves = append(moves, e.Manifest.Module)
|
|
}
|
|
sort.Strings(moves)
|
|
event, err := json.Marshal(keptMerge{SourceMoved: m, Moves: moves})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
heard := inventory.BatchedMerge{Repository: repository, Branch: m.Base, Commit: m.Commit, Merged: merged.UTC(),
|
|
Heard: now, Event: event}
|
|
|
|
// **A merge heard after a later merge of its branch** is answered by the walk holding that one.
|
|
// **A merge on the controller's own path never shares a batch with one that waits for the delivery's
|
|
// word** (decided during the build, 2026-10-10): batched together, the catalogue's deliveries would start
|
|
// with the controller's and skip their turn. Each kind has a batch of its own.
|
|
if p, ok, err := answeredByALaterMerge(ctx, inv, heard, touches, own, deliverySeatHeld(entries)); err != nil {
|
|
return notNow(err)
|
|
} else if ok {
|
|
heard.Plan = p.ID
|
|
if _, err := inv.AddMerge(ctx, heard); err != nil {
|
|
return notNow(err)
|
|
}
|
|
if p.Batch() {
|
|
if err := keepBatch(ctx, inv, &p, now, ""); err != nil {
|
|
return notNow(err)
|
|
}
|
|
} else if err := nameMerges(ctx, inv, &p); err != nil {
|
|
return notNow(err)
|
|
}
|
|
fmt.Printf("%s merged into %s (%.8s) after a later merge of its branch was put in %s (%s): that %s "+
|
|
"answers it\n", repository, m.Base, m.Commit, p.ID, p.State, walkOrBatch(p))
|
|
return cutBatchesHeld(ctx, f.open, now)
|
|
}
|
|
|
|
batch, why, err := batchToJoin(ctx, inv, heard, entries, read, now, own, waits)
|
|
if err != nil {
|
|
return notNow(err)
|
|
}
|
|
heard.Plan = batch.ID
|
|
if added, err := inv.AddMerge(ctx, heard); err != nil || !added {
|
|
return notNow(err)
|
|
}
|
|
if err := keepBatch(ctx, inv, &batch, now, ""); err != nil {
|
|
return notNow(err)
|
|
}
|
|
fmt.Printf("%s merged into %s (%.8s); %s, it joins %s — %s\n", repository, m.Base, m.Commit, why, batch.ID,
|
|
batchWords(batch, now))
|
|
return cutBatchesHeld(ctx, f.open, now)
|
|
}
|
|
|
|
// owedLate says a merge that reads as history is still the record's to answer (review of this change): a cut
|
|
// marks every module it moved seen at that moment, so a merge of that branch the bus lost and the catch-up
|
|
// hands over later — made before the cut, heard after it — reads as history and was dropped. One that touches
|
|
// a module the mesh holds, of a branch whose merges the controller keeps, made since it began keeping them,
|
|
// is kept; one made before it began is old news, as before.
|
|
func owedLate(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, merged time.Time,
|
|
touches []inventory.Entry) (bool, error) {
|
|
if len(touches) == 0 {
|
|
return false, nil
|
|
}
|
|
kept, first, err := inv.KeptOnBranch(ctx, m.Owner+"/"+m.Repo, m.Base)
|
|
if err != nil || !kept {
|
|
return false, err
|
|
}
|
|
return !merged.Before(first.Add(-mergeGrace)), nil
|
|
}
|
|
|
|
// answeredByALaterMerge is the plan that answers a merge heard after a later merge of its branch was put in
|
|
// one (ADR 0276 decision 4): a batch not yet cut, whose walk plans every file of the repository's merges; an
|
|
// open walk, or a walk done, that builds every module this merge would move — built from the branch after the
|
|
// later merge, which contains it. A failed or stopped walk answers nothing more, and a walk folded into
|
|
// another is followed to that one. False when none does: the merge joins the next batch.
|
|
func answeredByALaterMerge(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge,
|
|
moves []inventory.Entry, own, held bool) (inventory.Plan, bool, error) {
|
|
later, err := inv.LaterMergesOf(ctx, heard.Repository, heard.Branch, heard.Merged)
|
|
if err != nil || len(later) == 0 {
|
|
return inventory.Plan{}, false, err
|
|
}
|
|
for _, l := range later {
|
|
p, err := followTakeOver(ctx, inv, l.Plan)
|
|
if err != nil {
|
|
return inventory.Plan{}, false, err
|
|
}
|
|
switch {
|
|
case p.Batch():
|
|
// A batch of the other kind is not this merge's: it joins the batch of its own kind.
|
|
if p.OwnPath() == own {
|
|
return p, true, nil
|
|
}
|
|
case p.Open() || p.State == inventory.PlanDone:
|
|
// A walk that waited for nobody's word is not a catalogue merge's to be answered by while the delivery
|
|
// seat has a holder: its delivery would skip its turn (decided during the build, 2026-10-10). With no
|
|
// holder on record nothing waits, and any walk that built it answers.
|
|
waited := p.Delivery != nil && p.Delivery.Awaits != ""
|
|
if buildsEvery(p, moves) && (own || waited || !held) {
|
|
return p, true, nil
|
|
}
|
|
}
|
|
}
|
|
return inventory.Plan{}, false, nil
|
|
}
|
|
|
|
// followTakeOver is a plan, or the walk that took it over when it was folded before it started.
|
|
func followTakeOver(ctx context.Context, inv *inventory.Inventory, id string) (inventory.Plan, error) {
|
|
p, err := inv.PlanByID(ctx, id)
|
|
for hops := 0; err == nil && hops < 16 && p.State == inventory.PlanSuperseded && p.Delivery != nil &&
|
|
p.Delivery.TakenOverBy != ""; hops++ {
|
|
p, err = inv.PlanByID(ctx, p.Delivery.TakenOverBy)
|
|
}
|
|
return p, err
|
|
}
|
|
|
|
// buildsEvery says a walk builds every one of the modules, and that there is one: a merge that moves nothing
|
|
// is no walk's to name.
|
|
func buildsEvery(p inventory.Plan, modules []inventory.Entry) bool {
|
|
if len(modules) == 0 {
|
|
return false
|
|
}
|
|
for _, e := range modules {
|
|
if _, in := p.Modules[e.Manifest.Module]; !in {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// ownPath says a merge touches a module on the controller's own path, whose walk waits for nobody's word.
|
|
func ownPath(touches []inventory.Entry) bool {
|
|
for _, e := range touches {
|
|
if _, own := onTheControllersPath[e.Manifest.Module]; own {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, as ADR 0306
|
|
// decisions 2 and 3 read it). batchToJoin is the batch a merge heard now joins, and why in words:
|
|
//
|
|
// - an assembling or queued batch of its kind it is related to (related.go): it joins it and waits for the
|
|
// window, which its coming restarts. Related to two or more, the newer are folded into the oldest, which
|
|
// answers all their merges: a merge never waits in two windows, and what it shares with each walks in one.
|
|
// - none, and its delivery group has a member still to merge, or mesh-delivery could not say (groupWait, waits
|
|
// names why): a new batch, which waits for the window, beside the ones it shares nothing with; its words say
|
|
// why.
|
|
// - none, and no member of its group is still to merge: a new batch whose window is closed at once. Cut at
|
|
// once means only that: walks stay one at a time (ADR 0276 decision 4), so with a walk open the batch is
|
|
// queued behind it as any closed batch is.
|
|
func batchToJoin(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, entries []inventory.Entry,
|
|
read map[string][]inventory.ReadRepository, now time.Time, own bool, waits string) (inventory.Plan, string, error) {
|
|
edges, err := inv.Dependencies(ctx)
|
|
if err != nil {
|
|
return inventory.Plan{}, "", err
|
|
}
|
|
joins, whys, _, err := relatedBatches(ctx, inv, []inventory.BatchedMerge{heard}, own, "", entries, read, edges)
|
|
if err != nil {
|
|
return inventory.Plan{}, "", err
|
|
}
|
|
if len(joins) > 0 {
|
|
into := joins[0]
|
|
why := "related to " + into.ID + " (" + whys[0] + ")"
|
|
if len(joins) > 1 {
|
|
plans, moves, err := foldInto(ctx, inv, &into, joins[1:])
|
|
if err != nil {
|
|
return inventory.Plan{}, "", err
|
|
}
|
|
if err := inv.SavePlansMoving(ctx, plans, moves); err != nil {
|
|
return inventory.Plan{}, "", err
|
|
}
|
|
var folded []string
|
|
for i, b := range joins[1:] {
|
|
folded = append(folded, b.ID+" ("+whys[i+1]+")")
|
|
}
|
|
why += ", and to " + strings.Join(folded, ", ") + ", folded into it"
|
|
}
|
|
into.Delivery.Batch.CutAtOnce = false
|
|
return into, why, nil
|
|
}
|
|
if waits != "" {
|
|
// Its own window, beside the batches it shares nothing with: waiting is for its group's members, which
|
|
// are related to it by their head and join it (ADR 0306 decision 2).
|
|
b, err := newBatch(ctx, inv, heard, now, own, false, waits)
|
|
return b, waits + ", it waits for the window in a batch of its own", err
|
|
}
|
|
b, err := newBatch(ctx, inv, heard, now, own, true, "")
|
|
return b, "it shares nothing with a batch assembling and no member of a group is still to merge: it is cut at once", err
|
|
}
|
|
|
|
// relatedBatches are the batches not yet cut of a kind, oldest first, parted into those related to merges (with
|
|
// the first reason of each) and the others. Each batch is judged by what its cut would move (ADR 0306 decision
|
|
// 3): its merges' moves as they were kept and as they read now — the catalogue may have moved since a merge was
|
|
// heard — widened to their dependents, and, for a catalogue batch, every walk waiting for its word, which the
|
|
// cut folds in (cutBatch's carry). A merge kept with no moves leaves its batch unknown, related to everything.
|
|
// except is a walk not to count as folded: the one being stopped.
|
|
func relatedBatches(ctx context.Context, inv *inventory.Inventory, merges []inventory.BatchedMerge, own bool,
|
|
except string, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
|
edges []inventory.Edge) (joins []inventory.Plan, whys []string, others []inventory.Plan, err error) {
|
|
batches, err := inv.Batches(ctx)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
var kind []inventory.Plan
|
|
for _, b := range batches {
|
|
if b.OwnPath() == own {
|
|
kind = append(kind, b)
|
|
}
|
|
}
|
|
if len(kind) == 0 {
|
|
return nil, nil, nil, nil
|
|
}
|
|
carry := newFootprint()
|
|
if !own {
|
|
// An own-path batch folds no waiting walk (cutBatchesHeld); a catalogue batch may fold every one.
|
|
plans, err := inv.OpenPlans(ctx)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
for _, p := range plans {
|
|
if p.Release != nil || !p.Waiting() || p.ID == except {
|
|
continue
|
|
}
|
|
kept, err := inv.MergesOf(ctx, p.ID)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
carry = unionFootprint(carry, footprintOfWalk(p, kept))
|
|
}
|
|
}
|
|
mine := footprintAsCut(merges, entries, read, edges)
|
|
for _, b := range kind {
|
|
held, err := inv.MergesOf(ctx, b.ID)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
if b.Delivery == nil {
|
|
b.Delivery = &inventory.PlanDelivery{}
|
|
}
|
|
if b.Delivery.Batch == nil {
|
|
b.Delivery.Batch = &inventory.PlanBatch{Own: own}
|
|
}
|
|
if ok, why := related(mine, unionFootprint(footprintAsCut(held, entries, read, edges), carry), edges); ok {
|
|
joins, whys = append(joins, b), append(whys, why)
|
|
} else {
|
|
others = append(others, b)
|
|
}
|
|
}
|
|
return joins, whys, others, nil
|
|
}
|
|
|
|
// footprintAsCut is a batch's footprint as a cut now would read it: footprintOfBatch, widened by the modules each
|
|
// merge moves as the catalogue reads now (planBatch judges each merge again at the cut, so what it moves may
|
|
// have changed since it was heard). Both are kept: when in doubt, related.
|
|
func footprintAsCut(merges []inventory.BatchedMerge, entries []inventory.Entry,
|
|
read map[string][]inventory.ReadRepository, edges []inventory.Edge) footprint {
|
|
f := footprintOfBatch(merges, edges)
|
|
var moved []string
|
|
for _, m := range merges {
|
|
s := announcedOf(m).SourceMoved
|
|
if s.Owner == "" {
|
|
continue
|
|
}
|
|
s.MergedAt = "" // news, whatever a cut marked seen since (as planBatch reads it)
|
|
for _, e := range wouldMove(s, entries, read) {
|
|
moved = append(moved, e.Manifest.Module)
|
|
}
|
|
}
|
|
for _, name := range reachableFrom(moved, edges) {
|
|
f.modules[name] = true
|
|
}
|
|
return f
|
|
}
|
|
|
|
// unionFootprint is what two changes reach together.
|
|
func unionFootprint(a, b footprint) footprint {
|
|
f := newFootprint()
|
|
for _, from := range []footprint{a, b} {
|
|
for k := range from.repos {
|
|
f.repos[k] = true
|
|
}
|
|
for k := range from.modules {
|
|
f.modules[k] = true
|
|
}
|
|
for k := range from.heads {
|
|
f.heads[k] = true
|
|
}
|
|
}
|
|
f.unknown = a.unknown
|
|
if f.unknown == "" {
|
|
f.unknown = b.unknown
|
|
}
|
|
return f
|
|
}
|
|
|
|
// foldInto folds batches into the oldest one related to a merge: the plans to write (into first, so no merge
|
|
// names a plan the store does not hold) and the merges to move, for one SavePlansMoving. The folded batches name
|
|
// the batch that took them over; a stop's closed window and its withheld modules (ADR 0299) go with their merges.
|
|
func foldInto(ctx context.Context, inv *inventory.Inventory, into *inventory.Plan, from []inventory.Plan) ([]*inventory.Plan,
|
|
[]inventory.MergeMove, error) {
|
|
if into.Delivery == nil {
|
|
into.Delivery = &inventory.PlanDelivery{}
|
|
}
|
|
if into.Delivery.Batch == nil {
|
|
into.Delivery.Batch = &inventory.PlanBatch{Own: into.OwnPath()}
|
|
}
|
|
plans := []*inventory.Plan{into}
|
|
var moves []inventory.MergeMove
|
|
for i := range from {
|
|
f := from[i]
|
|
merges, err := inv.MergesOf(ctx, f.ID)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
for _, m := range merges {
|
|
moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: into.ID, Alone: m.Alone})
|
|
}
|
|
if f.Delivery == nil {
|
|
f.Delivery = &inventory.PlanDelivery{}
|
|
}
|
|
if w := f.Delivery.Batch; w != nil {
|
|
into.Delivery.Batch.Closed = into.Delivery.Batch.Closed || w.Closed
|
|
into.Delivery.Batch.Withheld = union(into.Delivery.Batch.Withheld, w.Withheld)
|
|
sort.Strings(into.Delivery.Batch.Withheld)
|
|
}
|
|
f.State = inventory.PlanSuperseded
|
|
f.Delivery.Batch = nil
|
|
f.Delivery.TakenOverBy = into.ID
|
|
f.Note = fmt.Sprintf("folded into %s before it was cut: a merge related to both joined it, and %s answers "+
|
|
"its merges", into.ID, into.ID)
|
|
plans = append(plans, &f)
|
|
}
|
|
return plans, moves, nil
|
|
}
|
|
|
|
// newBatch is a new batch of a kind, kept before its first merge names it: a merge never names a plan the store
|
|
// does not hold. cutAtOnce closes its window at once; waits says why it waits for its group, when it does.
|
|
func newBatch(ctx context.Context, inv *inventory.Inventory, first inventory.BatchedMerge, now time.Time, own,
|
|
cutAtOnce bool, waits string) (inventory.Plan, error) {
|
|
b := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
|
|
Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling,
|
|
Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{},
|
|
Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own, CutAtOnce: cutAtOnce, Waits: waits}}}
|
|
if err := inv.SavePlan(ctx, &b); err != nil {
|
|
return inventory.Plan{}, err
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
// keepBatch writes a batch as its merges and the clock say it now: the commits it holds, the merges they
|
|
// answer, its window, and assembling or queued behind a walk. Saved only when that changed.
|
|
func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind string) error {
|
|
merges, err := inv.MergesOf(ctx, b.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
window, atMost := mergeWindowOf()
|
|
// The note's seconds left are no change of the batch's: it is saved when what it holds or its state moved.
|
|
unNoted := func(p inventory.Plan) string {
|
|
p.Note = ""
|
|
return planSnapshot(p)
|
|
}
|
|
before := unNoted(*b)
|
|
commits, named := carriedOf(merges)
|
|
b.Commits = commits
|
|
if len(commits) > 0 {
|
|
newest := newestCommit(commits)
|
|
b.Repository, b.Branch, b.Commit, b.Merged = newest.Repository, newest.Branch, newest.Commit, newest.Merged
|
|
}
|
|
if b.Delivery == nil {
|
|
b.Delivery = &inventory.PlanDelivery{}
|
|
}
|
|
closes, latest := windowOf(merges, b.Created, window, atMost)
|
|
b.Delivery.Merges = named
|
|
var closed, cutAtOnce bool
|
|
var withheld []string
|
|
var waits string
|
|
if was := b.Delivery.Batch; was != nil {
|
|
closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299)
|
|
cutAtOnce = was.CutAtOnce // the merge's, until another joins it (ADR 0304 decision 1)
|
|
waits = was.Waits // its group's, as said when it was made (ADR 0306 decision 2)
|
|
}
|
|
b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, Own: b.OwnPath(),
|
|
Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits}
|
|
b.State = inventory.PlanAssembling
|
|
if behind != "" && windowClosed(*b, now) {
|
|
b.State = inventory.PlanQueued
|
|
} else {
|
|
b.Delivery.Batch.Behind = ""
|
|
}
|
|
b.Note = batchWords(*b, now)
|
|
if unNoted(*b) == before {
|
|
return nil
|
|
}
|
|
return inv.SavePlan(ctx, b)
|
|
}
|
|
|
|
// windowOf is when a batch's window closes, unless another merge comes, and when at the latest: the last
|
|
// merge heard plus the window, and the batch's first merge plus the maximum.
|
|
func windowOf(merges []inventory.BatchedMerge, created time.Time, window, atMost time.Duration) (closes, latest time.Time) {
|
|
var last time.Time
|
|
for _, m := range merges {
|
|
if m.Heard.After(last) {
|
|
last = m.Heard
|
|
}
|
|
}
|
|
if last.IsZero() {
|
|
last = created
|
|
}
|
|
return last.Add(window), created.Add(atMost)
|
|
}
|
|
|
|
// windowClosed says a batch's window is closed: closed by a stop, closed at once for a merge that shares
|
|
// nothing (ADR 0304 decision 1), no merge for the window's length, or its maximum reached.
|
|
func windowClosed(b inventory.Plan, now time.Time) bool {
|
|
if b.Delivery == nil || b.Delivery.Batch == nil {
|
|
return true
|
|
}
|
|
w := b.Delivery.Batch
|
|
return w.Closed || w.CutAtOnce || !now.Before(w.ClosesAt) || !now.Before(w.AtMost)
|
|
}
|
|
|
|
// carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and
|
|
// every merge named with the commit that carries it.
|
|
func carriedOf(merges []inventory.BatchedMerge) ([]inventory.PlanCommit, []inventory.PlanMerge) {
|
|
latest := map[string]inventory.BatchedMerge{}
|
|
spelled := map[string]string{}
|
|
for _, m := range merges {
|
|
k := strings.ToLower(m.Repository) + "\x00" + m.Branch
|
|
if l, ok := latest[k]; !ok || laterMerge(m, l) {
|
|
latest[k] = m
|
|
}
|
|
var e link.SourceMoved
|
|
if json.Unmarshal(m.Event, &e) == nil && e.Owner != "" {
|
|
spelled[k] = e.Owner + "/" + e.Repo
|
|
}
|
|
}
|
|
var commits []inventory.PlanCommit
|
|
for k, l := range latest {
|
|
repo := spelled[k]
|
|
if repo == "" {
|
|
repo = l.Repository
|
|
}
|
|
commits = append(commits, inventory.PlanCommit{Repository: repo, Branch: l.Branch, Commit: l.Commit, Merged: l.Merged})
|
|
}
|
|
sort.Slice(commits, func(i, j int) bool {
|
|
if !strings.EqualFold(commits[i].Repository, commits[j].Repository) {
|
|
return strings.ToLower(commits[i].Repository) < strings.ToLower(commits[j].Repository)
|
|
}
|
|
return commits[i].Branch < commits[j].Branch
|
|
})
|
|
var named []inventory.PlanMerge
|
|
for _, m := range merges {
|
|
k := strings.ToLower(m.Repository) + "\x00" + m.Branch
|
|
repo := spelled[k]
|
|
if repo == "" {
|
|
repo = m.Repository
|
|
}
|
|
n := namedOf(m, repo)
|
|
if l := latest[k]; l.Commit != m.Commit {
|
|
n.Carried = l.Commit
|
|
}
|
|
named = append(named, n)
|
|
}
|
|
sort.SliceStable(named, func(i, j int) bool {
|
|
return strings.ToLower(named[i].Repository) < strings.ToLower(named[j].Repository)
|
|
})
|
|
return commits, named
|
|
}
|
|
|
|
// keptMerge is a merge as the record keeps it: the forge's announcement, and the modules it moved when it was
|
|
// heard. The announcement reads back as a SourceMoved alone, which ignores the rest.
|
|
type keptMerge struct {
|
|
link.SourceMoved
|
|
Moves []string `json:"moves,omitempty"`
|
|
}
|
|
|
|
// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none.
|
|
func announcedOf(m inventory.BatchedMerge) keptMerge {
|
|
var k keptMerge
|
|
_ = json.Unmarshal(m.Event, &k)
|
|
return k
|
|
}
|
|
|
|
// spelledOf is a kept merge's repository as the forge spelled it; empty when its announcement does not say.
|
|
func spelledOf(m inventory.BatchedMerge) string {
|
|
if e := announcedOf(m); e.Owner != "" {
|
|
return e.Owner + "/" + e.Repo
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// namedOf is one merge as a walk or batch names it: its commit, and its pull request and moves as announced.
|
|
func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge {
|
|
k := announcedOf(m)
|
|
return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves,
|
|
Head: k.HeadSHA, Merged: m.Merged, Heard: m.Heard}
|
|
}
|
|
|
|
// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard.
|
|
func laterMerge(a, b inventory.BatchedMerge) bool {
|
|
if !a.Merged.Equal(b.Merged) {
|
|
return a.Merged.After(b.Merged)
|
|
}
|
|
return a.Heard.After(b.Heard)
|
|
}
|
|
|
|
// newestCommit is the commit merged last among those a walk carries.
|
|
func newestCommit(commits []inventory.PlanCommit) inventory.PlanCommit {
|
|
newest := commits[0]
|
|
for _, c := range commits[1:] {
|
|
if c.Merged.After(newest.Merged) {
|
|
newest = c
|
|
}
|
|
}
|
|
return newest
|
|
}
|
|
|
|
// nameMerges writes the merges a walk answers onto its record, after a merge heard late was given to it.
|
|
func nameMerges(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) error {
|
|
merges, err := inv.MergesOf(ctx, p.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// The walk's own commits stand: a merge heard late is carried by the one of its repository's branch.
|
|
var named []inventory.PlanMerge
|
|
for _, m := range merges {
|
|
repo := m.Repository
|
|
if e := spelledOf(m); e != "" {
|
|
repo = e
|
|
}
|
|
n := namedOf(m, repo)
|
|
if c := p.CommitOn(m.Repository, m.Branch); c != "" && c != m.Commit {
|
|
n.Carried = c
|
|
}
|
|
named = append(named, n)
|
|
}
|
|
sort.SliceStable(named, func(i, j int) bool {
|
|
return strings.ToLower(named[i].Repository) < strings.ToLower(named[j].Repository)
|
|
})
|
|
if p.Delivery == nil {
|
|
p.Delivery = &inventory.PlanDelivery{}
|
|
}
|
|
p.Delivery.Merges = named
|
|
return inv.SavePlan(ctx, p)
|
|
}
|
|
|
|
// batchCutter cuts what is due on a timer, until the context ends: a window closes with no merge to say so.
|
|
func batchCutter(ctx context.Context, open *stores) {
|
|
tick := time.NewTicker(batchCutEvery)
|
|
defer tick.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-tick.C:
|
|
}
|
|
release, err := open.inventory.HoldPlans(ctx, false)
|
|
if err != nil {
|
|
continue // whoever holds the plans moves them, and cuts at the end
|
|
}
|
|
if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil {
|
|
fmt.Printf("batches: %v\n", err)
|
|
}
|
|
release()
|
|
}
|
|
}
|
|
|
|
// cutBatchesHeld takes the step that is due, for a caller holding the plans: after a failed walk, the search
|
|
// for the merge that brought the failure; with no walk open, the next merge of that search walked alone, or a
|
|
// closed batch cut into one walk with the waiting walks folded in; with a walk open, a closed batch said
|
|
// queued behind it.
|
|
func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
|
inv := open.inventory
|
|
if err := searchAfterFailures(ctx, inv); err != nil {
|
|
return err
|
|
}
|
|
alone, err := inv.AloneMerges(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if alone, err = coverAlone(ctx, inv, alone); err != nil {
|
|
return err
|
|
}
|
|
plans, err := inv.OpenPlans(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var started *inventory.Plan
|
|
var waiting []inventory.Plan
|
|
for i := range plans {
|
|
p := plans[i]
|
|
if p.Release != nil {
|
|
continue // the backlog's walk through the gates: no merge's
|
|
}
|
|
if p.Waiting() {
|
|
waiting = append(waiting, p)
|
|
} else if started == nil {
|
|
started = &plans[i]
|
|
}
|
|
}
|
|
batches, err := inv.Batches(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
keepAll := func(behind string) error {
|
|
for i := range batches {
|
|
if err := keepBatch(ctx, inv, &batches[i], now, behind); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
if started != nil {
|
|
// **One walk at a time** (ADR 0276 decision 4): every batch waits behind it, whatever its window says.
|
|
return keepAll(started.ID)
|
|
}
|
|
if len(alone) > 0 {
|
|
if len(waiting) > 0 {
|
|
// The waiting walk is the one open; the search walks after it, and the batches wait behind both.
|
|
return keepAll(waiting[0].ID)
|
|
}
|
|
return walkAlone(ctx, open, alone[0], now)
|
|
}
|
|
if err := keepAll(""); err != nil {
|
|
return err
|
|
}
|
|
// The oldest batch whose window closed is cut, an own-path one before a catalogue one (it goes first, and
|
|
// the catalogue batch queues behind it rather than being cut and overtaken). Several of a kind may be open
|
|
// since a merge that shares nothing is cut at once (ADR 0304 decision 1): walks stay one at a time, so one
|
|
// is cut per look, and once its walk started the others are queued behind it.
|
|
sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() })
|
|
for i := range batches {
|
|
b := &batches[i]
|
|
if !windowClosed(*b, now) {
|
|
continue
|
|
}
|
|
if b.OwnPath() {
|
|
// A walk on the controller's own path folds no walk waiting for its word (decided during the build,
|
|
// 2026-10-10): that walk keeps waiting beside it, and starts once this one ended (advanceOnce).
|
|
waiting = nil
|
|
}
|
|
if err := cutBatch(ctx, open, b, waiting, now); err != nil {
|
|
return err
|
|
}
|
|
// The batches left wait behind the walk just cut when it started; the next look does the same.
|
|
if b.Open() && !b.Waiting() {
|
|
if batches, err = inv.Batches(ctx); err != nil {
|
|
return err
|
|
}
|
|
return keepAll(b.ID)
|
|
}
|
|
return nil
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// cutBatch makes a closed batch one walk, folding in the walks that wait for their word: it keeps its id, and
|
|
// the folded walks name it as the walk that took them over.
|
|
func cutBatch(ctx context.Context, open *stores, batch *inventory.Plan, waiting []inventory.Plan, now time.Time) error {
|
|
inv := open.inventory
|
|
var carry []string
|
|
for _, w := range waiting {
|
|
merges, err := inv.MergesOf(ctx, w.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, m := range merges {
|
|
if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, batch.ID, m.Alone); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// What the folded walk was to build is built by the batch's: its merges were acted on when it was cut,
|
|
// so they read as history now.
|
|
for name := range w.Modules {
|
|
carry = append(carry, name)
|
|
}
|
|
}
|
|
if err := planBatch(ctx, open, batch, carry, now); err != nil {
|
|
return err
|
|
}
|
|
// Closed after the batch's walk is kept, never before: a controller replaced between the two leaves both
|
|
// open, which the next cut settles, rather than neither.
|
|
for i := range waiting {
|
|
w := waiting[i]
|
|
w.State = inventory.PlanSuperseded
|
|
if w.Delivery == nil {
|
|
w.Delivery = &inventory.PlanDelivery{}
|
|
}
|
|
w.Delivery.TakenOverBy = batch.ID
|
|
w.Note = fmt.Sprintf("superseded at tier %d by %s (%s) before it started: that walk answers its merges",
|
|
w.Tier, batch.ID, batch.Named())
|
|
if err := inv.SavePlan(ctx, &w); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf(" %s is %s\n", w.ID, w.Note)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// walkAlone walks one merge on its own commit (ADR 0276 decision 3): a plan of its own, cut at once.
|
|
func walkAlone(ctx context.Context, open *stores, m inventory.BatchedMerge, now time.Time) error {
|
|
inv := open.inventory
|
|
p := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: m.Repository, Branch: m.Branch,
|
|
Commit: m.Commit, Merged: m.Merged, Created: now, State: inventory.PlanQueued, Tiers: [][]string{},
|
|
Modules: map[string]*inventory.PlanModule{}, Note: "walked alone after a failed walk carried it",
|
|
Delivery: &inventory.PlanDelivery{Alone: true}}
|
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
|
return err
|
|
}
|
|
if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, p.ID, true); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("%s %.8s, carried by a walk that failed, is walked on its own commit as %s\n", m.Repository, m.Commit, p.ID)
|
|
return planBatch(ctx, open, &p, nil, now)
|
|
}
|
|
|
|
// planBatch plans a batch's merges as one walk, under the batch's id: per repository the latest merge of its
|
|
// branch with every file the batch's merges of it changed, the moved modules and their dependents in tiers
|
|
// along the graph and the order of any delivery group among them.
|
|
func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry []string, now time.Time) error {
|
|
inv := open.inventory
|
|
merges, err := inv.MergesOf(ctx, batch.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(merges) == 0 {
|
|
batch.State = inventory.PlanSuperseded
|
|
batch.Note = "a batch that holds no merge: nothing to walk"
|
|
batch.Delivery.Batch = nil
|
|
return inv.SavePlan(ctx, batch)
|
|
}
|
|
entries, err := inv.Catalogued(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
read, err := readForPlanning(ctx, inv)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
edges, err := inv.Dependencies(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
commits, named := carriedOf(merges)
|
|
var moved []string
|
|
var members []batchMember
|
|
unchanged := map[string]string{}
|
|
for _, m := range combinedMerges(merges) {
|
|
// **News, whatever was seen since** (review of this change): each merge was judged when it was heard, and
|
|
// a cut, a fold or a failed walk since marks its modules seen — so a merge walked alone after a failed
|
|
// walk, or folded, would read as history and move nothing. When it was made is not asked again here.
|
|
m.MergedAt = ""
|
|
// **Unchanged, said** (novox/hq issue 611): read before the merge is acted on, from the same records.
|
|
for n, why := range unchangedOfMerge(m, entries, read) {
|
|
unchanged[n] = why
|
|
}
|
|
names, err := movesOfMerge(ctx, inv, m, entries, read)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
members = append(members, batchMember{merge: m, moved: names})
|
|
for _, n := range names {
|
|
if !slices.Contains(moved, n) {
|
|
moved = append(moved, n)
|
|
}
|
|
}
|
|
}
|
|
// **Whether it waits for its delivery's word** is read from what its merges moved (novox/hq ADR 0239), never
|
|
// from what a folded walk carried along, which holds dependents too.
|
|
awaits := awaitsFor(entries, moved)
|
|
held := map[string]bool{}
|
|
for _, e := range entries {
|
|
held[e.Manifest.Module] = true
|
|
}
|
|
var also []string
|
|
for _, n := range carry {
|
|
// One the catalogue no longer holds would fail the walk's ask; it is nobody's to build.
|
|
if held[n] && !slices.Contains(moved, n) {
|
|
moved = append(moved, n)
|
|
also = append(also, n)
|
|
}
|
|
}
|
|
sort.Strings(also)
|
|
ordered := groupOrderEdges(members, entries, read, edges)
|
|
var withheld []string
|
|
plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...))
|
|
// **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the
|
|
// merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it
|
|
// would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change,
|
|
// and the stop named it.
|
|
if batch.Delivery != nil && batch.Delivery.Batch != nil {
|
|
if out := withhold(&plan, batch.Delivery.Batch.Withheld, moved); len(out) > 0 {
|
|
withheld = out
|
|
fmt.Printf(" withheld, a stopped merge's modules built only as dependents: %s\n", strings.Join(out, ", "))
|
|
}
|
|
}
|
|
plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch
|
|
plan.Commits = commits
|
|
newest := newestCommit(commits)
|
|
plan.Repository, plan.Branch, plan.Commit, plan.Merged = newest.Repository, newest.Branch, newest.Commit, newest.Merged
|
|
plan.Delivery = awaits
|
|
if plan.Delivery == nil {
|
|
plan.Delivery = &inventory.PlanDelivery{}
|
|
}
|
|
plan.Delivery.Merges = named
|
|
plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
|
|
plan.Delivery.Withheld = withheld
|
|
// **Left where it is, a module whose program did not change** (novox/hq issue 721): its check found it the same as
|
|
// the build the mesh holds.
|
|
for n, why := range leaveSameAsRunning(ctx, inv, &plan, moved, append(append([]inventory.Edge{}, edges...), ordered...)) {
|
|
unchanged[n] = why
|
|
}
|
|
plan.Delivery.Unchanged = unchangedLeft(unchanged, plan.Modules)
|
|
// **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on.
|
|
plan.Times = walkTimesAtCut(*batch, plan, entries, now)
|
|
if len(moved) == 0 || len(plan.Modules) == 0 {
|
|
plan.State = inventory.PlanDone
|
|
plan.Tiers = [][]string{}
|
|
plan.Note = "moved nothing the mesh holds: nothing to walk"
|
|
if len(plan.Delivery.Unchanged) > 0 {
|
|
plan.Note = "moved nothing the mesh holds: nothing to walk; " + unchangedSaid(plan.Delivery.Unchanged)
|
|
}
|
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
return err
|
|
}
|
|
*batch = plan
|
|
fmt.Printf("%s (%s): its merges move nothing the mesh holds\n", plan.ID, plan.Named())
|
|
return nil
|
|
}
|
|
if hasCycle(plan.Tiers, edges) {
|
|
fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n",
|
|
strings.Join(plan.Tiers[len(plan.Tiers)-1], ", "))
|
|
}
|
|
if len(also) > 0 {
|
|
fmt.Printf(" %s, left unbuilt by a walk folded into it, are planned here again\n", strings.Join(also, ", "))
|
|
}
|
|
var tiers []string
|
|
for i, t := range plan.Tiers {
|
|
tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(t, ", ")))
|
|
}
|
|
var answers []string
|
|
for _, n := range named {
|
|
answers = append(answers, n.Repository+"@"+short(n.Commit))
|
|
}
|
|
fmt.Printf("%s cut: %s, answering %s; %d module(s) in %d tier(s)\n %s\n", plan.ID, plan.Named(),
|
|
strings.Join(answers, ", "), len(plan.Modules), len(plan.Tiers), strings.Join(tiers, "\n "))
|
|
if plan.Waiting() {
|
|
plan.Note = waitingNote(plan)
|
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
return err
|
|
}
|
|
*batch = plan
|
|
fmt.Printf(" %s waits for %s's word before its first tier is asked\n", plan.ID, plan.Delivery.Awaits)
|
|
return nil
|
|
}
|
|
// Written and its first tier asked as one act on the plans (novox/hq issue 213), under the caller's hold.
|
|
if err := askTier(ctx, inv, &plan, plan.Tiers[plan.Tier]); err != nil {
|
|
return err
|
|
}
|
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
return err
|
|
}
|
|
*batch = plan
|
|
return nil
|
|
}
|
|
|
|
// unchangedSaid names the modules a walk says matched what runs, for its note.
|
|
func unchangedSaid(unchanged map[string]string) string {
|
|
names := make([]string, 0, len(unchanged))
|
|
for n := range unchanged {
|
|
names = append(names, n)
|
|
}
|
|
sort.Strings(names)
|
|
return "unchanged, the build the mesh holds is theirs: " + strings.Join(names, ", ")
|
|
}
|
|
|
|
// unchangedLeft is what a walk says matched what runs: every module a merge left unchanged that the walk does not
|
|
// build anyway, as another merge's module or a dependent (novox/hq issue 611). Nil when none is.
|
|
func unchangedLeft(unchanged map[string]string, builds map[string]*inventory.PlanModule) map[string]string {
|
|
out := map[string]string{}
|
|
for n, why := range unchanged {
|
|
if _, built := builds[n]; !built {
|
|
out[n] = why
|
|
}
|
|
}
|
|
if len(out) == 0 {
|
|
return nil
|
|
}
|
|
return out
|
|
}
|
|
|
|
// withhold takes from a plan every module named that its merges do not move, and the tiers left empty: the
|
|
// modules taken.
|
|
func withhold(plan *inventory.Plan, names, moved []string) []string {
|
|
var out []string
|
|
for _, n := range names {
|
|
if _, in := plan.Modules[n]; in && !slices.Contains(moved, n) {
|
|
delete(plan.Modules, n)
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
if len(out) == 0 {
|
|
return nil
|
|
}
|
|
var tiers [][]string
|
|
for _, t := range plan.Tiers {
|
|
var kept []string
|
|
for _, n := range t {
|
|
if !slices.Contains(out, n) {
|
|
kept = append(kept, n)
|
|
}
|
|
}
|
|
if len(kept) > 0 {
|
|
tiers = append(tiers, kept)
|
|
}
|
|
}
|
|
if tiers == nil {
|
|
tiers = [][]string{}
|
|
}
|
|
plan.Tiers = tiers
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window
|
|
// closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its
|
|
// class. A merge walked alone had no window.
|
|
func walkTimesAtCut(batch, walk inventory.Plan, entries []inventory.Entry, now time.Time) *inventory.PlanTimes {
|
|
cut := now
|
|
t := &inventory.PlanTimes{Cut: &cut, Class: classOf(walk, entries)}
|
|
if batch.Delivery != nil && batch.Delivery.Batch != nil {
|
|
w := batch.Delivery.Batch
|
|
closed := w.ClosesAt
|
|
if !w.AtMost.IsZero() && w.AtMost.Before(closed) {
|
|
closed = w.AtMost
|
|
}
|
|
if w.CutAtOnce && !batch.Created.IsZero() {
|
|
closed = batch.Created // closed when it was made (ADR 0304 decision 1)
|
|
}
|
|
if !closed.IsZero() {
|
|
if closed.After(now) {
|
|
closed = now
|
|
}
|
|
t.WindowClosed = &closed
|
|
}
|
|
}
|
|
return t
|
|
}
|
|
|
|
// resolverSeat is the seat of the mesh's resolver: a module claiming it is a core module (ADR 0282 decision 1).
|
|
const resolverSeat = "mesh-dns-resolver"
|
|
|
|
// classOf is a walk's class (novox/hq ADR 0282 decision 1): core when it walks a module on the controller's own
|
|
// path or one holding the mesh's resolver, leaf otherwise.
|
|
func classOf(walk inventory.Plan, entries []inventory.Entry) string {
|
|
modules := make([]string, 0, len(walk.Modules))
|
|
for m := range walk.Modules {
|
|
modules = append(modules, m)
|
|
}
|
|
return classOfModules(modules, entries)
|
|
}
|
|
|
|
// classOfModules is the class of modules moved together: core when one is on the controller's own path or holds
|
|
// the mesh's resolver, leaf otherwise (ADR 0282 decision 1).
|
|
func classOfModules(modules []string, entries []inventory.Entry) string {
|
|
resolvers := map[string]bool{}
|
|
for _, e := range entries {
|
|
for _, c := range e.Manifest.Claims {
|
|
if c.Name == resolverSeat {
|
|
resolvers[e.Manifest.Module] = true
|
|
}
|
|
}
|
|
}
|
|
for _, m := range modules {
|
|
if _, own := onTheControllersPath[m]; own || resolvers[m] {
|
|
return inventory.ClassCore
|
|
}
|
|
}
|
|
return inventory.ClassLeaf
|
|
}
|
|
|
|
// combinedMerges is a batch's merges as one merge per repository's branch: the latest, with every file the
|
|
// merges of it changed and said to be a module's — on a linear trunk the latest contains the others. A file is
|
|
// removed when the last merge of the batch that changed it removed it. merges are in the order they were made.
|
|
func combinedMerges(merges []inventory.BatchedMerge) []link.SourceMoved {
|
|
type combined struct {
|
|
latest inventory.BatchedMerge
|
|
m link.SourceMoved
|
|
said bool
|
|
removed map[string]bool
|
|
paths []string
|
|
dirs []string
|
|
cut bool
|
|
}
|
|
by := map[string]*combined{}
|
|
var keys []string
|
|
for _, b := range merges {
|
|
var e link.SourceMoved
|
|
if err := json.Unmarshal(b.Event, &e); err != nil {
|
|
continue
|
|
}
|
|
k := strings.ToLower(b.Repository) + "\x00" + b.Branch
|
|
c, ok := by[k]
|
|
if !ok {
|
|
c = &combined{latest: b, m: e, said: true, removed: map[string]bool{}}
|
|
by[k] = c
|
|
keys = append(keys, k)
|
|
} else if laterMerge(b, c.latest) {
|
|
c.latest, c.m = b, e
|
|
}
|
|
c.said = c.said && e.ModuleDirsSaid
|
|
c.cut = c.cut || e.PathsTruncated
|
|
c.paths, c.dirs = union(c.paths, e.Paths), union(c.dirs, e.ModuleDirs)
|
|
for _, path := range e.Paths {
|
|
c.removed[path] = slices.Contains(e.Removed, path)
|
|
}
|
|
for _, path := range e.Removed {
|
|
c.removed[path] = true
|
|
}
|
|
}
|
|
sort.Strings(keys)
|
|
var out []link.SourceMoved
|
|
for _, k := range keys {
|
|
c := by[k]
|
|
m := c.m
|
|
m.Paths, m.ModuleDirs, m.PathsTruncated, m.ModuleDirsSaid = c.paths, c.dirs, c.cut, c.said
|
|
m.Removed = nil
|
|
for _, path := range c.paths {
|
|
if c.removed[path] {
|
|
m.Removed = append(m.Removed, path)
|
|
}
|
|
}
|
|
out = append(out, m)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// union is a and b without repeats, in order.
|
|
func union(a, b []string) []string {
|
|
out := append([]string{}, a...)
|
|
for _, s := range b {
|
|
if !slices.Contains(out, s) {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// batchMember is one repository's merge in a batch and what it moves.
|
|
type batchMember struct {
|
|
merge link.SourceMoved
|
|
moved []string
|
|
}
|
|
|
|
// afterLines are the `after: <repository>` lines of a pull request's description, as mesh-delivery reads them.
|
|
var afterLines = regexp.MustCompile(`(?im)^\s*after:\s*([A-Za-z0-9_.\-/]+)\s*$`)
|
|
|
|
// groupOrderEdges is the order of every delivery group among a batch's merges as edges of its walk's tiers
|
|
// (ADR 0276 decision 5, ADR 0249 unchanged): merges of two repositories or more from one head branch name are
|
|
// a group; each pair its order rules give puts the modules of the member after in a later tier than the
|
|
// member before's. A pair in a contradiction orders nothing: its group was refused before it merged.
|
|
func groupOrderEdges(members []batchMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
|
edges []inventory.Edge) []inventory.Edge {
|
|
byHead := map[string][]batchMember{}
|
|
for _, m := range members {
|
|
if m.merge.Head != "" {
|
|
byHead[m.merge.Head] = append(byHead[m.merge.Head], m)
|
|
}
|
|
}
|
|
var out []inventory.Edge
|
|
heads := make([]string, 0, len(byHead))
|
|
for h := range byHead {
|
|
heads = append(heads, h)
|
|
}
|
|
sort.Strings(heads)
|
|
for _, h := range heads {
|
|
group := byHead[h]
|
|
repos := map[string]bool{}
|
|
for _, m := range group {
|
|
repos[strings.ToLower(m.merge.Owner+"/"+m.merge.Repo)] = true
|
|
}
|
|
if len(repos) < 2 {
|
|
continue
|
|
}
|
|
var om []orderMember
|
|
moved := map[string][]string{}
|
|
for _, m := range group {
|
|
repository := m.merge.Owner + "/" + m.merge.Repo
|
|
id := repository + "@" + m.merge.Commit
|
|
var after []string
|
|
for _, l := range afterLines.FindAllStringSubmatch(m.merge.Body, -1) {
|
|
after = append(after, strings.TrimSuffix(l[1], ".git"))
|
|
}
|
|
om = append(om, orderMember{ID: id, Repository: repository, Base: m.merge.Base, Head: m.merge.Commit,
|
|
Number: m.merge.Number, Paths: m.merge.Paths, PathsTruncated: m.merge.PathsTruncated,
|
|
Removed: m.merge.Removed, ModuleDirs: m.merge.ModuleDirs, ModuleDirsSaid: m.merge.ModuleDirsSaid,
|
|
CloneURL: m.merge.CloneURL, After: after})
|
|
moved[id] = m.moved
|
|
}
|
|
order := orderOf(om, reachOfMembers(om, entries, read, edges), edges)
|
|
contradicted := map[string]bool{}
|
|
for _, c := range order.Contradictions {
|
|
for _, id := range c.Members {
|
|
contradicted[id] = true
|
|
}
|
|
}
|
|
for _, p := range order.Pairs {
|
|
if contradicted[p.Before] && contradicted[p.After] {
|
|
continue
|
|
}
|
|
for _, before := range moved[p.Before] {
|
|
for _, after := range moved[p.After] {
|
|
if before != after {
|
|
out = append(out, inventory.Edge{From: after, To: before, Kind: edgeGroupOrder})
|
|
}
|
|
}
|
|
}
|
|
fmt.Printf(" group %s: %s before %s (%s) inside the walk\n", h, p.Before, p.After, p.Why)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// searchAfterFailures starts the search a failed walk leaves (ADR 0276 decision 3): every merge it answered
|
|
// in a later commit of its repository waits to be walked on its own commit. A stopped walk starts none: a
|
|
// person ended it.
|
|
func searchAfterFailures(ctx context.Context, inv *inventory.Inventory) error {
|
|
plans, err := inv.RecentPlans(ctx, 20)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, p := range plans {
|
|
if p.State != inventory.PlanFailed || p.Delivery == nil || p.Delivery.Stopped != "" ||
|
|
strings.HasPrefix(p.Note, "stopped by hand") || strings.HasPrefix(p.Note, "closed by hand") {
|
|
continue
|
|
}
|
|
merges, err := inv.MergesOf(ctx, p.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, m := range merges {
|
|
if m.Alone || p.CommitOf(m.Repository) == m.Commit {
|
|
continue // its own commit was walked
|
|
}
|
|
if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, "", true); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("%s failed; %s %.8s, which it carried in %.8s, is walked on its own commit\n", p.ID,
|
|
m.Repository, m.Commit, p.CommitOf(m.Repository))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// coverAlone gives every merge waiting to be walked alone to a walk done that holds a later merge of its
|
|
// branch — one walked alone, or the failed walk retried by a person: the search stops at the first merge
|
|
// delivered, and that walk answers the older ones. What is left waits, the newest first.
|
|
func coverAlone(ctx context.Context, inv *inventory.Inventory, alone []inventory.BatchedMerge) ([]inventory.BatchedMerge, error) {
|
|
var left []inventory.BatchedMerge
|
|
for _, m := range alone {
|
|
later, err := inv.LaterMergesOf(ctx, m.Repository, m.Branch, m.Merged)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
covered := false
|
|
for _, l := range later {
|
|
p, err := inv.PlanByID(ctx, l.Plan)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if p.State != inventory.PlanDone {
|
|
continue
|
|
}
|
|
if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, p.ID, true); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := nameMerges(ctx, inv, &p); err != nil {
|
|
return nil, err
|
|
}
|
|
fmt.Printf("%s %.8s is answered by %s, which delivered a later merge of its branch on its own commit\n",
|
|
m.Repository, m.Commit, p.ID)
|
|
covered = true
|
|
break
|
|
}
|
|
if !covered {
|
|
left = append(left, m)
|
|
}
|
|
}
|
|
return left, nil
|
|
}
|
|
|
|
// batchWords is a batch in the operator's words (ADR 0276 decision 7): "assembling: 13 s left (at the latest
|
|
// 00:31:09); grouped: novox/mesh-catalog@48bda475 (answers 553b7191), …; plan not yet calculated".
|
|
func batchWords(b inventory.Plan, now time.Time) string {
|
|
grouped := groupedWords(b)
|
|
if b.Delivery == nil || b.Delivery.Batch == nil {
|
|
return "grouped: " + grouped + "; plan not yet calculated"
|
|
}
|
|
w := b.Delivery.Batch
|
|
if w.Own {
|
|
grouped += " (on the controller's own path)"
|
|
}
|
|
if b.State == inventory.PlanQueued {
|
|
return fmt.Sprintf("queued behind %s; grouped: %s; plan not yet calculated", w.Behind, grouped)
|
|
}
|
|
if w.CutAtOnce {
|
|
return fmt.Sprintf("cut at once, sharing nothing with a batch assembling; grouped: %s; plan not yet calculated",
|
|
grouped)
|
|
}
|
|
left := w.ClosesAt.Sub(now)
|
|
if w.AtMost.Before(w.ClosesAt) {
|
|
left = w.AtMost.Sub(now)
|
|
}
|
|
if left < 0 {
|
|
left = 0
|
|
}
|
|
waits := ""
|
|
if w.Waits != "" {
|
|
waits = " — " + w.Waits
|
|
}
|
|
return fmt.Sprintf("assembling: %s left (at the latest %s)%s; grouped: %s; plan not yet calculated",
|
|
secondsWords(left), w.AtMost.Local().Format("15:04:05"), waits, grouped)
|
|
}
|
|
|
|
// groupedWords is a batch's commits, each with the earlier merges it answers.
|
|
func groupedWords(b inventory.Plan) string {
|
|
var out []string
|
|
for _, c := range b.Carried() {
|
|
s := c.Repository + "@" + short(c.Commit)
|
|
var answers []string
|
|
if b.Delivery != nil {
|
|
for _, m := range b.Delivery.Merges {
|
|
if strings.EqualFold(m.Repository, c.Repository) && m.Carried == c.Commit {
|
|
answers = append(answers, short(m.Commit))
|
|
}
|
|
}
|
|
}
|
|
if len(answers) > 0 {
|
|
s += " (answers " + strings.Join(answers, ", ") + ")"
|
|
}
|
|
out = append(out, s)
|
|
}
|
|
if len(out) == 0 {
|
|
return "nothing yet"
|
|
}
|
|
return strings.Join(out, ", ")
|
|
}
|
|
|
|
// batchLines is a batch's grouped list as `plans` prints it under its line (asked by the operator, 2026-10-10):
|
|
// one line per repository, its pull request and title, the earlier merges it answers, and what its merges move.
|
|
func batchLines(b inventory.Plan) []string {
|
|
var out []string
|
|
for _, c := range b.Carried() {
|
|
var head *inventory.PlanMerge
|
|
var answers []string
|
|
moves := map[string]bool{}
|
|
if b.Delivery != nil {
|
|
for i := range b.Delivery.Merges {
|
|
m := &b.Delivery.Merges[i]
|
|
if !strings.EqualFold(m.Repository, c.Repository) {
|
|
continue
|
|
}
|
|
for _, n := range m.Moves {
|
|
moves[n] = true
|
|
}
|
|
switch {
|
|
case m.Commit == c.Commit:
|
|
head = m
|
|
case m.Carried == c.Commit:
|
|
answers = append(answers, pullWords(*m))
|
|
}
|
|
}
|
|
}
|
|
line := repoName(c.Repository) + " " + short(c.Commit)
|
|
if head != nil && head.Number > 0 {
|
|
line = repoName(c.Repository) + " " + pullWords(*head)
|
|
}
|
|
if len(answers) > 0 {
|
|
line += " (answers " + strings.Join(answers, ", ") + ")"
|
|
}
|
|
if len(moves) > 0 {
|
|
line += " · " + strings.Join(sortedKeysOf(boolsToStrings(moves)), ", ")
|
|
}
|
|
out = append(out, line)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// pullWords is a merge as its pull request: "#175 a tap shows its outcome", the title cut at fifty runes; the
|
|
// commit where the announcement named no pull request.
|
|
func pullWords(m inventory.PlanMerge) string {
|
|
if m.Number == 0 {
|
|
return short(m.Commit)
|
|
}
|
|
s := fmt.Sprintf("#%d", m.Number)
|
|
if t := cutTitle(m.Title, 50); t != "" {
|
|
s += " " + t
|
|
}
|
|
return s
|
|
}
|
|
|
|
// cutTitle is a title cut at n runes, with an ellipsis where it was cut.
|
|
func cutTitle(title string, n int) string {
|
|
r := []rune(strings.TrimSpace(title))
|
|
if len(r) <= n {
|
|
return string(r)
|
|
}
|
|
return strings.TrimSpace(string(r[:n-1])) + "…"
|
|
}
|
|
|
|
// boolsToStrings is a set's members, for sortedKeysOf.
|
|
func boolsToStrings(set map[string]bool) map[string]string {
|
|
out := make(map[string]string, len(set))
|
|
for k := range set {
|
|
out[k] = ""
|
|
}
|
|
return out
|
|
}
|
|
|
|
// secondsWords is a short wait as a person reads it.
|
|
func secondsWords(d time.Duration) string {
|
|
if d < 2*time.Minute {
|
|
return fmt.Sprintf("%d s", int(d.Round(time.Second)/time.Second))
|
|
}
|
|
return d.Round(time.Second).String()
|
|
}
|
|
|
|
// walkOrBatch is what a plan record is, in a word.
|
|
func walkOrBatch(p inventory.Plan) string {
|
|
if p.Batch() {
|
|
return "batch"
|
|
}
|
|
return "walk"
|
|
}
|
|
|
|
// orWords is s, or the words for when it is empty.
|
|
func orWords(s, none string) string {
|
|
if s == "" {
|
|
return none
|
|
}
|
|
return s
|
|
}
|