Assemble merges in a rolling window and walk each batch once (hq ADR 0276, issue 362)
Every merge opened a walk and the next merge of the branch superseded it: two catalogue merges 18 s apart left a walk no delivery held, and the operator started it by hand 58 minutes later. A merge now joins the open batch, kept in the store (migration 0089), which is cut into one walk when no merge came for merge-window (90 s) or at merge-window-at-most (10 min): one commit per repository, the latest of its branch, with every file the batch's merges changed. One walk at a time; a started walk is never superseded, a waiting one is folded into the next. The walk names every merge it answers on the wire (delivery.merges, taken_over_by, batch). A failed walk walks its earlier merges alone, newest first, until one is delivered. A delivery group's order becomes tier edges inside the walk. plans shows the batch assembling; S18 and S19 bound its waits; S16 names the merges a waiting walk answers.
This commit is contained in:
jochen
+25
-136
@@ -298,42 +298,20 @@ func notNow(err error) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// SourceMoved is the forge announcing a merge: every module recorded as built from that
|
||||
// repository and branch is marked as moved to the merge commit, and built — bases first, so a
|
||||
// module that stands on another's artifact is built after it and not against the old one
|
||||
// (novox/hq 04-ISSUES/131). Nothing is pushed here: what a finished build does to the machines
|
||||
// running the module is the upgrade's decision, taken when the catalogue announces it.
|
||||
// SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which
|
||||
// becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go.
|
||||
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
// One merge acted on at a time, whoever hands it over: the bus, or the catch-up that reads back
|
||||
// what the bus did not hand over (novox/hq issue 266). Each judges against what the other wrote.
|
||||
actingOnMerges.Lock()
|
||||
defer actingOnMerges.Unlock()
|
||||
return f.hearMerge(ctx, m, time.Now().UTC())
|
||||
}
|
||||
|
||||
inv := f.open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
|
||||
// **A merge older than the newest planned merge of its branch is planned at that merge's commit**
|
||||
// (novox/hq issue 349): the catch-up acts on a merge the bus did not hand over after the merges that
|
||||
// came after it, and planned at its own commit it built what a later merge had fixed — the forge's
|
||||
// security fix, on 2026-10-09 — from the commit before it, dependents and all. The later commit contains
|
||||
// this merge's change. Planned there, with that merge's time, a module the later merge already looked at
|
||||
// reads as history and is not built again; the rest are built from the newest commit.
|
||||
newest, known, err := inv.NewestMergeOf(ctx, m.Owner+"/"+m.Repo, m.Base)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
if known && laterOnTheBranch(m, newest) {
|
||||
fmt.Printf(" %s/%s %.8s was merged before %.8s (%s, %s): what it moved is built from %.8s, which "+
|
||||
"contains it\n", m.Owner, m.Repo, m.Commit, newest.Commit, newest.ID, newest.State, newest.Commit)
|
||||
m.Commit, m.MergedAt = newest.Commit, newest.Merged.Format(time.RFC3339Nano)
|
||||
}
|
||||
// movesOfMerge is what one merge moves, acted on: every module recorded as built from that repository and
|
||||
// branch is marked as moved to the merge commit; a module the merge deleted is forgotten or said; a new module
|
||||
// is asked for and sent nowhere. The names returned are what the walk builds, bases first along the graph
|
||||
// (novox/hq 04-ISSUES/131). Nothing is built or pushed here: the walk asks its tiers. Called when a batch is
|
||||
// cut, once per repository, with the latest merge of the repository's branch and every file the batch's
|
||||
// merges of it changed.
|
||||
func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) ([]string, error) {
|
||||
from, packaging, already := mergeCandidates(m, entries, read)
|
||||
if len(from) == 0 && len(packaging) == 0 {
|
||||
// "Already built from it" and "nothing reads it" are different facts, and reading the first
|
||||
@@ -341,11 +319,11 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
if already > 0 {
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); %d module(s) the mesh holds are already built "+
|
||||
"from it\n", m.Owner, m.Repo, m.Base, m.Commit, already)
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); nothing the mesh holds reads it\n",
|
||||
m.Owner, m.Repo, m.Base, m.Commit)
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
// Said, never silent (novox/hq 04-ISSUES/215): a module built from this repository that follows
|
||||
// another branch is not part of this merge, and whoever is waiting for its change should read why.
|
||||
@@ -365,7 +343,7 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
}
|
||||
for _, e := range touched {
|
||||
if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil {
|
||||
return notNow(err)
|
||||
return nil, notNow(err)
|
||||
}
|
||||
}
|
||||
// **A new module is built and registered, and sent nowhere** (novox/hq issue 300): the delivery plan
|
||||
@@ -377,94 +355,12 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); it changed no module the mesh holds, and adds %s: "+
|
||||
"built, registered when the build lands, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit,
|
||||
strings.Join(built, ", "))
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+
|
||||
"built from\n", m.Owner, m.Repo, m.Base, m.Commit)
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
// A merge produces a plan the mesh keeps (novox/hq ADR 0162): what moved and everything that
|
||||
// depends on it, along the catalogue's one dependency relation, sorted into tiers. The plan is
|
||||
// written before any build is asked; the first tier is asked; this returns. Outcomes advance it.
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
var movedNames []string
|
||||
for _, e := range moved {
|
||||
movedNames = append(movedNames, e.Manifest.Module)
|
||||
}
|
||||
// Written and its first tier asked as one act on the plans (novox/hq issue 213): a timer on
|
||||
// another controller reading it between the two would ask the tier again.
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
defer release()
|
||||
plan := planOfMerge(m, movedNames, edges)
|
||||
// **A newer plan supersedes the older open plans of this repository and branch** (novox/hq issue
|
||||
// 254, ADR 0218): what they had not built is planned here again, and they are closed, so one plan
|
||||
// works a repository's modules at a time and a stuck one ends at the next merge.
|
||||
working, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
rollsOut := func(module string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return err == nil && u.RollOut
|
||||
}
|
||||
folded, superseded := supersededBy(plan, working, rollsOut)
|
||||
if len(folded) > 0 {
|
||||
held := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
held[e.Manifest.Module] = true
|
||||
}
|
||||
names := map[string]bool{}
|
||||
for _, name := range movedNames {
|
||||
names[name] = true
|
||||
}
|
||||
var also []string
|
||||
for _, name := range folded {
|
||||
// One the catalogue no longer holds would fail the newer plan's ask; it is not this
|
||||
// merge's to build.
|
||||
if held[name] && !names[name] {
|
||||
names[name] = true
|
||||
movedNames = append(movedNames, name)
|
||||
also = append(also, name)
|
||||
}
|
||||
}
|
||||
if len(also) > 0 {
|
||||
again := planOfMerge(m, movedNames, edges)
|
||||
again.ID, again.Created = plan.ID, plan.Created
|
||||
plan = again
|
||||
fmt.Printf(" %s, left unbuilt by an older plan of %s, are planned here again\n",
|
||||
strings.Join(also, ", "), plan.Repository)
|
||||
}
|
||||
}
|
||||
// **Whether it waits for its delivery's word** (novox/hq ADR 0239): while the mesh-delivery seat has a
|
||||
// holder on record, a walk that moves no module on the controller's own path is opened and waits.
|
||||
plan.Delivery = awaitsFor(entries, movedNames)
|
||||
if hasCycle(plan.Tiers, edges) {
|
||||
fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n",
|
||||
strings.Join(plan.Tiers[len(plan.Tiers)-1], ", "))
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
// Closed after the newer plan is kept, never before: a controller replaced between the two leaves
|
||||
// both open, which the next merge settles, rather than neither.
|
||||
for _, old := range superseded {
|
||||
if err := inv.SavePlan(ctx, &old); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
fmt.Printf(" %s (%s at %s) is %s\n", old.ID, old.Repository, short(old.Commit), old.Note)
|
||||
}
|
||||
var tiers []string
|
||||
for i, t := range plan.Tiers {
|
||||
tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(t, ", ")))
|
||||
}
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); plan %s, %d module(s) in %d tier(s)\n %s\n",
|
||||
m.Owner, m.Repo, m.Base, m.Commit, plan.ID, len(plan.Modules), len(plan.Tiers), strings.Join(tiers, "\n "))
|
||||
// Why each is in it (issue 363): the files of its build source the merge changed, or why it is read whole.
|
||||
for _, e := range moved {
|
||||
fmt.Printf(" %s: %s\n", e.Manifest.Module, whyMoved(e, read[e.Manifest.Module], m))
|
||||
@@ -473,21 +369,11 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
fmt.Printf(" %s reads %s/%s through its build context: its own source record is left where it is\n",
|
||||
e.Manifest.Module, m.Owner, m.Repo)
|
||||
}
|
||||
if plan.Waiting() {
|
||||
plan.Note = waitingNote(plan)
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
fmt.Printf(" %s waits for %s's word before its first tier is asked\n", plan.ID, plan.Delivery.Awaits)
|
||||
return nil
|
||||
var names []string
|
||||
for _, e := range moved {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
if err := askTier(ctx, inv, &plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
return nil
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// askNewModules asks the build seat for every module a merge adds to the repository — a directory holding a
|
||||
@@ -893,7 +779,10 @@ func planningView(read map[string][]inventory.ReadRepository, plans []inventory.
|
||||
var commits []string
|
||||
for _, p := range plans {
|
||||
if st, in := p.Modules[name]; in && p.Commit != "" && (p.Open() || (st != nil && st.State == "built")) {
|
||||
commits = append(commits, p.Commit)
|
||||
// Every commit the walk carries (novox/hq ADR 0276): a batch's walk answers one per repository.
|
||||
for _, c := range p.Carried() {
|
||||
commits = append(commits, c.Commit)
|
||||
}
|
||||
}
|
||||
}
|
||||
var kept []inventory.ReadRepository
|
||||
|
||||
Reference in New Issue
Block a user