Say a walk that waits too long, and a delivery's own stalls (hq ADR 0239)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check fail: its merge-check.sh failed: FAIL
mesh/delivery delivered
mesh/delivery-group group feat/mesh-delivery-waits-said merged: every member is merged

A walk mesh-delivery never lets go waited for ever with nothing open: S16
says it at 30 minutes, urgent at 4 hours, naming plans go. Phase B: probe
D14 reads the delivery owner's stalled and raises delivery.<id>.stalled,
and H2 takes the table's transition through its close.
This commit is contained in:
jochen
2026-10-07 00:16:32 +02:00
parent 487aa040de
commit 75213b9091
11 changed files with 515 additions and 12 deletions
+19 -8
View File
@@ -51,6 +51,8 @@ type signalFacts struct {
plans []planFacts
plansErr error
// waits are the walks waiting for their delivery's word (S16, novox/hq ADR 0239).
waits []waitFacts
loop loopFacts
loopErr error
@@ -144,6 +146,12 @@ type planFacts struct {
paused bool
}
// waitFacts is one walk waiting for its delivery's word: since its merge opened it.
type waitFacts struct {
id, repository, commit, awaits string
since time.Time
}
type loopFacts struct {
took time.Time
pending uint64
@@ -307,7 +315,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
}
}
f.machines, f.machinesErr = w.gatherMachines(ctx, inv, now)
f.plans, f.plansErr = gatherPlans(ctx, inv, now)
f.plans, f.waits, f.plansErr = gatherPlans(ctx, inv, now)
f.loop, f.loopErr = w.gatherLoop()
f.mergesPassed, f.merges, f.mergesErr = watchedMerges.last()
if f.mergesErr == nil && !f.mergesPassed.IsZero() && now.Sub(f.mergesPassed) > 3*mergeCatchUpEvery {
@@ -423,17 +431,17 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
}
// gatherPlans is every open plan, its tier's bound from what was measured, and what it waits on.
func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]planFacts, error) {
func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]planFacts, []waitFacts, error) {
plans, err := inv.OpenPlans(ctx)
if err != nil {
return nil, fmt.Errorf("the open plans cannot be read: %w", err)
return nil, nil, fmt.Errorf("the open plans cannot be read: %w", err)
}
if len(plans) == 0 {
return nil, nil
return nil, nil, nil
}
tiers, err := inv.Durations(ctx, inventory.DurationPlanTier, now.Add(-14*24*time.Hour))
if err != nil {
return nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err)
return nil, nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err)
}
measured := map[string][]time.Duration{}
for _, d := range tiers {
@@ -441,10 +449,13 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) (
}
pause := buildSeatPause(ctx, inv, plans)
var out []planFacts
var waits []waitFacts
for _, p := range plans {
// A walk waiting for its delivery's word is not late (novox/hq ADR 0239): its owner keeps the bound
// of that wait, and a silent owner is its own condition (D3, holder-silent).
// A walk waiting for its delivery's word is no tier late (novox/hq ADR 0239); how long it has waited is
// S16's, whatever mesh-delivery says or does not say.
if p.Waiting() {
waits = append(waits, waitFacts{id: p.ID, repository: p.Repository, commit: p.Commit,
awaits: p.Delivery.Awaits, since: p.Created})
continue
}
_, paused := pausedWaiting(p, pause, now)
@@ -452,7 +463,7 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) (
tiers: len(p.Tiers), entered: p.TierEntered, bound: max(tierAtLeast, 3*p90(measured[p.Repository])),
waiting: planLineWith(p, now, pause), paused: paused})
}
return out, nil
return out, waits, nil
}
// p90 is the ninetieth percentile of measurements; zero for none.