jschoubben
6196a4815e
A merge waited the whole window even when nothing assembling shared anything with it. It now joins only a batch it is related to (ADR 0304 §1, read by ADR 0306 §3: repository, a module the cut would move, a graph edge, a group head), folding the newer into the oldest when it relates to two. A merge related to none starts a batch whose window is closed at once (PlanBatch.CutAtOnce), unless its delivery group may still have a member to merge (ADR 0306 §2). The controller cannot see a group's unmerged members (that is mesh-delivery's record), so a merge from any head still waits, in a batch of its own: the safe error is to wait. Walks stay one at a time; a batch cut at once queues behind an open walk. A stop's merges walked again join a related batch, or their own.
108 lines
4.4 KiB
Go
108 lines
4.4 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still
|
|
// assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18),
|
|
// and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19).
|
|
|
|
// The kinds S18 and S19 raise.
|
|
const (
|
|
kindBatchNotCut = "batch-not-cut"
|
|
kindBatchBehindWalk = "batch-behind-walk"
|
|
)
|
|
|
|
// batchFacts is one batch not yet cut, as the watchdogs read it.
|
|
type batchFacts struct {
|
|
id, state, grouped string
|
|
// atMost is when its window closes at the latest; closed when it closed.
|
|
atMost, closed time.Time
|
|
// behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its
|
|
// tiers.
|
|
behind string
|
|
walkBound time.Duration
|
|
}
|
|
|
|
// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind.
|
|
func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) {
|
|
batches, err := inv.Batches(ctx)
|
|
if err != nil || len(batches) == 0 {
|
|
return nil, err
|
|
}
|
|
bounds, err := measuredTierBounds(ctx, inv, now)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []batchFacts
|
|
for _, b := range batches {
|
|
f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b)}
|
|
if w := b.Delivery; w != nil && w.Batch != nil {
|
|
f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind
|
|
if f.atMost.Before(f.closed) {
|
|
f.closed = f.atMost
|
|
}
|
|
if w.Batch.CutAtOnce && !b.Created.IsZero() {
|
|
// Its window closed when it was made, which is its latest too (ADR 0304 decision 1).
|
|
f.atMost, f.closed = b.Created, b.Created
|
|
}
|
|
}
|
|
if f.behind != "" {
|
|
if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
|
|
f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers)))
|
|
}
|
|
}
|
|
out = append(out, f)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open.
|
|
func watchBatchesNotCut(f *signalFacts) []conditions.Observation {
|
|
var out []conditions.Observation
|
|
for _, b := range f.batches {
|
|
if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter {
|
|
continue
|
|
}
|
|
late := f.now.Sub(b.atMost)
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut,
|
|
Severity: conditions.Warning,
|
|
Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+
|
|
"the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped),
|
|
Said: fmt.Sprintf("assembling since %s past its maximum", ago(late)),
|
|
Headline: "Merged changes are not being delivered",
|
|
Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+
|
|
"Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)),
|
|
Resolved: "Merged changes are being delivered again"})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it.
|
|
func watchBatchesBehind(f *signalFacts) []conditions.Observation {
|
|
var out []conditions.Observation
|
|
for _, b := range f.batches {
|
|
if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound {
|
|
continue
|
|
}
|
|
in := f.now.Sub(b.closed)
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk,
|
|
Severity: conditions.Warning,
|
|
Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+
|
|
"grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound),
|
|
b.grouped, b.behind),
|
|
Said: fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)),
|
|
Headline: "Merged changes wait behind a slow delivery",
|
|
Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+
|
|
"them, which is taking longer than it should. Nothing is lost.", humanDuration(in)),
|
|
Resolved: "Merged changes no longer wait behind a slow delivery"})
|
|
}
|
|
return out
|
|
}
|