package main import ( "context" "fmt" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" ) // **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still // assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18), // and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19). // The kinds S18 and S19 raise. const ( kindBatchNotCut = "batch-not-cut" kindBatchBehindWalk = "batch-behind-walk" ) // batchFacts is one batch not yet cut, as the watchdogs read it. type batchFacts struct { id, state, grouped string // atMost is when its window closes at the latest; closed when it closed. atMost, closed time.Time // behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its // tiers. behind string walkBound time.Duration } // gatherBatches is every batch not yet cut, with the bound of the walk it waits behind. func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) { batches, err := inv.Batches(ctx) if err != nil || len(batches) == 0 { return nil, err } bounds, err := measuredTierBounds(ctx, inv, now) if err != nil { return nil, err } var out []batchFacts for _, b := range batches { f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b)} if w := b.Delivery; w != nil && w.Batch != nil { f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind if f.atMost.Before(f.closed) { f.closed = f.atMost } if w.Batch.CutAtOnce && !b.Created.IsZero() { // Its window closed when it was made, which is its latest too (ADR 0304 decision 1). f.atMost, f.closed = b.Created, b.Created } } if f.behind != "" { if walk, err := inv.PlanByID(ctx, f.behind); err == nil { f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers))) } } out = append(out, f) } return out, nil } // watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open. func watchBatchesNotCut(f *signalFacts) []conditions.Observation { var out []conditions.Observation for _, b := range f.batches { if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter { continue } late := f.now.Sub(b.atMost) out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut, Severity: conditions.Warning, Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+ "the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped), Said: fmt.Sprintf("assembling since %s past its maximum", ago(late)), Headline: "Merged changes are not being delivered", Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+ "Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)), Resolved: "Merged changes are being delivered again"}) } return out } // watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it. func watchBatchesBehind(f *signalFacts) []conditions.Observation { var out []conditions.Observation for _, b := range f.batches { if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound { continue } in := f.now.Sub(b.closed) out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk, Severity: conditions.Warning, Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+ "grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound), b.grouped, b.behind), Said: fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)), Headline: "Merged changes wait behind a slow delivery", Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+ "them, which is taking longer than it should. Nothing is lost.", humanDuration(in)), Resolved: "Merged changes no longer wait behind a slow delivery"}) } return out }