mesh-controller/cmd/mesh-controller/batches_watch.go

package main

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
)

// **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still
// assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18),
// and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19).

// The kinds S18 and S19 raise.
const (
	kindBatchNotCut     = "batch-not-cut"
	kindBatchBehindWalk = "batch-behind-walk"
	// kindGroupNotMerged is raised for a group whose last member has not merged by the merge window's maximum after
	// its first (novox/hq ADR 0418 clause 3).
	kindGroupNotMerged = "group-not-merged"
)

// batchFacts is one batch not yet cut, as the watchdogs read it.
type batchFacts struct {
	id, state, grouped string
	// atMost is when its window closes at the latest; closed when it closed.
	atMost, closed time.Time
	// behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its
	// tiers.
	behind    string
	walkBound time.Duration
	// holds are the groups the batch is held for, by head, with why; created is when the batch was made, the first
	// merge of the group heard; atMostWindow is the maximum a window may take (the controller's setting).
	holds        map[string]string
	created      time.Time
	atMostWindow time.Duration
}

// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind.
func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) {
	batches, err := inv.Batches(ctx)
	if err != nil || len(batches) == 0 {
		return nil, err
	}
	bounds, err := measuredTierBounds(ctx, inv, now)
	if err != nil {
		return nil, err
	}
	var out []batchFacts
	for _, b := range batches {
		_, atMostWindow := mergeWindowOf()
		f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b), created: b.Created, atMostWindow: atMostWindow}
		if w := b.Delivery; w != nil && w.Batch != nil {
			f.holds = w.Batch.Holds
			f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind
			if f.atMost.Before(f.closed) {
				f.closed = f.atMost
			}
			if w.Batch.CutAtOnce && !b.Created.IsZero() {
				// Its window closed when it was made, which is its latest too (ADR 0304 decision 1).
				f.atMost, f.closed = b.Created, b.Created
			}
		}
		if f.behind != "" {
			if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
				f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers)))
			}
		}
		out = append(out, f)
	}
	return out, nil
}

// watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open.
func watchBatchesNotCut(f *signalFacts) []conditions.Observation {
	var out []conditions.Observation
	for _, b := range f.batches {
		if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter ||
			len(b.holds) > 0 { // a batch held for a group waits on a member, which has its own condition
			continue
		}
		late := f.now.Sub(b.atMost)
		out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut,
			Severity: conditions.Warning,
			Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+
				"the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped),
			Said:     fmt.Sprintf("assembling since %s past its maximum", ago(late)),
			Headline: "Merged changes are not being delivered",
			Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+
				"Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)),
			Resolved: "Merged changes are being delivered again"})
	}
	return out
}

// watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it.
func watchBatchesBehind(f *signalFacts) []conditions.Observation {
	var out []conditions.Observation
	for _, b := range f.batches {
		if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound {
			continue
		}
		in := f.now.Sub(b.closed)
		out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk,
			Severity: conditions.Warning,
			Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+
				"grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound),
				b.grouped, b.behind),
			Said:     fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)),
			Headline: "Merged changes wait behind a slow delivery",
			Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+
				"them, which is taking longer than it should. Nothing is lost.", humanDuration(in)),
			Resolved: "Merged changes no longer wait behind a slow delivery"})
	}
	return out
}

// watchGroupsNotMerged is the condition of a batch held for a group whose last member has not merged by the merge
// window's maximum after the first member merged, naming the member and why (novox/hq ADR 0418 clause 3): nothing of
// the group is delivered while it waits.
func watchGroupsNotMerged(f *signalFacts) []conditions.Observation {
	var out []conditions.Observation
	for _, b := range f.batches {
		if len(b.holds) == 0 || b.created.IsZero() || f.now.Sub(b.created) <= b.atMostWindow {
			continue
		}
		heads := make([]string, 0, len(b.holds))
		for h := range b.holds {
			heads = append(heads, h)
		}
		sort.Strings(heads)
		var why []string
		for _, h := range heads {
			why = append(why, b.holds[h])
		}
		waited := f.now.Sub(b.created)
		out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id,
			Kind: kindGroupNotMerged, Severity: conditions.Warning,
			Summary: fmt.Sprintf("%s; its first member merged %s ago, past the merge window's maximum (%s), and nothing "+
				"of the group is delivered while it waits (batch %s)", strings.Join(why, "; "), ago(waited), ago(b.atMostWindow), b.id),
			Said:     "a group waits for a member that has not merged",
			Headline: "A change waits for the rest of its group to merge",
			Explanation: fmt.Sprintf("The changes of a group are delivered together, once the last one is merged. "+
				"One has not merged in %s. Nothing is lost; the merged ones are kept and delivered with it.",
				humanDuration(waited)),
			Resolved: "The group's last change merged"})
	}
	return out
}