package main import ( "context" "encoding/json" "errors" "fmt" "time" "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads // mesh-delivery's `times` and raises the warning `delivery..over-budget` when the newest delivery of a // class whose delivery time is known took longer than its class's budget — three minutes for a leaf module (ADR // 0304 decision 4), ten for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class // lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery. // probeBudgetsID is the probe that reads the delivery times. const probeBudgetsID = "D16" // kindOverBudget is what a delivery over its class's budget raises. const kindOverBudget = "over-budget" // deliveryBudgets are the budgets by class (ADR 0282 decision 1, the leaf's as ADR 0304 decision 4 changed it); // the probe raises nothing for another class. var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 3 * time.Minute, inventory.ClassCore: 10 * time.Minute} // timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it. type timesAnswer struct { Classes []timesClass `json:"classes"` } // timesClass is one class's line of `times`. type timesClass struct { Class string `json:"class"` Latest *timesLatest `json:"latest,omitempty"` } // timesLatest is the newest delivery of a class whose delivery time is known. type timesLatest struct { ID string `json:"id"` TookMS int64 `json:"took_ms"` Longest string `json:"longest,omitempty"` Landed string `json:"landed,omitempty"` } // deliveryTimes is what the delivery's owner says of its delivery times; nothing when no holder is on record or // none answers (D3 says that one). func deliveryTimes(ctx context.Context, conn *nats.Conn, held bool) (*timesAnswer, error) { if !held { return nil, nil } raw, err := askDeliveryOwner(ctx, conn, "times", map[string]any{}) if errors.Is(err, link.ErrNothingServes) { return nil, nil } if err != nil { return nil, err } var a timesAnswer if err := json.Unmarshal(raw, &a); err != nil { return nil, fmt.Errorf("%s.times answered something unreadable: %w", catalogue.DeliverySeat, err) } return &a, nil } // overBudgetObservations are the conditions of the classes whose newest delivery took longer than its budget: // strictly longer, so a delivery of exactly its budget is within it. func overBudgetObservations(a *timesAnswer) []conditions.Observation { if a == nil { return nil } var out []conditions.Observation for _, c := range a.Classes { budget, ok := deliveryBudgets[c.Class] if !ok || c.Latest == nil { continue } took := time.Duration(c.Latest.TookMS) * time.Millisecond if took <= budget { continue } longest := c.Latest.Longest if longest == "" { longest = "not known" } out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: c.Class, Kind: kindOverBudget, Severity: conditions.Warning, Summary: fmt.Sprintf("the %s delivery %s took %s from its merge to running everywhere, over its budget of %s; "+ "its longest phase: %s — `mesh-delivery.times`", c.Class, c.Latest.ID, humanDuration(took), humanDuration(budget), longest), Said: fmt.Sprintf("%s took %s (budget %s), longest phase %s", c.Latest.ID, took.Round(time.Second), budget, longest), Headline: fmt.Sprintf("A %s delivery took %s, over its %s budget", c.Class, humanDuration(took), humanDuration(budget)), Explanation: fmt.Sprintf("The delivery %s took %s from its merge until every machine ran it; a %s module is "+ "held to %s (ADR 0282). Most of the time went to %s. Nothing was held or changed because of this: it is "+ "a measurement.", c.Latest.ID, humanDuration(took), c.Class, humanDuration(budget), longest), Resolved: fmt.Sprintf("the next %s delivery lands within %s", c.Class, humanDuration(budget)), }) } return out } // probeBudgets is D16: the newest delivery of each class lands within its class's budget. func probeBudgets(ctx context.Context, d *doctor) ([]conditions.Observation, error) { entries, err := d.open.inventory.Catalogued(ctx) if err != nil { return nil, err } var conn *nats.Conn if d.js != nil { conn = d.js.Conn() } a, err := deliveryTimes(ctx, conn, deliverySeatHeld(entries)) if err != nil { return nil, err } return overBudgetObservations(a), nil }