mesh-controller/cmd/mesh-controller/over_budget.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"time"

	"github.com/nats-io/nats.go"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads
// mesh-delivery's `times` and raises the warning `delivery.<class>.over-budget` when the newest delivery of a
// class whose delivery time is known took longer than its class's budget — three minutes for a leaf module (ADR
// 0304 decision 4), ten for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class
// lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery.

// probeBudgetsID is the probe that reads the delivery times.
const probeBudgetsID = "D16"

// kindOverBudget is what a delivery over its class's budget raises.
const kindOverBudget = "over-budget"

// deliveryBudgets are the budgets by class (ADR 0282 decision 1, the leaf's as ADR 0304 decision 4 changed it);
// the probe raises nothing for another class.
var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 3 * time.Minute, inventory.ClassCore: 10 * time.Minute}

// timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it.
type timesAnswer struct {
	Classes []timesClass `json:"classes"`
}

// timesClass is one class's line of `times`.
type timesClass struct {
	Class  string       `json:"class"`
	Latest *timesLatest `json:"latest,omitempty"`
}

// timesLatest is the newest delivery of a class whose delivery time is known.
type timesLatest struct {
	ID      string `json:"id"`
	TookMS  int64  `json:"took_ms"`
	Longest string `json:"longest,omitempty"`
	Landed  string `json:"landed,omitempty"`
}

// deliveryTimes is what the delivery's owner says of its delivery times; nothing when no holder is on record or
// none answers (D3 says that one).
func deliveryTimes(ctx context.Context, conn *nats.Conn, held bool) (*timesAnswer, error) {
	if !held {
		return nil, nil
	}
	raw, err := askDeliveryOwner(ctx, conn, "times", map[string]any{})
	if errors.Is(err, link.ErrNothingServes) {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	var a timesAnswer
	if err := json.Unmarshal(raw, &a); err != nil {
		return nil, fmt.Errorf("%s.times answered something unreadable: %w", catalogue.DeliverySeat, err)
	}
	return &a, nil
}

// overBudgetObservations are the conditions of the classes whose newest delivery took longer than its budget:
// strictly longer, so a delivery of exactly its budget is within it.
func overBudgetObservations(a *timesAnswer) []conditions.Observation {
	if a == nil {
		return nil
	}
	var out []conditions.Observation
	for _, c := range a.Classes {
		budget, ok := deliveryBudgets[c.Class]
		if !ok || c.Latest == nil {
			continue
		}
		took := time.Duration(c.Latest.TookMS) * time.Millisecond
		if took <= budget {
			continue
		}
		longest := c.Latest.Longest
		if longest == "" {
			longest = "not known"
		}
		out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: c.Class, Kind: kindOverBudget,
			Severity: conditions.Warning,
			Summary: fmt.Sprintf("the %s delivery %s took %s from its merge to running everywhere, over its budget of %s; "+
				"its longest phase: %s — `mesh-delivery.times`", c.Class, c.Latest.ID, humanDuration(took),
				humanDuration(budget), longest),
			Said: fmt.Sprintf("%s took %s (budget %s), longest phase %s", c.Latest.ID, took.Round(time.Second), budget, longest),
			Headline: fmt.Sprintf("A %s delivery took %s, over its %s budget", c.Class, humanDuration(took),
				humanDuration(budget)),
			Explanation: fmt.Sprintf("The delivery %s took %s from its merge until every machine ran it; a %s module is "+
				"held to %s (ADR 0282). Most of the time went to %s. Nothing was held or changed because of this: it is "+
				"a measurement.", c.Latest.ID, humanDuration(took), c.Class, humanDuration(budget), longest),
			Resolved: fmt.Sprintf("the next %s delivery lands within %s", c.Class, humanDuration(budget)),
		})
	}
	return out
}

// probeBudgets is D16: the newest delivery of each class lands within its class's budget.
func probeBudgets(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
	entries, err := d.open.inventory.Catalogued(ctx)
	if err != nil {
		return nil, err
	}
	var conn *nats.Conn
	if d.js != nil {
		conn = d.js.Conn()
	}
	a, err := deliveryTimes(ctx, conn, deliverySeatHeld(entries))
	if err != nil {
		return nil, err
	}
	return overBudgetObservations(a), nil
}