mesh-controller/internal/inventory/plans.go

package inventory

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/jackc/pgx/v5"
)

// A Plan is what a merge produces (novox/hq ADR 0162): the modules it changed and everything
// standing on them, sorted into tiers, each module's state, and the tier the plan is at. Kept in
// the store so a controller replaced mid-plan resumes it, and so `status` can say what a merge
// still waits for.
type Plan struct {
	ID         string `json:"id"`
	Repository string `json:"repository"`
	// Branch is the branch the merge went into (novox/hq issue 254): a newer plan supersedes the open
	// ones of the same repository and branch. Empty for a plan from before it was kept.
	Branch string `json:"branch,omitempty"`
	Commit string `json:"commit"`
	// Merged is when the forge made the merge this plan answers (novox/hq issue 349): the order of a
	// branch's merges, which is not the order their plans were made in when one was acted on late. Zero
	// for a release, and for a plan kept before it was.
	Merged  time.Time              `json:"merged,omitzero"`
	Created time.Time              `json:"created"`
	Updated time.Time              `json:"updated"`
	State   string                 `json:"state"`
	Tier    int                    `json:"tier"`
	Tiers   [][]string             `json:"tiers"`
	Modules map[string]*PlanModule `json:"modules"`
	Note    string                 `json:"note,omitempty"`
	// TierEntered is when the plan entered the tier it is at (novox/hq to-be 45 Phase 0), read and
	// never written from here: a save measures the tier it leaves and stamps the next. What the
	// watchdog of a plan's progress (S3) reads.
	TierEntered time.Time `json:"tier_entered,omitempty"`
	// Revision is the plan's as it was read, and the one a save must find (novox/hq to-be 45 §6): a
	// plan is written by compare-and-set, so a write against a plan another writer moved since is
	// refused rather than laid over it. Zero is a plan never saved. SavePlan moves it.
	Revision int64 `json:"revision"`
	// Epoch is the controller lease epoch that wrote it last; zero for a write that claimed none.
	Epoch uint64 `json:"epoch,omitempty"`
	// Release is set on a release plan (novox/hq ADR 0236): not a merge's, but the builds waiting for a
	// gate, walked through the machines one at a time.
	Release *PlanRelease `json:"release,omitempty"`
	// Delivery is set on a walk that waits for its delivery's word (novox/hq ADR 0239), and on every walk and
	// batch that names the merges it answers (ADR 0276): Awaits empty for a walk on the controller's own path,
	// which starts when it is cut.
	Delivery *PlanDelivery `json:"delivery,omitempty"`
	// Commits is every repository's commit the walk carries, one per repository (novox/hq ADR 0276): a batch's
	// walk can carry several. Repository and Commit above keep one of them, for a reader that knows one. Empty
	// for a walk kept before it was: Repository and Commit are then the whole of it.
	Commits []PlanCommit `json:"commits,omitempty"`
}

// PlanCommit is one repository's commit a walk carries: the latest merge of its branch in the batch.
type PlanCommit struct {
	Repository string    `json:"repository"`
	Branch     string    `json:"branch,omitempty"`
	Commit     string    `json:"commit"`
	Merged     time.Time `json:"merged,omitzero"`
}

// PlanMerge is one merge a walk or batch answers (novox/hq ADR 0276), as mesh-delivery reads it: Carried is
// the commit of its repository the walk builds and sends when that is a later merge containing it, and
// empty when it is the merge itself.
type PlanMerge struct {
	Repository string `json:"repository"`
	Commit     string `json:"commit"`
	Carried    string `json:"carried,omitempty"`
}

// PlanBatch is a batch's window while it is one (novox/hq ADR 0276): when it closes unless another merge
// comes, when it closes at the latest, and the walk it waits behind once closed.
type PlanBatch struct {
	ClosesAt time.Time `json:"closes_at"`
	AtMost   time.Time `json:"at_most"`
	Behind   string    `json:"behind,omitempty"`
}

// CommitOf is the commit of a repository the walk carries; empty when it carries none of it.
func (p Plan) CommitOf(repository string) string {
	for _, c := range p.Commits {
		if strings.EqualFold(c.Repository, repository) {
			return c.Commit
		}
	}
	if strings.EqualFold(p.Repository, repository) {
		return p.Commit
	}
	return ""
}

// Carried is every repository's commit the walk carries: Commits, or its one repository and commit.
func (p Plan) Carried() []PlanCommit {
	if len(p.Commits) > 0 {
		return p.Commits
	}
	if p.Repository == "" && p.Commit == "" {
		return nil
	}
	return []PlanCommit{{Repository: p.Repository, Branch: p.Branch, Commit: p.Commit, Merged: p.Merged}}
}

// Named is the walk as a person reads it: each repository at its commit.
func (p Plan) Named() string {
	var out []string
	for _, c := range p.Carried() {
		out = append(out, c.Repository+" "+short(c.Commit))
	}
	return strings.Join(out, ", ")
}

// Batch says the record is a batch still being assembled or waiting to be cut, not a walk (ADR 0276).
func (p Plan) Batch() bool { return p.State == PlanAssembling || p.State == PlanQueued }

// PlanDelivery is what a walk waits for and what came of the wait (novox/hq ADR 0239).
type PlanDelivery struct {
	// Awaits is who must say the walk may start: the seat whose holder owns the delivery.
	Awaits string `json:"awaits"`
	// Go is when it was let go, By by whom (the seat's holder, or a person's name) and Why.
	Go  *time.Time `json:"go,omitempty"`
	By  string     `json:"by,omitempty"`
	Why string     `json:"why,omitempty"`
	// Stopped is who ended the walk through the delivery's owner, and StoppedWhy why: the walk is failed,
	// and the delivery reads it as stopped, not as a build that failed.
	Stopped    string `json:"stopped,omitempty"`
	StoppedWhy string `json:"stopped_why,omitempty"`
	// Merges is every merge the walk or batch answers (novox/hq ADR 0276): its own commits, and the earlier
	// merges of a repository its later commit contains.
	Merges []PlanMerge `json:"merges,omitempty"`
	// TakenOverBy names the walk a walk folded before it started was taken over by: its merges are that
	// walk's now.
	TakenOverBy string `json:"taken_over_by,omitempty"`
	// Batch is the window of a batch; nil once it is cut into a walk.
	Batch *PlanBatch `json:"batch,omitempty"`
}

// Waiting is whether the walk waits for its delivery's word.
func (p Plan) Waiting() bool {
	return p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go == nil
}

// PlanSaved is told every plan this process kept, after it is kept (novox/hq ADR 0239): the serving
// controller says it on the bus as `plan-moved`. Nil in a command, which says nothing; whoever follows a
// walk also asks for it, so a save made by a command is found by comparison.
var PlanSaved func(Plan)

// ErrPlanMoved is a save against a plan written by somebody else since it was read.
var ErrPlanMoved = errors.New("the plan was written by somebody else since it was read")

// PlanModule is one module's state within a plan.
type PlanModule struct {
	// State: asked, built, failed; empty for a module whose tier has not been asked yet.
	State   string     `json:"state,omitempty"`
	AskedAt *time.Time `json:"asked_at,omitempty"`
	BuiltAt *time.Time `json:"built_at,omitempty"`
	// SentAt is when the plan sent the machines running this module its new build, because a
	// later tier is built by it (ADR 0163's gate): the reports that open the gate are the ones
	// after this.
	SentAt *time.Time `json:"sent_at,omitempty"`
	// First is the machines the plan sent the new build to first, and FirstAt when (novox/hq issue
	// 249, ADR 0218): unless the module's policy rolls it out together, one machine takes it before
	// the rest, and the rest are sent once that one reports it applied. Kept so a controller
	// replaced while the plan waits on that report resumes the wait rather than sending again. The
	// machine holding the bus is among them when its user list had to go first.
	First   []string   `json:"first,omitempty"`
	FirstAt *time.Time `json:"first_at,omitempty"`
	Commit  string     `json:"commit,omitempty"`
	Why     string     `json:"why,omitempty"`
	// Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan
	// matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt
	// its module's name included. Empty in a plan from before it was kept, which is matched by
	// module, or by repository and path, as before.
	Build string `json:"build,omitempty"`
	// Previous is the build the first machine ran of this module before the plan sent it the new one —
	// the commit its last send carried (ADR 0221) — kept at the first send: what a rollback puts back
	// (novox/hq ADR 0236). Empty when the machine had never been sent the module, or what it was sent
	// is not known.
	Previous string `json:"previous,omitempty"`
	// Gate is the new build's judging on its first machine (novox/hq ADR 0236, to-be 45 §8), kept so a
	// controller replaced mid-judging resumes it, and read back through `plans` as the rollout's record.
	Gate *PlanGate `json:"gate,omitempty"`
	// GatedBy names the module of the same tier whose gate judges this one on its first machine: they
	// went there in one send (novox/hq issue 281), and one gate judges what one send moved. Empty for
	// the module the gate is kept on, and for a plan from before tiers were sent whole.
	GatedBy string `json:"gated_by,omitempty"`
}

// PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine,
// from and to which build, the verdict, how long it took to reach it, and whether it was rolled back.
type PlanGate struct {
	// Component is the core component the module is — mesh-controller, mesh-host, node-tools — or empty
	// for any other module, judged by its own health.
	Component string   `json:"component,omitempty"`
	Machines  []string `json:"machines"`
	From      string   `json:"from,omitempty"`
	To        string   `json:"to,omitempty"`
	// Since is when the judging began: the first machine reported the new build applied.
	Since *time.Time `json:"since,omitempty"`
	// Sent is, per machine, the declaration the gate's send carried there (novox/hq issue 352): what a
	// machine's report is held against. Absent on a gate kept before it was, which reads the report
	// against the send made last, as before.
	Sent map[string]SentDeclaration `json:"sent,omitempty"`
	// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
	// does not resets them.
	Passes   int        `json:"passes,omitempty"`
	LastPass *time.Time `json:"last_pass,omitempty"`
	// Last is what the newest judging found wanting, while it still may pass.
	Last string `json:"last,omitempty"`
	// Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.
	Verdict  string     `json:"verdict,omitempty"`
	Why      string     `json:"why,omitempty"`
	JudgedAt *time.Time `json:"judged_at,omitempty"`
	Took     string     `json:"took,omitempty"`
	// Rollback is how a failed build was put back: rolled-back, or not-rolled-back with why.
	Rollback string `json:"rollback,omitempty"`
	// Kept says a passing verdict was written to the gate's records.
	Kept bool `json:"kept,omitempty"`
	// Carried is every module whose build moved on the judged machines with the send — the plan's own
	// module and whatever else was waiting there for a gate (novox/hq ADR 0236): each is judged here, a
	// pass is its verdict too, and one that fails is put back.
	Carried []CarriedMove `json:"carried,omitempty"`
	// Failing names the modules the last judging found wanting.
	Failing []string `json:"failing,omitempty"`
	// Healthy counts, per module, the consecutive judgings that found it healthy on every machine judged,
	// or waiting for a person (novox/hq ADR 0254): each module's passes, apart from the send's.
	Healthy map[string]int `json:"healthy,omitempty"`
	// Waits is, per module, the wait for a person the newest judging read (ADR 0254): carried along, never
	// a reason to fail.
	Waits map[string]string `json:"waits,omitempty"`
	// Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its
	// pass, and is not put back.
	Passing []string `json:"passing,omitempty"`
	// HealthyAt is, per module, when its newest healthy judging was counted: a pass is counted only gateEvery
	// after the one before (issue 318 review).
	HealthyAt map[string]time.Time `json:"healthy_at,omitempty"`
	// Broken names the modules a judging found broken, kept for the rest of the judging, and BrokenWhy the
	// first reason: the send fails, and the modules beside them are judged to their own verdict first.
	Broken    []string `json:"broken,omitempty"`
	BrokenWhy string   `json:"broken_why,omitempty"`
	// Returned names the broken modules already put back, at once, while the rest of the send is judged.
	Returned []string `json:"returned,omitempty"`
}

// CarriedMove is one module's build moving on a machine with a gated send.
type CarriedMove struct {
	Module string `json:"module"`
	Node   string `json:"node"`
	From   string `json:"from,omitempty"`
	To     string `json:"to"`
	Build  string `json:"build,omitempty"`
	// Recreates is what the send does to the module's containers, in the mesh's words — how many it
	// recreates, and whether with a new image or only their declaration (novox/hq ADR 0245); empty when
	// it recreates none, or when the build the machine ran is not known.
	Recreates string `json:"recreates,omitempty"`
}

// PlanRelease is a release plan's walk through the machines (novox/hq ADR 0236): every module build
// that waits for a gate, sent one machine at a time, each judged before the next.
type PlanRelease struct {
	Order []string `json:"order"`
	Next  int      `json:"next"`
	// Gate is the machine being judged; nil between machines.
	Gate *PlanGate `json:"gate,omitempty"`
	Done []string  `json:"done,omitempty"`
	// Skipped are the machines not heard from when their turn came, left as they were.
	Skipped []string `json:"skipped,omitempty"`
	// By is the person who released it, empty when the mesh did.
	By string `json:"by,omitempty"`
}

// The states a plan passes through.
const (
	PlanBuilding = "building"
	PlanRolling  = "rolling"
	PlanDone     = "done"
	PlanFailed   = "failed"
	// PlanSuperseded is a plan a newer merge of the same repository and branch took over (novox/hq
	// issue 254, ADR 0218): what it had not built is in the newer plan, and its note names it.
	PlanSuperseded = "superseded"
	// PlanAssembling is a batch whose merge window is open (novox/hq ADR 0276), and PlanQueued one whose
	// window closed while a walk is open: neither is a walk yet. Cut, a batch keeps its id and is building.
	PlanAssembling = "assembling"
	PlanQueued     = "queued"
)

// Open says whether the plan is still being worked.
func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRolling }

// SavePlan writes a plan, new or changed, whole: the plan is small and read as one thing.
//
// **By compare-and-set on its revision, carrying the epoch** (novox/hq to-be 45 §6): written only if
// the plan is still at the revision it was read at — a new one only if it does not exist — and refused
// with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records.
// On success p's revision and epoch are the ones written, so the caller may save it again.
func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
	epoch, err := i.actingEpoch(ctx)
	if err != nil {
		return fmt.Errorf("the plan for %s %s is not written: %w", p.Repository, p.Commit, err)
	}
	tiers, err := json.Marshal(p.Tiers)
	if err != nil {
		return err
	}
	modules, err := json.Marshal(p.Modules)
	if err != nil {
		return err
	}
	var release, delivery, commits []byte
	if len(p.Commits) > 0 {
		if commits, err = json.Marshal(p.Commits); err != nil {
			return err
		}
	}
	if p.Release != nil {
		if release, err = json.Marshal(p.Release); err != nil {
			return err
		}
	}
	if p.Delivery != nil {
		if delivery, err = json.Marshal(p.Delivery); err != nil {
			return err
		}
	}
	// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
	// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
	// measure one twice.
	tx, err := i.store.Pool().Begin(ctx)
	if err != nil {
		return err
	}
	defer func() { _ = tx.Rollback(ctx) }()
	entered, err := planTierLeft(ctx, tx, *p, time.Now())
	if err != nil {
		return err
	}
	var revision int64
	err = tx.QueryRow(ctx,
		`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note,
		                           branch, tier_entered, revision, epoch, release, delivery, merged_at, commits)
		 values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14, $15, $16, $17)
		 on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
		        tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
		        tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch,
		        release = excluded.release, delivery = excluded.delivery, repository = excluded.repository,
		        commit_hash = excluded.commit_hash, merged_at = excluded.merged_at, commits = excluded.commits,
		        created = excluded.created
		  where release_plan.revision = $12
		 returning revision`,
		p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered,
		p.Revision, epoch, release, delivery, mergedAt(p.Merged), commits).Scan(&revision)
	if errors.Is(err, pgx.ErrNoRows) {
		// The row is there and at another revision — moved since this was read, or there already
		// when this one is new: either way not this writer's to overwrite. (A plan saved before plans
		// had revisions is at zero, and its first save here is from a read at zero.)
		return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
	}
	if err != nil {
		return err
	}
	if err := tx.Commit(ctx); err != nil {
		return err
	}
	p.Revision, p.TierEntered = revision, entered
	if epoch != nil {
		p.Epoch = uint64(*epoch)
	} else {
		p.Epoch = 0
	}
	if PlanSaved != nil {
		PlanSaved(*p)
	}
	return nil
}

// short is a commit as a person reads it.
func short(commit string) string {
	if len(commit) > 8 {
		return commit[:8]
	}
	return commit
}

// mergedAt is a plan's merge time as the store keeps it: null when not known.
func mergedAt(t time.Time) *time.Time {
	if t.IsZero() {
		return nil
	}
	return &t
}

// OpenPlans is every plan still being worked, oldest first.
func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) {
	return i.plans(ctx, `where state in ('building', 'rolling') order by created`)
}

// PlansSince is every plan made after a moment, and every plan still being worked, oldest first.
func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, error) {
	return i.plans(ctx, `where created > $1 or state in ('building', 'rolling') order by created`, since)
}

// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
	return i.plans(ctx, fmt.Sprintf(`order by created desc limit %d`, limit))
}

// NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh
// planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it
// recorded when its merge was made.
func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) {
	plans, err := i.plans(ctx, `where lower(repository) = lower($1) and branch = $2 and merged_at is not null
		 and release is null order by merged_at desc, created desc limit 1`, repository, branch)
	if err != nil || len(plans) == 0 {
		return Plan{}, false, err
	}
	return plans[0], true, nil
}

// PlanByID is one plan.
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
	plans, err := i.plans(ctx, `where id = '`+id+`'`)
	if err != nil {
		return Plan{}, err
	}
	if len(plans) == 0 {
		return Plan{}, fmt.Errorf("no plan %s", id)
	}
	return plans[0], nil
}

func (i *Inventory) plans(ctx context.Context, tail string, args ...any) ([]Plan, error) {
	rows, err := i.store.Pool().Query(ctx,
		`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
		        coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery, merged_at, commits
		   from release_plan `+tail, args...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var out []Plan
	for rows.Next() {
		var p Plan
		var tiers, modules, release, delivery, commits []byte
		var epoch int64
		var merged *time.Time
		if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
			&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch, &release,
			&delivery, &merged, &commits); err != nil {
			return nil, err
		}
		if len(commits) > 0 {
			if err := json.Unmarshal(commits, &p.Commits); err != nil {
				return nil, err
			}
		}
		if merged != nil {
			p.Merged = merged.UTC()
		}
		if len(release) > 0 {
			if err := json.Unmarshal(release, &p.Release); err != nil {
				return nil, err
			}
		}
		if len(delivery) > 0 {
			if err := json.Unmarshal(delivery, &p.Delivery); err != nil {
				return nil, err
			}
		}
		p.Epoch = uint64(epoch)
		if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
			return nil, err
		}
		if err := json.Unmarshal(modules, &p.Modules); err != nil {
			return nil, err
		}
		if p.Modules == nil {
			p.Modules = map[string]*PlanModule{}
		}
		out = append(out, p)
	}
	if errors.Is(rows.Err(), pgx.ErrNoRows) {
		return nil, nil
	}
	return out, rows.Err()
}