jochen
cdf30349a7
A plan's gated send for mail carried postgres's and mongodb's new builds (policy record) to the control node and the anchor on 2026-10-07: a send composes the machine's whole declaration from the builds the mesh holds, and the gate only ever looked at modules that roll out. Every send but a person's push now composes a recorded module from the manifest of the build the machine was last sent and records that it still carries it; the bus step moves the bus alone. And each move a gated send carries says how many of the module's containers it recreates, and whether with a new image or only their declaration.
344 lines
15 KiB
Go
344 lines
15 KiB
Go
package inventory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
// A Plan is what a merge produces (novox/hq ADR 0162): the modules it changed and everything
|
|
// standing on them, sorted into tiers, each module's state, and the tier the plan is at. Kept in
|
|
// the store so a controller replaced mid-plan resumes it, and so `status` can say what a merge
|
|
// still waits for.
|
|
type Plan struct {
|
|
ID string `json:"id"`
|
|
Repository string `json:"repository"`
|
|
// Branch is the branch the merge went into (novox/hq issue 254): a newer plan supersedes the open
|
|
// ones of the same repository and branch. Empty for a plan from before it was kept.
|
|
Branch string `json:"branch,omitempty"`
|
|
Commit string `json:"commit"`
|
|
Created time.Time `json:"created"`
|
|
Updated time.Time `json:"updated"`
|
|
State string `json:"state"`
|
|
Tier int `json:"tier"`
|
|
Tiers [][]string `json:"tiers"`
|
|
Modules map[string]*PlanModule `json:"modules"`
|
|
Note string `json:"note,omitempty"`
|
|
// TierEntered is when the plan entered the tier it is at (novox/hq to-be 45 Phase 0), read and
|
|
// never written from here: a save measures the tier it leaves and stamps the next. What the
|
|
// watchdog of a plan's progress (S3) reads.
|
|
TierEntered time.Time `json:"tier_entered,omitempty"`
|
|
// Revision is the plan's as it was read, and the one a save must find (novox/hq to-be 45 §6): a
|
|
// plan is written by compare-and-set, so a write against a plan another writer moved since is
|
|
// refused rather than laid over it. Zero is a plan never saved. SavePlan moves it.
|
|
Revision int64 `json:"revision"`
|
|
// Epoch is the controller lease epoch that wrote it last; zero for a write that claimed none.
|
|
Epoch uint64 `json:"epoch,omitempty"`
|
|
// Release is set on a release plan (novox/hq ADR 0236): not a merge's, but the builds waiting for a
|
|
// gate, walked through the machines one at a time.
|
|
Release *PlanRelease `json:"release,omitempty"`
|
|
// Delivery is set on a walk that waits for its delivery's word (novox/hq ADR 0239): nil for a walk on
|
|
// the controller's own path, which starts at the merge as every plan did before.
|
|
Delivery *PlanDelivery `json:"delivery,omitempty"`
|
|
}
|
|
|
|
// PlanDelivery is what a walk waits for and what came of the wait (novox/hq ADR 0239).
|
|
type PlanDelivery struct {
|
|
// Awaits is who must say the walk may start: the seat whose holder owns the delivery.
|
|
Awaits string `json:"awaits"`
|
|
// Go is when it was let go, By by whom (the seat's holder, or a person's name) and Why.
|
|
Go *time.Time `json:"go,omitempty"`
|
|
By string `json:"by,omitempty"`
|
|
Why string `json:"why,omitempty"`
|
|
// Stopped is who ended the walk through the delivery's owner, and StoppedWhy why: the walk is failed,
|
|
// and the delivery reads it as stopped, not as a build that failed.
|
|
Stopped string `json:"stopped,omitempty"`
|
|
StoppedWhy string `json:"stopped_why,omitempty"`
|
|
}
|
|
|
|
// Waiting is whether the walk waits for its delivery's word.
|
|
func (p Plan) Waiting() bool {
|
|
return p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go == nil
|
|
}
|
|
|
|
// PlanSaved is told every plan this process kept, after it is kept (novox/hq ADR 0239): the serving
|
|
// controller says it on the bus as `plan-moved`. Nil in a command, which says nothing; whoever follows a
|
|
// walk also asks for it, so a save made by a command is found by comparison.
|
|
var PlanSaved func(Plan)
|
|
|
|
// ErrPlanMoved is a save against a plan written by somebody else since it was read.
|
|
var ErrPlanMoved = errors.New("the plan was written by somebody else since it was read")
|
|
|
|
// PlanModule is one module's state within a plan.
|
|
type PlanModule struct {
|
|
// State: asked, built, failed; empty for a module whose tier has not been asked yet.
|
|
State string `json:"state,omitempty"`
|
|
AskedAt *time.Time `json:"asked_at,omitempty"`
|
|
BuiltAt *time.Time `json:"built_at,omitempty"`
|
|
// SentAt is when the plan sent the machines running this module its new build, because a
|
|
// later tier is built by it (ADR 0163's gate): the reports that open the gate are the ones
|
|
// after this.
|
|
SentAt *time.Time `json:"sent_at,omitempty"`
|
|
// First is the machines the plan sent the new build to first, and FirstAt when (novox/hq issue
|
|
// 249, ADR 0218): unless the module's policy rolls it out together, one machine takes it before
|
|
// the rest, and the rest are sent once that one reports it applied. Kept so a controller
|
|
// replaced while the plan waits on that report resumes the wait rather than sending again. The
|
|
// machine holding the bus is among them when its user list had to go first.
|
|
First []string `json:"first,omitempty"`
|
|
FirstAt *time.Time `json:"first_at,omitempty"`
|
|
Commit string `json:"commit,omitempty"`
|
|
Why string `json:"why,omitempty"`
|
|
// Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan
|
|
// matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt
|
|
// its module's name included. Empty in a plan from before it was kept, which is matched by
|
|
// module, or by repository and path, as before.
|
|
Build string `json:"build,omitempty"`
|
|
// Previous is the build the first machine ran of this module before the plan sent it the new one —
|
|
// the commit its last send carried (ADR 0221) — kept at the first send: what a rollback puts back
|
|
// (novox/hq ADR 0236). Empty when the machine had never been sent the module, or what it was sent
|
|
// is not known.
|
|
Previous string `json:"previous,omitempty"`
|
|
// Gate is the new build's judging on its first machine (novox/hq ADR 0236, to-be 45 §8), kept so a
|
|
// controller replaced mid-judging resumes it, and read back through `plans` as the rollout's record.
|
|
Gate *PlanGate `json:"gate,omitempty"`
|
|
// GatedBy names the module of the same tier whose gate judges this one on its first machine: they
|
|
// went there in one send (novox/hq issue 281), and one gate judges what one send moved. Empty for
|
|
// the module the gate is kept on, and for a plan from before tiers were sent whole.
|
|
GatedBy string `json:"gated_by,omitempty"`
|
|
}
|
|
|
|
// PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine,
|
|
// from and to which build, the verdict, how long it took to reach it, and whether it was rolled back.
|
|
type PlanGate struct {
|
|
// Component is the core component the module is — mesh-controller, mesh-host, node-tools — or empty
|
|
// for any other module, judged by its own health.
|
|
Component string `json:"component,omitempty"`
|
|
Machines []string `json:"machines"`
|
|
From string `json:"from,omitempty"`
|
|
To string `json:"to,omitempty"`
|
|
// Since is when the judging began: the first machine reported the new build applied.
|
|
Since *time.Time `json:"since,omitempty"`
|
|
// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
|
|
// does not resets them.
|
|
Passes int `json:"passes,omitempty"`
|
|
LastPass *time.Time `json:"last_pass,omitempty"`
|
|
// Last is what the newest judging found wanting, while it still may pass.
|
|
Last string `json:"last,omitempty"`
|
|
// Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.
|
|
Verdict string `json:"verdict,omitempty"`
|
|
Why string `json:"why,omitempty"`
|
|
JudgedAt *time.Time `json:"judged_at,omitempty"`
|
|
Took string `json:"took,omitempty"`
|
|
// Rollback is how a failed build was put back: rolled-back, or not-rolled-back with why.
|
|
Rollback string `json:"rollback,omitempty"`
|
|
// Kept says a passing verdict was written to the gate's records.
|
|
Kept bool `json:"kept,omitempty"`
|
|
// Carried is every module whose build moved on the judged machines with the send — the plan's own
|
|
// module and whatever else was waiting there for a gate (novox/hq ADR 0236): each is judged here, a
|
|
// pass is its verdict too, and one that fails is put back.
|
|
Carried []CarriedMove `json:"carried,omitempty"`
|
|
// Failing names the modules the last judging found wanting.
|
|
Failing []string `json:"failing,omitempty"`
|
|
}
|
|
|
|
// CarriedMove is one module's build moving on a machine with a gated send.
|
|
type CarriedMove struct {
|
|
Module string `json:"module"`
|
|
Node string `json:"node"`
|
|
From string `json:"from,omitempty"`
|
|
To string `json:"to"`
|
|
Build string `json:"build,omitempty"`
|
|
// Recreates is what the send does to the module's containers, in the mesh's words — how many it
|
|
// recreates, and whether with a new image or only their declaration (novox/hq ADR 0242); empty when
|
|
// it recreates none, or when the build the machine ran is not known.
|
|
Recreates string `json:"recreates,omitempty"`
|
|
}
|
|
|
|
// PlanRelease is a release plan's walk through the machines (novox/hq ADR 0236): every module build
|
|
// that waits for a gate, sent one machine at a time, each judged before the next.
|
|
type PlanRelease struct {
|
|
Order []string `json:"order"`
|
|
Next int `json:"next"`
|
|
// Gate is the machine being judged; nil between machines.
|
|
Gate *PlanGate `json:"gate,omitempty"`
|
|
Done []string `json:"done,omitempty"`
|
|
// Skipped are the machines not heard from when their turn came, left as they were.
|
|
Skipped []string `json:"skipped,omitempty"`
|
|
// By is the person who released it, empty when the mesh did.
|
|
By string `json:"by,omitempty"`
|
|
}
|
|
|
|
// The states a plan passes through.
|
|
const (
|
|
PlanBuilding = "building"
|
|
PlanRolling = "rolling"
|
|
PlanDone = "done"
|
|
PlanFailed = "failed"
|
|
// PlanSuperseded is a plan a newer merge of the same repository and branch took over (novox/hq
|
|
// issue 254, ADR 0218): what it had not built is in the newer plan, and its note names it.
|
|
PlanSuperseded = "superseded"
|
|
)
|
|
|
|
// Open says whether the plan is still being worked.
|
|
func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRolling }
|
|
|
|
// SavePlan writes a plan, new or changed, whole: the plan is small and read as one thing.
|
|
//
|
|
// **By compare-and-set on its revision, carrying the epoch** (novox/hq to-be 45 §6): written only if
|
|
// the plan is still at the revision it was read at — a new one only if it does not exist — and refused
|
|
// with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records.
|
|
// On success p's revision and epoch are the ones written, so the caller may save it again.
|
|
func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
|
|
epoch, err := i.actingEpoch(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("the plan for %s %s is not written: %w", p.Repository, p.Commit, err)
|
|
}
|
|
tiers, err := json.Marshal(p.Tiers)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
modules, err := json.Marshal(p.Modules)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var release, delivery []byte
|
|
if p.Release != nil {
|
|
if release, err = json.Marshal(p.Release); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if p.Delivery != nil {
|
|
if delivery, err = json.Marshal(p.Delivery); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
|
|
// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
|
|
// measure one twice.
|
|
tx, err := i.store.Pool().Begin(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = tx.Rollback(ctx) }()
|
|
entered, err := planTierLeft(ctx, tx, *p, time.Now())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var revision int64
|
|
err = tx.QueryRow(ctx,
|
|
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note,
|
|
branch, tier_entered, revision, epoch, release, delivery)
|
|
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14, $15)
|
|
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
|
|
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
|
|
tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch,
|
|
release = excluded.release, delivery = excluded.delivery
|
|
where release_plan.revision = $12
|
|
returning revision`,
|
|
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered,
|
|
p.Revision, epoch, release, delivery).Scan(&revision)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
// The row is there and at another revision — moved since this was read, or there already
|
|
// when this one is new: either way not this writer's to overwrite. (A plan saved before plans
|
|
// had revisions is at zero, and its first save here is from a read at zero.)
|
|
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return err
|
|
}
|
|
p.Revision, p.TierEntered = revision, entered
|
|
if epoch != nil {
|
|
p.Epoch = uint64(*epoch)
|
|
} else {
|
|
p.Epoch = 0
|
|
}
|
|
if PlanSaved != nil {
|
|
PlanSaved(*p)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// short is a commit as a person reads it.
|
|
func short(commit string) string {
|
|
if len(commit) > 8 {
|
|
return commit[:8]
|
|
}
|
|
return commit
|
|
}
|
|
|
|
// OpenPlans is every plan still being worked, oldest first.
|
|
func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) {
|
|
return i.plans(ctx, `where state in ('building', 'rolling') order by created`)
|
|
}
|
|
|
|
// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
|
|
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
|
|
return i.plans(ctx, fmt.Sprintf(`order by created desc limit %d`, limit))
|
|
}
|
|
|
|
// PlanByID is one plan.
|
|
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
|
|
plans, err := i.plans(ctx, `where id = '`+id+`'`)
|
|
if err != nil {
|
|
return Plan{}, err
|
|
}
|
|
if len(plans) == 0 {
|
|
return Plan{}, fmt.Errorf("no plan %s", id)
|
|
}
|
|
return plans[0], nil
|
|
}
|
|
|
|
func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
|
|
rows, err := i.store.Pool().Query(ctx,
|
|
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
|
|
coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery
|
|
from release_plan `+tail)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []Plan
|
|
for rows.Next() {
|
|
var p Plan
|
|
var tiers, modules, release, delivery []byte
|
|
var epoch int64
|
|
if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
|
|
&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch, &release,
|
|
&delivery); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(release) > 0 {
|
|
if err := json.Unmarshal(release, &p.Release); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if len(delivery) > 0 {
|
|
if err := json.Unmarshal(delivery, &p.Delivery); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
p.Epoch = uint64(epoch)
|
|
if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := json.Unmarshal(modules, &p.Modules); err != nil {
|
|
return nil, err
|
|
}
|
|
if p.Modules == nil {
|
|
p.Modules = map[string]*PlanModule{}
|
|
}
|
|
out = append(out, p)
|
|
}
|
|
if errors.Is(rows.Err(), pgx.ErrNoRows) {
|
|
return nil, nil
|
|
}
|
|
return out, rows.Err()
|
|
}
|