jochen
487aa040de
While the mesh-delivery seat has a holder on record, a merge that moves no core module opens its walk and asks nothing until mesh-delivery or a person says go; nothing of it is registered before its turn, so no other send carries it. The controller keeps the planner, the gate, sending and the walk, and gains the verbs the owner asks with: delivery-plan, -order, -check (a group composed as one future state), deliver, delivery-stop, delivery-walks; every walk kept is said as plan-moved.
340 lines
15 KiB
Go
340 lines
15 KiB
Go
package inventory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
// A Plan is what a merge produces (novox/hq ADR 0162): the modules it changed and everything
|
|
// standing on them, sorted into tiers, each module's state, and the tier the plan is at. Kept in
|
|
// the store so a controller replaced mid-plan resumes it, and so `status` can say what a merge
|
|
// still waits for.
|
|
type Plan struct {
|
|
ID string `json:"id"`
|
|
Repository string `json:"repository"`
|
|
// Branch is the branch the merge went into (novox/hq issue 254): a newer plan supersedes the open
|
|
// ones of the same repository and branch. Empty for a plan from before it was kept.
|
|
Branch string `json:"branch,omitempty"`
|
|
Commit string `json:"commit"`
|
|
Created time.Time `json:"created"`
|
|
Updated time.Time `json:"updated"`
|
|
State string `json:"state"`
|
|
Tier int `json:"tier"`
|
|
Tiers [][]string `json:"tiers"`
|
|
Modules map[string]*PlanModule `json:"modules"`
|
|
Note string `json:"note,omitempty"`
|
|
// TierEntered is when the plan entered the tier it is at (novox/hq to-be 45 Phase 0), read and
|
|
// never written from here: a save measures the tier it leaves and stamps the next. What the
|
|
// watchdog of a plan's progress (S3) reads.
|
|
TierEntered time.Time `json:"tier_entered,omitempty"`
|
|
// Revision is the plan's as it was read, and the one a save must find (novox/hq to-be 45 §6): a
|
|
// plan is written by compare-and-set, so a write against a plan another writer moved since is
|
|
// refused rather than laid over it. Zero is a plan never saved. SavePlan moves it.
|
|
Revision int64 `json:"revision"`
|
|
// Epoch is the controller lease epoch that wrote it last; zero for a write that claimed none.
|
|
Epoch uint64 `json:"epoch,omitempty"`
|
|
// Release is set on a release plan (novox/hq ADR 0236): not a merge's, but the builds waiting for a
|
|
// gate, walked through the machines one at a time.
|
|
Release *PlanRelease `json:"release,omitempty"`
|
|
// Delivery is set on a walk that waits for its delivery's word (novox/hq ADR 0239): nil for a walk on
|
|
// the controller's own path, which starts at the merge as every plan did before.
|
|
Delivery *PlanDelivery `json:"delivery,omitempty"`
|
|
}
|
|
|
|
// PlanDelivery is what a walk waits for and what came of the wait (novox/hq ADR 0239).
|
|
type PlanDelivery struct {
|
|
// Awaits is who must say the walk may start: the seat whose holder owns the delivery.
|
|
Awaits string `json:"awaits"`
|
|
// Go is when it was let go, By by whom (the seat's holder, or a person's name) and Why.
|
|
Go *time.Time `json:"go,omitempty"`
|
|
By string `json:"by,omitempty"`
|
|
Why string `json:"why,omitempty"`
|
|
// Stopped is who ended the walk through the delivery's owner, and StoppedWhy why: the walk is failed,
|
|
// and the delivery reads it as stopped, not as a build that failed.
|
|
Stopped string `json:"stopped,omitempty"`
|
|
StoppedWhy string `json:"stopped_why,omitempty"`
|
|
}
|
|
|
|
// Waiting is whether the walk waits for its delivery's word.
|
|
func (p Plan) Waiting() bool {
|
|
return p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go == nil
|
|
}
|
|
|
|
// PlanSaved is told every plan this process kept, after it is kept (novox/hq ADR 0239): the serving
|
|
// controller says it on the bus as `plan-moved`. Nil in a command, which says nothing; whoever follows a
|
|
// walk also asks for it, so a save made by a command is found by comparison.
|
|
var PlanSaved func(Plan)
|
|
|
|
// ErrPlanMoved is a save against a plan written by somebody else since it was read.
|
|
var ErrPlanMoved = errors.New("the plan was written by somebody else since it was read")
|
|
|
|
// PlanModule is one module's state within a plan.
|
|
type PlanModule struct {
|
|
// State: asked, built, failed; empty for a module whose tier has not been asked yet.
|
|
State string `json:"state,omitempty"`
|
|
AskedAt *time.Time `json:"asked_at,omitempty"`
|
|
BuiltAt *time.Time `json:"built_at,omitempty"`
|
|
// SentAt is when the plan sent the machines running this module its new build, because a
|
|
// later tier is built by it (ADR 0163's gate): the reports that open the gate are the ones
|
|
// after this.
|
|
SentAt *time.Time `json:"sent_at,omitempty"`
|
|
// First is the machines the plan sent the new build to first, and FirstAt when (novox/hq issue
|
|
// 249, ADR 0218): unless the module's policy rolls it out together, one machine takes it before
|
|
// the rest, and the rest are sent once that one reports it applied. Kept so a controller
|
|
// replaced while the plan waits on that report resumes the wait rather than sending again. The
|
|
// machine holding the bus is among them when its user list had to go first.
|
|
First []string `json:"first,omitempty"`
|
|
FirstAt *time.Time `json:"first_at,omitempty"`
|
|
Commit string `json:"commit,omitempty"`
|
|
Why string `json:"why,omitempty"`
|
|
// Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan
|
|
// matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt
|
|
// its module's name included. Empty in a plan from before it was kept, which is matched by
|
|
// module, or by repository and path, as before.
|
|
Build string `json:"build,omitempty"`
|
|
// Previous is the build the first machine ran of this module before the plan sent it the new one —
|
|
// the commit its last send carried (ADR 0221) — kept at the first send: what a rollback puts back
|
|
// (novox/hq ADR 0236). Empty when the machine had never been sent the module, or what it was sent
|
|
// is not known.
|
|
Previous string `json:"previous,omitempty"`
|
|
// Gate is the new build's judging on its first machine (novox/hq ADR 0236, to-be 45 §8), kept so a
|
|
// controller replaced mid-judging resumes it, and read back through `plans` as the rollout's record.
|
|
Gate *PlanGate `json:"gate,omitempty"`
|
|
// GatedBy names the module of the same tier whose gate judges this one on its first machine: they
|
|
// went there in one send (novox/hq issue 281), and one gate judges what one send moved. Empty for
|
|
// the module the gate is kept on, and for a plan from before tiers were sent whole.
|
|
GatedBy string `json:"gated_by,omitempty"`
|
|
}
|
|
|
|
// PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine,
|
|
// from and to which build, the verdict, how long it took to reach it, and whether it was rolled back.
|
|
type PlanGate struct {
|
|
// Component is the core component the module is — mesh-controller, mesh-host, node-tools — or empty
|
|
// for any other module, judged by its own health.
|
|
Component string `json:"component,omitempty"`
|
|
Machines []string `json:"machines"`
|
|
From string `json:"from,omitempty"`
|
|
To string `json:"to,omitempty"`
|
|
// Since is when the judging began: the first machine reported the new build applied.
|
|
Since *time.Time `json:"since,omitempty"`
|
|
// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
|
|
// does not resets them.
|
|
Passes int `json:"passes,omitempty"`
|
|
LastPass *time.Time `json:"last_pass,omitempty"`
|
|
// Last is what the newest judging found wanting, while it still may pass.
|
|
Last string `json:"last,omitempty"`
|
|
// Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.
|
|
Verdict string `json:"verdict,omitempty"`
|
|
Why string `json:"why,omitempty"`
|
|
JudgedAt *time.Time `json:"judged_at,omitempty"`
|
|
Took string `json:"took,omitempty"`
|
|
// Rollback is how a failed build was put back: rolled-back, or not-rolled-back with why.
|
|
Rollback string `json:"rollback,omitempty"`
|
|
// Kept says a passing verdict was written to the gate's records.
|
|
Kept bool `json:"kept,omitempty"`
|
|
// Carried is every module whose build moved on the judged machines with the send — the plan's own
|
|
// module and whatever else was waiting there for a gate (novox/hq ADR 0236): each is judged here, a
|
|
// pass is its verdict too, and one that fails is put back.
|
|
Carried []CarriedMove `json:"carried,omitempty"`
|
|
// Failing names the modules the last judging found wanting.
|
|
Failing []string `json:"failing,omitempty"`
|
|
}
|
|
|
|
// CarriedMove is one module's build moving on a machine with a gated send.
|
|
type CarriedMove struct {
|
|
Module string `json:"module"`
|
|
Node string `json:"node"`
|
|
From string `json:"from,omitempty"`
|
|
To string `json:"to"`
|
|
Build string `json:"build,omitempty"`
|
|
}
|
|
|
|
// PlanRelease is a release plan's walk through the machines (novox/hq ADR 0236): every module build
|
|
// that waits for a gate, sent one machine at a time, each judged before the next.
|
|
type PlanRelease struct {
|
|
Order []string `json:"order"`
|
|
Next int `json:"next"`
|
|
// Gate is the machine being judged; nil between machines.
|
|
Gate *PlanGate `json:"gate,omitempty"`
|
|
Done []string `json:"done,omitempty"`
|
|
// Skipped are the machines not heard from when their turn came, left as they were.
|
|
Skipped []string `json:"skipped,omitempty"`
|
|
// By is the person who released it, empty when the mesh did.
|
|
By string `json:"by,omitempty"`
|
|
}
|
|
|
|
// The states a plan passes through.
|
|
const (
|
|
PlanBuilding = "building"
|
|
PlanRolling = "rolling"
|
|
PlanDone = "done"
|
|
PlanFailed = "failed"
|
|
// PlanSuperseded is a plan a newer merge of the same repository and branch took over (novox/hq
|
|
// issue 254, ADR 0218): what it had not built is in the newer plan, and its note names it.
|
|
PlanSuperseded = "superseded"
|
|
)
|
|
|
|
// Open says whether the plan is still being worked.
|
|
func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRolling }
|
|
|
|
// SavePlan writes a plan, new or changed, whole: the plan is small and read as one thing.
|
|
//
|
|
// **By compare-and-set on its revision, carrying the epoch** (novox/hq to-be 45 §6): written only if
|
|
// the plan is still at the revision it was read at — a new one only if it does not exist — and refused
|
|
// with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records.
|
|
// On success p's revision and epoch are the ones written, so the caller may save it again.
|
|
func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
|
|
epoch, err := i.actingEpoch(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("the plan for %s %s is not written: %w", p.Repository, p.Commit, err)
|
|
}
|
|
tiers, err := json.Marshal(p.Tiers)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
modules, err := json.Marshal(p.Modules)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var release, delivery []byte
|
|
if p.Release != nil {
|
|
if release, err = json.Marshal(p.Release); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if p.Delivery != nil {
|
|
if delivery, err = json.Marshal(p.Delivery); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
|
|
// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
|
|
// measure one twice.
|
|
tx, err := i.store.Pool().Begin(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = tx.Rollback(ctx) }()
|
|
entered, err := planTierLeft(ctx, tx, *p, time.Now())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var revision int64
|
|
err = tx.QueryRow(ctx,
|
|
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note,
|
|
branch, tier_entered, revision, epoch, release, delivery)
|
|
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14, $15)
|
|
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
|
|
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
|
|
tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch,
|
|
release = excluded.release, delivery = excluded.delivery
|
|
where release_plan.revision = $12
|
|
returning revision`,
|
|
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered,
|
|
p.Revision, epoch, release, delivery).Scan(&revision)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
// The row is there and at another revision — moved since this was read, or there already
|
|
// when this one is new: either way not this writer's to overwrite. (A plan saved before plans
|
|
// had revisions is at zero, and its first save here is from a read at zero.)
|
|
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return err
|
|
}
|
|
p.Revision, p.TierEntered = revision, entered
|
|
if epoch != nil {
|
|
p.Epoch = uint64(*epoch)
|
|
} else {
|
|
p.Epoch = 0
|
|
}
|
|
if PlanSaved != nil {
|
|
PlanSaved(*p)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// short is a commit as a person reads it.
|
|
func short(commit string) string {
|
|
if len(commit) > 8 {
|
|
return commit[:8]
|
|
}
|
|
return commit
|
|
}
|
|
|
|
// OpenPlans is every plan still being worked, oldest first.
|
|
func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) {
|
|
return i.plans(ctx, `where state in ('building', 'rolling') order by created`)
|
|
}
|
|
|
|
// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
|
|
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
|
|
return i.plans(ctx, fmt.Sprintf(`order by created desc limit %d`, limit))
|
|
}
|
|
|
|
// PlanByID is one plan.
|
|
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
|
|
plans, err := i.plans(ctx, `where id = '`+id+`'`)
|
|
if err != nil {
|
|
return Plan{}, err
|
|
}
|
|
if len(plans) == 0 {
|
|
return Plan{}, fmt.Errorf("no plan %s", id)
|
|
}
|
|
return plans[0], nil
|
|
}
|
|
|
|
func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
|
|
rows, err := i.store.Pool().Query(ctx,
|
|
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
|
|
coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery
|
|
from release_plan `+tail)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []Plan
|
|
for rows.Next() {
|
|
var p Plan
|
|
var tiers, modules, release, delivery []byte
|
|
var epoch int64
|
|
if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
|
|
&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch, &release,
|
|
&delivery); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(release) > 0 {
|
|
if err := json.Unmarshal(release, &p.Release); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if len(delivery) > 0 {
|
|
if err := json.Unmarshal(delivery, &p.Delivery); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
p.Epoch = uint64(epoch)
|
|
if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := json.Unmarshal(modules, &p.Modules); err != nil {
|
|
return nil, err
|
|
}
|
|
if p.Modules == nil {
|
|
p.Modules = map[string]*PlanModule{}
|
|
}
|
|
out = append(out, p)
|
|
}
|
|
if errors.Is(rows.Err(), pgx.ErrNoRows) {
|
|
return nil, nil
|
|
}
|
|
return out, rows.Err()
|
|
}
|