Gate a release plan's first machine and roll a failed build back there (hq ADR 0236)
A build that reported applied was sent everywhere; one that then did nothing, served no tools or broke its machine's word reached every machine. Now the first machine is judged by the component's health (the core's definitions, as doctor probes H-*, or a module's own) three times over two minutes within ten; a failing gate puts the previous build back there once, marks the build, and says it as a condition and an event. Upgrades roll out by default; the bus is a planned step; a module deleted at its source is not built (the public-acme plan failure).
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// BusStep is one planned bus upgrade (novox/hq to-be 45 §8, ADR 0235).
|
||||
type BusStep struct {
|
||||
ID int64
|
||||
Module string
|
||||
Machines []string
|
||||
From, To string
|
||||
Snapshot string
|
||||
Reversible bool
|
||||
By, Why string
|
||||
Started time.Time
|
||||
Ended *time.Time
|
||||
Outcome string
|
||||
Found string
|
||||
}
|
||||
|
||||
// StartBusStep records a bus upgrade starting. Refused while another runs.
|
||||
func (i *Inventory) StartBusStep(ctx context.Context, s BusStep) (BusStep, error) {
|
||||
if _, err := i.actingEpoch(ctx); err != nil {
|
||||
return s, err
|
||||
}
|
||||
if open, found, err := i.LatestBusStep(ctx); err != nil {
|
||||
return s, err
|
||||
} else if found && open.Ended == nil {
|
||||
return s, ErrBusStepRunning
|
||||
}
|
||||
if s.Machines == nil {
|
||||
s.Machines = []string{}
|
||||
}
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`insert into bus_step (module, machines, from_build, to_build, snapshot, reversible, by_whom, why)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8) returning id, started`,
|
||||
s.Module, s.Machines, s.From, s.To, s.Snapshot, s.Reversible, s.By, s.Why).Scan(&s.ID, &s.Started)
|
||||
return s, err
|
||||
}
|
||||
|
||||
// ErrBusStepRunning is a bus upgrade asked while one runs.
|
||||
var ErrBusStepRunning = errors.New("a bus upgrade is already running")
|
||||
|
||||
// EndBusStep records how a bus upgrade ended: done or failed, with what was found.
|
||||
func (i *Inventory) EndBusStep(ctx context.Context, id int64, outcome, found string) error {
|
||||
if _, err := i.actingEpoch(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update bus_step set ended = now(), outcome = $2, found = $3 where id = $1 and ended is null`, id, outcome, found)
|
||||
return err
|
||||
}
|
||||
|
||||
// LatestBusStep is the newest bus upgrade, and whether there is any.
|
||||
func (i *Inventory) LatestBusStep(ctx context.Context) (BusStep, bool, error) {
|
||||
var s BusStep
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select id, module, machines, from_build, to_build, snapshot, reversible, by_whom, why, started, ended,
|
||||
outcome, found
|
||||
from bus_step order by id desc limit 1`).
|
||||
Scan(&s.ID, &s.Module, &s.Machines, &s.From, &s.To, &s.Snapshot, &s.Reversible, &s.By, &s.Why, &s.Started,
|
||||
&s.Ended, &s.Outcome, &s.Found)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return s, false, nil
|
||||
}
|
||||
return s, err == nil, err
|
||||
}
|
||||
+120
-19
@@ -1194,15 +1194,62 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
|
||||
// providedBy is what the source column says for a module the control plane ships.
|
||||
const providedBy = "the control plane"
|
||||
|
||||
// Upgrade is what the mesh decided to do when a module's current version moves.
|
||||
// Upgrade is what the mesh does when a module's current version moves (ADR 0162 §3, ADR 0235).
|
||||
type Upgrade struct {
|
||||
// RollOut is true when the machines running it should be sent the new version. False means
|
||||
// record it and stop — which needs no record of its own, because a machine not running what
|
||||
// the mesh would send it is already something the mesh reports.
|
||||
// RollOut is true when the machines running it are sent the new version: one machine first, judged
|
||||
// at the gate, then the rest (ADR 0218, ADR 0235). False means record it and stop — the machines
|
||||
// running it are behind until a person pushes, which the mesh already reports.
|
||||
RollOut bool
|
||||
// Together is true when every machine running it is sent the new version at once, rather than
|
||||
// one after another. Only meaningful when RollOut is.
|
||||
// Together is true when every machine running it is sent the new version at once. Only meaningful
|
||||
// when RollOut is.
|
||||
Together bool
|
||||
// From is where the policy came from: a person, the module, the bus, its irreplaceable data, or the
|
||||
// default (catalogue.From*); Why is the reason said with it.
|
||||
From string
|
||||
Why string
|
||||
// By is the person who chose it, when one did.
|
||||
By string
|
||||
}
|
||||
|
||||
// Policy is the policy as a word: roll, together or record.
|
||||
func (u Upgrade) Policy() string {
|
||||
switch {
|
||||
case !u.RollOut:
|
||||
return catalogue.PolicyRecord
|
||||
case u.Together:
|
||||
return catalogue.PolicyTogether
|
||||
}
|
||||
return catalogue.PolicyRoll
|
||||
}
|
||||
|
||||
// upgradeFrom is a module's policy from what the store holds of it: a person's choice, over the module's
|
||||
// own word, over the default — except that the bus is never rolled out, whoever says so (ADR 0235).
|
||||
func upgradeFrom(chosen *string, together bool, why, by string, manifest []byte) Upgrade {
|
||||
var m catalogue.Manifest
|
||||
// Leniently: a policy is read from what was registered, and a manifest registered before a field it
|
||||
// carries was known is still a manifest whose bus and data can be read.
|
||||
_ = json.Unmarshal(manifest, &m)
|
||||
policy, from, said := catalogue.DerivedUpgrade(m)
|
||||
if from != catalogue.FromBus && chosen != nil {
|
||||
policy, from, said = catalogue.PolicyRecord, catalogue.FromPerson, why
|
||||
if *chosen == "roll-out" {
|
||||
policy = catalogue.PolicyRoll
|
||||
if together {
|
||||
policy = catalogue.PolicyTogether
|
||||
}
|
||||
}
|
||||
}
|
||||
u := Upgrade{From: from, Why: said}
|
||||
if from == catalogue.FromPerson {
|
||||
u.By = by
|
||||
}
|
||||
switch policy {
|
||||
case catalogue.PolicyRoll:
|
||||
u.RollOut = true
|
||||
case catalogue.PolicyTogether:
|
||||
u.RollOut, u.Together = true, true
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// UpgradeOf is what to do when this module moves.
|
||||
@@ -1211,30 +1258,80 @@ type Upgrade struct {
|
||||
// mesh has never registered, and being told one of them moved is information, not a fault. The
|
||||
// answer is the safe one — record it — because there is nothing to roll out to.
|
||||
func (i *Inventory) UpgradeOf(ctx context.Context, module string) (Upgrade, error) {
|
||||
var u Upgrade
|
||||
var policy string
|
||||
var chosen *string
|
||||
var together bool
|
||||
var why, by string
|
||||
var manifest []byte
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select upgrade, upgrade_together from module where name = $1`, module).
|
||||
Scan(&policy, &u.Together)
|
||||
`select upgrade, upgrade_together, upgrade_why, upgrade_by, manifest from module where name = $1`, module).
|
||||
Scan(&chosen, &together, &why, &by, &manifest)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Upgrade{}, nil
|
||||
return Upgrade{From: catalogue.FromDefault, Why: "the mesh holds no such module"}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Upgrade{}, err
|
||||
}
|
||||
u.RollOut = policy == "roll-out"
|
||||
return u, nil
|
||||
return upgradeFrom(chosen, together, why, by, manifest), nil
|
||||
}
|
||||
|
||||
// SetUpgradeOf records what to do when this module moves.
|
||||
// Upgrades is every module's policy, by name: what `upgrade` lists.
|
||||
func (i *Inventory) Upgrades(ctx context.Context) (map[string]Upgrade, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select name, upgrade, upgrade_together, upgrade_why, upgrade_by, manifest from module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]Upgrade{}
|
||||
for rows.Next() {
|
||||
var name, why, by string
|
||||
var chosen *string
|
||||
var together bool
|
||||
var manifest []byte
|
||||
if err := rows.Scan(&name, &chosen, &together, &why, &by, &manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[name] = upgradeFrom(chosen, together, why, by, manifest)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ErrBusIsPlanned is a person asking the bus to be rolled out: its upgrade is a planned step (ADR 0235).
|
||||
var ErrBusIsPlanned = errors.New("the bus is never rolled out: its upgrade is a planned step a person starts " +
|
||||
"with `bus upgrade`, which snapshots its streams first and checks them after")
|
||||
|
||||
// SetUpgradeOf records a person's choice of what to do when this module moves, with why and who. A
|
||||
// record says why; the bus is refused a roll-out.
|
||||
func (i *Inventory) SetUpgradeOf(ctx context.Context, module string, u Upgrade) error {
|
||||
policy := "record"
|
||||
if u.RollOut {
|
||||
policy = "roll-out"
|
||||
var manifest []byte
|
||||
err := i.store.Pool().QueryRow(ctx, `select manifest from module where name = $1`, module).Scan(&manifest)
|
||||
if err == nil {
|
||||
var m catalogue.Manifest
|
||||
if json.Unmarshal(manifest, &m) == nil && catalogue.ProvidesBus(m) {
|
||||
return fmt.Errorf("%s provides the mesh's bus: %w", module, ErrBusIsPlanned)
|
||||
}
|
||||
}
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update module set upgrade = $2, upgrade_together = $3 where name = $1`,
|
||||
module, policy, u.Together)
|
||||
`update module set upgrade = $2, upgrade_together = $3, upgrade_why = $4, upgrade_by = $5 where name = $1`,
|
||||
module, policy, u.Together, u.Why, u.By)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("this mesh holds no module called %s", module)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearUpgradeOf takes a person's choice back: the module's own word, or the default, decides again.
|
||||
func (i *Inventory) ClearUpgradeOf(ctx context.Context, module string) error {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update module set upgrade = null, upgrade_together = false, upgrade_why = '', upgrade_by = '' where name = $1`,
|
||||
module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1258,18 +1355,22 @@ type CurrentBuild struct {
|
||||
// it carried, and what a push compares a machine's last send against.
|
||||
func (i *Inventory) CurrentBuilds(ctx context.Context) (map[string]CurrentBuild, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select name, coalesce(built_from, ''), upgrade = 'roll-out' from module`)
|
||||
`select name, coalesce(built_from, ''), upgrade, upgrade_together, upgrade_why, upgrade_by, manifest from module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]CurrentBuild{}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var name, why, by string
|
||||
var chosen *string
|
||||
var together bool
|
||||
var manifest []byte
|
||||
var b CurrentBuild
|
||||
if err := rows.Scan(&name, &b.Commit, &b.RollOut); err != nil {
|
||||
if err := rows.Scan(&name, &b.Commit, &chosen, &together, &why, &by, &manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b.RollOut = upgradeFrom(chosen, together, why, by, manifest).RollOut
|
||||
out[name] = b
|
||||
}
|
||||
return out, rows.Err()
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The gate's verdicts (novox/hq ADR 0235, to-be 45 §8): what a build did on its first machine, and,
|
||||
// for one that failed there, how it was put back. One row per build, written by the plan that rolled it
|
||||
// out, under the lease.
|
||||
|
||||
// The gate's verdicts and a failed build's rollback.
|
||||
const (
|
||||
GatePassed = "passed"
|
||||
GateFailed = "failed"
|
||||
|
||||
RollingBack = "rolling-back"
|
||||
RolledBack = "rolled-back"
|
||||
NotRolledBack = "not-rolled-back"
|
||||
)
|
||||
|
||||
// GateVerdict is one build's verdict at its gate.
|
||||
type GateVerdict struct {
|
||||
Build string
|
||||
Module string
|
||||
Commit string
|
||||
Previous string
|
||||
Plan string
|
||||
Machines []string
|
||||
Verdict string
|
||||
Rollback string
|
||||
Why string
|
||||
Component string
|
||||
// JudgingFrom is when the first machine reported the build applied and the judging began.
|
||||
JudgingFrom *time.Time
|
||||
JudgedAt time.Time
|
||||
Epoch uint64
|
||||
}
|
||||
|
||||
// RecordGate writes a build's verdict. **A failed build's row is written once**: a second failure for
|
||||
// the same build is refused with ErrGateKept, which is what keeps a rollback to one per build — the row
|
||||
// is written before the rollback's send, and a controller replaced in between finds it.
|
||||
func (i *Inventory) RecordGate(ctx context.Context, v GateVerdict) error {
|
||||
epoch, err := i.actingEpoch(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the gate's verdict on %s is not written: %w", v.Build, err)
|
||||
}
|
||||
if v.Machines == nil {
|
||||
v.Machines = []string{}
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`insert into build_gate (build, module, commit_hash, previous, plan, machines, verdict, rollback, why,
|
||||
component, judging_from, judged_at, epoch)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), $12)
|
||||
on conflict (build) do update set verdict = excluded.verdict, rollback = excluded.rollback,
|
||||
why = excluded.why, previous = excluded.previous, machines = excluded.machines,
|
||||
judged_at = now(), epoch = excluded.epoch
|
||||
where build_gate.verdict = 'passed' and excluded.verdict = 'passed'`,
|
||||
v.Build, v.Module, v.Commit, v.Previous, v.Plan, v.Machines, v.Verdict, v.Rollback, v.Why, v.Component,
|
||||
v.JudgingFrom, epoch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%w: %s", ErrGateKept, v.Build)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrGateKept is a verdict already kept for the build, which is not written over.
|
||||
var ErrGateKept = errors.New("this build's verdict at its gate is already kept")
|
||||
|
||||
// SetRollback records how a failed build's rollback went.
|
||||
func (i *Inventory) SetRollback(ctx context.Context, build, rollback, why string) error {
|
||||
if _, err := i.actingEpoch(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update build_gate set rollback = $2, why = $3, judged_at = now() where build = $1 and verdict = 'failed'`,
|
||||
build, rollback, why)
|
||||
return err
|
||||
}
|
||||
|
||||
// GateOf is a build's verdict, and whether it has one.
|
||||
func (i *Inventory) GateOf(ctx context.Context, build string) (GateVerdict, bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, gateSelect+` where build = $1`, build)
|
||||
if err != nil {
|
||||
return GateVerdict{}, false, err
|
||||
}
|
||||
list, err := scanGates(rows)
|
||||
if err != nil || len(list) == 0 {
|
||||
return GateVerdict{}, false, err
|
||||
}
|
||||
return list[0], true, nil
|
||||
}
|
||||
|
||||
// GateFailed is whether a build failed its gate: one the mesh never registers or sends again on its own.
|
||||
func (i *Inventory) GateFailed(ctx context.Context, build string) (bool, error) {
|
||||
v, found, err := i.GateOf(ctx, build)
|
||||
return found && v.Verdict == GateFailed, err
|
||||
}
|
||||
|
||||
// LatestGates is the newest verdict of every module that has one: what the gate probe (DG) reads.
|
||||
func (i *Inventory) LatestGates(ctx context.Context) ([]GateVerdict, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select distinct on (module) build, module, commit_hash, previous, plan,
|
||||
machines, verdict, rollback, why, component, judging_from, judged_at, coalesce(epoch, 0)
|
||||
from build_gate order by module, judged_at desc`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanGates(rows)
|
||||
}
|
||||
|
||||
// Gates is the newest verdicts, newest first: what `plans gates` lists.
|
||||
func (i *Inventory) Gates(ctx context.Context, limit int) ([]GateVerdict, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, gateSelect+` order by judged_at desc limit $1`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanGates(rows)
|
||||
}
|
||||
|
||||
const gateSelect = `select build, module, commit_hash, previous, plan, machines, verdict, rollback, why, component,
|
||||
judging_from, judged_at, coalesce(epoch, 0) from build_gate`
|
||||
|
||||
func scanGates(rows pgx.Rows) ([]GateVerdict, error) {
|
||||
defer rows.Close()
|
||||
var out []GateVerdict
|
||||
for rows.Next() {
|
||||
var v GateVerdict
|
||||
var epoch int64
|
||||
if err := rows.Scan(&v.Build, &v.Module, &v.Commit, &v.Previous, &v.Plan, &v.Machines, &v.Verdict,
|
||||
&v.Rollback, &v.Why, &v.Component, &v.JudgingFrom, &v.JudgedAt, &epoch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
v.Epoch = uint64(epoch)
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// PreviousBuild is the build a module goes back to when a build of it fails its gate: the newest build
|
||||
// that worked, made from the commit the first machine ran before, asked before the failed one, and not
|
||||
// itself failed at a gate. Among the builds whose artifacts the mesh keeps (KeptBuilds): an older one
|
||||
// may already be gone from the artifact store. False when there is none to go back to.
|
||||
func (i *Inventory) PreviousBuild(ctx context.Context, module, commit string, failed Build) (Build, bool, error) {
|
||||
builds, err := i.Builds(ctx, module, 50)
|
||||
if err != nil {
|
||||
return Build{}, false, err
|
||||
}
|
||||
kept := 0
|
||||
for _, b := range builds {
|
||||
if !b.Worked() {
|
||||
continue
|
||||
}
|
||||
kept++
|
||||
if kept > KeptBuilds {
|
||||
break
|
||||
}
|
||||
if b.ID == failed.ID || (commit != "" && b.Commit != commit) {
|
||||
continue
|
||||
}
|
||||
if !failed.AskedOrAt().IsZero() && !b.AskedOrAt().Before(failed.AskedOrAt()) {
|
||||
continue
|
||||
}
|
||||
if bad, err := i.GateFailed(ctx, b.ID); err != nil {
|
||||
return Build{}, false, err
|
||||
} else if bad {
|
||||
continue
|
||||
}
|
||||
var manifest []byte
|
||||
if err := i.store.Pool().QueryRow(ctx, `select manifest from build where id = $1`, b.ID).Scan(&manifest); err != nil {
|
||||
return Build{}, false, err
|
||||
}
|
||||
if len(manifest) == 0 || string(manifest) == "null" {
|
||||
continue
|
||||
}
|
||||
b.Manifest = manifest
|
||||
return b, true, nil
|
||||
}
|
||||
return Build{}, false, nil
|
||||
}
|
||||
|
||||
// RestoreModule puts a module's registered build back to an earlier one: its manifest, the commit it
|
||||
// was built from, and when it was asked — as now, so the build that failed its gate, asked before, can
|
||||
// never register over it again (issue 219's order). The source's head is left where the merge moved it:
|
||||
// the module IS behind its source, and `status` says so.
|
||||
func (i *Inventory) RestoreModule(ctx context.Context, b Build) error {
|
||||
if _, err := i.actingEpoch(ctx); err != nil {
|
||||
return fmt.Errorf("%s is not put back: %w", b.Module, err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(b.Manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's build %s is not a manifest the mesh can register again: %w", b.Module, b.ID, err)
|
||||
}
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update module set manifest = $2, version = nullif($3, ''), built_from = nullif($4, ''),
|
||||
built_asked = now(), registered = now()
|
||||
where name = $1`, b.Module, raw, m.Version, b.Commit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%w: %s", ErrNoSuchModule, b.Module)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
-- A module's build rolls out one machine first, judged at a gate, and rolls back there when the gate
|
||||
-- fails (novox/hq ADR 0235, to-be 45 §8, Phase 4).
|
||||
--
|
||||
-- 1. The upgrade policy becomes a person's choice over the module's own word. Until now every module
|
||||
-- held a policy here, 'record' unless a person had said 'roll-out', and nothing distinguished a
|
||||
-- 'record' somebody chose from the default it always was. From here a null policy is no choice: the
|
||||
-- module's manifest decides (its `upgrade`, its data, whether it is the bus), and its default is to
|
||||
-- roll out. A 'roll-out' a person chose is kept as their choice. A 'record' is the old default and
|
||||
-- becomes no choice — ADR 0235 decides it, and lists every module's resulting policy; a person who
|
||||
-- wants one held again says so with `upgrade <module> record --why`, which is kept with its why.
|
||||
alter table module alter column upgrade drop not null;
|
||||
alter table module alter column upgrade drop default;
|
||||
alter table module drop constraint if exists module_upgrade_check;
|
||||
alter table module add constraint module_upgrade_chosen check (upgrade is null or upgrade in ('record', 'roll-out'));
|
||||
update module set upgrade = null where upgrade = 'record';
|
||||
-- Why the person chose it, and who: said back by `upgrade`, so a held module says why it is held.
|
||||
alter table module add column upgrade_why text not null default '';
|
||||
alter table module add column upgrade_by text not null default '';
|
||||
|
||||
-- 2. The gate's verdict on each build a plan rolled out, one row per build: passed on its first machine,
|
||||
-- or failed there and rolled back. **A build that failed its gate is marked, and is never sent again
|
||||
-- automatically**: registration refuses it, and the rollback is attempted once per build — the row is
|
||||
-- written before the rollback's send, so a controller replaced in between does not send it twice.
|
||||
create table build_gate (
|
||||
build text primary key,
|
||||
module text not null,
|
||||
-- The commit the build was made from, and the one the module was put back to.
|
||||
commit_hash text not null default '',
|
||||
previous text not null default '',
|
||||
plan text not null default '',
|
||||
-- The machines it was judged on: the first machine, and the bus holder when it went with it.
|
||||
machines text[] not null default '{}',
|
||||
-- 'passed', or 'failed'; and for a failed one how the rollback went: 'rolling-back', 'rolled-back',
|
||||
-- or 'not-rolled-back' (no previous build to put back, or the send refused), said in `why`.
|
||||
verdict text not null check (verdict in ('passed', 'failed')),
|
||||
rollback text not null default '' check (rollback in ('', 'rolling-back', 'rolled-back', 'not-rolled-back')),
|
||||
why text not null default '',
|
||||
-- The core component it is, when it is one: mesh-controller, mesh-host, node-tools.
|
||||
component text not null default '',
|
||||
judging_from timestamptz,
|
||||
judged_at timestamptz not null default now(),
|
||||
epoch bigint
|
||||
);
|
||||
create index build_gate_module on build_gate (module, judged_at desc);
|
||||
|
||||
-- 3. The bus's planned step (to-be 45 §8): a bus upgrade is never rolled out; a person starts it, with
|
||||
-- why, after its streams are snapshotted, and it is checked after. One row per step; the open one is
|
||||
-- the step running, which the self-check says as `bus-maintenance` until the bus is healthy again or
|
||||
-- the step's bound passes and it is said failed, with its snapshot as the way back.
|
||||
create table bus_step (
|
||||
id bigserial primary key,
|
||||
module text not null,
|
||||
machines text[] not null default '{}',
|
||||
from_build text not null default '',
|
||||
to_build text not null default '',
|
||||
-- Where the streams' snapshot is: taken by the mesh, or one a person says they took.
|
||||
snapshot text not null,
|
||||
-- Whether the new version can be reverted by putting the old one back, as the person said it.
|
||||
reversible boolean not null,
|
||||
by_whom text not null default '',
|
||||
why text not null,
|
||||
started timestamptz not null default now(),
|
||||
ended timestamptz,
|
||||
-- '', then 'done' or 'failed', with what was found.
|
||||
outcome text not null default '' check (outcome in ('', 'done', 'failed')),
|
||||
found text not null default ''
|
||||
);
|
||||
@@ -67,6 +67,42 @@ type PlanModule struct {
|
||||
// its module's name included. Empty in a plan from before it was kept, which is matched by
|
||||
// module, or by repository and path, as before.
|
||||
Build string `json:"build,omitempty"`
|
||||
// Previous is the build the first machine ran of this module before the plan sent it the new one —
|
||||
// the commit its last send carried (ADR 0221) — kept at the first send: what a rollback puts back
|
||||
// (novox/hq ADR 0235). Empty when the machine had never been sent the module, or what it was sent
|
||||
// is not known.
|
||||
Previous string `json:"previous,omitempty"`
|
||||
// Gate is the new build's judging on its first machine (novox/hq ADR 0235, to-be 45 §8), kept so a
|
||||
// controller replaced mid-judging resumes it, and read back through `plans` as the rollout's record.
|
||||
Gate *PlanGate `json:"gate,omitempty"`
|
||||
}
|
||||
|
||||
// PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine,
|
||||
// from and to which build, the verdict, how long it took to reach it, and whether it was rolled back.
|
||||
type PlanGate struct {
|
||||
// Component is the core component the module is — mesh-controller, mesh-host, node-tools — or empty
|
||||
// for any other module, judged by its own health.
|
||||
Component string `json:"component,omitempty"`
|
||||
Machines []string `json:"machines"`
|
||||
From string `json:"from,omitempty"`
|
||||
To string `json:"to,omitempty"`
|
||||
// Since is when the judging began: the first machine reported the new build applied.
|
||||
Since *time.Time `json:"since,omitempty"`
|
||||
// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
|
||||
// does not resets them.
|
||||
Passes int `json:"passes,omitempty"`
|
||||
LastPass *time.Time `json:"last_pass,omitempty"`
|
||||
// Last is what the newest judging found wanting, while it still may pass.
|
||||
Last string `json:"last,omitempty"`
|
||||
// Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.
|
||||
Verdict string `json:"verdict,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
JudgedAt *time.Time `json:"judged_at,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
// Rollback is how a failed build was put back: rolled-back, or not-rolled-back with why.
|
||||
Rollback string `json:"rollback,omitempty"`
|
||||
// Kept says a passing verdict was written to the gate's records.
|
||||
Kept bool `json:"kept,omitempty"`
|
||||
}
|
||||
|
||||
// The states a plan passes through.
|
||||
|
||||
@@ -72,7 +72,8 @@ func TestTheCurrentBuildsAreTheCatalogues(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := current["resolver"]; got != (CurrentBuild{Commit: "c1"}) {
|
||||
// Rolled out by default, one machine first and gated (novox/hq ADR 0235).
|
||||
if got := current["resolver"]; got != (CurrentBuild{Commit: "c1", RollOut: true}) {
|
||||
t.Errorf("resolver is at %+v", got)
|
||||
}
|
||||
if got := current["by-hand"]; got != (CurrentBuild{RollOut: true}) {
|
||||
|
||||
Reference in New Issue
Block a user