Gate a release plan's first machine and roll a failed build back there (hq ADR 0236)
A build that reported applied was sent everywhere; one that then did nothing, served no tools or broke its machine's word reached every machine. Now the first machine is judged by the component's health (the core's definitions, as doctor probes H-*, or a module's own) three times over two minutes within ten; a failing gate puts the previous build back there once, marks the build, and says it as a condition and an event. Upgrades roll out by default; the bus is a planned step; a module deleted at its source is not built (the public-acme plan failure).
This commit is contained in:
jochen
@@ -0,0 +1,327 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0235).
|
||||
//
|
||||
// **A bus upgrade is never rolled out.** The bus carries every declaration, every report and the
|
||||
// controller's own lease; a new bus build that does not come up is a mesh nobody can tell anything,
|
||||
// and a version whose data format moved (2.10 → 2.11) cannot be undone by putting the old one back.
|
||||
// So nothing sends a new bus build on its own: its policy is `record` whatever anyone says (the
|
||||
// catalogue's DerivedUpgrade, and `upgrade` refuses a roll-out), a plan builds it and sends nothing, a
|
||||
// cascade holds the machine (ADR 0221), and a push naming that machine is refused while a new bus build
|
||||
// waits for it (busHeld). The one way is this verb: a person, with why, the streams snapshotted first,
|
||||
// whether it can be reverted said before it starts, the step said as `bus-maintenance` while it runs,
|
||||
// and every stream, durable consumer and a round trip checked after (H-bus) — or the step said failed,
|
||||
// with its snapshot as the way back.
|
||||
|
||||
// busStepProbe is the registry's row for the step; its kinds.
|
||||
const (
|
||||
busStepProbe = "DB"
|
||||
kindBusMaintenance = "bus-maintenance"
|
||||
kindBusUpgradeFailed = "bus-upgrade-failed"
|
||||
)
|
||||
|
||||
// busStepBound is how long after its start a bus upgrade must be followed by a healthy bus.
|
||||
var busStepBound = 15 * time.Minute
|
||||
|
||||
// takeBusSnapshot snapshots every stream before a bus upgrade and answers where the snapshot is. Nil
|
||||
// until the controller's JetStream snapshot is built (to-be 45 §8: "the streams are snapshotted"); until
|
||||
// then a person takes it by hand and says where with --snapshot-taken.
|
||||
var takeBusSnapshot func(ctx context.Context) (string, error)
|
||||
|
||||
// busPending is what a bus upgrade would do: the bus's module, the machines running it, and, per
|
||||
// machine, the build it was last sent against the build the mesh holds. Empty machines: the mesh holds
|
||||
// no bus module.
|
||||
type busPending struct {
|
||||
module string
|
||||
machines []string
|
||||
from map[string]string
|
||||
to string
|
||||
}
|
||||
|
||||
// moves is whether sending the machine would replace its bus.
|
||||
func (b busPending) moves(machine string) bool {
|
||||
from, known := b.from[machine]
|
||||
return b.module != "" && b.to != "" && (!known || !sameCommit(from, b.to))
|
||||
}
|
||||
|
||||
// pendingBus reads what a bus upgrade would do.
|
||||
func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, error) {
|
||||
var b busPending
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
for name, m := range shelf {
|
||||
if catalogue.ProvidesBus(m) {
|
||||
b.module = name
|
||||
}
|
||||
}
|
||||
if b.module == "" {
|
||||
return b, nil
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
b.to = current[b.module].Commit
|
||||
if b.machines, err = inv.Running(ctx, b.module); err != nil {
|
||||
return b, err
|
||||
}
|
||||
b.from = map[string]string{}
|
||||
for _, n := range b.machines {
|
||||
sent, known, err := inv.SentBuilds(ctx, n)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
if known {
|
||||
if c, carried := sent[b.module]; carried {
|
||||
b.from[n] = c
|
||||
}
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// busHeld names the machines a push may not send because sending them would replace the bus: the
|
||||
// planned step's, not a push's (ADR 0235). Said with the remedy.
|
||||
func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) {
|
||||
b, err := pendingBus(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, n := range machines {
|
||||
for _, holder := range b.machines {
|
||||
if n == holder && b.moves(n) {
|
||||
out[n] = fmt.Sprintf("sending %s would replace the bus (%s %s → %s), which is a planned step: "+
|
||||
"`bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0235)",
|
||||
n, b.module, short(orNotKnown(b.from[n])), short(b.to))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// busCommand is `bus` — what a bus upgrade would do and how the last went — and `bus upgrade`.
|
||||
func busCommand(ctx context.Context, args []string) error {
|
||||
sub := ""
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
sub, args = args[0], args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("bus", flag.ContinueOnError)
|
||||
snapshot := set.String("snapshot-taken", "", "where the streams' snapshot a person took is, while the mesh takes none itself")
|
||||
reversible := set.Bool("reversible", false, "the new version can be undone by putting the old one back")
|
||||
irreversible := set.Bool("irreversible", false, "the new version cannot be undone by putting the old one back, "+
|
||||
"and this is the person's explicit word that it runs anyway")
|
||||
why := addHandActFlags(set)
|
||||
if rest, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
} else if len(rest) > 0 {
|
||||
return errors.New("bus [upgrade --why … --reversible|--irreversible [--snapshot-taken <where>]]")
|
||||
}
|
||||
switch sub {
|
||||
case "":
|
||||
return busStatus(ctx)
|
||||
case "upgrade":
|
||||
default:
|
||||
return fmt.Errorf("bus says what a bus upgrade would do, or `bus upgrade` — not %q", sub)
|
||||
}
|
||||
// Everything refused before anything is done.
|
||||
if err := why.require("bus upgrade"); err != nil {
|
||||
return err
|
||||
}
|
||||
if *reversible == *irreversible {
|
||||
return errors.New("bus upgrade says, before it starts, whether the new version can be undone by putting the " +
|
||||
"old one back: --reversible, or --irreversible as your explicit word that it runs anyway (to-be 45 §8). " +
|
||||
"Nothing was done")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
b, err := pendingBus(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b.module == "" {
|
||||
return errors.New("the mesh holds no module that provides its bus: there is nothing to upgrade")
|
||||
}
|
||||
var moving []string
|
||||
for _, n := range b.machines {
|
||||
if b.moves(n) {
|
||||
moving = append(moving, n)
|
||||
}
|
||||
}
|
||||
if len(moving) == 0 {
|
||||
fmt.Printf("every machine running %s runs the build the mesh holds (%s): nothing to upgrade\n", b.module, short(b.to))
|
||||
return nil
|
||||
}
|
||||
where := strings.TrimSpace(*snapshot)
|
||||
switch {
|
||||
case takeBusSnapshot != nil:
|
||||
if where, err = takeBusSnapshot(ctx); err != nil {
|
||||
return fmt.Errorf("the streams could not be snapshotted, so the bus is not replaced: %w", err)
|
||||
}
|
||||
case where == "":
|
||||
return errors.New("the bus is replaced only after its streams are snapshotted. The mesh does not take the " +
|
||||
"snapshot itself yet (to-be 45 §8: the controller's JetStream snapshot); take one by hand and say where " +
|
||||
"with --snapshot-taken <where>. Nothing was done")
|
||||
}
|
||||
from := map[string]bool{}
|
||||
for _, n := range moving {
|
||||
from[orNotKnown(b.from[n])] = true
|
||||
}
|
||||
step, err := inv.StartBusStep(ctx, inventory.BusStep{Module: b.module, Machines: moving,
|
||||
From: strings.Join(sortedKeys(from), ", "), To: b.to, Snapshot: where, Reversible: *reversible,
|
||||
By: link.Caller(), Why: strings.TrimSpace(*why.why)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cause := "bus-upgrade"
|
||||
if strings.TrimSpace(*why.cause) == "" {
|
||||
why.cause = &cause
|
||||
}
|
||||
why.record(ctx, "bus upgrade", append([]string{b.module}, moving...))
|
||||
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
|
||||
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
|
||||
false: "NOT reversible: the snapshot is the only way back"}[*reversible])
|
||||
sent, err := sendRollout(ctx, open, moving)
|
||||
if err != nil {
|
||||
_ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error())
|
||||
return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err)
|
||||
}
|
||||
fmt.Printf("sent %s; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+
|
||||
"consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot "+
|
||||
"as the way back. `bus` says how it went\n", strings.Join(sent, ", "), busStepBound)
|
||||
return nil
|
||||
}
|
||||
|
||||
// busStatus is `bus`: what an upgrade would do, and the last step.
|
||||
func busStatus(ctx context.Context) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
b, err := pendingBus(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if b.module == "" {
|
||||
fmt.Println("the mesh holds no module that provides its bus")
|
||||
} else {
|
||||
fmt.Printf("the bus is %s, built %s, on %s; never rolled out — a planned step (`bus upgrade`)\n", b.module,
|
||||
short(b.to), orNone(strings.Join(b.machines, ", ")))
|
||||
for _, n := range b.machines {
|
||||
state := "runs it"
|
||||
if b.moves(n) {
|
||||
state = "runs " + short(orNotKnown(b.from[n])) + ": `bus upgrade` replaces it"
|
||||
}
|
||||
fmt.Printf(" %-10s %s\n", n, state)
|
||||
}
|
||||
}
|
||||
if takeBusSnapshot == nil {
|
||||
fmt.Println(" the mesh takes no snapshot of the streams itself yet: `bus upgrade` asks where yours is (--snapshot-taken)")
|
||||
}
|
||||
s, found, err := open.inventory.LatestBusStep(ctx)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
state := "running since " + s.Started.Local().Format("2006-01-02 15:04")
|
||||
if s.Ended != nil {
|
||||
state = s.Outcome + " at " + s.Ended.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
fmt.Printf("last step %d: %s → %s on %s by %s (%s): %s; snapshot %s\n", s.ID, s.From, short(s.To),
|
||||
strings.Join(s.Machines, ", "), orNone(s.By), s.Why, state, s.Snapshot)
|
||||
if s.Found != "" {
|
||||
fmt.Printf(" %s\n", s.Found)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeBusStep is DB: a bus upgrade running is said as `bus-maintenance`; the bus healthy again after
|
||||
// the machines reported the new build ends it done; past its bound, unhealthy, it ends failed and is
|
||||
// said — urgent, with its snapshot — while the bus is still not healthy.
|
||||
func probeBusStep(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
inv := d.open.inventory
|
||||
s, found, err := inv.LatestBusStep(ctx)
|
||||
if err != nil || !found {
|
||||
return nil, err
|
||||
}
|
||||
if s.Ended != nil && s.Outcome != "failed" {
|
||||
return nil, nil
|
||||
}
|
||||
problems, err := busHealth(ctx, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
applied := true
|
||||
for _, r := range reports {
|
||||
for _, n := range s.Machines {
|
||||
if r.Node == n && (!r.Current || r.Outcome != inventory.OutcomeApplied || r.At == nil || r.At.Before(s.Started)) {
|
||||
applied = false
|
||||
problems = append(problems, n+" has not reported the new bus applied")
|
||||
}
|
||||
}
|
||||
}
|
||||
id := s.Module
|
||||
if s.Ended == nil {
|
||||
switch {
|
||||
case applied && len(problems) == 0:
|
||||
return nil, inv.EndBusStep(ctx, s.ID, "done", "the bus answered healthy after the upgrade")
|
||||
case time.Since(s.Started) > busStepBound:
|
||||
found := strings.Join(problems, "; ")
|
||||
if err := inv.EndBusStep(ctx, s.ID, "failed", found); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Found = found
|
||||
default:
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "maintenance",
|
||||
Kind: kindBusMaintenance, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the bus is being upgraded (step %d, %s → %s on %s, by %s: %s); its snapshot is %s",
|
||||
s.ID, s.From, short(s.To), strings.Join(s.Machines, ", "), orNone(s.By), s.Why, s.Snapshot),
|
||||
Said: orNone(strings.Join(problems, "; "))}}, nil
|
||||
}
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil, nil // failed, and healthy since: nothing wrong now
|
||||
}
|
||||
way := "put the old build back"
|
||||
if !s.Reversible {
|
||||
way = "restore the snapshot"
|
||||
}
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "upgrade-failed",
|
||||
Kind: kindBusUpgradeFailed, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the bus upgrade (step %d, %s → %s) did not end healthy within %s: %s — the way back is to %s (%s)",
|
||||
s.ID, s.From, short(s.To), busStepBound, strings.Join(problems, "; "), way, s.Snapshot)}}, nil
|
||||
}
|
||||
|
||||
func sortedKeys(set map[string]bool) []string {
|
||||
out := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user