Gate a release plan's first machine and roll a failed build back there (hq ADR 0236)
A build that reported applied was sent everywhere; one that then did nothing, served no tools or broke its machine's word reached every machine. Now the first machine is judged by the component's health (the core's definitions, as doctor probes H-*, or a module's own) three times over two minutes within ten; a failing gate puts the previous build back there once, marks the build, and says it as a condition and an event. Upgrades roll out by default; the bus is a planned step; a module deleted at its source is not built (the public-acme plan failure).
This commit is contained in:
jochen
@@ -0,0 +1,692 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The gate on a release plan's first machine, and the rollback after it (novox/hq ADR 0235, to-be 45
|
||||
// §8, ADR 0227 rule 8).
|
||||
//
|
||||
// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once
|
||||
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
|
||||
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
|
||||
// the component's health — the core's health definitions, or a module's own — passing three times over
|
||||
// at least two minutes, within ten minutes of the apply. Only then are the rest sent.
|
||||
//
|
||||
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
|
||||
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
|
||||
// build goes back to the one the first machine ran before, and that machine is sent it: the ordinary
|
||||
// path, again. Once per build: the verdict is written before the send, and a verdict once written is
|
||||
// not written over. Said as a condition, urgent for the core and when it could not be put back, and as
|
||||
// the event `rolled-back`.
|
||||
|
||||
// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
|
||||
var (
|
||||
// gateSettle is how long the first machine must stay healthy, at the least.
|
||||
gateSettle = 2 * time.Minute
|
||||
// gateBound is how long after the apply the build may take to become healthy.
|
||||
gateBound = 10 * time.Minute
|
||||
// gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least.
|
||||
gatePasses = 3
|
||||
gateEvery = 40 * time.Second
|
||||
)
|
||||
|
||||
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
|
||||
// are raised by a plan as it judges, and kept or cleared by the probe on every run.
|
||||
const gateProbe = "DG"
|
||||
|
||||
// The kinds a gate raises.
|
||||
const (
|
||||
kindRolledBack = "rolled-back"
|
||||
kindRollbackFailed = "rollback-failed"
|
||||
)
|
||||
|
||||
// coreComponent is the core component a module is, as a witness names it — controller, node-engine,
|
||||
// node-tools — or empty: the core is judged by its health definitions, anything else by its own health.
|
||||
func coreComponent(module string) string {
|
||||
switch module {
|
||||
case catalogue.ControllerSeatName:
|
||||
return lease.ComponentController
|
||||
case hostModule:
|
||||
return lease.ComponentEngine
|
||||
case broker.RuntimeModule:
|
||||
return lease.ComponentNodeTools
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// health is a judging's word on one machine.
|
||||
type health int
|
||||
|
||||
const (
|
||||
healthGood health = iota
|
||||
healthNotYet
|
||||
healthBroken
|
||||
)
|
||||
|
||||
// served is what one machine's node tools answered the bus's discovery with.
|
||||
type served struct {
|
||||
runtime bool
|
||||
tools map[string]bool
|
||||
}
|
||||
|
||||
// gateFacts is what one judging reads, gathered once for every machine it judges.
|
||||
type gateFacts struct {
|
||||
now time.Time
|
||||
reports map[string]inventory.Reported
|
||||
// engines is each machine's node-engine build as it last reported it.
|
||||
engines map[string]string
|
||||
// served is what the bus's discovery answered; servedErr why it could not be asked.
|
||||
served map[string]served
|
||||
servedErr error
|
||||
// open are the open conditions; openErr why they could not be read (nil keeper: not judged).
|
||||
open []conditions.Condition
|
||||
openErr error
|
||||
judged bool
|
||||
// rolledBack is what each machine's witnesses say they decided.
|
||||
rolledBack map[string][]lease.Rollback
|
||||
// holder is who holds the controller lease, for judging the controller.
|
||||
holder *lease.Holder
|
||||
holderErr error
|
||||
}
|
||||
|
||||
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
|
||||
// variable so a test can hand a judging its facts.
|
||||
var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
inv := open.inventory
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: witnessed.all()}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, n := range nodes {
|
||||
f.engines[n.Name] = n.HostVersion
|
||||
}
|
||||
if d := doctorFrom; d != nil {
|
||||
if d.keeper != nil {
|
||||
f.judged = true
|
||||
f.open, f.openErr = d.keeper.Open(ctx)
|
||||
}
|
||||
if d.js != nil {
|
||||
f.served, f.servedErr = servedOnTheBus(ctx, d.js.Conn())
|
||||
} else {
|
||||
f.servedErr = errors.New("this controller has no bus to ask")
|
||||
}
|
||||
} else {
|
||||
f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what")
|
||||
}
|
||||
if component == lease.ComponentController {
|
||||
h, found, err := theLease.holder(ctx)
|
||||
switch {
|
||||
case err != nil:
|
||||
f.holderErr = err
|
||||
case found:
|
||||
f.holder = &h
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// judgeHealth is one machine's health for a module's new build, from what one judging read: healthy,
|
||||
// not yet (with what is wanting), or healthBroken — a witness put it back, or the machine refused or failed
|
||||
// what it was sent. Pure.
|
||||
func judgeHealth(module, component string, m catalogue.Manifest, machine string, since time.Time, f gateFacts) (health, string) {
|
||||
// A witness's verdict made since the build was sent: the build failed its health there. One that
|
||||
// could not judge at all (unwitnessed) is not a verdict on the build.
|
||||
for _, r := range f.rolledBack[machine] {
|
||||
if component == "" || r.Component != component || r.Outcome == lease.OutcomeUnwitnessed || r.At.Before(since) {
|
||||
continue
|
||||
}
|
||||
return healthBroken, fmt.Sprintf("the witness on %s judged the %s %s and %s: %s", machine, r.Component,
|
||||
short(r.From), r.Outcome, r.Why)
|
||||
}
|
||||
r, said := f.reports[machine]
|
||||
switch {
|
||||
case !said || r.At == nil || !r.Current:
|
||||
return healthNotYet, fmt.Sprintf("%s has not reported on what it was sent", machine)
|
||||
case r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused:
|
||||
return healthBroken, fmt.Sprintf("%s %s what it was sent", machine, r.Outcome)
|
||||
case r.Outcome != inventory.OutcomeApplied:
|
||||
return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome)
|
||||
}
|
||||
// **No new condition about it**: about the machine itself, or naming the module on that machine,
|
||||
// raised since the judging began. The gate's own are not evidence about the build.
|
||||
if f.judged {
|
||||
if f.openErr != nil {
|
||||
return healthNotYet, "what is wrong cannot be read, so whether the build made anything wrong is not known: " +
|
||||
firstLine(f.openErr.Error())
|
||||
}
|
||||
for _, c := range f.open {
|
||||
if c.Source == gateProbe || c.Raised.Before(since) {
|
||||
continue
|
||||
}
|
||||
onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
||||
(c.Subject.Scope == conditions.ScopeMachine && c.Subject.ID == machine)
|
||||
if !onIt {
|
||||
continue
|
||||
}
|
||||
if c.Subject.Scope == conditions.ScopeMachine || slices.Contains(strings.Split(c.Subject.ID, "."), module) {
|
||||
return healthNotYet, fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch component {
|
||||
case lease.ComponentEngine:
|
||||
// The node-engine has reported its current declaration under its own build.
|
||||
want := deliveredVersions(m)
|
||||
if len(want) > 0 && !slices.Contains(want, f.engines[machine]) {
|
||||
return healthNotYet, fmt.Sprintf("%s's node-engine reports build %s, not the new %s", machine,
|
||||
orNotKnown(f.engines[machine]), strings.Join(want, " or "))
|
||||
}
|
||||
case lease.ComponentNodeTools:
|
||||
// The node tools are announced and answer.
|
||||
if f.servedErr != nil {
|
||||
return healthNotYet, "whether the node tools answer cannot be asked: " + firstLine(f.servedErr.Error())
|
||||
}
|
||||
if !f.served[machine].runtime {
|
||||
return healthNotYet, fmt.Sprintf("the node tools on %s do not answer the bus", machine)
|
||||
}
|
||||
case lease.ComponentController:
|
||||
// The new controller holds the lease and says it is ready.
|
||||
switch {
|
||||
case f.holderErr != nil:
|
||||
return healthNotYet, "who holds the controller lease cannot be read: " + firstLine(f.holderErr.Error())
|
||||
case f.holder == nil:
|
||||
return healthNotYet, "no controller holds the lease"
|
||||
case f.holder.Taken.Before(since.Add(-time.Minute)):
|
||||
return healthNotYet, fmt.Sprintf("the lease is held since %s, by a controller older than the new build",
|
||||
f.holder.Taken.UTC().Format(time.RFC3339))
|
||||
case f.holder.Health == nil || !f.holder.Health.Ready:
|
||||
why := "the controller holding the lease does not say it is ready"
|
||||
if f.holder.Health != nil && f.holder.Health.Why != "" {
|
||||
why += ": " + f.holder.Health.Why
|
||||
}
|
||||
return healthNotYet, why
|
||||
}
|
||||
default:
|
||||
// A module's tools answer, where it has any and the machine runs the node tools that serve them.
|
||||
if len(m.Tools) > 0 {
|
||||
if f.servedErr != nil {
|
||||
return healthNotYet, "whether its tools are served cannot be asked: " + firstLine(f.servedErr.Error())
|
||||
}
|
||||
if s := f.served[machine]; s.runtime && !s.tools[module] {
|
||||
return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return healthGood, ""
|
||||
}
|
||||
|
||||
// judgeGate takes one judging of a module's first machines and records it in the plan's gate: a pass
|
||||
// counted, a pass missed (and why), or the verdict. Answers the verdict once there is one.
|
||||
func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
|
||||
running []string, now time.Time) (string, error) {
|
||||
g := state.Gate
|
||||
if g == nil {
|
||||
// Judged from the send: a witness's verdict, a condition, the bound — all counted from when the
|
||||
// first machine was sent the build.
|
||||
start := now
|
||||
if state.FirstAt != nil {
|
||||
start = *state.FirstAt
|
||||
}
|
||||
var machines []string
|
||||
for _, n := range state.First {
|
||||
if slices.Contains(running, n) {
|
||||
machines = append(machines, n)
|
||||
}
|
||||
}
|
||||
g = &inventory.PlanGate{Component: coreComponent(module), Machines: machines, From: state.Previous,
|
||||
To: state.Commit, Since: &start}
|
||||
state.Gate = g
|
||||
}
|
||||
if g.Verdict != "" {
|
||||
return g.Verdict, nil
|
||||
}
|
||||
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
|
||||
return "", nil
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
facts, err := gatherGateFacts(ctx, open, g.Component)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
worst, why := healthGood, ""
|
||||
for _, n := range g.Machines {
|
||||
h, said := judgeHealth(module, g.Component, shelf[module], n, *g.Since, facts)
|
||||
if h > worst {
|
||||
worst, why = h, said
|
||||
} else if h == worst && h != healthGood && why == "" {
|
||||
why = said
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case worst == healthBroken:
|
||||
decide(g, inventory.GateFailed, why, now)
|
||||
case worst == healthNotYet:
|
||||
g.Passes, g.LastPass, g.Last = 0, nil, why
|
||||
if now.Sub(*g.Since) > gateBound {
|
||||
decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
|
||||
}
|
||||
default:
|
||||
g.Passes++
|
||||
g.LastPass, g.Last = &now, ""
|
||||
if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
|
||||
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
|
||||
now.Sub(*g.Since).Round(time.Second)), now)
|
||||
}
|
||||
}
|
||||
return g.Verdict, nil
|
||||
}
|
||||
|
||||
// decide sets a gate's verdict.
|
||||
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
||||
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
||||
g.Took = now.Sub(*g.Since).Round(time.Second).String()
|
||||
}
|
||||
|
||||
// gatePassed keeps a passing build's verdict, so `plans` and the gate's probe can read it.
|
||||
func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule) {
|
||||
g := state.Gate
|
||||
err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
|
||||
Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines,
|
||||
Verdict: inventory.GatePassed, Why: g.Why, Component: g.Component, JudgingFrom: g.Since})
|
||||
if err != nil && state.Build != "" {
|
||||
fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
fmt.Printf("%s: %s passed its gate on %s (%s); the rest are sent\n", p.ID, module,
|
||||
strings.Join(g.Machines, ", "), g.Why)
|
||||
if module == catalogue.ControllerSeatName {
|
||||
carryUserList(ctx, open, p)
|
||||
}
|
||||
}
|
||||
|
||||
// carryUserList sends the machine holding the bus the user list a new controller composes, once that
|
||||
// controller passed its gate (ADR 0235). The controller's own grants travel in that list, and the old
|
||||
// controller composed the list the plan sent; on 2026-10-06 eight pushes by hand carried a new
|
||||
// controller's grant into it. Not when a build its policy or a plan holds back would go with it (ADR
|
||||
// 0221): then it is said, as a push would say it.
|
||||
func carryUserList(ctx context.Context, open *stores, p *inventory.Plan) {
|
||||
holder, behind, err := brokerBehind(ctx, open, nil)
|
||||
if err != nil || holder == "" || !behind {
|
||||
if err != nil {
|
||||
fmt.Printf("%s: whether the bus's user list is behind the new controller cannot be read: %v\n", p.ID, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
held, err := heldMachines(ctx, open, []string{holder})
|
||||
if err != nil {
|
||||
fmt.Printf("%s: whether %s may be sent the new user list cannot be read: %v\n", p.ID, holder, err)
|
||||
return
|
||||
}
|
||||
if why, isHeld := held[holder]; isHeld {
|
||||
fmt.Printf("%s: the new controller's user list is not carried to %s, which holds the bus: %s — `push %s` "+
|
||||
"carries it\n", p.ID, holder, strings.Join(why, "; "), holder)
|
||||
return
|
||||
}
|
||||
if _, err := sendRollout(ctx, open, []string{holder}); err != nil {
|
||||
fmt.Printf("%s: the new controller's user list could not be carried to %s: %v\n", p.ID, holder, err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("%s: carried the new controller's user list to %s, which holds the bus\n", p.ID, holder)
|
||||
}
|
||||
|
||||
// gateFailed stops the plan at a build that failed its gate and puts the previous build back on the
|
||||
// machines it was judged on — once per build, said as a condition and an event. The plan is saved
|
||||
// before the send: a controller that is itself the build being put back does not outlive it.
|
||||
func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
|
||||
machines []string, why string) {
|
||||
inv := open.inventory
|
||||
now := time.Now().UTC()
|
||||
if state.Gate == nil {
|
||||
state.Gate = &inventory.PlanGate{Component: coreComponent(module), Machines: machines,
|
||||
From: state.Previous, To: state.Commit, Since: state.FirstAt}
|
||||
}
|
||||
g := state.Gate
|
||||
if g.Verdict == "" {
|
||||
if g.Since == nil {
|
||||
g.Since = &now
|
||||
}
|
||||
decide(g, inventory.GateFailed, why, now)
|
||||
}
|
||||
if len(g.Machines) == 0 {
|
||||
g.Machines = machines
|
||||
}
|
||||
state.Why = "failed its gate: " + g.Why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s failed its gate on %s in tier %d: %s", module, strings.Join(g.Machines, ", "), p.Tier, g.Why)
|
||||
|
||||
verdict := inventory.GateVerdict{Build: state.Build, Module: module, Commit: state.Commit, Previous: state.Previous,
|
||||
Plan: p.ID, Machines: g.Machines, Verdict: inventory.GateFailed, Rollback: inventory.RollingBack, Why: g.Why,
|
||||
Component: g.Component, JudgingFrom: g.Since}
|
||||
if state.Build == "" {
|
||||
// A plan from before builds were asked by id: nothing to mark, so nothing is put back by the
|
||||
// mesh — said, for a person.
|
||||
g.Rollback = inventory.NotRolledBack
|
||||
p.Note += "; not put back: the plan does not know which build it sent"
|
||||
sayRollback(ctx, open, module, g, "")
|
||||
return
|
||||
}
|
||||
if err := inv.RecordGate(ctx, verdict); err != nil {
|
||||
if errors.Is(err, inventory.ErrGateKept) {
|
||||
// Already judged and acted on, by this controller before a restart or by another: never twice.
|
||||
if kept, found, _ := inv.GateOf(ctx, state.Build); found {
|
||||
g.Rollback = kept.Rollback
|
||||
}
|
||||
p.Note += "; its rollback was already made once and is not made again"
|
||||
return
|
||||
}
|
||||
g.Rollback = inventory.NotRolledBack
|
||||
p.Note += "; not put back: its verdict could not be kept, and a rollback that cannot be counted is not made — " + err.Error()
|
||||
sayRollback(ctx, open, module, g, "")
|
||||
return
|
||||
}
|
||||
// The previous build: the one the first machine ran, from the build records.
|
||||
notBack := func(why string) {
|
||||
g.Rollback = inventory.NotRolledBack
|
||||
p.Note += "; NOT put back: " + why
|
||||
if err := inv.SetRollback(ctx, state.Build, inventory.NotRolledBack, g.Why+"; not put back: "+why); err != nil {
|
||||
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
sayRollback(ctx, open, module, g, why)
|
||||
}
|
||||
if state.Previous == "" {
|
||||
notBack(fmt.Sprintf("%s had not been sent %s before, or what it was sent is not known — `unassign` takes it "+
|
||||
"off, or a newer merge replaces it", strings.Join(g.Machines, ", "), module))
|
||||
return
|
||||
}
|
||||
failed, _, err := inv.BuildByID(ctx, state.Build)
|
||||
if err != nil {
|
||||
notBack("the failed build's record cannot be read: " + err.Error())
|
||||
return
|
||||
}
|
||||
previous, found, err := inv.PreviousBuild(ctx, module, state.Previous, failed)
|
||||
if err != nil {
|
||||
notBack("the build records cannot be read: " + err.Error())
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
notBack(fmt.Sprintf("no build of %s from %s is still kept to put back", module, short(state.Previous)))
|
||||
return
|
||||
}
|
||||
if err := inv.RestoreModule(ctx, previous); err != nil {
|
||||
notBack(err.Error())
|
||||
return
|
||||
}
|
||||
g.Rollback = inventory.RollingBack
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err)
|
||||
}
|
||||
sent, err := sendRollout(ctx, open, g.Machines)
|
||||
if err != nil {
|
||||
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
|
||||
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
|
||||
return
|
||||
}
|
||||
g.Rollback = inventory.RolledBack
|
||||
p.Note += fmt.Sprintf("; put back to %s on %s", short(previous.Commit), strings.Join(sent, ", "))
|
||||
if err := inv.SetRollback(ctx, state.Build, inventory.RolledBack, g.Why); err != nil {
|
||||
fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err)
|
||||
}
|
||||
sayRollback(ctx, open, module, g, "")
|
||||
}
|
||||
|
||||
// rolledBackEvent is the body of `rolled-back` (ADR 0235): a contract, like a condition's events.
|
||||
type rolledBackEvent struct {
|
||||
Event string `json:"event"`
|
||||
At time.Time `json:"at"`
|
||||
Module string `json:"module"`
|
||||
Component string `json:"component,omitempty"`
|
||||
Machines []string `json:"machines"`
|
||||
From string `json:"from,omitempty"`
|
||||
To string `json:"to,omitempty"`
|
||||
Why string `json:"why"`
|
||||
// Rollback is rolled-back, or not-rolled-back with NotWhy.
|
||||
Rollback string `json:"rollback"`
|
||||
NotWhy string `json:"not_why,omitempty"`
|
||||
Show string `json:"show"`
|
||||
}
|
||||
|
||||
// sayRollback raises the gate's condition at once — the probe keeps it from then — and says the event.
|
||||
func sayRollback(ctx context.Context, open *stores, module string, g *inventory.PlanGate, notWhy string) {
|
||||
o := gateObservation(module, g.Component, g.Machines, g.Rollback, g.Why, notWhy, g.To, g.From)
|
||||
fmt.Println(o.Summary)
|
||||
d := doctorFrom
|
||||
if d == nil {
|
||||
return
|
||||
}
|
||||
if d.keeper != nil {
|
||||
o.Source = gateProbe
|
||||
if _, err := d.keeper.Observe(ctx, o); err != nil {
|
||||
fmt.Printf("the gate's condition %s could not be kept: %v\n", o.Key(), err)
|
||||
}
|
||||
}
|
||||
if d.teller == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(rolledBackEvent{Event: link.KeyRolledBack, At: time.Now().UTC(), Module: module,
|
||||
Component: g.Component, Machines: g.Machines, From: g.To, To: g.From, Why: g.Why, Rollback: g.Rollback,
|
||||
NotWhy: notWhy, Show: conditions.Condition{Key: o.Key()}.Show()})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
saying, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := d.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyRolledBack, body); err != nil {
|
||||
fmt.Printf("%s's rollback could NOT be said on the bus: %v\n", module, err)
|
||||
}
|
||||
}
|
||||
|
||||
// gateObservation is a failed gate as a condition: `core.<component>.<machine>.rolled-back` for the
|
||||
// core (to-be 45 §2), `build.<module>.<machine>.rolled-back` for any other module; `rollback-failed`
|
||||
// when it could not be put back. Urgent for the core and for a build left in place; a warning for a
|
||||
// module put back, which runs what it ran before.
|
||||
func gateObservation(module, component string, machines []string, rollback, why, notWhy, failed, previous string) conditions.Observation {
|
||||
machine := strings.Join(machines, ",")
|
||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: module + "." + machine, Kind: kindRolledBack,
|
||||
Token: kindRolledBack, Machine: firstOf(machines), Severity: conditions.Warning, Source: gateProbe,
|
||||
Summary: fmt.Sprintf("%s's build %s failed its gate on %s and was put back to %s: %s", module, short(failed),
|
||||
machine, short(previous), why)}
|
||||
if component != "" {
|
||||
o.Scope, o.ID, o.Severity = conditions.ScopeCore, component+"."+machine, conditions.Urgent
|
||||
}
|
||||
if len(machines) > 1 {
|
||||
o.Also = machines[1:]
|
||||
}
|
||||
if rollback != inventory.RolledBack && rollback != inventory.RollingBack {
|
||||
o.Kind, o.Token, o.Severity = kindRollbackFailed, kindRollbackFailed, conditions.Urgent
|
||||
o.Resolver = conditions.ResolverOperator
|
||||
o.Summary = fmt.Sprintf("%s's build %s failed its gate on %s and was NOT put back: %s — %s", module, short(failed),
|
||||
machine, why, orNone(notWhy))
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// probeGates is DG: no build that failed its gate is left without its condition, and no witness's
|
||||
// rollback goes unsaid. Each module whose newest verdict is a failure keeps its condition; a newer build
|
||||
// that passes clears it. Each core component a machine's witness says it put back is `core.<component>.
|
||||
// <machine>.rolled-back`, urgent, until the machine's reports stop saying it.
|
||||
func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
latest, err := d.open.inventory.LatestGates(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Observation
|
||||
for _, v := range latest {
|
||||
if v.Verdict != inventory.GateFailed {
|
||||
continue
|
||||
}
|
||||
notWhy := ""
|
||||
if v.Rollback == inventory.NotRolledBack {
|
||||
notWhy = v.Why
|
||||
}
|
||||
out = append(out, gateObservation(v.Module, v.Component, v.Machines, v.Rollback, v.Why, notWhy, v.Commit, v.Previous))
|
||||
}
|
||||
for node, list := range witnessed.all() {
|
||||
for _, r := range list {
|
||||
out = append(out, witnessObservation(node, r))
|
||||
}
|
||||
}
|
||||
return sortedFound(dedupeObservations(out)), nil
|
||||
}
|
||||
|
||||
// dedupeObservations keeps one observation per key, the first.
|
||||
func dedupeObservations(list []conditions.Observation) []conditions.Observation {
|
||||
seen := map[string]bool{}
|
||||
var out []conditions.Observation
|
||||
for _, o := range list {
|
||||
if seen[o.Key()] {
|
||||
continue
|
||||
}
|
||||
seen[o.Key()] = true
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// servedOnTheBus asks the bus's discovery what every machine's node tools answer and which modules'
|
||||
// tools are served where. A variable so a test needs no runtime.
|
||||
var servedOnTheBus = func(ctx context.Context, conn *nats.Conn) (map[string]served, error) {
|
||||
inbox := conn.NewRespInbox()
|
||||
sub, err := conn.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
|
||||
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
|
||||
}
|
||||
out := map[string]served{}
|
||||
add := func(node string) served {
|
||||
s, ok := out[node]
|
||||
if !ok {
|
||||
s = served{tools: map[string]bool{}}
|
||||
}
|
||||
return s
|
||||
}
|
||||
deadline := time.Now().Add(discoveryPatience)
|
||||
for time.Now().Before(deadline) {
|
||||
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
|
||||
msg, err := sub.NextMsgWithContext(wait)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
break
|
||||
}
|
||||
var info micro.Info
|
||||
if json.Unmarshal(msg.Data, &info) != nil {
|
||||
continue
|
||||
}
|
||||
if info.Name == broker.RuntimeModule {
|
||||
node := info.Metadata["node"]
|
||||
if node == "" {
|
||||
node = info.ID
|
||||
}
|
||||
s := add(node)
|
||||
s.runtime = true
|
||||
out[node] = s
|
||||
}
|
||||
for _, e := range info.Endpoints {
|
||||
if e.Metadata["kind"] != "tool" || e.Metadata["module"] == "" {
|
||||
continue
|
||||
}
|
||||
node := e.Metadata["node"]
|
||||
if node == "" {
|
||||
node = info.ID
|
||||
}
|
||||
s := add(node)
|
||||
s.tools[e.Metadata["module"]] = true
|
||||
out[node] = s
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// gateLines is a plan's gates as `plans <id>` says them: the rollout's record.
|
||||
func gateLine(g *inventory.PlanGate) string {
|
||||
if g == nil {
|
||||
return ""
|
||||
}
|
||||
what := "judging"
|
||||
switch g.Verdict {
|
||||
case inventory.GatePassed:
|
||||
what = "passed"
|
||||
case inventory.GateFailed:
|
||||
what = "FAILED"
|
||||
}
|
||||
line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what)
|
||||
if g.Component != "" {
|
||||
line += " (core: " + g.Component + ")"
|
||||
}
|
||||
if g.From != "" || g.To != "" {
|
||||
line += fmt.Sprintf(", %s → %s", short(orNone(g.From)), short(g.To))
|
||||
}
|
||||
if g.Took != "" {
|
||||
line += ", after " + g.Took
|
||||
}
|
||||
if g.Why != "" {
|
||||
line += ": " + g.Why
|
||||
} else if g.Last != "" {
|
||||
line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last)
|
||||
} else if g.Verdict == "" {
|
||||
line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
|
||||
}
|
||||
if g.Rollback != "" {
|
||||
line += "; " + g.Rollback
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
// witnessObservation is a witness's verdict as a condition (ADR 0236, the host's contract):
|
||||
// `core.<component>.<node>.<outcome>`, urgent for rolled-back, not-reversible, restore-failed and
|
||||
// halted, a warning for nothing-to-restore and unwitnessed.
|
||||
func witnessObservation(node string, r lease.Rollback) conditions.Observation {
|
||||
severity := conditions.Warning
|
||||
if lease.Urgent(r.Outcome) {
|
||||
severity = conditions.Urgent
|
||||
}
|
||||
outcome := r.Outcome
|
||||
if outcome == "" {
|
||||
outcome = lease.OutcomeRolledBack
|
||||
}
|
||||
what := map[string]string{
|
||||
lease.OutcomeRolledBack: "and put back " + short(orNone(r.To)),
|
||||
lease.OutcomeNotReversible: "and left it: it is not reversible",
|
||||
lease.OutcomeNothingToRestore: "and had nothing to put back",
|
||||
lease.OutcomeRestoreFailed: "and could not put the previous build back",
|
||||
lease.OutcomeUnwitnessed: "and could not judge it at all",
|
||||
lease.OutcomeHalted: "and gave up: the build before it fails too",
|
||||
}[outcome]
|
||||
return conditions.Observation{Scope: conditions.ScopeCore, ID: r.Component + "." + node, Kind: kindRolledBack,
|
||||
Token: outcome, Machine: node, Severity: severity,
|
||||
Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component,
|
||||
short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)}
|
||||
}
|
||||
Reference in New Issue
Block a user