No build reaches a machine without a gate; a release plan walks what waits (hq ADR 0236)

A send carries the machine's whole declaration, so at the switch to roll the next send of
anything would have carried the old default's backlog, unjudged, to every machine. A gated
send now carries and judges everything waiting on its machine; every other send is refused
or leaves the machine; a release plan walks what waits one machine at a time, the control
node last, and one that fails holds the next until a person releases it.
This commit is contained in:
jochen
2026-10-06 19:14:25 +02:00
parent 41f7b2c152
commit 2bfa6ae4a0
15 changed files with 1163 additions and 40 deletions
+1 -1
View File
@@ -218,7 +218,7 @@ func busCommand(ctx context.Context, args []string) error {
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From, fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it", short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
false: "NOT reversible: the snapshot is the only way back"}[*reversible]) false: "NOT reversible: the snapshot is the only way back"}[*reversible])
sent, err := sendRollout(withBusStep(ctx), open, moving) sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving)
if err != nil { if err != nil {
_ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error()) _ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error())
return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err) return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err)
+41 -6
View File
@@ -139,7 +139,7 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
} else { } else {
f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what") f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what")
} }
if component == lease.ComponentController { if theLease != nil {
h, found, err := theLease.holder(ctx) h, found, err := theLease.holder(ctx)
switch { switch {
case err != nil: case err != nil:
@@ -147,6 +147,9 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
case found: case found:
f.holder = &h f.holder = &h
} }
if component != lease.ComponentController && f.holderErr != nil {
f.holderErr = nil // read only for the controller's own judging
}
} }
return f, nil return f, nil
} }
@@ -263,12 +266,31 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
To: state.Commit, Since: &start} To: state.Commit, Since: &start}
state.Gate = g state.Gate = g
} }
pairs := []judged{}
for _, n := range g.Machines {
pairs = append(pairs, judged{module: module, node: n})
}
return judgeMoves(ctx, open, g, pairs, now)
}
// judged is one module on one machine, as a gate judges it.
type judged struct{ module, node string }
// judgeMoves takes one judging of a gate over the modules it judges on their machines — its own, and
// everything the send carried (Carried) — and records it: a pass counted, a pass missed (what is
// wanting, and which modules), or the verdict. Answers the verdict once there is one.
func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs []judged, now time.Time) (string, error) {
if g.Verdict != "" { if g.Verdict != "" {
return g.Verdict, nil return g.Verdict, nil
} }
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery { if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
return "", nil return "", nil
} }
for _, c := range g.Carried {
if !slices.Contains(pairs, judged{module: c.Module, node: c.Node}) {
pairs = append(pairs, judged{module: c.Module, node: c.Node})
}
}
shelf, err := open.inventory.Catalogue(ctx) shelf, err := open.inventory.Catalogue(ctx)
if err != nil { if err != nil {
return "", err return "", err
@@ -278,8 +300,16 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
return "", err return "", err
} }
worst, why := healthGood, "" worst, why := healthGood, ""
for _, n := range g.Machines { var failing []string
h, said := judgeHealth(module, g.Component, shelf[module], n, *g.Since, facts) broken := map[string]bool{}
for _, j := range pairs {
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, facts)
if h != healthGood && !slices.Contains(failing, j.module) {
failing = append(failing, j.module)
}
if h == healthBroken {
broken[j.module] = true
}
if h > worst { if h > worst {
worst, why = h, said worst, why = h, said
} else if h == worst && h != healthGood && why == "" { } else if h == worst && h != healthGood && why == "" {
@@ -288,15 +318,17 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
} }
switch { switch {
case worst == healthBroken: case worst == healthBroken:
// What broke is put back; what was only not yet healthy beside it is too — they moved together.
g.Failing = failing
decide(g, inventory.GateFailed, why, now) decide(g, inventory.GateFailed, why, now)
case worst == healthNotYet: case worst == healthNotYet:
g.Passes, g.LastPass, g.Last = 0, nil, why g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
if now.Sub(*g.Since) > gateBound { if now.Sub(*g.Since) > gateBound {
decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now) decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
} }
default: default:
g.Passes++ g.Passes++
g.LastPass, g.Last = &now, "" g.LastPass, g.Last, g.Failing = &now, "", nil
if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle { if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes, decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
now.Sub(*g.Since).Round(time.Second)), now) now.Sub(*g.Since).Round(time.Second)), now)
@@ -320,6 +352,7 @@ func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module str
if err != nil && state.Build != "" { if err != nil && state.Build != "" {
fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err) fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err)
} }
passCarried(ctx, open, p, g, module)
fmt.Printf("%s: %s passed its gate on %s (%s); the rest are sent\n", p.ID, module, fmt.Printf("%s: %s passed its gate on %s (%s); the rest are sent\n", p.ID, module,
strings.Join(g.Machines, ", "), g.Why) strings.Join(g.Machines, ", "), g.Why)
if module == catalogue.ControllerSeatName { if module == catalogue.ControllerSeatName {
@@ -443,7 +476,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
if err := inv.SavePlan(ctx, p); err != nil { if err := inv.SavePlan(ctx, p); err != nil {
fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err) fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err)
} }
sent, err := sendRollout(ctx, open, g.Machines) sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
if err != nil { if err != nil {
notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0])) "sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
@@ -553,6 +586,8 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
out = append(out, witnessObservation(node, r)) out = append(out, witnessObservation(node, r))
} }
} }
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
out = append(out, backlogObservation()...)
return sortedFound(dedupeObservations(out)), nil return sortedFound(dedupeObservations(out)), nil
} }
+3
View File
@@ -614,6 +614,9 @@ func lastReportOf(ctx context.Context, inv *inventory.Inventory, node string) (i
// planStale is why a plan's wait is superseded or finished, and the state closing it leaves it in; empty // planStale is why a plan's wait is superseded or finished, and the state closing it leaves it in; empty
// when it is neither, which is not H2's to repair. // when it is neither, which is not H2's to repair.
func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (state, why string, err error) { func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (state, why string, err error) {
if p.Release != nil {
return "", "", nil // a release plan walks machines, and its own gate says when it is done (ADR 0236)
}
recent, err := inv.RecentPlans(ctx, 50) recent, err := inv.RecentPlans(ctx, 50)
if err != nil { if err != nil {
return "", "", err return "", "", err
+23 -1
View File
@@ -120,6 +120,10 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
if err != nil { if err != nil {
return nil, err return nil, err
} }
f, err := readMoveFacts(ctx, inv)
if err != nil {
return nil, err
}
for _, node := range names { for _, node := range names {
plan, _, err := planFor(ctx, open, node) plan, _, err := planFor(ctx, open, node)
if err != nil { if err != nil {
@@ -133,7 +137,25 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
if err != nil { if err != nil {
return nil, err return nil, err
} }
if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 { // A rebuild that put the same thing on the machine is no move (ADR 0236): read as the build it
// runs.
same := map[string]string{}
for m, was := range sent {
same[m] = was
if f.identical(m, was, current[m].Commit) {
same[m] = current[m].Commit
}
}
why := heldBack(node, modules, same, known, current, plans)
// And a build no gate has seen is not carried by a send that does not judge it (ADR 0236).
for _, mv := range f.moves(node, modules, same, known, false) {
if planStillToSend(plans, mv.Module, node) == "" {
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
}
}
if len(why) > 0 {
sort.Strings(why)
out[node] = why out[node] = why
} }
} }
+15 -1
View File
@@ -264,7 +264,8 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
t.Fatalf("the push did not say both:\n%s", said.String()) t.Fatalf("the push did not say both:\n%s", said.String())
} }
// A policy that rolls out: the laptop is a consequence like any other, and sent. // A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it
// (novox/hq ADR 0236).
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil { if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -273,6 +274,19 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
compose, d, "", &said); err != nil { compose, d, "", &said); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String())
}
// Once it passed a gate on some machine, the laptop is a consequence like any other, and sent.
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2",
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
t.Fatal(err)
}
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", &said); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before { if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String()) t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
} }
+32
View File
@@ -1,6 +1,7 @@
package main package main
import ( import (
"slices"
"context" "context"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
@@ -466,6 +467,28 @@ func pushCommand(ctx context.Context, args []string) error {
which, len(asked), strings.Join(asked, ", ")) which, len(asked), strings.Join(asked, ", "))
} }
// **A whole-mesh push sends no build a gate has not seen** (novox/hq ADR 0236): a machine where one
// waits is left, named, for the release plan — `push <node>` still sends one machine by a person's word.
if len(args) == 0 {
f, err := readMoveFacts(ctx, inv)
if err != nil {
return err
}
var kept []string
for _, n := range asked {
moves, err := machineMoves(ctx, open, f, n, false)
if err != nil {
return err
}
if len(moves) > 0 {
fmt.Printf(" %s is left: %d build(s) wait there for a gate, which a release plan sends one machine "+
"at a time (`upgrade backlog` lists them; `push %s` sends it by name)\n", n, len(moves), n)
continue
}
kept = append(kept, n)
}
asked = kept
}
// **The bus is replaced only as a planned step** (novox/hq ADR 0236, to-be 45 §8): a machine whose bus // **The bus is replaced only as a planned step** (novox/hq ADR 0236, to-be 45 §8): a machine whose bus
// would move is not sent by a push — named, it is refused; otherwise it is left and said. // would move is not sent by a push — named, it is refused; otherwise it is left and said.
busKept, err := busHeld(ctx, inv, asked) busKept, err := busHeld(ctx, inv, asked)
@@ -1030,7 +1053,16 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
if err != nil { if err != nil {
return nil, err return nil, err
} }
added := ""
if behind && !slices.Contains(names, holder) {
added = holder
}
names = brokerFirst(names, holder, behind) names = brokerFirst(names, holder, behind)
// **No build moves on a machine without a gate** (novox/hq ADR 0236): a send outside its scope that
// would carry one is refused, said with what waits; the bus's machine added for its user list is left.
if names, err = ungatedIn(ctx, open, names, added); err != nil {
return nil, err
}
// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already — // Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
// converge, which flips the node and then sends it — is not made to wait on itself. // converge, which flips the node and then sends it — is not made to wait on itself.
+603
View File
@@ -0,0 +1,603 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"slices"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// No build reaches a machine without a gate (novox/hq ADR 0236).
//
// **A send carries the machine's whole declaration.** A plan sending one module to its first machine
// sends every other module whose build moved there too; a cascade, a healer's resend and a whole-mesh
// push do the same. Under the old default (`record`) builds were registered and sent nowhere, so on the
// day the default became `roll` every machine was behind on dozens of builds no gate had seen — and the
// next send of anything would have restarted all of them at once, on every machine.
//
// So **a build moves on a machine only through a send that judges it there**, or a person's:
//
// - a gated send — a plan's first machine, a release plan's machine — carries every move waiting on that
// machine, and its gate judges each of them; a pass is each build's verdict, a failure puts back what
// failed;
// - a module a send exists for may move anywhere it is sent: a policy of *together*, a rollback, a build
// whose gate already passed;
// - a send naming a machine by a person (`push <node>`, the bus step) carries what it carries;
// - every other send — a plan's "rest", a cascade, a healer's, the bus's user list carried — is refused,
// or leaves the machine, while a move there waits for a gate. A rebuild that made the same artifacts
// from the same manifest is no move.
//
// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last,
// each sent and judged before the next. A release plan that fails stops the next one opening on its own:
// a person releases it again (`upgrade release-backlog --why`), after the condition says why.
// sendScope is what a send may carry that no gate has seen.
type sendScope struct {
// judged are the machines whose every move this send's gate judges.
judged map[string]bool
// modules are those a send exists for, which may move wherever it goes.
modules map[string]bool
// person is a person's act: it carries what it carries.
person bool
}
type sendScopeKey struct{}
func withScope(ctx context.Context, s sendScope) context.Context {
return context.WithValue(ctx, sendScopeKey{}, s)
}
func scopeOf(ctx context.Context) sendScope {
s, _ := ctx.Value(sendScopeKey{}).(sendScope)
return s
}
// errUngated is a send refused because it would carry a build no gate has seen.
var errUngated = errors.New("a build waits there for its gate")
// errWalkedElsewhere is a gated send refused because a plan that has started walks a move there.
var errWalkedElsewhere = errors.New("a plan already walking a build there sends it")
// moveFacts is what tells a move from a rebuild, and a gated build from one no gate has seen.
type moveFacts struct {
current map[string]inventory.CurrentBuild
fps map[string]map[string]string
passed map[string]map[string]bool
plans []inventory.Plan
}
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
var f moveFacts
var err error
if f.current, err = inv.CurrentBuilds(ctx); err != nil {
return f, err
}
if f.fps, err = inv.Fingerprints(ctx); err != nil {
return f, err
}
if f.passed, err = inv.PassedCommits(ctx); err != nil {
return f, err
}
f.plans, err = inv.OpenPlans(ctx)
return f, err
}
// identical is whether two builds of a module put the same thing on a machine: the same commit, or
// builds that made the same artifacts from the same manifest.
func (f moveFacts) identical(module, a, b string) bool {
if a == b || sameCommit(a, b) {
return true
}
fa := f.fps[module][a]
return fa != "" && fa == f.fps[module][b]
}
// gated is whether a build of a module from this commit — or one identical to it — passed a gate.
func (f moveFacts) gated(module, commit string) bool {
for c := range f.passed[module] {
if f.identical(module, c, commit) {
return true
}
}
return false
}
// moves is what a machine's next send would move that no gate has seen: modules whose policy rolls out
// (a recorded one is a person's push), that the machine was sent before, moving to a build not identical
// to the one it runs and that has passed no gate. A machine whose last send's builds are not known names
// none: the cascade holds it whole (ADR 0221).
//
// With all, a move to a build that passed a gate elsewhere counts too: what a gated send carries.
func (f moveFacts) moves(node string, modules []string, sent map[string]string, known, all bool) []inventory.CarriedMove {
if !known {
return nil
}
var out []inventory.CarriedMove
for _, m := range modules {
now := f.current[m]
was, carried := sent[m]
if !now.RollOut || !carried || f.identical(m, was, now.Commit) || (!all && f.gated(m, now.Commit)) {
continue
}
out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: now.Commit})
}
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
return out
}
// walkedBy is the open plan that has started walking a module's build — sent it to a first machine,
// not yet passed — other than to this machine; empty when none does.
func (f moveFacts) walkedBy(module, node string) string {
for _, p := range f.plans {
s, holds := p.Modules[module]
if !p.Open() || !holds || s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(s.First, node) {
continue
}
if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed {
continue
}
return p.ID
}
return ""
}
// machineMoves is the moves no gate has seen on one machine, read from what it would be sent now.
var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
plan, _, err := planFor(ctx, open, node)
if err != nil {
return nil, nil // it cannot be worked out: the send says why
}
modules := make([]string, 0, len(plan.Modules))
for _, m := range plan.Modules {
modules = append(modules, m.Module)
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
// ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the
// machines named, and why; the machine holding the bus, added only for its user list, is left instead.
func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) {
scope := scopeOf(ctx)
if scope.person {
return names, nil
}
f, err := readMoveFacts(ctx, open.inventory)
if err != nil {
return nil, err
}
var kept, refused []string
for _, n := range names {
moves, err := machineMoves(ctx, open, f, n, false)
if err != nil {
return nil, err
}
var waiting []string
for _, mv := range moves {
if !scope.judged[n] && !scope.modules[mv.Module] {
waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
}
}
switch {
case len(waiting) == 0:
kept = append(kept, n)
case n == addedHolder:
fmt.Printf("%s holds the bus and its user list changed, and it is not sent now: %s wait there for a gate "+
"— the bus may refuse what was newly granted until it is\n", n, strings.Join(waiting, ", "))
default:
refused = append(refused, fmt.Sprintf("%s (%s)", n, strings.Join(waiting, ", ")))
}
}
if len(refused) > 0 {
return nil, fmt.Errorf("%w: %s — a release plan sends them, one machine at a time, each judged (novox/hq ADR "+
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
}
return kept, nil
}
// gatedSend sends one machine everything waiting there, under a gate that judges it all: what moved is
// answered, with the build each moved to, for the gate to judge and to put back. Refused while a plan that
// has started walks one of those builds elsewhere: that plan sends it here once its gate passed.
func gatedSend(ctx context.Context, open *stores, node string, own *inventory.CarriedMove) ([]inventory.CarriedMove, []string, error) {
inv := open.inventory
f, err := readMoveFacts(ctx, inv)
if err != nil {
return nil, nil, err
}
moves, err := machineMoves(ctx, open, f, node, true)
if err != nil {
return nil, nil, err
}
for _, mv := range moves {
if own != nil && mv.Module == own.Module {
continue
}
if id := f.walkedBy(mv.Module, node); id != "" {
return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
mv.Module, short(mv.To), node, id)
}
}
if own != nil && !slices.ContainsFunc(moves, func(mv inventory.CarriedMove) bool { return mv.Module == own.Module }) {
moves = append(moves, *own)
}
if len(moves) == 0 && own == nil {
return nil, nil, nil
}
for i := range moves {
if moves[i].Build == "" {
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
return nil, nil, err
}
}
}
sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node})
if err != nil {
return nil, nil, err
}
return moves, sent, nil
}
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
seen := map[string]bool{}
for _, c := range g.Carried {
if c.Module == except || c.Build == "" || seen[c.Build] {
continue
}
seen[c.Build] = true
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: g.Why,
Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
fmt.Printf("%s: %s passed its gate on %s, and the verdict could not be kept: %v\n", p.ID, c.Module, c.Node, err)
}
}
}
// failCarried puts back every build the gate carried that it found wanting, on the machines that were
// sent it, once each.
func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
var notes []string
if p.Note != "" {
notes = append(notes, p.Note)
}
done := map[string]bool{except: true}
for _, c := range g.Carried {
if done[c.Module] || (len(g.Failing) > 0 && !slices.Contains(g.Failing, c.Module)) {
continue
}
done[c.Module] = true
machines, err := sentTheBuild(ctx, open, c.Module, c.To)
if err != nil || len(machines) == 0 {
machines = []string{c.Node}
}
state := &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To}
p.Note = ""
gateFailed(ctx, open, p, c.Module, state, machines, g.Why)
notes = append(notes, p.Note)
}
p.State = inventory.PlanFailed
p.Note = strings.Join(notes, "; ")
}
// sentTheBuild is every machine running a module that was last sent this build of it.
func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) {
running, err := open.inventory.Running(ctx, module)
if err != nil {
return nil, err
}
var out []string
for _, n := range running {
sent, known, err := open.inventory.SentBuilds(ctx, n)
if err != nil {
return nil, err
}
if known && sameCommit(sent[module], commit) {
out = append(out, n)
}
}
return out, nil
}
// releaseRepository is what a release plan says it is for, where a merge's says its repository.
const releaseRepository = "the builds waiting for a gate"
// releaseHeard is the machines a release plan may send: heard within their heartbeat's bound. A machine
// away is left, not judged against a bound it cannot meet. A variable so a test says who is heard.
var releaseHeard = func(ctx context.Context, open *stores) (map[string]bool, error) {
if d := doctorFrom; d != nil && d.watchdogs != nil {
return heardMachines(d), nil
}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return nil, err
}
out := map[string]bool{}
for _, r := range reports {
if r.At != nil && time.Since(*r.At) < 15*time.Minute {
out[r.Node] = true
}
}
return out, nil
}
// backlogNow is what the newest look found waiting, for `upgrade backlog` and the gate's probe.
var backlogNow struct {
held string
waiting map[string][]inventory.CarriedMove
}
// waitingMoves is every machine's moves no gate has seen and no started plan walks.
func waitingMoves(ctx context.Context, open *stores, all bool) (map[string][]inventory.CarriedMove, error) {
f, err := readMoveFacts(ctx, open.inventory)
if err != nil {
return nil, err
}
nodes, err := open.inventory.Nodes(ctx)
if err != nil {
return nil, err
}
out := map[string][]inventory.CarriedMove{}
for _, n := range nodes {
moves, err := machineMoves(ctx, open, f, n.Name, all)
if err != nil {
return nil, err
}
for _, mv := range moves {
if f.walkedBy(mv.Module, n.Name) == "" {
out[n.Name] = append(out[n.Name], mv)
}
}
}
return out, nil
}
// releaseBacklog opens a release plan when builds wait for a gate and none is open; not after a release
// plan failed, until a person releases one (by). Called with the plans held.
func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Plan, error) {
inv := open.inventory
plans, err := inv.OpenPlans(ctx)
if err != nil {
return nil, err
}
for _, p := range plans {
if p.Release != nil {
if by != "" {
return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID)
}
return nil, nil
}
}
waiting, err := waitingMoves(ctx, open, false)
if err != nil {
return nil, err
}
backlogNow.waiting, backlogNow.held = waiting, ""
if len(waiting) == 0 {
return nil, nil
}
// Opened by what no gate has seen; it walks every machine where anything of the release waits — a
// build that passed on the first machine still goes to the next one by this plan, judged there too.
if waiting, err = waitingMoves(ctx, open, true); err != nil {
return nil, err
}
if by == "" {
recent, err := inv.RecentPlans(ctx, 50)
if err != nil {
return nil, err
}
for _, p := range recent {
if p.Release == nil {
continue
}
if p.State == inventory.PlanFailed {
backlogNow.held = fmt.Sprintf("%s failed (%s); what waits is released again by a person", p.ID, p.Note)
return nil, nil
}
break
}
}
heard, err := releaseHeard(ctx, open)
if err != nil {
return nil, err
}
controllers, err := inv.Running(ctx, catalogue.ControllerSeatName)
if err != nil {
return nil, err
}
var order, last []string
modules := map[string]bool{}
for node, moves := range waiting {
if !heard[node] {
continue
}
for _, mv := range moves {
modules[mv.Module] = true
}
if slices.Contains(controllers, node) {
last = append(last, node)
} else {
order = append(order, node)
}
}
if len(order)+len(last) == 0 {
return nil, nil
}
sort.Strings(order)
sort.Strings(last)
order = append(order, last...)
names := make([]string, 0, len(modules))
for m := range modules {
names = append(names, m)
}
sort.Strings(names)
now := time.Now().UTC()
p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
Release: &inventory.PlanRelease{Order: order, By: by},
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names),
strings.Join(order, ", "))}
if err := inv.SavePlan(ctx, &p); err != nil {
return nil, err
}
fmt.Printf("%s: %s\n", p.ID, p.Note)
return &p, nil
}
// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a
// gate, or the machine being judged judged once more; the plan done when every machine is.
func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
r := p.Release
now := time.Now().UTC()
if r.Gate == nil {
heard, err := releaseHeard(ctx, open)
if err != nil {
return false, err
}
for r.Next < len(r.Order) {
node := r.Order[r.Next]
if !heard[node] {
r.Skipped = append(r.Skipped, node)
r.Next++
continue
}
moves, sent, err := gatedSend(ctx, open, node, nil)
if errors.Is(err, errWalkedElsewhere) {
note := fmt.Sprintf("waiting before %s: %v", node, err)
changed := p.Note != note
p.Note = note
return changed, nil
}
if err != nil {
return false, fmt.Errorf("sending %s what waits there: %w", node, err)
}
if len(moves) == 0 {
r.Done = append(r.Done, node)
r.Next++
continue
}
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves}
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
p.Note = fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
if len(r.Skipped) > 0 {
p.Note += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
}
return true, nil
}
g := r.Gate
verdict, err := judgeMoves(ctx, open, g, nil, now)
if err != nil {
return false, err
}
switch verdict {
case "":
note := fmt.Sprintf("judging %s: %s", strings.Join(g.Machines, ", "), gateLine(g))
changed := p.Note != note
p.Note = note
return changed, nil
case inventory.GatePassed:
passCarried(ctx, open, p, g, "")
r.Done = append(r.Done, firstOf(g.Machines))
r.Next++
r.Gate = nil
return true, nil
}
p.Note = ""
failCarried(ctx, open, p, g, "")
p.Note = fmt.Sprintf("failed its gate on %s: %s — %s", strings.Join(g.Machines, ", "), g.Why, p.Note)
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
// backlogObservation is what the gate's probe says of a release held after a failure.
func backlogObservation() []conditions.Observation {
if backlogNow.held == "" || len(backlogNow.waiting) == 0 {
return nil
}
n := 0
var machines []string
for node, moves := range backlogNow.waiting {
n += len(moves)
machines = append(machines, node)
}
sort.Strings(machines)
return []conditions.Observation{{Scope: conditions.ScopeMesh, ID: "release", Token: "held", Kind: "release-held",
Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%d build move(s) on %s wait for a gate and are not released: %s — `upgrade backlog` lists "+
"them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)}}
}
// backlogCommand is `upgrade backlog`, read-only, and `upgrade release-backlog --why`.
func backlogCommand(ctx context.Context, sub string, args []string) error {
set := flag.NewFlagSet("upgrade "+sub, flag.ContinueOnError)
why := addHandActFlags(set)
if rest, err := parseAround(set, args); err != nil {
return err
} else if len(rest) > 0 {
return errors.New("upgrade backlog | upgrade release-backlog --why <text>")
}
if sub == "release-backlog" {
if err := why.require("upgrade release-backlog"); err != nil {
return err
}
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
if sub == "release-backlog" {
release, err := open.inventory.HoldPlans(ctx, true)
if err != nil {
return err
}
defer release()
why.record(ctx, "upgrade release-backlog", nil)
p, err := releaseBacklog(ctx, open, link.Caller())
if err != nil {
return err
}
if p == nil {
fmt.Println("nothing waits for a gate on any machine heard from: nothing to release")
return nil
}
fmt.Printf("%s releases it; `plans %s` says where it is\n", p.ID, p.ID)
return nil
}
waiting, err := waitingMoves(ctx, open, false)
if err != nil {
return err
}
if len(waiting) == 0 {
fmt.Println("no build waits for a gate on any machine")
return nil
}
nodes := make([]string, 0, len(waiting))
for n := range waiting {
nodes = append(nodes, n)
}
sort.Strings(nodes)
for _, n := range nodes {
fmt.Printf("%s: %d build(s) wait for a gate\n", n, len(waiting[n]))
for _, mv := range waiting[n] {
fmt.Printf(" %-28s %s → %s\n", mv.Module, short(mv.From), short(mv.To))
}
}
return nil
}
+41 -1
View File
@@ -476,6 +476,10 @@ func advancePlans(ctx context.Context, open *stores) {
return return
} }
defer release() defer release()
// What waits for a gate, released one machine at a time (ADR 0236).
if _, err := releaseBacklog(ctx, open, ""); err != nil {
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
}
advanceHeld(ctx, open) advanceHeld(ctx, open)
} }
@@ -531,6 +535,9 @@ func advanceHeld(ctx context.Context, open *stores) {
func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
edges []inventory.Edge, rollsOut func(string) bool) (bool, error) { edges []inventory.Edge, rollsOut func(string) bool) (bool, error) {
inv := open.inventory inv := open.inventory
if p.Release != nil {
return advanceRelease(ctx, open, p)
}
if p.Tier >= len(p.Tiers) { if p.Tier >= len(p.Tiers) {
p.State = inventory.PlanDone p.State = inventory.PlanDone
fmt.Printf("%s: done — %s at %s, %d tier(s)\n", p.ID, p.Repository, short(p.Commit), len(p.Tiers)) fmt.Printf("%s: done — %s at %s, %d tier(s)\n", p.ID, p.Repository, short(p.Commit), len(p.Tiers))
@@ -631,6 +638,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
// The first machine refused or failed what it was sent, or never said: the gate failed, and // The first machine refused or failed what it was sent, or never said: the gate failed, and
// the build is put back there (novox/hq ADR 0236); the rest are left as they were. // the build is put back there (novox/hq ADR 0236); the rest are left as they were.
gateFailed(ctx, open, p, m, state, firstRunning(state.First, running), step.failed) gateFailed(ctx, open, p, m, state, firstRunning(state.First, running), step.failed)
if state.Gate != nil && len(state.Gate.Carried) > 0 {
failCarried(ctx, open, p, state.Gate, m)
}
p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", "))) p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", ")))
fmt.Printf("%s: %s\n", p.ID, p.Note) fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil return true, nil
@@ -653,6 +663,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
continue continue
case inventory.GateFailed: case inventory.GateFailed:
gateFailed(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why) gateFailed(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why)
if len(state.Gate.Carried) > 0 {
failCarried(ctx, open, p, state.Gate, m)
}
p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", "))) p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", ")))
fmt.Printf("%s: %s\n", p.ID, p.Note) fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil return true, nil
@@ -677,7 +690,20 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
} }
// What sendToEach answers, not what was asked: the machine holding the bus is sent before // What sendToEach answers, not what was asked: the machine holding the bus is sent before
// the first when its user list must change (issue 249), and the plan waits for it too. // the first when its user list must change (issue 249), and the plan waits for it too.
sent, err := sendRollout(ctx, open, step.send) var sent []string
var carried []inventory.CarriedMove
switch {
case step.first:
// **A gated send** (ADR 0236): everything waiting on the first machine goes with the build,
// and the gate judges all of it there.
own := inventory.CarriedMove{Module: m, Node: step.send[0], From: before[m], To: state.Commit, Build: state.Build}
carried, sent, err = gatedSend(ctx, open, step.send[0], &own)
case policy.Together:
sent, err = sendRollout(withScope(ctx, sendScope{modules: map[string]bool{m: true}}), open, step.send)
default:
// The rest, after the gate passed: nothing else may move with it that no gate has seen.
sent, err = sendRollout(ctx, open, step.send)
}
if err != nil { if err != nil {
// Not marked sent, so the next step tries again (issue 249): a grant that could not be // Not marked sent, so the next step tries again (issue 249): a grant that could not be
// issued is a send that did not happen. // issued is a send that did not happen.
@@ -690,6 +716,8 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
} }
state.First = sent state.First = sent
state.FirstAt = &now state.FirstAt = &now
state.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, running),
From: state.Previous, To: state.Commit, Since: &now, Carried: carried}
p.State = inventory.PlanRolling p.State = inventory.PlanRolling
p.Note = fmt.Sprintf("tier %d built; sent %s to %s first", p.Tier, m, strings.Join(sent, ", ")) p.Note = fmt.Sprintf("tier %d built; sent %s to %s first", p.Tier, m, strings.Join(sent, ", "))
fmt.Printf("%s: tier %d built; sent %s to %s first, the rest once it reports it applied\n", fmt.Printf("%s: tier %d built; sent %s to %s first, the rest once it reports it applied\n",
@@ -1059,6 +1087,18 @@ func plansCommand(ctx context.Context, args []string) error {
return err return err
} }
fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}))) fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p})))
if r := p.Release; r != nil {
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", ")))
if r.Gate != nil {
fmt.Printf(" %s\n", gateLine(r.Gate))
for _, c := range r.Gate.Carried {
fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
}
}
return nil
}
for i, tier := range p.Tiers { for i, tier := range p.Tiers {
marker := " " marker := " "
if i == p.Tier && p.Open() { if i == p.Tier && p.Open() {
+257
View File
@@ -0,0 +1,257 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// The backlog the old default left — builds registered and sent nowhere — is released by a plan, one
// machine at a time, each judged, never by the next send of something else (novox/hq ADR 0236).
type backlogMesh struct {
open *stores
sent [][]string
broken map[string]bool
}
// aBacklog is a mesh where `app` moved c1 → c2 on anchor and laptop, `late` moved on laptop only, and
// `same` was rebuilt with the very artifacts it had: two machines heard, every send applied at once.
func aBacklog(t *testing.T) *backlogMesh {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
b := &backlogMesh{open: open, broken: map[string]bool{}}
withConditionsInMemory(t)
was := doctorFrom
doctorFrom = nil
t.Cleanup(func() { doctorFrom = was })
build := func(module, commit, made string, asked time.Time) {
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: made}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
Asked: asked}); err != nil {
t.Fatal(err)
}
}
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
on := map[string][]string{"app": {"anchor", "laptop"}, "late": {"laptop"}, "same": {"anchor", "laptop"}}
for _, m := range []string{"app", "late", "same"} {
build(m, "c1", "sha256:"+m+"-1", old)
for _, n := range on[m] {
if _, err := inv.Assign(ctx, n, m); err != nil {
t.Fatal(err)
}
}
}
for _, n := range []string{"anchor", "laptop"} {
sent := map[string]string{}
for m, nodes := range on {
for _, x := range nodes {
if x == n {
sent[m] = "c1"
}
}
}
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil {
t.Fatal(err)
}
}
build("app", "c2", "sha256:app-2", now)
build("late", "c2", "sha256:late-2", now)
build("same", "c2", "sha256:same-1", now) // rebuilt, the same bytes
assigned := func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, err := open.inventory.Assigned(ctx, node)
if err != nil {
return nil, err
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
machineMoves = assigned
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
return map[string]bool{"anchor": true, "laptop": true}, nil
}
n := 0
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
b.sent = append(b.sent, append([]string(nil), names...))
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
modules, _ := open.inventory.Assigned(ctx, node)
carried := map[string]string{}
for _, m := range modules {
carried[m] = current[m].Commit
}
n++
digest := fmt.Sprintf("d-%s-%d", node, n)
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
return nil, err
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
return names, nil
}
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return f, err
}
for _, r := range reports {
if b.broken[r.Node] {
r.Outcome = inventory.OutcomeFailed
}
f.reports[r.Node] = r
}
return f, nil
}
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
})
return b
}
func (b *backlogMesh) release(t *testing.T) inventory.Plan {
t.Helper()
plans, err := b.open.inventory.RecentPlans(t.Context(), 10)
if err != nil {
t.Fatal(err)
}
for _, p := range plans {
if p.Release != nil {
return p
}
}
t.Fatal("no release plan")
return inventory.Plan{}
}
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
f, err := readMoveFacts(ctx, inv)
if err != nil {
t.Fatal(err)
}
moves, _ := machineMoves(ctx, b.open, f, "laptop", false)
var names []string
for _, mv := range moves {
names = append(names, mv.Module)
}
if !reflect.DeepEqual(names, []string{"app", "late"}) {
t.Fatalf("laptop waits for %v; a rebuild with the same bytes is no move", names)
}
// Nothing else may carry them: a send that judges nothing is refused.
if _, err := ungatedIn(ctx, b.open, []string{"laptop"}, ""); !errors.Is(err, errUngated) {
t.Fatalf("a send that judges nothing would carry them: %v", err)
}
advancePlans(ctx, b.open)
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) {
t.Fatalf("sent %v: the first machine alone, before it is judged", b.sent)
}
p := b.release(t)
if !reflect.DeepEqual(p.Release.Order, []string{"anchor", "laptop"}) || p.Release.Gate == nil {
t.Fatalf("the release plan is %+v", p.Release)
}
gateEvery = 0
for i := 0; i < 4; i++ {
advancePlans(ctx, b.open)
}
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"laptop"}}) {
t.Fatalf("sent %v", b.sent)
}
p = b.release(t)
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) {
t.Fatalf("the release plan is %s: %s %+v", p.State, p.Note, p.Release)
}
for _, build := range []string{"build-app-c2", "build-late-c2"} {
if v, found, err := inv.GateOf(ctx, build); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("%s's pass was not kept: %+v %v %v", build, v, found, err)
}
}
// Nothing waits now, and nothing opens again.
if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil {
t.Fatalf("a release opened with nothing waiting: %+v %v", p, err)
}
}
// A release that fails on its first machine puts back what it carried there, goes no further, and the
// next one waits for a person, said as a condition; a person releases it with why.
func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) {
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
gateEvery = 0
b.broken["anchor"] = true
advancePlans(ctx, b.open)
p := b.release(t)
if p.State != inventory.PlanFailed {
t.Fatalf("the release is %s: %s", p.State, p.Note)
}
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"anchor"}}) {
t.Fatalf("sent %v: the first machine, then the put-back to it, and nothing to laptop", b.sent)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
t.Fatalf("app is at %s, not put back", current["app"].Commit)
}
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
t.Fatal("app's build is not marked failed at its gate")
}
// late still waits on laptop, and is not released on its own after a failure.
if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil {
t.Fatalf("a release opened after a failed one: %+v %v", p, err)
}
if obs := backlogObservation(); len(obs) != 1 || !strings.Contains(obs[0].Summary, "release-backlog") {
t.Fatalf("the held release is not said: %+v", obs)
}
b.broken["anchor"] = false
if err := backlogCommand(ctx, "release-backlog", []string{"--why", "anchor is fixed"}); err != nil {
t.Fatal(err)
}
for i := 0; i < 4; i++ {
advancePlans(ctx, b.open)
}
if p := b.release(t); p.State != inventory.PlanDone || p.Release.By == "" {
t.Fatalf("the person's release is %s (%+v)", p.State, p.Release)
}
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
t.Fatalf("late was not released to laptop: %v", sent)
}
}
+10
View File
@@ -509,6 +509,16 @@ func (a *verbArguments) commandLine() ([]string, error) {
} }
return argv, nil return argv, nil
case "upgrade": case "upgrade":
if on("backlog") {
return []string{"upgrade", "backlog"}, nil
}
if on("release-backlog") {
argv := []string{"upgrade", "release-backlog", "--why", str("why")}
if c := str("cause"); c != "" {
argv = append(argv, "--cause", c)
}
return argv, nil
}
m, policy := str("module"), str("policy") m, policy := str("module"), str("policy")
if m == "" { if m == "" {
if policy != "" { if policy != "" {
+10 -18
View File
@@ -74,24 +74,12 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
u.Module, shortCommit(u.Commit), id, readableList(on)) u.Module, shortCommit(u.Commit), id, readableList(on))
return nil return nil
} }
if decision.Together { // **No plan holds it: a release plan sends it** (novox/hq ADR 0236). A build asked outside a plan — a
fmt.Printf("%s moved to %s; sending %s together\n", // `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it
u.Module, shortCommit(u.Commit), readableList(on)) // now waits for a gate like any other build, and the release plan walks it through the machines, one
return askAgainOnGrants(sendTo(ctx, f.open, on)) // at a time, each judged.
} fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine at a time, each judged "+
// One at a time, and stopping at the first that fails. "(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on))
//
// **Stopping is the point.** The machines are done one after another precisely so that a
// version that breaks the first one does not reach the rest; carrying on past a failure would
// make this the same as sending them together, only slower.
fmt.Printf("%s moved to %s; sending %s one at a time\n",
u.Module, shortCommit(u.Commit), readableList(on))
for _, node := range on {
if err := sendTo(ctx, f.open, []string{node}); err != nil {
return askAgainOnGrants(fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w",
node, u.Module, err))
}
}
return nil return nil
} }
@@ -147,6 +135,10 @@ func isAre(n int) string {
// upgradeCommand says what should happen when a module's current version moves (ADR 0162 §3, ADR // upgradeCommand says what should happen when a module's current version moves (ADR 0162 §3, ADR
// 0235): every module's policy and where it comes from; one module's; or a person's choice for one. // 0235): every module's policy and where it comes from; one module's; or a person's choice for one.
func upgradeCommand(ctx context.Context, args []string) error { func upgradeCommand(ctx context.Context, args []string) error {
// What waits for a gate, and releasing it by a person's word (ADR 0236).
if len(args) > 0 && (args[0] == "backlog" || args[0] == "release-backlog") {
return backlogCommand(ctx, args[0], args[1:])
}
set := flag.NewFlagSet("upgrade", flag.ContinueOnError) set := flag.NewFlagSet("upgrade", flag.ContinueOnError)
together := set.Bool("together", false, together := set.Bool("together", false,
"send every machine running it at once, instead of one machine first") "send every machine running it at once, instead of one machine first")
+6 -2
View File
@@ -277,8 +277,12 @@ var ControllerVerbs = []Verb{
"module": "one module", "module": "one module",
"policy": "with module: roll-out, record or default", "policy": "with module: roll-out, record or default",
"together": "\"true\": with roll-out, every machine at once instead of one machine first", "together": "\"true\": with roll-out, every machine at once instead of one machine first",
"why": "with policy: why — required for record, kept and said with the policy", "why": "with policy or release-backlog: why — required for record and for a release, kept",
}, nil, "together")}, "backlog": "\"true\": every build waiting for a gate, per machine — what a release plan would send",
"release-backlog": "\"true\": release what waits now, one machine at a time, each judged — after a release " +
"plan failed, the mesh waits for this; with why",
"cause": "with release-backlog: the cause in a word (optional)",
}, nil, "together", "backlog", "release-backlog")},
{Name: "bus", Description: "The bus as a planned step (novox/hq to-be 45 §8, ADR 0236): what a bus upgrade " + {Name: "bus", Description: "The bus as a planned step (novox/hq to-be 45 §8, ADR 0236): what a bus upgrade " +
"would do — the bus's build on each machine against the one the mesh holds — and how the last step went. " + "would do — the bus's build on each machine against the one the mesh holds — and how the last step went. " +
"With upgrade, start one: a person's act with why, after the streams are snapshotted (snapshot-taken says " + "With upgrade, start one: a person's act with why, after the streams are snapshotted (snapshot-taken says " +
+65 -3
View File
@@ -46,8 +46,8 @@ type GateVerdict struct {
Epoch uint64 Epoch uint64
} }
// RecordGate writes a build's verdict. **A failed build's row is written once**: a second failure for // RecordGate writes a build's verdict. A build that passed on one machine may still fail on the next one
// the same build is refused with ErrGateKept, which is what keeps a rollback to one per build — the row // judged; **a failed build's row is written once**: any later verdict for it is refused with ErrGateKept, which is what keeps a rollback to one per build — the row
// is written before the rollback's send, and a controller replaced in between finds it. // is written before the rollback's send, and a controller replaced in between finds it.
func (i *Inventory) RecordGate(ctx context.Context, v GateVerdict) error { func (i *Inventory) RecordGate(ctx context.Context, v GateVerdict) error {
epoch, err := i.actingEpoch(ctx) epoch, err := i.actingEpoch(ctx)
@@ -64,7 +64,7 @@ func (i *Inventory) RecordGate(ctx context.Context, v GateVerdict) error {
on conflict (build) do update set verdict = excluded.verdict, rollback = excluded.rollback, on conflict (build) do update set verdict = excluded.verdict, rollback = excluded.rollback,
why = excluded.why, previous = excluded.previous, machines = excluded.machines, why = excluded.why, previous = excluded.previous, machines = excluded.machines,
judged_at = now(), epoch = excluded.epoch judged_at = now(), epoch = excluded.epoch
where build_gate.verdict = 'passed' and excluded.verdict = 'passed'`, where build_gate.verdict = 'passed'`,
v.Build, v.Module, v.Commit, v.Previous, v.Plan, v.Machines, v.Verdict, v.Rollback, v.Why, v.Component, v.Build, v.Module, v.Commit, v.Previous, v.Plan, v.Machines, v.Verdict, v.Rollback, v.Why, v.Component,
v.JudgingFrom, epoch) v.JudgingFrom, epoch)
if err != nil { if err != nil {
@@ -245,3 +245,65 @@ func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[s
} }
return out, rows.Err() return out, rows.Err()
} }
// Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint.
func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) {
rows, err := i.store.Pool().Query(ctx,
`select module, commit_hash, made, coalesce(manifest::text, '') from build
where module is not null and failed = '' and commit_hash <> '' order by at desc`)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]map[string]string{}
for rows.Next() {
var module, commit, manifest string
var made []byte
if err := rows.Scan(&module, &commit, &made, &manifest); err != nil {
return nil, err
}
if out[module] == nil {
out[module] = map[string]string{}
}
if _, seen := out[module][commit]; seen {
continue
}
sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...))
out[module][commit] = hex.EncodeToString(sum[:])
}
return out, rows.Err()
}
// PassedCommits is, per module, the commits a build of which passed its gate on some machine.
func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bool, error) {
rows, err := i.store.Pool().Query(ctx, `select module, commit_hash from build_gate where verdict = 'passed'`)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]map[string]bool{}
for rows.Next() {
var module, commit string
if err := rows.Scan(&module, &commit); err != nil {
return nil, err
}
if out[module] == nil {
out[module] = map[string]bool{}
}
out[module][commit] = true
}
return out, rows.Err()
}
// BuildOf is the id of the newest successful build of a module from a commit; empty when none is
// recorded (a manifest handed over by hand).
func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) {
var id string
err := i.store.Pool().QueryRow(ctx,
`select id from build where module = $1 and commit_hash = $2 and failed = '' order by at desc limit 1`,
module, commit).Scan(&id)
if errors.Is(err, pgx.ErrNoRows) {
return "", nil
}
return id, err
}
@@ -65,3 +65,8 @@ create table bus_step (
outcome text not null default '' check (outcome in ('', 'done', 'failed')), outcome text not null default '' check (outcome in ('', 'done', 'failed')),
found text not null default '' found text not null default ''
); );
-- 4. A release plan (ADR 0236): the builds that wait for a gate — the backlog the old default left, and
-- whatever a plan built and did not send — walked through the machines one at a time, each judged
-- before the next. Its walk is kept with the plan.
alter table release_plan add column release jsonb;
+51 -7
View File
@@ -38,6 +38,9 @@ type Plan struct {
Revision int64 `json:"revision"` Revision int64 `json:"revision"`
// Epoch is the controller lease epoch that wrote it last; zero for a write that claimed none. // Epoch is the controller lease epoch that wrote it last; zero for a write that claimed none.
Epoch uint64 `json:"epoch,omitempty"` Epoch uint64 `json:"epoch,omitempty"`
// Release is set on a release plan (novox/hq ADR 0236): not a merge's, but the builds waiting for a
// gate, walked through the machines one at a time.
Release *PlanRelease `json:"release,omitempty"`
} }
// ErrPlanMoved is a save against a plan written by somebody else since it was read. // ErrPlanMoved is a save against a plan written by somebody else since it was read.
@@ -103,6 +106,35 @@ type PlanGate struct {
Rollback string `json:"rollback,omitempty"` Rollback string `json:"rollback,omitempty"`
// Kept says a passing verdict was written to the gate's records. // Kept says a passing verdict was written to the gate's records.
Kept bool `json:"kept,omitempty"` Kept bool `json:"kept,omitempty"`
// Carried is every module whose build moved on the judged machines with the send — the plan's own
// module and whatever else was waiting there for a gate (novox/hq ADR 0236): each is judged here, a
// pass is its verdict too, and one that fails is put back.
Carried []CarriedMove `json:"carried,omitempty"`
// Failing names the modules the last judging found wanting.
Failing []string `json:"failing,omitempty"`
}
// CarriedMove is one module's build moving on a machine with a gated send.
type CarriedMove struct {
Module string `json:"module"`
Node string `json:"node"`
From string `json:"from,omitempty"`
To string `json:"to"`
Build string `json:"build,omitempty"`
}
// PlanRelease is a release plan's walk through the machines (novox/hq ADR 0236): every module build
// that waits for a gate, sent one machine at a time, each judged before the next.
type PlanRelease struct {
Order []string `json:"order"`
Next int `json:"next"`
// Gate is the machine being judged; nil between machines.
Gate *PlanGate `json:"gate,omitempty"`
Done []string `json:"done,omitempty"`
// Skipped are the machines not heard from when their turn came, left as they were.
Skipped []string `json:"skipped,omitempty"`
// By is the person who released it, empty when the mesh did.
By string `json:"by,omitempty"`
} }
// The states a plan passes through. // The states a plan passes through.
@@ -138,6 +170,12 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
if err != nil { if err != nil {
return err return err
} }
var release []byte
if p.Release != nil {
if release, err = json.Marshal(p.Release); err != nil {
return err
}
}
// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the // **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or // plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
// measure one twice. // measure one twice.
@@ -153,15 +191,16 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
var revision int64 var revision int64
err = tx.QueryRow(ctx, err = tx.QueryRow(ctx,
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, `insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note,
branch, tier_entered, revision, epoch) branch, tier_entered, revision, epoch, release)
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13) values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14)
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier, on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch, tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch,
release = excluded.release
where release_plan.revision = $12 where release_plan.revision = $12
returning revision`, returning revision`,
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered, p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered,
p.Revision, epoch).Scan(&revision) p.Revision, epoch, release).Scan(&revision)
if errors.Is(err, pgx.ErrNoRows) { if errors.Is(err, pgx.ErrNoRows) {
// The row is there and at another revision — moved since this was read, or there already // The row is there and at another revision — moved since this was read, or there already
// when this one is new: either way not this writer's to overwrite. (A plan saved before plans // when this one is new: either way not this writer's to overwrite. (A plan saved before plans
@@ -216,7 +255,7 @@ func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) { func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
rows, err := i.store.Pool().Query(ctx, rows, err := i.store.Pool().Query(ctx,
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch, `select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
coalesce(tier_entered, created), revision, coalesce(epoch, 0) coalesce(tier_entered, created), revision, coalesce(epoch, 0), release
from release_plan `+tail) from release_plan `+tail)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -225,12 +264,17 @@ func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
var out []Plan var out []Plan
for rows.Next() { for rows.Next() {
var p Plan var p Plan
var tiers, modules []byte var tiers, modules, release []byte
var epoch int64 var epoch int64
if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State, if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch); err != nil { &p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch, &release); err != nil {
return nil, err return nil, err
} }
if len(release) > 0 {
if err := json.Unmarshal(release, &p.Release); err != nil {
return nil, err
}
}
p.Epoch = uint64(epoch) p.Epoch = uint64(epoch)
if err := json.Unmarshal(tiers, &p.Tiers); err != nil { if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
return nil, err return nil, err