Refuse every send that would replace the bus outside its planned step (hq ADR 0236)

A plan's send to the bus's machine for another module carried the bus's new build and
restarted it under every machine with nobody asking (2026-10-06). The guard is in the one
send everything uses; only the bus step passes it. A rebuild that made the same artifacts
is no move.
This commit is contained in:
jochen
2026-10-06 18:56:54 +02:00
parent d7bf1bae83
commit 37229b4db5
3 changed files with 90 additions and 4 deletions
+55 -4
View File
@@ -54,12 +54,19 @@ type busPending struct {
machines []string
from map[string]string
to string
// same are the commits whose build made the same artifacts as the build the mesh holds.
same map[string]bool
}
// moves is whether sending the machine would replace its bus.
// moves is whether sending the machine would replace its bus: a build it was not last sent, unless the
// two builds made the same artifacts — a rebuild of the same source for another module's merge changes
// nothing the machine runs.
func (b busPending) moves(machine string) bool {
from, known := b.from[machine]
return b.module != "" && b.to != "" && (!known || !sameCommit(from, b.to))
if b.module == "" || b.to == "" || (known && sameCommit(from, b.to)) {
return false
}
return !known || !b.same[from]
}
// pendingBus reads what a bus upgrade would do.
@@ -85,7 +92,14 @@ func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, erro
if b.machines, err = inv.Running(ctx, b.module); err != nil {
return b, err
}
b.from = map[string]string{}
b.from, b.same = map[string]string{}, map[string]bool{}
made, err := madeBy(ctx, inv, b.module)
if err != nil {
return b, err
}
for commit, refs := range made {
b.same[commit] = refs != "" && refs == made[b.to]
}
for _, n := range b.machines {
sent, known, err := inv.SentBuilds(ctx, n)
if err != nil {
@@ -204,7 +218,7 @@ func busCommand(ctx context.Context, args []string) error {
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
false: "NOT reversible: the snapshot is the only way back"}[*reversible])
sent, err := sendRollout(ctx, open, moving)
sent, err := sendRollout(withBusStep(ctx), open, moving)
if err != nil {
_ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error())
return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err)
@@ -371,3 +385,40 @@ func snapshotTheBusNow(ctx context.Context, module, node string) (string, error)
})
return where, err
}
// errBusWaits is a send refused because it would replace the bus outside its planned step.
var errBusWaits = errors.New("a new bus build waits for its planned step")
type busStepKey struct{}
// withBusStep marks a send as the bus's planned step: the one send that may replace the bus.
func withBusStep(ctx context.Context) context.Context {
return context.WithValue(ctx, busStepKey{}, true)
}
func busStepSending(ctx context.Context) bool { on, _ := ctx.Value(busStepKey{}).(bool); return on }
// madeBy is, per commit, the artifacts the newest worked build of a module from it made, as one sorted
// string: what tells a rebuild that changes nothing from one that does.
func madeBy(ctx context.Context, inv *inventory.Inventory, module string) (map[string]string, error) {
builds, err := inv.Builds(ctx, module, 50)
if err != nil {
return nil, err
}
out := map[string]string{}
for _, b := range builds {
if !b.Worked() || b.Commit == "" {
continue
}
if _, seen := out[b.Commit]; seen {
continue
}
var refs []string
for _, a := range b.Made {
refs = append(refs, a.Name+"="+a.Reference)
}
sort.Strings(refs)
out[b.Commit] = strings.Join(refs, " ")
}
return out, nil
}
+20
View File
@@ -403,6 +403,26 @@ func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) {
if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" {
t.Fatalf("a push may send the bus's machine: %v %v", held, err)
}
// Nor may any other send — a plan's for another module on that machine carried the bus with it.
if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) {
t.Fatalf("a send to the bus's machine was not refused: %v", err)
}
// A rebuild that made the same artifacts is no move.
for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} {
b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}}
b.Asked, b.At = time.Now(), time.Now()
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err)
}
if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second),
At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image",
Reference: "registry/nats@sha256:new"}}}); err != nil {
t.Fatal(err)
}
// The planned step refuses to start without its word on reversibility, and without a snapshot taken
// first by the bus machine's backup holder.
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") {
+15
View File
@@ -1001,6 +1001,21 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
// the machines it sent waits for that one too.
func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) {
inv := open.inventory
// **No send replaces the bus but its planned step** (novox/hq ADR 0236). A plan's send to the bus's
// machine for some other module carried the bus's new build with it on 2026-10-06, and the bus
// restarted under every machine with nobody having asked. Refused whole — the send cannot leave the
// bus behind and carry the rest (ADR 0221 option 2) — and said with the remedy; the plan tries again.
if !busStepSending(ctx) {
held, err := busHeld(ctx, inv, names)
if err != nil {
return nil, err
}
for _, n := range names {
if why, h := held[n]; h {
return nil, fmt.Errorf("%w: %s", errBusWaits, why)
}
}
}
ident, err := openIdentity(ctx)
if err != nil {
return nil, err