diff --git a/cmd/mesh-controller/bus_step.go b/cmd/mesh-controller/bus_step.go index 4ef2149..95a4880 100644 --- a/cmd/mesh-controller/bus_step.go +++ b/cmd/mesh-controller/bus_step.go @@ -54,12 +54,19 @@ type busPending struct { machines []string from map[string]string to string + // same are the commits whose build made the same artifacts as the build the mesh holds. + same map[string]bool } -// moves is whether sending the machine would replace its bus. +// moves is whether sending the machine would replace its bus: a build it was not last sent, unless the +// two builds made the same artifacts — a rebuild of the same source for another module's merge changes +// nothing the machine runs. func (b busPending) moves(machine string) bool { from, known := b.from[machine] - return b.module != "" && b.to != "" && (!known || !sameCommit(from, b.to)) + if b.module == "" || b.to == "" || (known && sameCommit(from, b.to)) { + return false + } + return !known || !b.same[from] } // pendingBus reads what a bus upgrade would do. @@ -85,7 +92,14 @@ func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, erro if b.machines, err = inv.Running(ctx, b.module); err != nil { return b, err } - b.from = map[string]string{} + b.from, b.same = map[string]string{}, map[string]bool{} + made, err := madeBy(ctx, inv, b.module) + if err != nil { + return b, err + } + for commit, refs := range made { + b.same[commit] = refs != "" && refs == made[b.to] + } for _, n := range b.machines { sent, known, err := inv.SentBuilds(ctx, n) if err != nil { @@ -204,7 +218,7 @@ func busCommand(ctx context.Context, args []string) error { fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From, short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it", false: "NOT reversible: the snapshot is the only way back"}[*reversible]) - sent, err := sendRollout(ctx, open, moving) + sent, err := sendRollout(withBusStep(ctx), open, moving) if err != nil { _ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error()) return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err) @@ -371,3 +385,40 @@ func snapshotTheBusNow(ctx context.Context, module, node string) (string, error) }) return where, err } + +// errBusWaits is a send refused because it would replace the bus outside its planned step. +var errBusWaits = errors.New("a new bus build waits for its planned step") + +type busStepKey struct{} + +// withBusStep marks a send as the bus's planned step: the one send that may replace the bus. +func withBusStep(ctx context.Context) context.Context { + return context.WithValue(ctx, busStepKey{}, true) +} + +func busStepSending(ctx context.Context) bool { on, _ := ctx.Value(busStepKey{}).(bool); return on } + +// madeBy is, per commit, the artifacts the newest worked build of a module from it made, as one sorted +// string: what tells a rebuild that changes nothing from one that does. +func madeBy(ctx context.Context, inv *inventory.Inventory, module string) (map[string]string, error) { + builds, err := inv.Builds(ctx, module, 50) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, b := range builds { + if !b.Worked() || b.Commit == "" { + continue + } + if _, seen := out[b.Commit]; seen { + continue + } + var refs []string + for _, a := range b.Made { + refs = append(refs, a.Name+"="+a.Reference) + } + sort.Strings(refs) + out[b.Commit] = strings.Join(refs, " ") + } + return out, nil +} diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index 6c6dc55..0ad1bd4 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -403,6 +403,26 @@ func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) { if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" { t.Fatalf("a push may send the bus's machine: %v %v", held, err) } + // Nor may any other send — a plan's for another module on that machine carried the bus with it. + if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) { + t.Fatalf("a send to the bus's machine was not refused: %v", err) + } + // A rebuild that made the same artifacts is no move. + for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} { + b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}} + b.Asked, b.At = time.Now(), time.Now() + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 { + t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second), + At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image", + Reference: "registry/nats@sha256:new"}}}); err != nil { + t.Fatal(err) + } // The planned step refuses to start without its word on reversibility, and without a snapshot taken // first by the bus machine's backup holder. if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") { diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 882cc39..f8aa3f4 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -1001,6 +1001,21 @@ func sendTo(ctx context.Context, open *stores, names []string) error { // the machines it sent waits for that one too. func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) { inv := open.inventory + // **No send replaces the bus but its planned step** (novox/hq ADR 0236). A plan's send to the bus's + // machine for some other module carried the bus's new build with it on 2026-10-06, and the bus + // restarted under every machine with nobody having asked. Refused whole — the send cannot leave the + // bus behind and carry the rest (ADR 0221 option 2) — and said with the remedy; the plan tries again. + if !busStepSending(ctx) { + held, err := busHeld(ctx, inv, names) + if err != nil { + return nil, err + } + for _, n := range names { + if why, h := held[n]; h { + return nil, fmt.Errorf("%w: %s", errBusWaits, why) + } + } + } ident, err := openIdentity(ctx) if err != nil { return nil, err