From d9289ef6d4d06d309f6963e510380f7499d0bc2b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 6 Oct 2026 18:29:29 +0200 Subject: [PATCH 1/7] Gate a release plan's first machine and roll a failed build back there (hq ADR 0236) A build that reported applied was sent everywhere; one that then did nothing, served no tools or broke its machine's word reached every machine. Now the first machine is judged by the component's health (the core's definitions, as doctor probes H-*, or a module's own) three times over two minutes within ten; a failing gate puts the previous build back there once, marks the build, and says it as a condition and an event. Upgrades roll out by default; the bus is a planned step; a module deleted at its source is not built (the public-acme plan failure). --- cmd/mesh-controller/acting.go | 2 +- cmd/mesh-controller/build.go | 10 + cmd/mesh-controller/bus_step.go | 327 +++++++++ cmd/mesh-controller/doctor.go | 21 + cmd/mesh-controller/gate.go | 692 ++++++++++++++++++ cmd/mesh-controller/gate_test.go | 523 +++++++++++++ cmd/mesh-controller/health_probes.go | 226 ++++++ cmd/mesh-controller/held_back_test.go | 6 +- cmd/mesh-controller/main.go | 3 + cmd/mesh-controller/plan_retry.go | 9 + cmd/mesh-controller/push.go | 21 + cmd/mesh-controller/queue_test.go | 4 + cmd/mesh-controller/release_plan.go | 106 ++- cmd/mesh-controller/rollout.go | 9 +- cmd/mesh-controller/seatverbs.go | 37 + cmd/mesh-controller/seatverbs_test.go | 2 +- cmd/mesh-controller/status_summary.go | 5 + cmd/mesh-controller/upgrades.go | 142 +++- cmd/mesh-controller/witness.go | 119 +++ internal/broker/nats.go | 24 + internal/broker/states_agreement_test.go | 2 + internal/broker/streams.go | 4 +- internal/broker/testdata/composed.conf | 4 +- internal/broker/users.go | 11 +- internal/broker/witness_grants_test.go | 37 + internal/catalogue/manifest.go | 6 + internal/catalogue/seats.go | 4 +- internal/catalogue/upgrade.go | 120 +++ internal/catalogue/upgrade_test.go | 52 ++ internal/catalogue/verbs.go | 25 + internal/inventory/busstep.go | 72 ++ internal/inventory/catalogue.go | 139 +++- internal/inventory/gate.go | 218 ++++++ ...a-build-rolls-out-gated-and-rolls-back.sql | 67 ++ internal/inventory/plans.go | 36 + internal/inventory/sent_builds_test.go | 3 +- internal/lease/lease.go | 12 + internal/lease/witness.go | 180 +++++ internal/lease/witness_test.go | 59 ++ internal/link/events.go | 8 + internal/link/protocol.go | 11 + 41 files changed, 3297 insertions(+), 61 deletions(-) create mode 100644 cmd/mesh-controller/bus_step.go create mode 100644 cmd/mesh-controller/gate.go create mode 100644 cmd/mesh-controller/gate_test.go create mode 100644 cmd/mesh-controller/health_probes.go create mode 100644 cmd/mesh-controller/witness.go create mode 100644 internal/broker/witness_grants_test.go create mode 100644 internal/catalogue/upgrade.go create mode 100644 internal/catalogue/upgrade_test.go create mode 100644 internal/inventory/busstep.go create mode 100644 internal/inventory/gate.go create mode 100644 internal/inventory/migrations/0073-a-build-rolls-out-gated-and-rolls-back.sql create mode 100644 internal/lease/witness.go create mode 100644 internal/lease/witness_test.go diff --git a/cmd/mesh-controller/acting.go b/cmd/mesh-controller/acting.go index 5fbb01f..e796a6a 100644 --- a/cmd/mesh-controller/acting.go +++ b/cmd/mesh-controller/acting.go @@ -206,7 +206,7 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory } say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) } l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance), - Floor: inv.HighestEpoch, Say: say, Moved: func(was, floor uint64) { + Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) { a.mu.Lock() defer a.mu.Unlock() a.reset = time.Now() diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 8a78ae3..d7fb7da 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -570,6 +570,16 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w", result.On, result.Repository, short(result.Commit), err) } + // **A build that failed its gate is never registered again** (novox/hq ADR 0235): an outcome heard + // twice, or replayed, would otherwise make the build a rollback put back what the module is again, + // and the next push would send it. + if failed, err := inv.GateFailed(ctx, kept.ID); err != nil { + return manifest, kept, err + } else if failed { + return manifest, kept, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+ + "back: it is recorded and not registered again — a newer build is", result.On, manifest.Module, + short(result.Commit)) + } if err := inv.RegisterModule(ctx, manifest, recorded); err != nil { if errors.Is(err, inventory.ErrSuperseded) { return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", diff --git a/cmd/mesh-controller/bus_step.go b/cmd/mesh-controller/bus_step.go new file mode 100644 index 0000000..415230a --- /dev/null +++ b/cmd/mesh-controller/bus_step.go @@ -0,0 +1,327 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The bus as a planned step (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0235). +// +// **A bus upgrade is never rolled out.** The bus carries every declaration, every report and the +// controller's own lease; a new bus build that does not come up is a mesh nobody can tell anything, +// and a version whose data format moved (2.10 → 2.11) cannot be undone by putting the old one back. +// So nothing sends a new bus build on its own: its policy is `record` whatever anyone says (the +// catalogue's DerivedUpgrade, and `upgrade` refuses a roll-out), a plan builds it and sends nothing, a +// cascade holds the machine (ADR 0221), and a push naming that machine is refused while a new bus build +// waits for it (busHeld). The one way is this verb: a person, with why, the streams snapshotted first, +// whether it can be reverted said before it starts, the step said as `bus-maintenance` while it runs, +// and every stream, durable consumer and a round trip checked after (H-bus) — or the step said failed, +// with its snapshot as the way back. + +// busStepProbe is the registry's row for the step; its kinds. +const ( + busStepProbe = "DB" + kindBusMaintenance = "bus-maintenance" + kindBusUpgradeFailed = "bus-upgrade-failed" +) + +// busStepBound is how long after its start a bus upgrade must be followed by a healthy bus. +var busStepBound = 15 * time.Minute + +// takeBusSnapshot snapshots every stream before a bus upgrade and answers where the snapshot is. Nil +// until the controller's JetStream snapshot is built (to-be 45 §8: "the streams are snapshotted"); until +// then a person takes it by hand and says where with --snapshot-taken. +var takeBusSnapshot func(ctx context.Context) (string, error) + +// busPending is what a bus upgrade would do: the bus's module, the machines running it, and, per +// machine, the build it was last sent against the build the mesh holds. Empty machines: the mesh holds +// no bus module. +type busPending struct { + module string + machines []string + from map[string]string + to string +} + +// moves is whether sending the machine would replace its bus. +func (b busPending) moves(machine string) bool { + from, known := b.from[machine] + return b.module != "" && b.to != "" && (!known || !sameCommit(from, b.to)) +} + +// pendingBus reads what a bus upgrade would do. +func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, error) { + var b busPending + shelf, err := inv.Catalogue(ctx) + if err != nil { + return b, err + } + for name, m := range shelf { + if catalogue.ProvidesBus(m) { + b.module = name + } + } + if b.module == "" { + return b, nil + } + current, err := inv.CurrentBuilds(ctx) + if err != nil { + return b, err + } + b.to = current[b.module].Commit + if b.machines, err = inv.Running(ctx, b.module); err != nil { + return b, err + } + b.from = map[string]string{} + for _, n := range b.machines { + sent, known, err := inv.SentBuilds(ctx, n) + if err != nil { + return b, err + } + if known { + if c, carried := sent[b.module]; carried { + b.from[n] = c + } + } + } + return b, nil +} + +// busHeld names the machines a push may not send because sending them would replace the bus: the +// planned step's, not a push's (ADR 0235). Said with the remedy. +func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) { + b, err := pendingBus(ctx, inv) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, n := range machines { + for _, holder := range b.machines { + if n == holder && b.moves(n) { + out[n] = fmt.Sprintf("sending %s would replace the bus (%s %s → %s), which is a planned step: "+ + "`bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0235)", + n, b.module, short(orNotKnown(b.from[n])), short(b.to)) + } + } + } + return out, nil +} + +// busCommand is `bus` — what a bus upgrade would do and how the last went — and `bus upgrade`. +func busCommand(ctx context.Context, args []string) error { + sub := "" + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + sub, args = args[0], args[1:] + } + set := flag.NewFlagSet("bus", flag.ContinueOnError) + snapshot := set.String("snapshot-taken", "", "where the streams' snapshot a person took is, while the mesh takes none itself") + reversible := set.Bool("reversible", false, "the new version can be undone by putting the old one back") + irreversible := set.Bool("irreversible", false, "the new version cannot be undone by putting the old one back, "+ + "and this is the person's explicit word that it runs anyway") + why := addHandActFlags(set) + if rest, err := parseAround(set, args); err != nil { + return err + } else if len(rest) > 0 { + return errors.New("bus [upgrade --why … --reversible|--irreversible [--snapshot-taken ]]") + } + switch sub { + case "": + return busStatus(ctx) + case "upgrade": + default: + return fmt.Errorf("bus says what a bus upgrade would do, or `bus upgrade` — not %q", sub) + } + // Everything refused before anything is done. + if err := why.require("bus upgrade"); err != nil { + return err + } + if *reversible == *irreversible { + return errors.New("bus upgrade says, before it starts, whether the new version can be undone by putting the " + + "old one back: --reversible, or --irreversible as your explicit word that it runs anyway (to-be 45 §8). " + + "Nothing was done") + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + b, err := pendingBus(ctx, inv) + if err != nil { + return err + } + if b.module == "" { + return errors.New("the mesh holds no module that provides its bus: there is nothing to upgrade") + } + var moving []string + for _, n := range b.machines { + if b.moves(n) { + moving = append(moving, n) + } + } + if len(moving) == 0 { + fmt.Printf("every machine running %s runs the build the mesh holds (%s): nothing to upgrade\n", b.module, short(b.to)) + return nil + } + where := strings.TrimSpace(*snapshot) + switch { + case takeBusSnapshot != nil: + if where, err = takeBusSnapshot(ctx); err != nil { + return fmt.Errorf("the streams could not be snapshotted, so the bus is not replaced: %w", err) + } + case where == "": + return errors.New("the bus is replaced only after its streams are snapshotted. The mesh does not take the " + + "snapshot itself yet (to-be 45 §8: the controller's JetStream snapshot); take one by hand and say where " + + "with --snapshot-taken . Nothing was done") + } + from := map[string]bool{} + for _, n := range moving { + from[orNotKnown(b.from[n])] = true + } + step, err := inv.StartBusStep(ctx, inventory.BusStep{Module: b.module, Machines: moving, + From: strings.Join(sortedKeys(from), ", "), To: b.to, Snapshot: where, Reversible: *reversible, + By: link.Caller(), Why: strings.TrimSpace(*why.why)}) + if err != nil { + return err + } + cause := "bus-upgrade" + if strings.TrimSpace(*why.cause) == "" { + why.cause = &cause + } + why.record(ctx, "bus upgrade", append([]string{b.module}, moving...)) + fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From, + short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it", + false: "NOT reversible: the snapshot is the only way back"}[*reversible]) + sent, err := sendRollout(ctx, open, moving) + if err != nil { + _ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error()) + return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err) + } + fmt.Printf("sent %s; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+ + "consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot "+ + "as the way back. `bus` says how it went\n", strings.Join(sent, ", "), busStepBound) + return nil +} + +// busStatus is `bus`: what an upgrade would do, and the last step. +func busStatus(ctx context.Context) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + b, err := pendingBus(ctx, open.inventory) + if err != nil { + return err + } + if b.module == "" { + fmt.Println("the mesh holds no module that provides its bus") + } else { + fmt.Printf("the bus is %s, built %s, on %s; never rolled out — a planned step (`bus upgrade`)\n", b.module, + short(b.to), orNone(strings.Join(b.machines, ", "))) + for _, n := range b.machines { + state := "runs it" + if b.moves(n) { + state = "runs " + short(orNotKnown(b.from[n])) + ": `bus upgrade` replaces it" + } + fmt.Printf(" %-10s %s\n", n, state) + } + } + if takeBusSnapshot == nil { + fmt.Println(" the mesh takes no snapshot of the streams itself yet: `bus upgrade` asks where yours is (--snapshot-taken)") + } + s, found, err := open.inventory.LatestBusStep(ctx) + if err != nil || !found { + return err + } + state := "running since " + s.Started.Local().Format("2006-01-02 15:04") + if s.Ended != nil { + state = s.Outcome + " at " + s.Ended.Local().Format("2006-01-02 15:04") + } + fmt.Printf("last step %d: %s → %s on %s by %s (%s): %s; snapshot %s\n", s.ID, s.From, short(s.To), + strings.Join(s.Machines, ", "), orNone(s.By), s.Why, state, s.Snapshot) + if s.Found != "" { + fmt.Printf(" %s\n", s.Found) + } + return nil +} + +// probeBusStep is DB: a bus upgrade running is said as `bus-maintenance`; the bus healthy again after +// the machines reported the new build ends it done; past its bound, unhealthy, it ends failed and is +// said — urgent, with its snapshot — while the bus is still not healthy. +func probeBusStep(ctx context.Context, d *doctor) ([]conditions.Observation, error) { + inv := d.open.inventory + s, found, err := inv.LatestBusStep(ctx) + if err != nil || !found { + return nil, err + } + if s.Ended != nil && s.Outcome != "failed" { + return nil, nil + } + problems, err := busHealth(ctx, d) + if err != nil { + return nil, err + } + reports, err := inv.LastReports(ctx) + if err != nil { + return nil, err + } + applied := true + for _, r := range reports { + for _, n := range s.Machines { + if r.Node == n && (!r.Current || r.Outcome != inventory.OutcomeApplied || r.At == nil || r.At.Before(s.Started)) { + applied = false + problems = append(problems, n+" has not reported the new bus applied") + } + } + } + id := s.Module + if s.Ended == nil { + switch { + case applied && len(problems) == 0: + return nil, inv.EndBusStep(ctx, s.ID, "done", "the bus answered healthy after the upgrade") + case time.Since(s.Started) > busStepBound: + found := strings.Join(problems, "; ") + if err := inv.EndBusStep(ctx, s.ID, "failed", found); err != nil { + return nil, err + } + s.Found = found + default: + return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "maintenance", + Kind: kindBusMaintenance, Severity: conditions.Warning, + Summary: fmt.Sprintf("the bus is being upgraded (step %d, %s → %s on %s, by %s: %s); its snapshot is %s", + s.ID, s.From, short(s.To), strings.Join(s.Machines, ", "), orNone(s.By), s.Why, s.Snapshot), + Said: orNone(strings.Join(problems, "; "))}}, nil + } + } + if len(problems) == 0 { + return nil, nil // failed, and healthy since: nothing wrong now + } + way := "put the old build back" + if !s.Reversible { + way = "restore the snapshot" + } + return []conditions.Observation{{Scope: conditions.ScopeBus, ID: id, Token: "upgrade-failed", + Kind: kindBusUpgradeFailed, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("the bus upgrade (step %d, %s → %s) did not end healthy within %s: %s — the way back is to %s (%s)", + s.ID, s.From, short(s.To), busStepBound, strings.Join(problems, "; "), way, s.Snapshot)}}, nil +} + +func sortedKeys(set map[string]bool) []string { + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index 199d7bb..ad98778 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -113,6 +113,27 @@ var probeRegistry = []probe{ Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}}, {ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals", From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs}, + // The core's health definitions (novox/hq to-be 45 §8, ADR 0235): what a core component's new build is + // judged by on its first machine, run against every machine between upgrades too. + {ID: "H-controller", Asserts: "the controller lease is held, renewed in time, by a controller that says it is " + + "ready: its self-check ran and status answered in full within ten seconds", From: "ADR 0235, to-be 45 §8", + Kind: kindCoreUnhealthy, Phase: 4, run: probeControllerHealth}, + {ID: "H-engine", Asserts: "every machine heard from has reported its current declaration, under a node-engine " + + "build it names", From: "ADR 0235, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeEngineHealth}, + {ID: "H-tools", Asserts: "every machine heard from that runs the node tools has them answering the bus", + From: "ADR 0235, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeToolsHealth}, + {ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses " + + "it to the machines' node tools and back", From: "ADR 0235, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, + run: probeBusHealth}, + // The gate's verdicts and the witnesses' rollbacks (ADR 0235): each build that failed its gate keeps its + // condition until a newer build passes; each rollback a witness stands by is said. + {ID: gateProbe, Asserts: "no build that failed its gate, and no core component a witness put back, goes unsaid; " + + "a newer build that passes its gate clears it", From: "ADR 0235, to-be 45 §8", Kind: kindRolledBack, + Raises: []string{kindRollbackFailed}, Phase: 4, run: probeGates}, + // The bus's planned step (ADR 0235): open while a person's bus upgrade runs, then checked by H-bus. + {ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " + + "health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0235, to-be 45 §8", + Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep}, } // probeVerdict is one probe's outcome in a run. diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go new file mode 100644 index 0000000..6a9fdbe --- /dev/null +++ b/cmd/mesh-controller/gate.go @@ -0,0 +1,692 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/micro" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" + "github.com/novox/mesh-controller/internal/link" +) + +// The gate on a release plan's first machine, and the rollback after it (novox/hq ADR 0235, to-be 45 +// §8, ADR 0227 rule 8). +// +// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once +// that machine reported it applied. A build that applies and then does nothing, crashes, serves no +// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by +// the component's health — the core's health definitions, or a module's own — passing three times over +// at least two minutes, within ten minutes of the apply. Only then are the rest sent. +// +// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed +// at its gate — registration refuses it and nothing sends it again on its own — the module's registered +// build goes back to the one the first machine ran before, and that machine is sent it: the ordinary +// path, again. Once per build: the verdict is written before the send, and a verdict once written is +// not written over. Said as a condition, urgent for the core and when it could not be put back, and as +// the event `rolled-back`. + +// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes. +var ( + // gateSettle is how long the first machine must stay healthy, at the least. + gateSettle = 2 * time.Minute + // gateBound is how long after the apply the build may take to become healthy. + gateBound = 10 * time.Minute + // gatePasses is how many consecutive judgings must find it healthy, gateEvery apart at the least. + gatePasses = 3 + gateEvery = 40 * time.Second +) + +// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions +// are raised by a plan as it judges, and kept or cleared by the probe on every run. +const gateProbe = "DG" + +// The kinds a gate raises. +const ( + kindRolledBack = "rolled-back" + kindRollbackFailed = "rollback-failed" +) + +// coreComponent is the core component a module is, as a witness names it — controller, node-engine, +// node-tools — or empty: the core is judged by its health definitions, anything else by its own health. +func coreComponent(module string) string { + switch module { + case catalogue.ControllerSeatName: + return lease.ComponentController + case hostModule: + return lease.ComponentEngine + case broker.RuntimeModule: + return lease.ComponentNodeTools + } + return "" +} + +// health is a judging's word on one machine. +type health int + +const ( + healthGood health = iota + healthNotYet + healthBroken +) + +// served is what one machine's node tools answered the bus's discovery with. +type served struct { + runtime bool + tools map[string]bool +} + +// gateFacts is what one judging reads, gathered once for every machine it judges. +type gateFacts struct { + now time.Time + reports map[string]inventory.Reported + // engines is each machine's node-engine build as it last reported it. + engines map[string]string + // served is what the bus's discovery answered; servedErr why it could not be asked. + served map[string]served + servedErr error + // open are the open conditions; openErr why they could not be read (nil keeper: not judged). + open []conditions.Condition + openErr error + judged bool + // rolledBack is what each machine's witnesses say they decided. + rolledBack map[string][]lease.Rollback + // holder is who holds the controller lease, for judging the controller. + holder *lease.Holder + holderErr error +} + +// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A +// variable so a test can hand a judging its facts. +var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + inv := open.inventory + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: witnessed.all()} + reports, err := inv.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + f.reports[r.Node] = r + } + nodes, err := inv.Nodes(ctx) + if err != nil { + return f, err + } + for _, n := range nodes { + f.engines[n.Name] = n.HostVersion + } + if d := doctorFrom; d != nil { + if d.keeper != nil { + f.judged = true + f.open, f.openErr = d.keeper.Open(ctx) + } + if d.js != nil { + f.served, f.servedErr = servedOnTheBus(ctx, d.js.Conn()) + } else { + f.servedErr = errors.New("this controller has no bus to ask") + } + } else { + f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what") + } + if component == lease.ComponentController { + h, found, err := theLease.holder(ctx) + switch { + case err != nil: + f.holderErr = err + case found: + f.holder = &h + } + } + return f, nil +} + +// judgeHealth is one machine's health for a module's new build, from what one judging read: healthy, +// not yet (with what is wanting), or healthBroken — a witness put it back, or the machine refused or failed +// what it was sent. Pure. +func judgeHealth(module, component string, m catalogue.Manifest, machine string, since time.Time, f gateFacts) (health, string) { + // A witness's verdict made since the build was sent: the build failed its health there. One that + // could not judge at all (unwitnessed) is not a verdict on the build. + for _, r := range f.rolledBack[machine] { + if component == "" || r.Component != component || r.Outcome == lease.OutcomeUnwitnessed || r.At.Before(since) { + continue + } + return healthBroken, fmt.Sprintf("the witness on %s judged the %s %s and %s: %s", machine, r.Component, + short(r.From), r.Outcome, r.Why) + } + r, said := f.reports[machine] + switch { + case !said || r.At == nil || !r.Current: + return healthNotYet, fmt.Sprintf("%s has not reported on what it was sent", machine) + case r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused: + return healthBroken, fmt.Sprintf("%s %s what it was sent", machine, r.Outcome) + case r.Outcome != inventory.OutcomeApplied: + return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome) + } + // **No new condition about it**: about the machine itself, or naming the module on that machine, + // raised since the judging began. The gate's own are not evidence about the build. + if f.judged { + if f.openErr != nil { + return healthNotYet, "what is wrong cannot be read, so whether the build made anything wrong is not known: " + + firstLine(f.openErr.Error()) + } + for _, c := range f.open { + if c.Source == gateProbe || c.Raised.Before(since) { + continue + } + onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) || + (c.Subject.Scope == conditions.ScopeMachine && c.Subject.ID == machine) + if !onIt { + continue + } + if c.Subject.Scope == conditions.ScopeMachine || slices.Contains(strings.Split(c.Subject.ID, "."), module) { + return healthNotYet, fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary) + } + } + } + switch component { + case lease.ComponentEngine: + // The node-engine has reported its current declaration under its own build. + want := deliveredVersions(m) + if len(want) > 0 && !slices.Contains(want, f.engines[machine]) { + return healthNotYet, fmt.Sprintf("%s's node-engine reports build %s, not the new %s", machine, + orNotKnown(f.engines[machine]), strings.Join(want, " or ")) + } + case lease.ComponentNodeTools: + // The node tools are announced and answer. + if f.servedErr != nil { + return healthNotYet, "whether the node tools answer cannot be asked: " + firstLine(f.servedErr.Error()) + } + if !f.served[machine].runtime { + return healthNotYet, fmt.Sprintf("the node tools on %s do not answer the bus", machine) + } + case lease.ComponentController: + // The new controller holds the lease and says it is ready. + switch { + case f.holderErr != nil: + return healthNotYet, "who holds the controller lease cannot be read: " + firstLine(f.holderErr.Error()) + case f.holder == nil: + return healthNotYet, "no controller holds the lease" + case f.holder.Taken.Before(since.Add(-time.Minute)): + return healthNotYet, fmt.Sprintf("the lease is held since %s, by a controller older than the new build", + f.holder.Taken.UTC().Format(time.RFC3339)) + case f.holder.Health == nil || !f.holder.Health.Ready: + why := "the controller holding the lease does not say it is ready" + if f.holder.Health != nil && f.holder.Health.Why != "" { + why += ": " + f.holder.Health.Why + } + return healthNotYet, why + } + default: + // A module's tools answer, where it has any and the machine runs the node tools that serve them. + if len(m.Tools) > 0 { + if f.servedErr != nil { + return healthNotYet, "whether its tools are served cannot be asked: " + firstLine(f.servedErr.Error()) + } + if s := f.served[machine]; s.runtime && !s.tools[module] { + return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module) + } + } + } + return healthGood, "" +} + +// judgeGate takes one judging of a module's first machines and records it in the plan's gate: a pass +// counted, a pass missed (and why), or the verdict. Answers the verdict once there is one. +func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule, + running []string, now time.Time) (string, error) { + g := state.Gate + if g == nil { + // Judged from the send: a witness's verdict, a condition, the bound — all counted from when the + // first machine was sent the build. + start := now + if state.FirstAt != nil { + start = *state.FirstAt + } + var machines []string + for _, n := range state.First { + if slices.Contains(running, n) { + machines = append(machines, n) + } + } + g = &inventory.PlanGate{Component: coreComponent(module), Machines: machines, From: state.Previous, + To: state.Commit, Since: &start} + state.Gate = g + } + if g.Verdict != "" { + return g.Verdict, nil + } + if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery { + return "", nil + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return "", err + } + facts, err := gatherGateFacts(ctx, open, g.Component) + if err != nil { + return "", err + } + worst, why := healthGood, "" + for _, n := range g.Machines { + h, said := judgeHealth(module, g.Component, shelf[module], n, *g.Since, facts) + if h > worst { + worst, why = h, said + } else if h == worst && h != healthGood && why == "" { + why = said + } + } + switch { + case worst == healthBroken: + decide(g, inventory.GateFailed, why, now) + case worst == healthNotYet: + g.Passes, g.LastPass, g.Last = 0, nil, why + if now.Sub(*g.Since) > gateBound { + decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now) + } + default: + g.Passes++ + g.LastPass, g.Last = &now, "" + if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle { + decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes, + now.Sub(*g.Since).Round(time.Second)), now) + } + } + return g.Verdict, nil +} + +// decide sets a gate's verdict. +func decide(g *inventory.PlanGate, verdict, why string, now time.Time) { + g.Verdict, g.Why, g.JudgedAt = verdict, why, &now + g.Took = now.Sub(*g.Since).Round(time.Second).String() +} + +// gatePassed keeps a passing build's verdict, so `plans` and the gate's probe can read it. +func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule) { + g := state.Gate + err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module, + Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines, + Verdict: inventory.GatePassed, Why: g.Why, Component: g.Component, JudgingFrom: g.Since}) + if err != nil && state.Build != "" { + fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err) + } + fmt.Printf("%s: %s passed its gate on %s (%s); the rest are sent\n", p.ID, module, + strings.Join(g.Machines, ", "), g.Why) + if module == catalogue.ControllerSeatName { + carryUserList(ctx, open, p) + } +} + +// carryUserList sends the machine holding the bus the user list a new controller composes, once that +// controller passed its gate (ADR 0235). The controller's own grants travel in that list, and the old +// controller composed the list the plan sent; on 2026-10-06 eight pushes by hand carried a new +// controller's grant into it. Not when a build its policy or a plan holds back would go with it (ADR +// 0221): then it is said, as a push would say it. +func carryUserList(ctx context.Context, open *stores, p *inventory.Plan) { + holder, behind, err := brokerBehind(ctx, open, nil) + if err != nil || holder == "" || !behind { + if err != nil { + fmt.Printf("%s: whether the bus's user list is behind the new controller cannot be read: %v\n", p.ID, err) + } + return + } + held, err := heldMachines(ctx, open, []string{holder}) + if err != nil { + fmt.Printf("%s: whether %s may be sent the new user list cannot be read: %v\n", p.ID, holder, err) + return + } + if why, isHeld := held[holder]; isHeld { + fmt.Printf("%s: the new controller's user list is not carried to %s, which holds the bus: %s — `push %s` "+ + "carries it\n", p.ID, holder, strings.Join(why, "; "), holder) + return + } + if _, err := sendRollout(ctx, open, []string{holder}); err != nil { + fmt.Printf("%s: the new controller's user list could not be carried to %s: %v\n", p.ID, holder, err) + return + } + fmt.Printf("%s: carried the new controller's user list to %s, which holds the bus\n", p.ID, holder) +} + +// gateFailed stops the plan at a build that failed its gate and puts the previous build back on the +// machines it was judged on — once per build, said as a condition and an event. The plan is saved +// before the send: a controller that is itself the build being put back does not outlive it. +func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule, + machines []string, why string) { + inv := open.inventory + now := time.Now().UTC() + if state.Gate == nil { + state.Gate = &inventory.PlanGate{Component: coreComponent(module), Machines: machines, + From: state.Previous, To: state.Commit, Since: state.FirstAt} + } + g := state.Gate + if g.Verdict == "" { + if g.Since == nil { + g.Since = &now + } + decide(g, inventory.GateFailed, why, now) + } + if len(g.Machines) == 0 { + g.Machines = machines + } + state.Why = "failed its gate: " + g.Why + p.State = inventory.PlanFailed + p.Note = fmt.Sprintf("%s failed its gate on %s in tier %d: %s", module, strings.Join(g.Machines, ", "), p.Tier, g.Why) + + verdict := inventory.GateVerdict{Build: state.Build, Module: module, Commit: state.Commit, Previous: state.Previous, + Plan: p.ID, Machines: g.Machines, Verdict: inventory.GateFailed, Rollback: inventory.RollingBack, Why: g.Why, + Component: g.Component, JudgingFrom: g.Since} + if state.Build == "" { + // A plan from before builds were asked by id: nothing to mark, so nothing is put back by the + // mesh — said, for a person. + g.Rollback = inventory.NotRolledBack + p.Note += "; not put back: the plan does not know which build it sent" + sayRollback(ctx, open, module, g, "") + return + } + if err := inv.RecordGate(ctx, verdict); err != nil { + if errors.Is(err, inventory.ErrGateKept) { + // Already judged and acted on, by this controller before a restart or by another: never twice. + if kept, found, _ := inv.GateOf(ctx, state.Build); found { + g.Rollback = kept.Rollback + } + p.Note += "; its rollback was already made once and is not made again" + return + } + g.Rollback = inventory.NotRolledBack + p.Note += "; not put back: its verdict could not be kept, and a rollback that cannot be counted is not made — " + err.Error() + sayRollback(ctx, open, module, g, "") + return + } + // The previous build: the one the first machine ran, from the build records. + notBack := func(why string) { + g.Rollback = inventory.NotRolledBack + p.Note += "; NOT put back: " + why + if err := inv.SetRollback(ctx, state.Build, inventory.NotRolledBack, g.Why+"; not put back: "+why); err != nil { + fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err) + } + sayRollback(ctx, open, module, g, why) + } + if state.Previous == "" { + notBack(fmt.Sprintf("%s had not been sent %s before, or what it was sent is not known — `unassign` takes it "+ + "off, or a newer merge replaces it", strings.Join(g.Machines, ", "), module)) + return + } + failed, _, err := inv.BuildByID(ctx, state.Build) + if err != nil { + notBack("the failed build's record cannot be read: " + err.Error()) + return + } + previous, found, err := inv.PreviousBuild(ctx, module, state.Previous, failed) + if err != nil { + notBack("the build records cannot be read: " + err.Error()) + return + } + if !found { + notBack(fmt.Sprintf("no build of %s from %s is still kept to put back", module, short(state.Previous))) + return + } + if err := inv.RestoreModule(ctx, previous); err != nil { + notBack(err.Error()) + return + } + g.Rollback = inventory.RollingBack + if err := inv.SavePlan(ctx, p); err != nil { + fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err) + } + sent, err := sendRollout(ctx, open, g.Machines) + if err != nil { + notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ + "sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0])) + return + } + g.Rollback = inventory.RolledBack + p.Note += fmt.Sprintf("; put back to %s on %s", short(previous.Commit), strings.Join(sent, ", ")) + if err := inv.SetRollback(ctx, state.Build, inventory.RolledBack, g.Why); err != nil { + fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err) + } + sayRollback(ctx, open, module, g, "") +} + +// rolledBackEvent is the body of `rolled-back` (ADR 0235): a contract, like a condition's events. +type rolledBackEvent struct { + Event string `json:"event"` + At time.Time `json:"at"` + Module string `json:"module"` + Component string `json:"component,omitempty"` + Machines []string `json:"machines"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` + Why string `json:"why"` + // Rollback is rolled-back, or not-rolled-back with NotWhy. + Rollback string `json:"rollback"` + NotWhy string `json:"not_why,omitempty"` + Show string `json:"show"` +} + +// sayRollback raises the gate's condition at once — the probe keeps it from then — and says the event. +func sayRollback(ctx context.Context, open *stores, module string, g *inventory.PlanGate, notWhy string) { + o := gateObservation(module, g.Component, g.Machines, g.Rollback, g.Why, notWhy, g.To, g.From) + fmt.Println(o.Summary) + d := doctorFrom + if d == nil { + return + } + if d.keeper != nil { + o.Source = gateProbe + if _, err := d.keeper.Observe(ctx, o); err != nil { + fmt.Printf("the gate's condition %s could not be kept: %v\n", o.Key(), err) + } + } + if d.teller == nil { + return + } + body, err := json.Marshal(rolledBackEvent{Event: link.KeyRolledBack, At: time.Now().UTC(), Module: module, + Component: g.Component, Machines: g.Machines, From: g.To, To: g.From, Why: g.Why, Rollback: g.Rollback, + NotWhy: notWhy, Show: conditions.Condition{Key: o.Key()}.Show()}) + if err != nil { + return + } + saying, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if err := d.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyRolledBack, body); err != nil { + fmt.Printf("%s's rollback could NOT be said on the bus: %v\n", module, err) + } +} + +// gateObservation is a failed gate as a condition: `core...rolled-back` for the +// core (to-be 45 §2), `build...rolled-back` for any other module; `rollback-failed` +// when it could not be put back. Urgent for the core and for a build left in place; a warning for a +// module put back, which runs what it ran before. +func gateObservation(module, component string, machines []string, rollback, why, notWhy, failed, previous string) conditions.Observation { + machine := strings.Join(machines, ",") + o := conditions.Observation{Scope: conditions.ScopeBuild, ID: module + "." + machine, Kind: kindRolledBack, + Token: kindRolledBack, Machine: firstOf(machines), Severity: conditions.Warning, Source: gateProbe, + Summary: fmt.Sprintf("%s's build %s failed its gate on %s and was put back to %s: %s", module, short(failed), + machine, short(previous), why)} + if component != "" { + o.Scope, o.ID, o.Severity = conditions.ScopeCore, component+"."+machine, conditions.Urgent + } + if len(machines) > 1 { + o.Also = machines[1:] + } + if rollback != inventory.RolledBack && rollback != inventory.RollingBack { + o.Kind, o.Token, o.Severity = kindRollbackFailed, kindRollbackFailed, conditions.Urgent + o.Resolver = conditions.ResolverOperator + o.Summary = fmt.Sprintf("%s's build %s failed its gate on %s and was NOT put back: %s — %s", module, short(failed), + machine, why, orNone(notWhy)) + } + return o +} + +// probeGates is DG: no build that failed its gate is left without its condition, and no witness's +// rollback goes unsaid. Each module whose newest verdict is a failure keeps its condition; a newer build +// that passes clears it. Each core component a machine's witness says it put back is `core.. +// .rolled-back`, urgent, until the machine's reports stop saying it. +func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error) { + latest, err := d.open.inventory.LatestGates(ctx) + if err != nil { + return nil, err + } + var out []conditions.Observation + for _, v := range latest { + if v.Verdict != inventory.GateFailed { + continue + } + notWhy := "" + if v.Rollback == inventory.NotRolledBack { + notWhy = v.Why + } + out = append(out, gateObservation(v.Module, v.Component, v.Machines, v.Rollback, v.Why, notWhy, v.Commit, v.Previous)) + } + for node, list := range witnessed.all() { + for _, r := range list { + out = append(out, witnessObservation(node, r)) + } + } + return sortedFound(dedupeObservations(out)), nil +} + +// dedupeObservations keeps one observation per key, the first. +func dedupeObservations(list []conditions.Observation) []conditions.Observation { + seen := map[string]bool{} + var out []conditions.Observation + for _, o := range list { + if seen[o.Key()] { + continue + } + seen[o.Key()] = true + out = append(out, o) + } + return out +} + +// servedOnTheBus asks the bus's discovery what every machine's node tools answer and which modules' +// tools are served where. A variable so a test needs no runtime. +var servedOnTheBus = func(ctx context.Context, conn *nats.Conn) (map[string]served, error) { + inbox := conn.NewRespInbox() + sub, err := conn.SubscribeSync(inbox) + if err != nil { + return nil, err + } + defer func() { _ = sub.Unsubscribe() }() + if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil { + return nil, fmt.Errorf("asking the bus who serves what: %w", err) + } + out := map[string]served{} + add := func(node string) served { + s, ok := out[node] + if !ok { + s = served{tools: map[string]bool{}} + } + return s + } + deadline := time.Now().Add(discoveryPatience) + for time.Now().Before(deadline) { + wait, cancel := context.WithTimeout(ctx, discoveryQuiet) + msg, err := sub.NextMsgWithContext(wait) + cancel() + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + break + } + var info micro.Info + if json.Unmarshal(msg.Data, &info) != nil { + continue + } + if info.Name == broker.RuntimeModule { + node := info.Metadata["node"] + if node == "" { + node = info.ID + } + s := add(node) + s.runtime = true + out[node] = s + } + for _, e := range info.Endpoints { + if e.Metadata["kind"] != "tool" || e.Metadata["module"] == "" { + continue + } + node := e.Metadata["node"] + if node == "" { + node = info.ID + } + s := add(node) + s.tools[e.Metadata["module"]] = true + out[node] = s + } + } + return out, nil +} + +// gateLines is a plan's gates as `plans ` says them: the rollout's record. +func gateLine(g *inventory.PlanGate) string { + if g == nil { + return "" + } + what := "judging" + switch g.Verdict { + case inventory.GatePassed: + what = "passed" + case inventory.GateFailed: + what = "FAILED" + } + line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what) + if g.Component != "" { + line += " (core: " + g.Component + ")" + } + if g.From != "" || g.To != "" { + line += fmt.Sprintf(", %s → %s", short(orNone(g.From)), short(g.To)) + } + if g.Took != "" { + line += ", after " + g.Took + } + if g.Why != "" { + line += ": " + g.Why + } else if g.Last != "" { + line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gatePasses, g.Last) + } else if g.Verdict == "" { + line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses) + } + if g.Rollback != "" { + line += "; " + g.Rollback + } + return line +} + +// witnessObservation is a witness's verdict as a condition (ADR 0236, the host's contract): +// `core...`, urgent for rolled-back, not-reversible, restore-failed and +// halted, a warning for nothing-to-restore and unwitnessed. +func witnessObservation(node string, r lease.Rollback) conditions.Observation { + severity := conditions.Warning + if lease.Urgent(r.Outcome) { + severity = conditions.Urgent + } + outcome := r.Outcome + if outcome == "" { + outcome = lease.OutcomeRolledBack + } + what := map[string]string{ + lease.OutcomeRolledBack: "and put back " + short(orNone(r.To)), + lease.OutcomeNotReversible: "and left it: it is not reversible", + lease.OutcomeNothingToRestore: "and had nothing to put back", + lease.OutcomeRestoreFailed: "and could not put the previous build back", + lease.OutcomeUnwitnessed: "and could not judge it at all", + lease.OutcomeHalted: "and gave up: the build before it fails too", + }[outcome] + return conditions.Observation{Scope: conditions.ScopeCore, ID: r.Component + "." + node, Kind: kindRolledBack, + Token: outcome, Machine: node, Severity: severity, + Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component, + short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)} +} diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go new file mode 100644 index 0000000..ff2135b --- /dev/null +++ b/cmd/mesh-controller/gate_test.go @@ -0,0 +1,523 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" + "github.com/novox/mesh-controller/internal/link" +) + +// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0235, to-be 45 §8). + +// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered, +// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is +// sent and reports it — with the gate's bounds shortened to judge in three steps. +type gateMesh struct { + open *stores + sent [][]string + health map[string]health // per machine, what a judging finds; healthy when unsaid + keeper *conditions.Keeper + told *conditions.Told +} + +func aGateMesh(t *testing.T) *gateMesh { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + g := &gateMesh{open: open, health: map[string]health{}} + g.keeper, _ = withConditionsInMemory(t) + g.told = &conditions.Told{} + was := doctorFrom + doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told} + t.Cleanup(func() { doctorFrom = was }) + + for _, b := range []inventory.Build{ + {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", + Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, + {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", + Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, + } { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) + b.Manifest = manifest + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + register := func(commit string, asked time.Time) { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, + Head: commit, Asked: asked}); err != nil { + t.Fatal(err) + } + } + register("c1", time.Now().Add(-2*time.Hour)) + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + } + register("c2", time.Now().Add(-time.Minute)) + + // Every send: recorded with the build the module is at, applied and reported by the machine. + n := 0 + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + g.sent = append(g.sent, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { + return nil, err + } + if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + + // What a judging reads: the store's reports, and a health the test says per machine. + wasGather := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + if g.health[r.Node] == healthBroken { + r.Outcome = inventory.OutcomeFailed + } + f.reports[r.Node] = r + } + for node, h := range g.health { + if h == healthNotYet { + f.rolledBack[node] = nil + r := f.reports[node] + r.Current = false + f.reports[node] = r + } + } + return f, nil + } + t.Cleanup(func() { gatherGateFacts = wasGather }) + + wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. + gateSettle, gateEvery = 0, time.Hour + t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound }) + + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", + Build: "build-2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + return g +} + +func (g *gateMesh) plan(t *testing.T) inventory.Plan { + t.Helper() + p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate") + if err != nil { + t.Fatal(err) + } + return p +} + +// A module's build that fails its gate on the first machine is put back there — the previous build +// registered and sent to it again — and never reaches the second machine; it is marked, said as a +// condition and an event, never registered or rolled back again. +func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) { + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + + advancePlans(ctx, g.open) // the first machine is sent the new build + if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v, not the first machine alone", g.sent) + } + if p := g.plan(t); p.Modules["app"].Previous != "c1" { + t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"]) + } + + g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging + gateEvery = 0 + advancePlans(ctx, g.open) + + p := g.plan(t) + gate := p.Modules["app"].Gate + if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed || + gate.Rollback != inventory.RolledBack { + t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) + } + if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) { + t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { + t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) + } + if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" { + t.Fatalf("anchor was last sent %v, not the previous build", sent) + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" { + t.Fatalf("laptop was sent the failed build: %v", sent) + } + if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed { + t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err) + } + + // Said: a condition for the operator, and the event. + open, err := g.keeper.Open(ctx) + if err != nil { + t.Fatal(err) + } + var key string + for _, c := range open { + if c.Kind == kindRolledBack { + key = c.Key + } + } + if key != "build.app.anchor.rolled-back" { + t.Fatalf("no rolled-back condition for app on anchor: %+v", open) + } + saidIt := false + for _, e := range g.told.Said() { + saidIt = saidIt || e.Event == link.KeyRolledBack + } + if !saidIt { + t.Fatalf("the rollback was not said as an event: %+v", g.told.Said()) + } + + // Never again: more passes send nothing, a second judging rolls nothing back, and the failed build + // heard again is not registered. + advancePlans(ctx, g.open) + state := p.Modules["app"] + gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again") + if len(g.sent) != 2 { + t.Fatalf("sent again after the rollback: %v", g.sent) + } + _, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app", + Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"})}) + if err == nil || !strings.Contains(err.Error(), "failed its gate") { + t.Fatalf("the failed build was registered again: %v", err) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { + t.Fatalf("the module moved to %s", current["app"].Commit) + } + if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") { + t.Fatalf("a plan stopped at a failed gate was retried: %v", err) + } + + // The probe keeps the condition while the newest verdict is the failure. + obs, err := probeGates(ctx, doctorFrom) + if err != nil || len(obs) != 1 || obs[0].Key() != key { + t.Fatalf("the gate's probe found %+v %v", obs, err) + } +} + +// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then +// the rest are sent, the verdict kept, and the plan done. +func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { + g := aGateMesh(t) + ctx := t.Context() + gateEvery = 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, g.open) + } + p := g.plan(t) + if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("sent %v", g.sent) + } + gate := p.Modules["app"].Gate + if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gatePasses { + t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) + } + if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err) + } + if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { + t.Fatalf("a passing build left a condition: %+v %v", obs, err) + } +} + +// A first machine that never becomes healthy fails its gate at the bound, and is put back. +func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { + g := aGateMesh(t) + ctx := t.Context() + advancePlans(ctx, g.open) + g.health["anchor"] = healthNotYet + gateEvery = 0 + advancePlans(ctx, g.open) + if p := g.plan(t); !p.Open() || len(g.sent) != 1 { + t.Fatalf("judged before the bound: %s %v", p.State, g.sent) + } + gateBound = -time.Second + advancePlans(ctx, g.open) + p := g.plan(t) + if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || + !strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack { + t.Fatalf("the plan is %s, its gate %+v", p.State, gate) + } +} + +// The health a judging finds, per core component and for a module. +func TestTheHealthDefinitions(t *testing.T) { + now := time.Now() + since := now.Add(-time.Minute) + applied := func(node string) gateFacts { + at := now + return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node, + Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, + served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}} + } + m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}} + + f := applied("anchor") + f.served["anchor"] = served{runtime: true, tools: map[string]bool{}} + if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") { + t.Errorf("a module whose tools are not served is %v (%s)", h, why) + } + f.served["anchor"].tools["app"] = true + if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood { + t.Errorf("a module applied with its tools served is %v (%s)", h, why) + } + // A condition raised about it on that machine since it was sent: not yet healthy. + f.judged = true + f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{ + Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}} + if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet { + t.Errorf("a module with a new condition about it is %v", h) + } + f.open[0].Raised = since.Add(-time.Hour) + if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood { + t.Errorf("a condition older than the send counted against it: %v", h) + } + // A witness that put it back: broken. + f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb", + Outcome: lease.OutcomeRolledBack, Why: "x", At: now}} + if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken { + t.Errorf("a component a witness put back is %v", h) + } + // The node tools answer, or not. + f = applied("anchor") + if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet { + t.Errorf("node tools not answering are %v", h) + } + f.served["anchor"] = served{runtime: true} + if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood { + t.Errorf("node tools answering are %v", h) + } + // The controller: the lease held since the send, by a controller that says it is ready. + f = applied("control") + f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}} + if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet { + t.Errorf("a lease held by a controller older than the build is %v", h) + } + f.holder.Taken = now + if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood { + t.Errorf("a new controller holding the lease and ready is %v", h) + } + f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"} + if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet || + !strings.Contains(why, "first run") { + t.Errorf("a controller not ready is %v (%s)", h, why) + } + // A machine that refused what it was sent: broken. + f = applied("anchor") + r := f.reports["anchor"] + r.Outcome = inventory.OutcomeRefused + f.reports["anchor"] = r + if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken { + t.Errorf("a refusal is %v", h) + } +} + +// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused, +// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits. +func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}}, + Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}} + if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil { + t.Fatal(err) + } + u, err := inv.UpgradeOf(ctx, "nats") + if err != nil || u.RollOut || u.From != catalogue.FromBus { + t.Fatalf("the bus's policy is %+v %v", u, err) + } + if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) { + t.Fatalf("a person rolled the bus out: %v", err) + } + var sent [][]string + was := sendRollout + sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { + sent = append(sent, names) + return names, nil + } + t.Cleanup(func() { sendRollout = was }) + now := time.Now().UTC() + plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now, + State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}}, + Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + advancePlans(ctx, open) + if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 { + t.Fatalf("a plan sent the bus: %s %v", p.State, sent) + } + held, err := busHeld(ctx, inv, []string{"anchor", "laptop"}) + if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" { + t.Fatalf("a push may send the bus's machine: %v %v", held, err) + } + // The planned step refuses to start without its word on reversibility, and without a snapshot. + if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") { + t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err) + } + if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil || + !strings.Contains(err.Error(), "snapshot") { + t.Fatalf("a bus upgrade started without a snapshot: %v", err) + } + if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible", "--snapshot-taken", "nightly"}); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) { + t.Fatalf("the step sent %v, not the bus's machine", sent) + } + s, found, err := inv.LatestBusStep(ctx) + if err != nil || !found || s.Snapshot != "nightly" || s.Ended != nil || !reflect.DeepEqual(s.Machines, []string{"anchor"}) { + t.Fatalf("the step is %+v %v %v", s, found, err) + } +} + +// A merge that deletes a module's directory builds nothing for it: the module is forgotten where +// nothing holds it, and a plan whose build finds no manifest goes on instead of failing. +func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + asked := asksRecorded(t) + for _, name := range []string{"gone", "kept"} { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + } + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef", + Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"}, + Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + if len(*asked) != 0 { + t.Fatalf("a deleted module was asked to build: %v", *asked) + } + if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 { + t.Fatalf("a merge that only deleted a module made a plan: %+v", plans) + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + if _, still := shelf["gone"]; still { + t.Fatal("a deleted module nothing holds was not forgotten") + } + + // Without the announcer saying what went: the build finds no manifest, and the plan goes on. + asked0 := time.Now().UTC().Add(-time.Minute) + plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0, + State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}}, + Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+ + "there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k") + p, _ := inv.PlanByID(ctx, "plan-deleted") + if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted { + t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"]) + } +} + +func mustJSON(t *testing.T, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return b +} + +func nodeID(t *testing.T, open *stores, name string) string { + t.Helper() + n, err := open.inventory.NodeByName(context.Background(), name) + if err != nil { + t.Fatal(err) + } + return n.ID +} + +// What a machine's witness says in its reports is a condition while it says it, by the host's words: +// `core...`, urgent for a rollback, a warning when it could not judge — and +// gone with the first report that no longer carries it. +func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) { + open := aMesh(t) + d := &doctor{open: open} + at := time.Now().UTC() + witnessed.heard("control", []lease.Rollback{ + {Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack, + Why: "the new controller did not take the lease within 60s", At: at}, + {Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at}, + }, at) + t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) }) + obs, err := probeGates(t.Context(), d) + if err != nil { + t.Fatal(err) + } + got := map[string]conditions.Severity{} + for _, o := range obs { + got[o.Key()] = o.Severity + } + want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent, + "core.node-tools.control.unwitnessed": conditions.Warning} + if !reflect.DeepEqual(got, want) { + t.Fatalf("the witness's verdicts are %v", got) + } + witnessed.heard("control", nil, time.Now()) + if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 { + t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err) + } +} diff --git a/cmd/mesh-controller/health_probes.go b/cmd/mesh-controller/health_probes.go new file mode 100644 index 0000000..c0e5976 --- /dev/null +++ b/cmd/mesh-controller/health_probes.go @@ -0,0 +1,226 @@ +package main + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// The core components' health definitions, as probes in the self-check's registry (novox/hq to-be 45 +// §8, §4 `H-*`). The same definitions judge a core component's new build on its first machine (the +// gate, gate.go); here they are run against every machine on the self-check's schedule, so a core +// component that stops being healthy between upgrades is said too. +// +// controller holds the lease, and says it is ready: its self-check ran, `status` answered in bound +// node-engine has reported its current declaration, under a build the mesh delivered +// node tools announced, and answer the bus's discovery +// bus every stream and durable consumer the mesh defines is there, and a request crosses the +// bus to every machine's node tools and back +const kindCoreUnhealthy = "core-unhealthy" + +// probeControllerHealth is H-controller: the lease is held, fresh, by a controller that says it is +// ready — or one that started less than the witness's bound ago. +func probeControllerHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) { + h, found, err := theLease.holder(ctx) + if err != nil { + return nil, err + } + unhealthy := func(why string) []conditions.Observation { + node := d.host + if found && h.Host != "" { + node = h.Host + } + return []conditions.Observation{{Scope: conditions.ScopeCore, ID: lease.ComponentController + "." + node, + Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Urgent, + Summary: "the controller is not healthy: " + why}} + } + switch { + case !found: + return unhealthy("nobody holds the controller lease"), nil + case time.Since(h.Renewed) > lease.FreshWithin: + return unhealthy(fmt.Sprintf("the lease was last renewed %s ago", time.Since(h.Renewed).Round(time.Second))), nil + case (h.Health == nil || !h.Health.Ready) && time.Since(h.Taken) > lease.ReadyWithin: + why := "the controller holding the lease does not say it is ready" + if h.Health != nil && h.Health.Why != "" { + why += ": " + h.Health.Why + } + return unhealthy(why), nil + } + return nil, nil +} + +// probeEngineHealth is H-engine: every machine heard from has reported its current declaration — the +// last one it was sent — under a node-engine build the mesh delivered, or is inside the bound of a send. +func probeEngineHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) { + return machinesHealth(ctx, d, hostModule) +} + +// probeToolsHealth is H-tools: every machine heard from that is assigned the node tools has them +// announced, answering the bus's discovery. +func probeToolsHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) { + return machinesHealth(ctx, d, broker.RuntimeModule) +} + +// machinesHealth judges a component on every machine running it and heard from, by its health +// definition, as the gate does — with no condition taken as the build's doing. +func machinesHealth(ctx context.Context, d *doctor, component string) ([]conditions.Observation, error) { + inv := d.open.inventory + running, err := inv.Running(ctx, component) + if err != nil { + return nil, err + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + return nil, err + } + f, err := gatherGateFacts(ctx, d.open, coreComponent(component)) + if err != nil { + return nil, err + } + f.judged = false + heard := heardMachines(d) + var out []conditions.Observation + for _, node := range running { + if !heard[node] { + continue // a machine not heard from is S1's + } + if r, said := f.reports[node]; said && !r.Current && r.Sent != nil && time.Since(*r.Sent) < gateBound { + continue // inside the bound of a send: S2's, and the gate's + } + // What the machine did with what it was sent is not the component's health: the node-engine's is + // that it reported its current declaration at all, the node tools' that they answer. + if r := f.reports[node]; r.Current || component == broker.RuntimeModule { + now := time.Now() + r.Current, r.Outcome = true, inventory.OutcomeApplied + if r.At == nil { + r.At = &now + } + f.reports[node] = r + } + if component == hostModule { + // A node-engine reporting a build the mesh delivered, current or previous, is the version + // split's (D10), not ill health; a build the mesh did not deliver is. + f.engines[node] = deliveredOr(shelf[component], f.engines[node]) + } + h, why := judgeHealth(component, coreComponent(component), shelf[component], node, time.Time{}, f) + if h == healthGood { + continue + } + out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: coreComponent(component) + "." + node, + Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Warning, + Summary: fmt.Sprintf("the %s on %s is not healthy: %s", componentWords(component), node, why)}) + } + return sortedFound(out), nil +} + +// deliveredOr is the reported engine build, or the current delivered one when the report names any +// build at all: what H-engine judges is that it reports, not which. +func deliveredOr(m catalogue.Manifest, reported string) string { + if reported == "" { + return reported + } + if v := deliveredVersions(m); len(v) > 0 { + return v[0] + } + return reported +} + +// componentWords is a core component as a sentence names it. +func componentWords(component string) string { + switch component { + case hostModule: + return "node-engine" + case broker.RuntimeModule: + return "node tools" + case catalogue.ControllerSeatName: + return "controller" + } + return component +} + +// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a +// request crosses it to every machine's node tools and back. +func probeBusHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) { + problems, err := busHealth(ctx, d) + if err != nil { + return nil, err + } + if len(problems) == 0 { + return nil, nil + } + return []conditions.Observation{{Scope: conditions.ScopeBus, ID: "mesh", Token: "unhealthy", + Kind: kindCoreUnhealthy, Severity: conditions.Urgent, + Summary: fmt.Sprintf("the bus is not healthy: %s", strings.Join(problems, "; ")), + Said: strings.Join(problems, "; ")}}, nil +} + +// busHealth is the bus's health definition, as what is wanting: nothing when healthy. What the bus's +// planned step checks after the bus is replaced, too. +var busHealth = func(ctx context.Context, d *doctor) ([]string, error) { + if d.js == nil { + return nil, errors.New("this controller has no bus to ask") + } + streams, consumers, err := expectedBusObjects(ctx, d.open.inventory) + if err != nil { + return nil, err + } + js := d.js.Context() + var problems []string + for _, s := range streams { + if _, err := js.StreamInfo(s.Name, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) { + problems = append(problems, "the stream "+s.Name+" is missing") + } else if err != nil { + return nil, fmt.Errorf("the stream %s cannot be read: %w", s.Name, err) + } + } + for _, c := range consumers { + _, err := js.ConsumerInfo(c.Stream, c.Name, nats.Context(ctx)) + if errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound) { + problems = append(problems, consumerWords(c)+" is missing") + } else if err != nil { + return nil, fmt.Errorf("%s cannot be read: %w", consumerWords(c), err) + } + } + // The round trip: every machine heard from that runs the node tools answers across the bus. + running, err := d.open.inventory.Running(ctx, broker.RuntimeModule) + if err != nil { + return nil, err + } + heard := heardMachines(d) + var expected []string + for _, n := range running { + if heard[n] { + expected = append(expected, n) + } + } + if len(expected) > 0 { + answered, err := servedOnTheBus(ctx, d.js.Conn()) + if err != nil { + return nil, err + } + var silent []string + for _, n := range expected { + if !answered[n].runtime { + silent = append(silent, n) + } + } + // One machine's tools silent is that machine's (H-tools); none answering is the bus. + if len(silent) == len(expected) { + slices.Sort(silent) + problems = append(problems, "no request crossed the bus and came back: no machine's node tools answered ("+ + strings.Join(silent, ", ")+")") + } + } + return problems, nil +} diff --git a/cmd/mesh-controller/held_back_test.go b/cmd/mesh-controller/held_back_test.go index 483097c..28fb555 100644 --- a/cmd/mesh-controller/held_back_test.go +++ b/cmd/mesh-controller/held_back_test.go @@ -193,6 +193,10 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { t.Fatal(err) } } + // Recorded by a person's choice: the default rolls out since novox/hq ADR 0235. + if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil { + t.Fatal(err) + } gens, err := generators(ctx, open) if err != nil { t.Fatal(err) @@ -214,7 +218,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { } before := digestOfLaptop() - // The change merges; the policy is the default, record. `push anchor` sends the anchor... + // The change merges; the policy is record. `push anchor` sends the anchor... aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute)) d.declared = nil if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil { diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index c63f756..7c43803 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -116,6 +116,9 @@ func run() error { return serve(ctx) case "upgrade": return upgradeCommand(ctx, args[1:]) + // The bus as a planned step (novox/hq to-be 45 §8, ADR 0235). + case "bus": + return busCommand(ctx, args[1:]) case "declare": return declare(ctx, args[1:]) case "overlay": diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index 1e2d1fd..f4e7751 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -178,6 +178,15 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error { return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s", p.Tier, p.ID, p.Note) } + // **A build that failed its gate is not sent again** (novox/hq ADR 0235): it was put back on its first + // machine, and retrying would judge the build the mesh put back, or send the failed one by hand. + for _, m := range stopped { + if g := p.Modules[m].Gate; g != nil && g.Verdict == inventory.GateFailed { + return fmt.Errorf("%s failed its gate on %s (%s) and was put back: a build that failed its gate is not "+ + "sent again — a newer merge, or `rebuild %s`, makes a new build, judged at the gate again", + m, strings.Join(g.Machines, ", "), g.Why, m) + } + } // **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or // sent — a newer build, and sending this one again would put the older build back on its machines. for _, m := range stopped { diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 11b2808..0dce90c 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -466,6 +466,27 @@ func pushCommand(ctx context.Context, args []string) error { which, len(asked), strings.Join(asked, ", ")) } + // **The bus is replaced only as a planned step** (novox/hq ADR 0235, to-be 45 §8): a machine whose bus + // would move is not sent by a push — named, it is refused; otherwise it is left and said. + busKept, err := busHeld(ctx, inv, asked) + if err != nil { + return err + } + if len(busKept) > 0 { + if len(args) == 1 { + return fmt.Errorf("%s. Nothing was sent", busKept[args[0]]) + } + var kept []string + for _, n := range asked { + if why, h := busKept[n]; h { + fmt.Printf(" %s is left: %s\n", n, why) + continue + } + kept = append(kept, n) + } + asked = kept + } + // **The machine holding the bus first** (novox/hq issue 249): its declaration carries the bus's // user list, and a module's new grants are refused by the bus until that list says them. Among // the machines asked it goes first; not among them and behind, it is added — a named push whose diff --git a/cmd/mesh-controller/queue_test.go b/cmd/mesh-controller/queue_test.go index a157932..ab97aac 100644 --- a/cmd/mesh-controller/queue_test.go +++ b/cmd/mesh-controller/queue_test.go @@ -625,6 +625,10 @@ func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) { } t.Cleanup(func() { sendRollout = was }) + // a records: a person's choice, since the default rolls out (novox/hq ADR 0235). + if err := open.inventory.SetUpgradeOf(ctx, "a", inventory.Upgrade{Why: "test"}); err != nil { + t.Fatal(err) + } long := time.Now().UTC().Add(-2 * time.Hour) stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee", Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index ee912d5..06d2ae2 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -5,6 +5,7 @@ import ( "errors" "flag" "fmt" + "slices" "sort" "strings" "time" @@ -223,6 +224,9 @@ func supersededBy(newer inventory.Plan, open []inventory.Plan, rollsOut func(str } var took []string for name, s := range old.Modules { + if s != nil && s.State == planDeleted { + continue // deleted at its source: nothing to plan again + } if s == nil || s.State != "built" || (s.SentAt == nil && rollsOut(name)) { folded[name] = true took = append(took, name) @@ -427,7 +431,11 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string, } else if askedBefore(asked, state.AskedAt) { continue } - if failed != "" { + if failed != "" && deletedAtSource(failed) { + // Deleted at its source by the merge, not broken (novox/hq ADR 0235): the plan goes on. + state.State, state.Why = planDeleted, "deleted at its source: "+firstLine(failed) + forgetDeleted(ctx, inv, module, p) + } else if failed != "" { state.State = "failed" state.Why = failed p.State = inventory.PlanFailed @@ -575,7 +583,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, var latest time.Time for _, m := range tier { s := p.Modules[m] - if s == nil || s.State != "built" { + if s == nil || (s.State != "built" && s.State != planDeleted) { return false, nil } if s.BuiltAt != nil && s.BuiltAt.After(latest) { @@ -598,7 +606,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, var pending []string for _, m := range tier { state := p.Modules[m] - if state == nil || state.SentAt != nil || !rollsOut(m) { + if state == nil || state.SentAt != nil || state.State == planDeleted || !rollsOut(m) { continue } running, err := inv.Running(ctx, m) @@ -620,29 +628,66 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound) switch { case step.failed != "": - state.Why = step.failed - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s stopped at its first machine in tier %d: %s; %s left as it was", - m, p.Tier, step.failed, orNone(strings.Join(step.rest, ", "))) + // The first machine refused or failed what it was sent, or never said: the gate failed, and + // the build is put back there (novox/hq ADR 0235); the rest are left as they were. + gateFailed(ctx, open, p, m, state, firstRunning(state.First, running), step.failed) + p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", "))) fmt.Printf("%s: %s\n", p.ID, p.Note) return true, nil case step.waiting != "": pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, state.FirstAt.Local().Format("15:04"))) continue + } + // **The gate** (novox/hq ADR 0235, to-be 45 §8): the first machine reported the build applied; + // it is judged by its health before anything else is sent — the rest, or, where it is the only + // machine, the plan's next step. + if !policy.Together && state.FirstAt != nil && len(state.First) > 0 { + verdict, err := judgeGate(ctx, open, p, m, state, running, now) + if err != nil { + return false, err + } + switch verdict { + case "": + pending = append(pending, fmt.Sprintf("%s judged on %s: %s", m, + strings.Join(state.Gate.Machines, ", "), gateLine(state.Gate))) + continue + case inventory.GateFailed: + gateFailed(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why) + p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", "))) + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil + case inventory.GatePassed: + if !state.Gate.Kept { + gatePassed(ctx, open, p, m, state) + state.Gate.Kept = true + } + } + } + switch { case len(step.send) == 0: // No machine runs it: nothing to send, and nothing to wait for. state.SentAt = &now continue } + // Read before the send, which records what it carries. + var before map[string]string + var beforeKnown bool + if step.first { + before, beforeKnown, _ = inv.SentBuilds(ctx, step.send[0]) + } // What sendToEach answers, not what was asked: the machine holding the bus is sent before // the first when its user list must change (issue 249), and the plan waits for it too. - sent, err := sendToEach(ctx, open, step.send) + sent, err := sendRollout(ctx, open, step.send) if err != nil { // Not marked sent, so the next step tries again (issue 249): a grant that could not be // issued is a send that did not happen. return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(step.send, ", "), p.Tier, err) } if step.first { + // What the first machine ran before: what a failed gate puts back (ADR 0235). + if beforeKnown { + state.Previous = before[m] + } state.First = sent state.FirstAt = &now p.State = inventory.PlanRolling @@ -681,6 +726,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, state = &inventory.PlanModule{} p.Modules[m] = state } + if state.State == planDeleted { + continue + } // The plan sends what it waits for. A rebuild from the same source commit is not a // move the catalogue announces — the build machine rebuilt for a controller change // is one — so the roll-out that opens this gate is the plan's to make, once, and @@ -1033,6 +1081,11 @@ func plansCommand(ctx context.Context, args []string) error { } } fmt.Printf(" %-22s %s\n", m, state) + // The rollout's record at its gate (novox/hq to-be 45 §8): first machine, from and to, + // verdict, time to it, rolled back or not. + if s != nil && s.Gate != nil { + fmt.Printf(" %-22s %s\n", "", gateLine(s.Gate)) + } } } return nil @@ -1231,6 +1284,12 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i continue } at := outcome.At + if !outcome.Worked() && deletedAtSource(outcome.Failed) { + s.State, s.Why = planDeleted, "deleted at its source: "+firstLine(outcome.Failed) + fmt.Printf("%s: %s was deleted at its source (%s): not built, and the plan goes on\n", p.ID, m, outcome.ID) + changed = true + continue + } if outcome.Worked() { s.State = "built" s.BuiltAt = &at @@ -1252,3 +1311,34 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i func askedBefore(asked time.Time, planAsked *time.Time) bool { return !asked.IsZero() && planAsked != nil && asked.Before(*planAsked) } + +// firstRunning is the machines sent first that run the module — not the machine holding the bus, sent +// with them only for its user list. +func firstRunning(first, running []string) []string { + var out []string + for _, n := range first { + if slices.Contains(running, n) { + out = append(out, n) + } + } + if len(out) == 0 { + return first + } + return out +} + +// planDeleted is a plan's module that the merge deleted at its source: not built, not sent, and no +// failure of the plan (novox/hq ADR 0235). +const planDeleted = "deleted" + +// forgetDeleted forgets a module a plan found deleted at its source, where nothing holds it, and says +// it otherwise. +func forgetDeleted(ctx context.Context, inv *inventory.Inventory, module string, p *inventory.Plan) { + switch err := inv.ForgetModule(ctx, module); { + case err == nil: + fmt.Printf("%s: %s was deleted at its source: forgotten, and the plan goes on\n", p.ID, module) + default: + fmt.Printf("%s: %s was deleted at its source and is not built; it is kept until a person unassigns it and "+ + "`module forget %s`: %v\n", p.ID, module, module, firstLine(err.Error())) + } +} diff --git a/cmd/mesh-controller/rollout.go b/cmd/mesh-controller/rollout.go index c4608ee..4e459c5 100644 --- a/cmd/mesh-controller/rollout.go +++ b/cmd/mesh-controller/rollout.go @@ -388,14 +388,7 @@ func rolloutMint(ctx context.Context, again bool) error { } // providesBus is whether a manifest provides the mesh's bus. -func providesBus(m catalogue.Manifest) bool { - for _, o := range m.Provides { - if o.Name == "mesh-bus" { - return true - } - } - return false -} +func providesBus(m catalogue.Manifest) bool { return catalogue.ProvidesBus(m) } // rolloutHand mints a machine its credential for the new bus afresh and prints its membership // once, for an operator to carry by hand — the rescue for a machine that cannot be reached over diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index a802c43..d4a04f0 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -508,6 +508,43 @@ func (a *verbArguments) commandLine() ([]string, error) { argv = append(argv, "--retired") } return argv, nil + case "upgrade": + m, policy := str("module"), str("policy") + if m == "" { + if policy != "" { + return nil, errors.New("upgrade takes a policy only for a module") + } + return []string{"upgrade"}, nil + } + argv := []string{"upgrade", m} + if policy != "" { + argv = append(argv, policy) + if on("together") { + argv = append(argv, "--together") + } + if w := str("why"); w != "" { + argv = append(argv, "--why", w) + } + } + return argv, nil + case "bus": + if !on("upgrade") { + return []string{"bus"}, nil + } + argv := []string{"bus", "upgrade", "--why", str("why")} + if c := str("cause"); c != "" { + argv = append(argv, "--cause", c) + } + if on("reversible") { + argv = append(argv, "--reversible") + } + if on("irreversible") { + argv = append(argv, "--irreversible") + } + if w := str("snapshot-taken"); w != "" { + argv = append(argv, "--snapshot-taken", w) + } + return argv, nil case "doctor": which := 0 argv := []string{"doctor"} diff --git a/cmd/mesh-controller/seatverbs_test.go b/cmd/mesh-controller/seatverbs_test.go index 4709f5b..4e8b1c6 100644 --- a/cmd/mesh-controller/seatverbs_test.go +++ b/cmd/mesh-controller/seatverbs_test.go @@ -99,7 +99,7 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) { if _, err := argvFor("node", map[string]any{}); err == nil || !strings.Contains(err.Error(), `node needs "node"`) { t.Fatalf("node without a machine was accepted: %v", err) } - if _, err := argvFor("upgrade", map[string]any{}); err == nil { + if _, err := argvFor("no-such-verb", map[string]any{}); err == nil { t.Fatal("a verb the seat does not serve was accepted") } } diff --git a/cmd/mesh-controller/status_summary.go b/cmd/mesh-controller/status_summary.go index c9c9085..74bc280 100644 --- a/cmd/mesh-controller/status_summary.go +++ b/cmd/mesh-controller/status_summary.go @@ -195,6 +195,11 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e if report.Ordered() { link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now) } + // What the machine's witnesses put back and stand by (novox/hq ADR 0235): read by the gate and its + // probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey. + if report.Superseded == "" && report.Rekey == nil && report.Node != "" { + witnessed.heard(report.Node, report.Rollbacks, now) + } news, err := l.Enrolment.Heard(ctx, report) if news { l.summary.nudge() diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index 267126c..d6707ab 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -7,6 +7,7 @@ import ( "fmt" "path" "regexp" + "sort" "strings" "sync" "time" @@ -143,20 +144,17 @@ func isAre(n int) string { return "are" } -// upgradeCommand says what should happen when a module's current version moves. +// upgradeCommand says what should happen when a module's current version moves (ADR 0162 §3, ADR +// 0235): every module's policy and where it comes from; one module's; or a person's choice for one. func upgradeCommand(ctx context.Context, args []string) error { set := flag.NewFlagSet("upgrade", flag.ContinueOnError) together := set.Bool("together", false, - "send every machine running it at once, instead of one after another") + "send every machine running it at once, instead of one machine first") + why := set.String("why", "", "why this choice — required for record; kept and said with the policy") positionals, err := parseAround(set, args) if err != nil { return err } - if len(positionals) == 0 { - return errors.New("upgrade [roll-out|record] [--together]") - } - module := positionals[0] - open, err := openStores(ctx) if err != nil { return err @@ -164,6 +162,27 @@ func upgradeCommand(ctx context.Context, args []string) error { defer open.Close() inv := open.inventory + if len(positionals) == 0 { + all, err := inv.Upgrades(ctx) + if err != nil { + return err + } + names := make([]string, 0, len(all)) + for n := range all { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + u := all[n] + line := fmt.Sprintf("%-28s %-9s %s", n, u.Policy(), u.From) + if u.Why != "" { + line += ": " + u.Why + } + fmt.Println(line) + } + return nil + } + module := positionals[0] if len(positionals) == 1 { decision, err := inv.UpgradeOf(ctx, module) if err != nil { @@ -173,10 +192,10 @@ func upgradeCommand(ctx context.Context, args []string) error { return nil } - var decision inventory.Upgrade + decision := inventory.Upgrade{Why: strings.TrimSpace(*why), By: link.Caller()} switch positionals[1] { - case "roll-out": - decision = inventory.Upgrade{RollOut: true, Together: *together} + case "roll-out", "roll": + decision.RollOut, decision.Together = true, *together case "record": if *together { // Refused rather than ignored: --together only means anything for a roll-out, and @@ -184,28 +203,53 @@ func upgradeCommand(ctx context.Context, args []string) error { return errors.New("`--together` says how to roll out, so it cannot be given with " + "`record`, which is the choice not to") } - decision = inventory.Upgrade{} + if decision.Why == "" { + return fmt.Errorf("holding %s back from every merge is a choice a person reads later: --why "+ + "(novox/hq ADR 0235). Nothing was changed", module) + } + case "default": + if err := inv.ClearUpgradeOf(ctx, module); err != nil { + return err + } + decision, err := inv.UpgradeOf(ctx, module) + if err != nil { + return err + } + fmt.Println(sayUpgrade(module, decision)) + return nil default: - return fmt.Errorf("upgrade roll-out|record — not %q", positionals[1]) + return fmt.Errorf("upgrade roll-out|record|default — not %q", positionals[1]) } if err := inv.SetUpgradeOf(ctx, module, decision); err != nil { return err } + decision, err = inv.UpgradeOf(ctx, module) + if err != nil { + return err + } fmt.Println(sayUpgrade(module, decision)) return nil } func sayUpgrade(module string, u inventory.Upgrade) string { + from := " (" + u.From + if u.By != "" { + from += ", " + u.By + } + if u.Why != "" { + from += ": " + u.Why + } + from += ")" if !u.RollOut { return fmt.Sprintf("when %s moves, the mesh records it and the machines running it are "+ - "reported as behind", module) + "reported as behind until a person pushes them%s", module, from) } if u.Together { return fmt.Sprintf("when %s moves, every machine running it is sent the new version "+ - "together", module) + "together%s", module, from) } - return fmt.Sprintf("when %s moves, the machines running it are sent the new version one at "+ - "a time, stopping at the first that fails", module) + return fmt.Sprintf("when %s moves, one machine running it is sent the new version first and judged at "+ + "the gate; the rest follow once it passes, and a build that fails is put back there%s", module, from) } // Announceable is every build this mesh recorded, in the shape the builder announces one. @@ -310,6 +354,13 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error { } } touched := whatTheMergeTouched(from, entries, m) + // **A module the merge deleted is not built** (novox/hq ADR 0235): its manifest is gone, so the build + // seat finds nothing saying what it is, and the plan failed on it (`has no module.json at …`) with + // every other module of its tier left unsent. It is forgotten where nothing holds it, said otherwise. + touched, deleted := splitDeleted(touched, m) + for _, e := range deleted { + retireDeleted(ctx, inv, e, m) + } for _, e := range touched { if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil { return notNow(err) @@ -465,7 +516,52 @@ func mergeCandidates(m link.SourceMoved, entries []inventory.Entry, func wouldMove(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository) []inventory.Entry { from, _, _ := mergeCandidates(m, entries, read) - return whatTheMergeTouched(from, entries, m) + touched, _ := splitDeleted(whatTheMergeTouched(from, entries, m), m) + return touched +} + +// splitDeleted parts the modules a merge touched into those it changed and those whose manifest it +// removed — deleted at their source (ADR 0235). +func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted []inventory.Entry) { + removed := map[string]bool{} + for _, p := range m.Removed { + removed[strings.Trim(p, "/")] = true + } + for _, e := range touched { + dir := strings.Trim(e.Source.Path, "/") + manifest := moduleManifestFile + if dir != "" { + manifest = dir + "/" + manifest + } + if len(removed) > 0 && removed[manifest] { + deleted = append(deleted, e) + continue + } + kept = append(kept, e) + } + return kept, deleted +} + +// retireDeleted is what the mesh does with a module deleted at its source: its record says the merge +// was looked at, so it is not acted on again; it is forgotten where nothing holds it; where a machine +// runs it or the mesh holds something for it, that is said, and nothing is built. +func retireDeleted(ctx context.Context, inv *inventory.Inventory, e inventory.Entry, m link.SourceMoved) { + name := e.Manifest.Module + if err := inv.SourceMoved(ctx, name, m.Commit); err != nil { + fmt.Printf(" %s was deleted from %s/%s at %.8s, and that could not be recorded: %v\n", name, m.Owner, m.Repo, m.Commit, err) + } + err := inv.ForgetModule(ctx, name) + switch { + case err == nil: + fmt.Printf(" %s was deleted from %s/%s at %.8s: forgotten, nothing built\n", name, m.Owner, m.Repo, m.Commit) + case errors.Is(err, inventory.ErrStillAssigned) || errors.Is(err, inventory.ErrStillHolds): + fmt.Printf(" %s was deleted from %s/%s at %.8s and is not built; the mesh still runs or holds it, so it is "+ + "kept until a person unassigns it and `module forget %s`: %v\n", name, m.Owner, m.Repo, m.Commit, name, + firstLine(err.Error())) + default: + fmt.Printf(" %s was deleted from %s/%s at %.8s and is not built; forgetting it failed: %v\n", name, m.Owner, + m.Repo, m.Commit, err) + } } // sourceIs is whether a recorded source is the repository and branch a merge announced. A source on @@ -749,3 +845,15 @@ func dependentsOf(moved, entries []inventory.Entry, against map[string][]string) } return out } + +// moduleManifestFile is the file that says what a module is, in its directory (the build seat's +// ManifestName). +const moduleManifestFile = "module.json" + +// deletedAtSource is whether a build failed because its module's manifest is not at its source any +// more — the build seat's own words (internal/builder) — which a merge that deleted the module causes +// when its announcer did not say which files went (ADR 0235). Such a module is not a failure of the plan. +func deletedAtSource(failed string) bool { + return strings.Contains(failed, "has no "+moduleManifestFile+" at ") && + strings.Contains(failed, "so there is nothing saying what it is") +} diff --git a/cmd/mesh-controller/witness.go b/cmd/mesh-controller/witness.go new file mode 100644 index 0000000..544956b --- /dev/null +++ b/cmd/mesh-controller/witness.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/novox/mesh-controller/internal/lease" +) + +// The controller's side of its own rollback witness (novox/hq to-be 45 §8, ADR 0235; the contract is +// internal/lease/witness.go): what the serving controller says of itself in every write of the lease's +// key, for the node-engine on the control node to judge a new controller build by. + +// processStarted is when this process started. +var processStarted = time.Now().UTC() + +// readiness remembers when this process first became ready. +var readiness struct { + sync.Mutex + at time.Time +} + +// controllerHealth is the controller's health definition (to-be 45 §8) as this process meets it now: +// it holds the lease — it is asked only while writing the key — its self-check has finished a run, and +// in that run `status` answered in full within ten seconds (D9). Anything short of that is said, never +// read as ready. +func controllerHealth() *lease.Health { + h := &lease.Health{Started: processStarted} + d := doctorFrom + if d == nil { + h.Why = "the self-check is not running in this process yet" + return h + } + run := d.lastRun() + if run == nil { + h.Why = "the self-check has not finished its first run" + return h + } + h.DoctorRan = run.At + ran := false + for _, p := range run.Probes { + if p.ID != "D9" { + continue + } + ran = true + if p.Verdict != verdictPass { + h.Why = "in the last self-check, status did not answer in full within ten seconds (D9 " + p.Verdict + ")" + return h + } + } + if !ran { + h.Why = "the last self-check did not judge status (D9)" + return h + } + readiness.Lock() + if readiness.at.IsZero() { + readiness.at = time.Now().UTC() + } + h.Ready, h.ReadyAt = true, readiness.at + readiness.Unlock() + return h +} + +// witnessed is every rollback the machines' witnesses say they made and still stand by, as their last +// report carried it (Report.RolledBack). Kept in the serving controller's memory only: a witness says it +// on every report until it is resolved, so a controller started afresh hears it again with the next +// report — the host's reconcile is minutes apart, and a restored controller hears the report that +// follows its own restoring. +var witnessed = &rollbacksHeard{byNode: map[string]heardRollbacks{}} + +type rollbacksHeard struct { + mu sync.Mutex + byNode map[string]heardRollbacks +} + +type heardRollbacks struct { + at time.Time + list []lease.Rollback +} + +// heard keeps what one machine's account said, replacing what it said before: an account without any +// says the machine's witnesses stand by none. +func (w *rollbacksHeard) heard(node string, list []lease.Rollback, at time.Time) { + w.mu.Lock() + defer w.mu.Unlock() + w.byNode[node] = heardRollbacks{at: at, list: append([]lease.Rollback(nil), list...)} +} + +// all is what every machine heard from said, by machine. +func (w *rollbacksHeard) all() map[string][]lease.Rollback { + w.mu.Lock() + defer w.mu.Unlock() + out := map[string][]lease.Rollback{} + for node, h := range w.byNode { + if len(h.list) > 0 { + out[node] = append([]lease.Rollback(nil), h.list...) + } + } + return out +} + +// holder is who holds the controller lease now, read from the bucket: through the serving controller's +// own lease, or the bucket a command opened. +func (a *actor) holder(ctx context.Context) (lease.Holder, bool, error) { + a.mu.Lock() + held, kv := a.held, a.kv + a.mu.Unlock() + reading, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + switch { + case held != nil: + return held.Current(reading) + case kv != nil: + return lease.Current(reading, kv) + } + return lease.Holder{}, false, errors.New("this process has no view of the controller lease") +} diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 173b0c5..84284ab 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -19,6 +19,8 @@ import ( "sort" "strings" "time" + + "github.com/novox/mesh-controller/internal/lease" ) // A Kind is what a principal is, which decides the shape of its authority rather than its @@ -115,6 +117,12 @@ type Principal struct { // else — not its declarations, which the node's tool runtime serves for it. SnapshotsTheBus bool + // WitnessesController is a machine principal whose node-engine witnesses the controller's upgrades: + // the machine runs the controller (novox/hq ADR 0236, lease/witness.go). It may read the lease's one + // key, and nothing else of the bucket, so it can judge a new controller build and put the previous + // one back. + WitnessesController bool + // PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal // and never appears here: this file is written to a node's disk and read by a server, and a // secret that can be read from a configuration file is a secret with a wider blast radius @@ -432,6 +440,10 @@ func PermissionsFor(p Principal) (Permissions, error) { // `report` verb healer H1 asks): its own machine's, on core NATS and off any stream. It // answers through its report, the one thing it already says — no reply to anybody's inbox. AskReportSubject(p.Node)} + // The node-engine witnesses the core builds it places (novox/hq to-be 45 §8, ADR 0236; the + // contract is lease/witness.go): it asks its own machine's node tools PING, and, where the + // machine runs the controller, reads the lease's one key — read, never written. + pub = append(pub, WitnessSubjects(p)...) case KindModule: // 1. Its own namespace: it publishes its events there and serves its tools there. Nothing @@ -979,3 +991,15 @@ func announcing(names ...string) []string { func discovering() []string { return []string{"$SRV.PING", "$SRV.PING.>", "$SRV.INFO", "$SRV.INFO.>"} } + +// WitnessSubjects are the subjects a machine's node-engine publishes to witness the core builds it +// places (novox/hq ADR 0236, lease/witness.go): its own node tools' PING, and the lease's key where it +// runs the controller. Reads only: a write to the bucket is `$KV..>`, the controller's alone +// (the writers table). The answers come to the machine's own inbox. +func WitnessSubjects(p Principal) []string { + out := []string{lease.PingSubject(p.Node)} + if p.WitnessesController { + out = append(out, lease.LeaseReadSubject(LeaseBucket)) + } + return out +} diff --git a/internal/broker/states_agreement_test.go b/internal/broker/states_agreement_test.go index 9d11f88..ce86809 100644 --- a/internal/broker/states_agreement_test.go +++ b/internal/broker/states_agreement_test.go @@ -28,6 +28,8 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) { states = append(states, link.KeySecretReplaced) // And every act a healer takes (novox/hq to-be 45 §7). states = append(states, link.KeyHealerActed) + // And a build put back after its gate failed (novox/hq ADR 0235). + states = append(states, link.KeyRolledBack) for _, event := range states { if !slices.Contains(broker.ControllerStates, event) { t.Errorf("the mesh states %q and its account may not publish it", event) diff --git a/internal/broker/streams.go b/internal/broker/streams.go index 8e34fa5..7030e77 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -213,7 +213,9 @@ var ControllerStates = []string{"applied", "refused", "built-before", "secret-replaced", // And every act a healer takes on a condition (novox/hq to-be 45 §7, Phase 3): a repair the mesh // made by itself is said like one a person made, never quietly. - "healer-acted"} + "healer-acted", + // And a build put back after its gate failed on its first machine (novox/hq ADR 0235, to-be 45 §8). + "rolled-back"} // BusAdvisories are what the bus server says about the mesh's own account that the controller // reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index e7ab6c2..e53cc8a 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] } allow_responses: { max: 1, ttl: "1m" } } } @@ -33,7 +33,7 @@ accounts { subscribe: { allow: ["_INBOX.enrol.one.>"] } } } { user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: { - publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "mesh.control.one.>"] } + publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "$SRV.PING.node-tools.one", "mesh.control.one.>"] } subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.report", "mesh.node.one.declare"] } } } { user: "one.nats", password: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb", permissions: { diff --git a/internal/broker/users.go b/internal/broker/users.go index 0bcc8c6..d4534b0 100644 --- a/internal/broker/users.go +++ b/internal/broker/users.go @@ -72,7 +72,13 @@ func Users(r Records) ([]Principal, error) { out := []Principal{{Kind: KindController}} for _, node := range sortedCopy(r.Nodes) { - out = append(out, Principal{Kind: KindNode, Node: node}) + witness := false + for _, d := range r.Assigned[node] { + if d.Module == controllerModule { + witness = true + } + } + out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness}) // **Where the runtime is assigned, the machine gets one runtime principal in place of the // runtime module's own** (novox/hq ADR 0175, to-be 38). It carries every module on the // node: its serving grants are the union of theirs. Every other module keeps its own @@ -171,3 +177,6 @@ func sortedNames(in map[string][]string) []string { sort.Strings(out) return out } + +// controllerModule is the controller's module: the machine assigned it witnesses its upgrades. +const controllerModule = "mesh-controller" diff --git a/internal/broker/witness_grants_test.go b/internal/broker/witness_grants_test.go new file mode 100644 index 0000000..e2b1538 --- /dev/null +++ b/internal/broker/witness_grants_test.go @@ -0,0 +1,37 @@ +package broker + +import ( + "slices" + "testing" +) + +// Every machine's node-engine may ask its own node tools PING; the one running the controller may read +// the lease's key, and nothing else of the bucket (novox/hq ADR 0236). +func TestTheWitnessIsGrantedWhatItReadsAndNoMore(t *testing.T) { + users, err := Users(Records{Nodes: []string{"control", "edge"}, + Assigned: map[string][]Declared{"control": {{Module: "mesh-controller"}}}}) + if err != nil { + t.Fatal(err) + } + lease := "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder" + for _, u := range users { + if u.Kind != KindNode { + continue + } + perms, err := PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(perms.Publish, "$SRV.PING.node-tools."+u.Node) { + t.Errorf("%s may not ask its node tools: %v", u.Node, perms.Publish) + } + if got := slices.Contains(perms.Publish, lease); got != (u.Node == "control") { + t.Errorf("%s may read the lease: %v", u.Node, got) + } + for _, p := range perms.Publish { + if p == "$KV.mesh-controller_lease.>" || p == "$KV.mesh-controller_lease.holder" { + t.Errorf("%s may write the lease", u.Node) + } + } + } +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 630e70b..3c214dc 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -453,6 +453,11 @@ type Manifest struct { // unassignment retires and what the self-check measures are all derived from it. Data *Data `json:"data,omitempty"` + // Upgrade is how this module's new builds reach its machines (novox/hq ADR 0235): rolled out one + // machine first and gated when unsaid; `together`, or `record` — wait for a person's push — with + // why. A person's choice through the `upgrade` verb stands over it; the bus records whatever it says. + Upgrade *UpgradePolicy `json:"upgrade,omitempty"` + // Reads are other modules' state this module reads and watches, each `.` // (novox/hq ADR 0201). Read-only: only the owner's instances write. Reads []string `json:"reads,omitempty"` @@ -2018,6 +2023,7 @@ func ParseManifest(raw []byte) (Manifest, error) { problems = append(problems, m.contributionPlaceholderProblems()...) problems = append(problems, m.seatContributionProblems()...) problems = append(problems, m.dataProblems()...) + problems = append(problems, m.upgradeProblems()...) for i, r := range m.Resources { id, _ := r["id"].(string) diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index e5928ab..14b1926 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -89,7 +89,9 @@ var defaultSeats = append([]Seat{ // A value given by hand, replaced after its module's first good start (novox/hq ADR 0228). "secret-replaced", // Every act a healer takes (novox/hq to-be 45 §7). - "healer-acted"}, + "healer-acted", + // A build put back after its gate failed (novox/hq ADR 0235, to-be 45 §8). + "rolled-back"}, Serves: ControllerVerbs}, // The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable, // served by whichever module holds the seat with tools of these names. diff --git a/internal/catalogue/upgrade.go b/internal/catalogue/upgrade.go new file mode 100644 index 0000000..5580a7f --- /dev/null +++ b/internal/catalogue/upgrade.go @@ -0,0 +1,120 @@ +package catalogue + +import ( + "fmt" + "strings" +) + +// What the mesh does when a module's build moves (novox/hq ADR 0235, extending ADR 0162 §3 and ADR +// 0218 §2). +// +// **Rolled out by default, one machine first and gated.** With the gate on the first machine and the +// rollback after it (to-be 45 §8), a build that moves is sent to one machine, judged there by its own +// health, and only then to the rest — or put back there, said, and sent nowhere else. Recording an +// upgrade and waiting for a person to push it is kept where a module says why, and where the mesh knows +// a rollback cannot undo what a new build does. + +// The policies a module may declare. +const ( + // PolicyRoll sends one machine first, judges it at the gate, then the rest. + PolicyRoll = "roll" + // PolicyTogether sends every machine running the module at once — for a module that must change + // everywhere in the same minute. Still judged, on every machine, after. + PolicyTogether = "together" + // PolicyRecord builds and sends nothing: the machines running it are behind until a person pushes. + PolicyRecord = "record" +) + +// UpgradePolicy is what a module says about how its new builds reach its machines: `upgrade` in its +// manifest. +type UpgradePolicy struct { + Policy string `json:"policy"` + // Why is required for anything but roll: a person reading the catalogue sees why this module waits + // for them, or why it changes everywhere at once. + Why string `json:"why,omitempty"` +} + +func (m Manifest) upgradeProblems() []string { + if m.Upgrade == nil { + return nil + } + switch m.Upgrade.Policy { + case PolicyRoll: + case PolicyTogether, PolicyRecord: + if strings.TrimSpace(m.Upgrade.Why) == "" { + return []string{fmt.Sprintf("upgrade %q says why: a module that does not roll out one machine first "+ + "names the reason a person reads", m.Upgrade.Policy)} + } + default: + return []string{fmt.Sprintf("upgrade is %q, %q or %q, not %q", PolicyRoll, PolicyTogether, PolicyRecord, + m.Upgrade.Policy)} + } + return nil +} + +// Where a policy came from, as `upgrade` says it. +const ( + FromPerson = "person" + FromModule = "module" + FromBus = "the bus" + FromData = "irreplaceable data" + FromDefault = "default" +) + +// DerivedUpgrade is the policy a module's manifest gives it when no person has chosen one, with where +// it came from and why (ADR 0235): +// +// - **the bus is never rolled**: a module that provides the mesh's bus records, whatever it says — its +// upgrade is a planned step a person starts (to-be 45 §8); +// - a module that says its policy has it; +// - a module that keeps irreplaceable data records — sending the previous build cannot undo what a new +// one did to data that cannot be had again, so a person takes it, after a backup; +// - everything else rolls out, one machine first. +func DerivedUpgrade(m Manifest) (policy, from, why string) { + if ProvidesBus(m) { + return PolicyRecord, FromBus, "the bus is replaced only as a planned step a person starts (`bus upgrade`): " + + "its streams are snapshotted first and checked after" + } + if m.Upgrade != nil && m.Upgrade.Policy != "" { + return m.Upgrade.Policy, FromModule, m.Upgrade.Why + } + if item := m.irreplaceable(); item != "" { + return PolicyRecord, FromData, "it keeps irreplaceable data (" + item + "): a rollback cannot undo what a new " + + "build does to it, so a person takes each build, after a backup" + } + return PolicyRoll, FromDefault, "" +} + +// ProvidesBus is whether a manifest provides the mesh's bus. +func ProvidesBus(m Manifest) bool { + for _, o := range m.Provides { + if o.Name == "mesh-bus" { + return true + } + } + return false +} + +// irreplaceable names the first irreplaceable data the module keeps, its own or its consumers', or +// nothing. +func (m Manifest) irreplaceable() string { + if m.Data == nil { + return "" + } + for _, it := range m.Data.Own { + if it.Class == ClassIrreplaceable { + return it.ID + } + } + for provision, c := range m.Data.Consumers { + if c.Class == ClassIrreplaceable { + return "its consumers' " + provision + } + } + for provision, k := range m.Data.KeptBy { + if k.Class == ClassIrreplaceable { + return "what it keeps with " + provision + } + } + return "" +} diff --git a/internal/catalogue/upgrade_test.go b/internal/catalogue/upgrade_test.go new file mode 100644 index 0000000..6e41a73 --- /dev/null +++ b/internal/catalogue/upgrade_test.go @@ -0,0 +1,52 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// A module rolls out by default; it records where it says so with why, where it keeps irreplaceable +// data, and always where it is the bus (novox/hq ADR 0236). +func TestWhereAModulesUpgradePolicyComesFrom(t *testing.T) { + cases := []struct { + name string + m Manifest + policy, from string + }{ + {"default", Manifest{Module: "app"}, PolicyRoll, FromDefault}, + {"says record", Manifest{Module: "db", Upgrade: &UpgradePolicy{Policy: PolicyRecord, Why: "a restart costs"}}, + PolicyRecord, FromModule}, + {"says together", Manifest{Module: "dns", Upgrade: &UpgradePolicy{Policy: PolicyTogether, Why: "one zone"}}, + PolicyTogether, FromModule}, + {"irreplaceable data", Manifest{Module: "media", Data: &Data{Own: []DataItem{{ID: "library", Class: ClassIrreplaceable}}}}, + PolicyRecord, FromData}, + {"valuable data rolls", Manifest{Module: "notes", Data: &Data{Own: []DataItem{{ID: "db", Class: ClassValuable}}}}, + PolicyRoll, FromDefault}, + {"irreplaceable but says roll", Manifest{Module: "photos", Upgrade: &UpgradePolicy{Policy: PolicyRoll}, + Data: &Data{Own: []DataItem{{ID: "originals", Class: ClassIrreplaceable}}}}, PolicyRoll, FromModule}, + {"the bus, whatever it says", Manifest{Module: "nats", Provides: []Offer{{Name: "mesh-bus"}}, + Upgrade: &UpgradePolicy{Policy: PolicyRoll}}, PolicyRecord, FromBus}, + } + for _, c := range cases { + policy, from, _ := DerivedUpgrade(c.m) + if policy != c.policy || from != c.from { + t.Errorf("%s: %s from %s, want %s from %s", c.name, policy, from, c.policy, c.from) + } + } +} + +// A policy other than roll says why, and an unknown one is refused. +func TestAnUpgradePolicySaysWhy(t *testing.T) { + for _, u := range []UpgradePolicy{{Policy: PolicyRecord}, {Policy: PolicyTogether}, {Policy: "sometimes", Why: "x"}} { + if problems := (Manifest{Module: "m", Upgrade: &u}).upgradeProblems(); len(problems) == 0 { + t.Errorf("%+v was accepted", u) + } + } + if problems := (Manifest{Module: "m", Upgrade: &UpgradePolicy{Policy: PolicyRecord, Why: "a restart costs"}}).upgradeProblems(); len(problems) != 0 { + t.Errorf("a record with why was refused: %v", problems) + } + if _, err := ParseManifest([]byte(`{"module":"m","upgrade":{"policy":"record"}}`)); err == nil || + !strings.Contains(err.Error(), "says why") { + t.Errorf("a manifest recording without why parsed: %v", err) + } +} diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 33d5f43..7781eb7 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -267,6 +267,31 @@ var ControllerVerbs = []Verb{ "probes": "\"true\": the registry — what each probe asserts, and the condition it raises", "signals": "\"true\": the signals table, each row with the age of its newest signal", }, nil, "run", "probes", "signals")}, + // How a module's new builds reach its machines, and the bus's planned step (novox/hq ADR 0235). + {Name: "upgrade", Description: "How each module's new builds reach its machines (novox/hq ADR 0235): rolled " + + "out one machine first and judged there at the gate, then the rest — or recorded, waiting for a person's " + + "push — with where that comes from (a person, the module, the bus, its irreplaceable data, the default) and " + + "why. With module, that one; with policy, a person's choice for it — roll-out, record (with why) or default " + + "to take the choice back. The bus is never rolled out.", + Input: schema(map[string]string{ + "module": "one module", + "policy": "with module: roll-out, record or default", + "together": "\"true\": with roll-out, every machine at once instead of one machine first", + "why": "with policy: why — required for record, kept and said with the policy", + }, nil, "together")}, + {Name: "bus", Description: "The bus as a planned step (novox/hq to-be 45 §8, ADR 0235): what a bus upgrade " + + "would do — the bus's build on each machine against the one the mesh holds — and how the last step went. " + + "With upgrade, start one: a person's act with why, after the streams are snapshotted (snapshot-taken says " + + "where, while the mesh takes none itself), saying first whether it can be reverted; bus-maintenance is open " + + "while it runs and every stream, consumer and a round trip are checked after.", + Input: schema(map[string]string{ + "upgrade": "\"true\": start the bus's upgrade", + "why": "with upgrade: why — required, recorded in the hand-act log", + "cause": "with upgrade: the cause in a word (default bus-upgrade)", + "reversible": "\"true\": with upgrade, the new version can be undone by putting the old one back", + "irreversible": "\"true\": with upgrade, it cannot — your explicit word that it runs anyway", + "snapshot-taken": "with upgrade: where the streams' snapshot you took is", + }, nil, "upgrade", "reversible", "irreversible")}, // A consumer the mesh stopped asking for: retired, waiting for a person, deleted only by one // (novox/hq ADR 0230). {Name: "retire", Description: "A consumer the mesh stops asking for is retired by its provider — access " + diff --git a/internal/inventory/busstep.go b/internal/inventory/busstep.go new file mode 100644 index 0000000..4659d5e --- /dev/null +++ b/internal/inventory/busstep.go @@ -0,0 +1,72 @@ +package inventory + +import ( + "context" + "errors" + "time" + + "github.com/jackc/pgx/v5" +) + +// BusStep is one planned bus upgrade (novox/hq to-be 45 §8, ADR 0235). +type BusStep struct { + ID int64 + Module string + Machines []string + From, To string + Snapshot string + Reversible bool + By, Why string + Started time.Time + Ended *time.Time + Outcome string + Found string +} + +// StartBusStep records a bus upgrade starting. Refused while another runs. +func (i *Inventory) StartBusStep(ctx context.Context, s BusStep) (BusStep, error) { + if _, err := i.actingEpoch(ctx); err != nil { + return s, err + } + if open, found, err := i.LatestBusStep(ctx); err != nil { + return s, err + } else if found && open.Ended == nil { + return s, ErrBusStepRunning + } + if s.Machines == nil { + s.Machines = []string{} + } + err := i.store.Pool().QueryRow(ctx, + `insert into bus_step (module, machines, from_build, to_build, snapshot, reversible, by_whom, why) + values ($1, $2, $3, $4, $5, $6, $7, $8) returning id, started`, + s.Module, s.Machines, s.From, s.To, s.Snapshot, s.Reversible, s.By, s.Why).Scan(&s.ID, &s.Started) + return s, err +} + +// ErrBusStepRunning is a bus upgrade asked while one runs. +var ErrBusStepRunning = errors.New("a bus upgrade is already running") + +// EndBusStep records how a bus upgrade ended: done or failed, with what was found. +func (i *Inventory) EndBusStep(ctx context.Context, id int64, outcome, found string) error { + if _, err := i.actingEpoch(ctx); err != nil { + return err + } + _, err := i.store.Pool().Exec(ctx, + `update bus_step set ended = now(), outcome = $2, found = $3 where id = $1 and ended is null`, id, outcome, found) + return err +} + +// LatestBusStep is the newest bus upgrade, and whether there is any. +func (i *Inventory) LatestBusStep(ctx context.Context) (BusStep, bool, error) { + var s BusStep + err := i.store.Pool().QueryRow(ctx, + `select id, module, machines, from_build, to_build, snapshot, reversible, by_whom, why, started, ended, + outcome, found + from bus_step order by id desc limit 1`). + Scan(&s.ID, &s.Module, &s.Machines, &s.From, &s.To, &s.Snapshot, &s.Reversible, &s.By, &s.Why, &s.Started, + &s.Ended, &s.Outcome, &s.Found) + if errors.Is(err, pgx.ErrNoRows) { + return s, false, nil + } + return s, err == nil, err +} diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index ff95e62..58bd52b 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -1194,15 +1194,62 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) { // providedBy is what the source column says for a module the control plane ships. const providedBy = "the control plane" -// Upgrade is what the mesh decided to do when a module's current version moves. +// Upgrade is what the mesh does when a module's current version moves (ADR 0162 §3, ADR 0235). type Upgrade struct { - // RollOut is true when the machines running it should be sent the new version. False means - // record it and stop — which needs no record of its own, because a machine not running what - // the mesh would send it is already something the mesh reports. + // RollOut is true when the machines running it are sent the new version: one machine first, judged + // at the gate, then the rest (ADR 0218, ADR 0235). False means record it and stop — the machines + // running it are behind until a person pushes, which the mesh already reports. RollOut bool - // Together is true when every machine running it is sent the new version at once, rather than - // one after another. Only meaningful when RollOut is. + // Together is true when every machine running it is sent the new version at once. Only meaningful + // when RollOut is. Together bool + // From is where the policy came from: a person, the module, the bus, its irreplaceable data, or the + // default (catalogue.From*); Why is the reason said with it. + From string + Why string + // By is the person who chose it, when one did. + By string +} + +// Policy is the policy as a word: roll, together or record. +func (u Upgrade) Policy() string { + switch { + case !u.RollOut: + return catalogue.PolicyRecord + case u.Together: + return catalogue.PolicyTogether + } + return catalogue.PolicyRoll +} + +// upgradeFrom is a module's policy from what the store holds of it: a person's choice, over the module's +// own word, over the default — except that the bus is never rolled out, whoever says so (ADR 0235). +func upgradeFrom(chosen *string, together bool, why, by string, manifest []byte) Upgrade { + var m catalogue.Manifest + // Leniently: a policy is read from what was registered, and a manifest registered before a field it + // carries was known is still a manifest whose bus and data can be read. + _ = json.Unmarshal(manifest, &m) + policy, from, said := catalogue.DerivedUpgrade(m) + if from != catalogue.FromBus && chosen != nil { + policy, from, said = catalogue.PolicyRecord, catalogue.FromPerson, why + if *chosen == "roll-out" { + policy = catalogue.PolicyRoll + if together { + policy = catalogue.PolicyTogether + } + } + } + u := Upgrade{From: from, Why: said} + if from == catalogue.FromPerson { + u.By = by + } + switch policy { + case catalogue.PolicyRoll: + u.RollOut = true + case catalogue.PolicyTogether: + u.RollOut, u.Together = true, true + } + return u } // UpgradeOf is what to do when this module moves. @@ -1211,30 +1258,80 @@ type Upgrade struct { // mesh has never registered, and being told one of them moved is information, not a fault. The // answer is the safe one — record it — because there is nothing to roll out to. func (i *Inventory) UpgradeOf(ctx context.Context, module string) (Upgrade, error) { - var u Upgrade - var policy string + var chosen *string + var together bool + var why, by string + var manifest []byte err := i.store.Pool().QueryRow(ctx, - `select upgrade, upgrade_together from module where name = $1`, module). - Scan(&policy, &u.Together) + `select upgrade, upgrade_together, upgrade_why, upgrade_by, manifest from module where name = $1`, module). + Scan(&chosen, &together, &why, &by, &manifest) if errors.Is(err, pgx.ErrNoRows) { - return Upgrade{}, nil + return Upgrade{From: catalogue.FromDefault, Why: "the mesh holds no such module"}, nil } if err != nil { return Upgrade{}, err } - u.RollOut = policy == "roll-out" - return u, nil + return upgradeFrom(chosen, together, why, by, manifest), nil } -// SetUpgradeOf records what to do when this module moves. +// Upgrades is every module's policy, by name: what `upgrade` lists. +func (i *Inventory) Upgrades(ctx context.Context) (map[string]Upgrade, error) { + rows, err := i.store.Pool().Query(ctx, + `select name, upgrade, upgrade_together, upgrade_why, upgrade_by, manifest from module`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]Upgrade{} + for rows.Next() { + var name, why, by string + var chosen *string + var together bool + var manifest []byte + if err := rows.Scan(&name, &chosen, &together, &why, &by, &manifest); err != nil { + return nil, err + } + out[name] = upgradeFrom(chosen, together, why, by, manifest) + } + return out, rows.Err() +} + +// ErrBusIsPlanned is a person asking the bus to be rolled out: its upgrade is a planned step (ADR 0235). +var ErrBusIsPlanned = errors.New("the bus is never rolled out: its upgrade is a planned step a person starts " + + "with `bus upgrade`, which snapshots its streams first and checks them after") + +// SetUpgradeOf records a person's choice of what to do when this module moves, with why and who. A +// record says why; the bus is refused a roll-out. func (i *Inventory) SetUpgradeOf(ctx context.Context, module string, u Upgrade) error { policy := "record" if u.RollOut { policy = "roll-out" + var manifest []byte + err := i.store.Pool().QueryRow(ctx, `select manifest from module where name = $1`, module).Scan(&manifest) + if err == nil { + var m catalogue.Manifest + if json.Unmarshal(manifest, &m) == nil && catalogue.ProvidesBus(m) { + return fmt.Errorf("%s provides the mesh's bus: %w", module, ErrBusIsPlanned) + } + } } tag, err := i.store.Pool().Exec(ctx, - `update module set upgrade = $2, upgrade_together = $3 where name = $1`, - module, policy, u.Together) + `update module set upgrade = $2, upgrade_together = $3, upgrade_why = $4, upgrade_by = $5 where name = $1`, + module, policy, u.Together, u.Why, u.By) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return fmt.Errorf("this mesh holds no module called %s", module) + } + return nil +} + +// ClearUpgradeOf takes a person's choice back: the module's own word, or the default, decides again. +func (i *Inventory) ClearUpgradeOf(ctx context.Context, module string) error { + tag, err := i.store.Pool().Exec(ctx, + `update module set upgrade = null, upgrade_together = false, upgrade_why = '', upgrade_by = '' where name = $1`, + module) if err != nil { return err } @@ -1258,18 +1355,22 @@ type CurrentBuild struct { // it carried, and what a push compares a machine's last send against. func (i *Inventory) CurrentBuilds(ctx context.Context) (map[string]CurrentBuild, error) { rows, err := i.store.Pool().Query(ctx, - `select name, coalesce(built_from, ''), upgrade = 'roll-out' from module`) + `select name, coalesce(built_from, ''), upgrade, upgrade_together, upgrade_why, upgrade_by, manifest from module`) if err != nil { return nil, err } defer rows.Close() out := map[string]CurrentBuild{} for rows.Next() { - var name string + var name, why, by string + var chosen *string + var together bool + var manifest []byte var b CurrentBuild - if err := rows.Scan(&name, &b.Commit, &b.RollOut); err != nil { + if err := rows.Scan(&name, &b.Commit, &chosen, &together, &why, &by, &manifest); err != nil { return nil, err } + b.RollOut = upgradeFrom(chosen, together, why, by, manifest).RollOut out[name] = b } return out, rows.Err() diff --git a/internal/inventory/gate.go b/internal/inventory/gate.go new file mode 100644 index 0000000..729ac29 --- /dev/null +++ b/internal/inventory/gate.go @@ -0,0 +1,218 @@ +package inventory + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The gate's verdicts (novox/hq ADR 0235, to-be 45 §8): what a build did on its first machine, and, +// for one that failed there, how it was put back. One row per build, written by the plan that rolled it +// out, under the lease. + +// The gate's verdicts and a failed build's rollback. +const ( + GatePassed = "passed" + GateFailed = "failed" + + RollingBack = "rolling-back" + RolledBack = "rolled-back" + NotRolledBack = "not-rolled-back" +) + +// GateVerdict is one build's verdict at its gate. +type GateVerdict struct { + Build string + Module string + Commit string + Previous string + Plan string + Machines []string + Verdict string + Rollback string + Why string + Component string + // JudgingFrom is when the first machine reported the build applied and the judging began. + JudgingFrom *time.Time + JudgedAt time.Time + Epoch uint64 +} + +// RecordGate writes a build's verdict. **A failed build's row is written once**: a second failure for +// the same build is refused with ErrGateKept, which is what keeps a rollback to one per build — the row +// is written before the rollback's send, and a controller replaced in between finds it. +func (i *Inventory) RecordGate(ctx context.Context, v GateVerdict) error { + epoch, err := i.actingEpoch(ctx) + if err != nil { + return fmt.Errorf("the gate's verdict on %s is not written: %w", v.Build, err) + } + if v.Machines == nil { + v.Machines = []string{} + } + tag, err := i.store.Pool().Exec(ctx, + `insert into build_gate (build, module, commit_hash, previous, plan, machines, verdict, rollback, why, + component, judging_from, judged_at, epoch) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), $12) + on conflict (build) do update set verdict = excluded.verdict, rollback = excluded.rollback, + why = excluded.why, previous = excluded.previous, machines = excluded.machines, + judged_at = now(), epoch = excluded.epoch + where build_gate.verdict = 'passed' and excluded.verdict = 'passed'`, + v.Build, v.Module, v.Commit, v.Previous, v.Plan, v.Machines, v.Verdict, v.Rollback, v.Why, v.Component, + v.JudgingFrom, epoch) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return fmt.Errorf("%w: %s", ErrGateKept, v.Build) + } + return nil +} + +// ErrGateKept is a verdict already kept for the build, which is not written over. +var ErrGateKept = errors.New("this build's verdict at its gate is already kept") + +// SetRollback records how a failed build's rollback went. +func (i *Inventory) SetRollback(ctx context.Context, build, rollback, why string) error { + if _, err := i.actingEpoch(ctx); err != nil { + return err + } + _, err := i.store.Pool().Exec(ctx, + `update build_gate set rollback = $2, why = $3, judged_at = now() where build = $1 and verdict = 'failed'`, + build, rollback, why) + return err +} + +// GateOf is a build's verdict, and whether it has one. +func (i *Inventory) GateOf(ctx context.Context, build string) (GateVerdict, bool, error) { + rows, err := i.store.Pool().Query(ctx, gateSelect+` where build = $1`, build) + if err != nil { + return GateVerdict{}, false, err + } + list, err := scanGates(rows) + if err != nil || len(list) == 0 { + return GateVerdict{}, false, err + } + return list[0], true, nil +} + +// GateFailed is whether a build failed its gate: one the mesh never registers or sends again on its own. +func (i *Inventory) GateFailed(ctx context.Context, build string) (bool, error) { + v, found, err := i.GateOf(ctx, build) + return found && v.Verdict == GateFailed, err +} + +// LatestGates is the newest verdict of every module that has one: what the gate probe (DG) reads. +func (i *Inventory) LatestGates(ctx context.Context) ([]GateVerdict, error) { + rows, err := i.store.Pool().Query(ctx, `select distinct on (module) build, module, commit_hash, previous, plan, + machines, verdict, rollback, why, component, judging_from, judged_at, coalesce(epoch, 0) + from build_gate order by module, judged_at desc`) + if err != nil { + return nil, err + } + return scanGates(rows) +} + +// Gates is the newest verdicts, newest first: what `plans gates` lists. +func (i *Inventory) Gates(ctx context.Context, limit int) ([]GateVerdict, error) { + rows, err := i.store.Pool().Query(ctx, gateSelect+` order by judged_at desc limit $1`, limit) + if err != nil { + return nil, err + } + return scanGates(rows) +} + +const gateSelect = `select build, module, commit_hash, previous, plan, machines, verdict, rollback, why, component, + judging_from, judged_at, coalesce(epoch, 0) from build_gate` + +func scanGates(rows pgx.Rows) ([]GateVerdict, error) { + defer rows.Close() + var out []GateVerdict + for rows.Next() { + var v GateVerdict + var epoch int64 + if err := rows.Scan(&v.Build, &v.Module, &v.Commit, &v.Previous, &v.Plan, &v.Machines, &v.Verdict, + &v.Rollback, &v.Why, &v.Component, &v.JudgingFrom, &v.JudgedAt, &epoch); err != nil { + return nil, err + } + v.Epoch = uint64(epoch) + out = append(out, v) + } + return out, rows.Err() +} + +// PreviousBuild is the build a module goes back to when a build of it fails its gate: the newest build +// that worked, made from the commit the first machine ran before, asked before the failed one, and not +// itself failed at a gate. Among the builds whose artifacts the mesh keeps (KeptBuilds): an older one +// may already be gone from the artifact store. False when there is none to go back to. +func (i *Inventory) PreviousBuild(ctx context.Context, module, commit string, failed Build) (Build, bool, error) { + builds, err := i.Builds(ctx, module, 50) + if err != nil { + return Build{}, false, err + } + kept := 0 + for _, b := range builds { + if !b.Worked() { + continue + } + kept++ + if kept > KeptBuilds { + break + } + if b.ID == failed.ID || (commit != "" && b.Commit != commit) { + continue + } + if !failed.AskedOrAt().IsZero() && !b.AskedOrAt().Before(failed.AskedOrAt()) { + continue + } + if bad, err := i.GateFailed(ctx, b.ID); err != nil { + return Build{}, false, err + } else if bad { + continue + } + var manifest []byte + if err := i.store.Pool().QueryRow(ctx, `select manifest from build where id = $1`, b.ID).Scan(&manifest); err != nil { + return Build{}, false, err + } + if len(manifest) == 0 || string(manifest) == "null" { + continue + } + b.Manifest = manifest + return b, true, nil + } + return Build{}, false, nil +} + +// RestoreModule puts a module's registered build back to an earlier one: its manifest, the commit it +// was built from, and when it was asked — as now, so the build that failed its gate, asked before, can +// never register over it again (issue 219's order). The source's head is left where the merge moved it: +// the module IS behind its source, and `status` says so. +func (i *Inventory) RestoreModule(ctx context.Context, b Build) error { + if _, err := i.actingEpoch(ctx); err != nil { + return fmt.Errorf("%s is not put back: %w", b.Module, err) + } + m, err := catalogue.ParseManifest(b.Manifest) + if err != nil { + return fmt.Errorf("%s's build %s is not a manifest the mesh can register again: %w", b.Module, b.ID, err) + } + raw, err := json.Marshal(m) + if err != nil { + return err + } + tag, err := i.store.Pool().Exec(ctx, + `update module set manifest = $2, version = nullif($3, ''), built_from = nullif($4, ''), + built_asked = now(), registered = now() + where name = $1`, b.Module, raw, m.Version, b.Commit) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return fmt.Errorf("%w: %s", ErrNoSuchModule, b.Module) + } + return nil +} diff --git a/internal/inventory/migrations/0073-a-build-rolls-out-gated-and-rolls-back.sql b/internal/inventory/migrations/0073-a-build-rolls-out-gated-and-rolls-back.sql new file mode 100644 index 0000000..6deb22c --- /dev/null +++ b/internal/inventory/migrations/0073-a-build-rolls-out-gated-and-rolls-back.sql @@ -0,0 +1,67 @@ +-- A module's build rolls out one machine first, judged at a gate, and rolls back there when the gate +-- fails (novox/hq ADR 0235, to-be 45 §8, Phase 4). +-- +-- 1. The upgrade policy becomes a person's choice over the module's own word. Until now every module +-- held a policy here, 'record' unless a person had said 'roll-out', and nothing distinguished a +-- 'record' somebody chose from the default it always was. From here a null policy is no choice: the +-- module's manifest decides (its `upgrade`, its data, whether it is the bus), and its default is to +-- roll out. A 'roll-out' a person chose is kept as their choice. A 'record' is the old default and +-- becomes no choice — ADR 0235 decides it, and lists every module's resulting policy; a person who +-- wants one held again says so with `upgrade record --why`, which is kept with its why. +alter table module alter column upgrade drop not null; +alter table module alter column upgrade drop default; +alter table module drop constraint if exists module_upgrade_check; +alter table module add constraint module_upgrade_chosen check (upgrade is null or upgrade in ('record', 'roll-out')); +update module set upgrade = null where upgrade = 'record'; +-- Why the person chose it, and who: said back by `upgrade`, so a held module says why it is held. +alter table module add column upgrade_why text not null default ''; +alter table module add column upgrade_by text not null default ''; + +-- 2. The gate's verdict on each build a plan rolled out, one row per build: passed on its first machine, +-- or failed there and rolled back. **A build that failed its gate is marked, and is never sent again +-- automatically**: registration refuses it, and the rollback is attempted once per build — the row is +-- written before the rollback's send, so a controller replaced in between does not send it twice. +create table build_gate ( + build text primary key, + module text not null, + -- The commit the build was made from, and the one the module was put back to. + commit_hash text not null default '', + previous text not null default '', + plan text not null default '', + -- The machines it was judged on: the first machine, and the bus holder when it went with it. + machines text[] not null default '{}', + -- 'passed', or 'failed'; and for a failed one how the rollback went: 'rolling-back', 'rolled-back', + -- or 'not-rolled-back' (no previous build to put back, or the send refused), said in `why`. + verdict text not null check (verdict in ('passed', 'failed')), + rollback text not null default '' check (rollback in ('', 'rolling-back', 'rolled-back', 'not-rolled-back')), + why text not null default '', + -- The core component it is, when it is one: mesh-controller, mesh-host, node-tools. + component text not null default '', + judging_from timestamptz, + judged_at timestamptz not null default now(), + epoch bigint +); +create index build_gate_module on build_gate (module, judged_at desc); + +-- 3. The bus's planned step (to-be 45 §8): a bus upgrade is never rolled out; a person starts it, with +-- why, after its streams are snapshotted, and it is checked after. One row per step; the open one is +-- the step running, which the self-check says as `bus-maintenance` until the bus is healthy again or +-- the step's bound passes and it is said failed, with its snapshot as the way back. +create table bus_step ( + id bigserial primary key, + module text not null, + machines text[] not null default '{}', + from_build text not null default '', + to_build text not null default '', + -- Where the streams' snapshot is: taken by the mesh, or one a person says they took. + snapshot text not null, + -- Whether the new version can be reverted by putting the old one back, as the person said it. + reversible boolean not null, + by_whom text not null default '', + why text not null, + started timestamptz not null default now(), + ended timestamptz, + -- '', then 'done' or 'failed', with what was found. + outcome text not null default '' check (outcome in ('', 'done', 'failed')), + found text not null default '' +); diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index 1a1261d..791f0d0 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -67,6 +67,42 @@ type PlanModule struct { // its module's name included. Empty in a plan from before it was kept, which is matched by // module, or by repository and path, as before. Build string `json:"build,omitempty"` + // Previous is the build the first machine ran of this module before the plan sent it the new one — + // the commit its last send carried (ADR 0221) — kept at the first send: what a rollback puts back + // (novox/hq ADR 0235). Empty when the machine had never been sent the module, or what it was sent + // is not known. + Previous string `json:"previous,omitempty"` + // Gate is the new build's judging on its first machine (novox/hq ADR 0235, to-be 45 §8), kept so a + // controller replaced mid-judging resumes it, and read back through `plans` as the rollout's record. + Gate *PlanGate `json:"gate,omitempty"` +} + +// PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine, +// from and to which build, the verdict, how long it took to reach it, and whether it was rolled back. +type PlanGate struct { + // Component is the core component the module is — mesh-controller, mesh-host, node-tools — or empty + // for any other module, judged by its own health. + Component string `json:"component,omitempty"` + Machines []string `json:"machines"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` + // Since is when the judging began: the first machine reported the new build applied. + Since *time.Time `json:"since,omitempty"` + // Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that + // does not resets them. + Passes int `json:"passes,omitempty"` + LastPass *time.Time `json:"last_pass,omitempty"` + // Last is what the newest judging found wanting, while it still may pass. + Last string `json:"last,omitempty"` + // Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since. + Verdict string `json:"verdict,omitempty"` + Why string `json:"why,omitempty"` + JudgedAt *time.Time `json:"judged_at,omitempty"` + Took string `json:"took,omitempty"` + // Rollback is how a failed build was put back: rolled-back, or not-rolled-back with why. + Rollback string `json:"rollback,omitempty"` + // Kept says a passing verdict was written to the gate's records. + Kept bool `json:"kept,omitempty"` } // The states a plan passes through. diff --git a/internal/inventory/sent_builds_test.go b/internal/inventory/sent_builds_test.go index 9a1d150..eb9ae85 100644 --- a/internal/inventory/sent_builds_test.go +++ b/internal/inventory/sent_builds_test.go @@ -72,7 +72,8 @@ func TestTheCurrentBuildsAreTheCatalogues(t *testing.T) { if err != nil { t.Fatal(err) } - if got := current["resolver"]; got != (CurrentBuild{Commit: "c1"}) { + // Rolled out by default, one machine first and gated (novox/hq ADR 0235). + if got := current["resolver"]; got != (CurrentBuild{Commit: "c1", RollOut: true}) { t.Errorf("resolver is at %+v", got) } if got := current["by-hand"]; got != (CurrentBuild{RollOut: true}) { diff --git a/internal/lease/lease.go b/internal/lease/lease.go index cd0bf71..a305094 100644 --- a/internal/lease/lease.go +++ b/internal/lease/lease.go @@ -46,6 +46,9 @@ type Holder struct { Epoch uint64 `json:"epoch,omitempty"` Taken time.Time `json:"taken"` Renewed time.Time `json:"renewed"` + // Health is the holder's word about itself, written with every renewal (witness.go): nil from a + // process that says none, which the controller's gate reads as not ready. The host ignores it. + Health *Health `json:"health,omitempty"` } // Defaults, as to-be 45 §6 sets them. The age is the bucket's, read from it (Open), so the bucket @@ -85,6 +88,9 @@ type Options struct { Moved func(was, floor uint64) // Now is the clock; nil is time.Now. Now func() time.Time + // Health is asked at every take and renewal for the holder's word about itself (witness.go); nil + // writes none. + Health func() *Health } // Lease is one instance's hold, or its wait for one. @@ -241,6 +247,9 @@ func (l *Lease) TryTake(ctx context.Context) (uint64, error) { now := l.o.Now() h := l.o.Holder h.Taken, h.Renewed, h.Epoch = now, now, 0 + if l.o.Health != nil { + h.Health = l.o.Health() + } value, err := json.Marshal(h) if err != nil { return 0, err @@ -390,6 +399,9 @@ func (l *Lease) Renew(ctx context.Context) error { h.Epoch, h.Taken = epoch, taken anchor := l.o.Now() h.Renewed = anchor + if l.o.Health != nil { + h.Health = l.o.Health() + } value, err := json.Marshal(h) if err != nil { return l.lose(err) diff --git a/internal/lease/witness.go b/internal/lease/witness.go new file mode 100644 index 0000000..9672c3f --- /dev/null +++ b/internal/lease/witness.go @@ -0,0 +1,180 @@ +package lease + +import ( + "fmt" + "strings" + "time" +) + +// The controller's rollback witness: the contract between the controller and the node-engine that +// placed it (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0236). Its other half is mesh-host's +// internal/witness/contract.go and the Report's `rollbacks` and `witness` (mesh-host internal/link); +// the two are held field for field, and a change on either side is a change to both. +// +// **The component being replaced is never the only witness of its successor.** A new controller that +// starts and does nothing, crashes, or cannot reach the bus cannot say so, and cannot put back the build +// before it. The node-engine on the machine running the controller can: it placed the new bundle, it +// keeps the previous one beside it, and it reads one key on the bus to judge the new one by. +// +// **What the host reads.** A direct get of the lease bucket's one key — LeaseReadSubject — answered with +// the Holder below as JSON (unknown fields ignored, `build` not read). The node principal of every +// machine assigned the controller is granted that one subject, and every machine's node principal the +// runtime's PING on its own machine (broker.WitnessSubjects); replies come to its own inbox. +// +// **When the host calls a new controller healthy** (HeldBySince, as the host has it): the holder's host +// is this machine, it took the key at or after the moment the host started the new build (less Skew), +// and it renewed it within the key's age. Asked every WitnessEvery, within ControllerWithin of the +// start; not met by then, the host stops the new build, starts the one it kept, and says so in its +// reports (Rollback). A key absent, deleted or purged is held by nobody. +// +// **What the controller adds, and the host does not read.** Health: whether the holder says it is +// ready — its self-check ran and `status` answered in bound. The host's bound is the lease alone, sixty +// seconds; the controller's own gate (gate.go) also asks for ready within ten minutes and, failing that, +// sends the previous controller build itself, which the host applies as any declaration. +// +// **What the controller does with what the host says.** Each Rollback a report carries is a condition +// `core...` — urgent for rolled-back, not-reversible, restore-failed and halted; +// a warning for nothing-to-restore and unwitnessed — kept while reports carry it and cleared by the first +// report from that machine without it. A rolled-back controller or node tools build is marked failed at +// its gate and the module's registered build is put back to the one running, so nothing sends it again. + +// Witness bounds, as the host has them (mesh-host internal/witness). +const ( + // ControllerWithin: the new controller holds the lease within this of starting. + ControllerWithin = 60 * time.Second + // NodeToolsWithin: the node tools answer PING within this of starting, each PING within PingWithin. + NodeToolsWithin = 60 * time.Second + PingWithin = 5 * time.Second + // WitnessEvery is how often the host asks. + WitnessEvery = 5 * time.Second + // Skew is how far the controller's clock and the host's may disagree about when it took the lease: + // one machine, one clock — a margin, not a tolerance. + Skew = 2 * time.Second + // FreshWithin is the key's age: a holder not renewed within it is not holding it. + FreshWithin = 15 * time.Second + // ReadyWithin is the controller's own bound for saying it is ready (Health), which its gate judges. + ReadyWithin = 10 * time.Minute +) + +// LeaseReadSubject is the one subject the witness of the controller publishes to read the lease: a +// direct get of the key `holder`. +func LeaseReadSubject(bucket string) string { + return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + Key +} + +// PingSubject is the subject a machine's witness asks its own node tools on: the services protocol's +// PING to the runtime, whose instance is the machine's name. +func PingSubject(node string) string { return "$SRV.PING." + NodeToolsService + "." + node } + +// NodeToolsService is the runtime's name on the services protocol. +const NodeToolsService = "node-tools" + +// Health is what the holding controller says of itself in every write of the lease's key. The host +// does not read it; the controller's gate does. A controller that says nothing is not ready. +type Health struct { + // Ready is the controller's health definition met (to-be 45 §8): it holds the lease, its self-check + // has run once, and in that run `status` answered in full within ten seconds (D9). + Ready bool `json:"ready"` + // ReadyAt is when it first became ready; zero while it is not. + ReadyAt time.Time `json:"ready_at,omitempty"` + // Started is when this process started. + Started time.Time `json:"started"` + // DoctorRan is when its self-check last finished a run; zero before the first. + DoctorRan time.Time `json:"doctor_ran,omitempty"` + // Why says what is missing while it is not ready, in the mesh's words. + Why string `json:"why,omitempty"` +} + +// The witness contract version a host keeps (Report.Witness): its presence says the host reads a +// process's `witness` and `not-reversible`, which an older, strict host refuses — so the controller +// sends them only to a machine whose report carries it. +const WitnessContract = 1 + +// The core components a witness judges, as a Rollback names them. +const ( + ComponentEngine = "node-engine" + ComponentController = "controller" + ComponentNodeTools = "node-tools" +) + +// What a witness concluded, as a Rollback says it. +const ( + OutcomeRolledBack = "rolled-back" + OutcomeNotReversible = "not-reversible" + OutcomeNothingToRestore = "nothing-to-restore" + OutcomeRestoreFailed = "restore-failed" + OutcomeUnwitnessed = "unwitnessed" + OutcomeHalted = "halted" +) + +// Urgent says whether a witness's outcome needs the operator now. +func Urgent(outcome string) bool { + switch outcome { + case OutcomeRolledBack, OutcomeNotReversible, OutcomeRestoreFailed, OutcomeHalted: + return true + } + return false +} + +// Rollback is one witness's verdict on one core build, as the node-engine says it in its report +// (`rollbacks`) — on every report while it stands, until a newer build of that component is declared to +// it and proves itself. +type Rollback struct { + // Component is node-engine, controller or node-tools. + Component string `json:"component"` + // From is the build judged: a host version for the node-engine, a bundle's digest for a process. + From string `json:"from"` + // To is the build restored; empty when none was. + To string `json:"to,omitempty"` + // Outcome is one of the Outcome words. + Outcome string `json:"outcome"` + Why string `json:"why"` + At time.Time `json:"at"` +} + +// ModuleOf is the module a core component is delivered as. +func ModuleOf(component string) string { + switch component { + case ComponentController: + return "mesh-controller" + case ComponentEngine: + return "mesh-host" + case ComponentNodeTools: + return NodeToolsService + } + return "" +} + +// HeldBySince is the host's judgement of a new controller (mesh-host witness.ControllerLease.HeldBySince, +// kept here so both sides test one rule): the lease is held by a controller on machine `host` that took +// it at or after `since`, less Skew, and renewed it within the key's age. +func (h Holder) HeldBySince(host string, since, now time.Time) (bool, string) { + last := h.Taken + if h.Renewed.After(last) { + last = h.Renewed + } + switch { + case h.Instance == "": + return false, "the lease names no holder" + case host != "" && !sameMachine(h.Host, host): + return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", h.Instance, h.Host) + case h.Taken.Before(since.Add(-Skew)): + return false, fmt.Sprintf("the lease is held by %s, taken before the new build started", h.Instance) + case now.Sub(last) > FreshWithin: + return false, fmt.Sprintf("the lease names %s and was last renewed %s ago", h.Instance, + now.Sub(last).Round(time.Second)) + } + return true, fmt.Sprintf("%s holds the lease, epoch %d", h.Instance, h.Epoch) +} + +// sameMachine compares two hostnames as names, so a short name and its fully qualified form agree. +func sameMachine(a, b string) bool { + short := func(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + if i := strings.IndexByte(s, '.'); i > 0 { + s = s[:i] + } + return s + } + return short(a) == short(b) +} diff --git a/internal/lease/witness_test.go b/internal/lease/witness_test.go new file mode 100644 index 0000000..6422b05 --- /dev/null +++ b/internal/lease/witness_test.go @@ -0,0 +1,59 @@ +package lease + +import ( + "testing" + "time" +) + +// The host's judgement of a new controller, held here as the host holds it (mesh-host internal/witness): +// the lease held by a controller on this machine, taken since the new build started, and fresh. +func TestTheWitnessJudgesANewControllerByTheLease(t *testing.T) { + started := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + now := started.Add(30 * time.Second) + fresh := Holder{Instance: "controller@control pid 2", Host: "control.example", Taken: started.Add(5 * time.Second), + Renewed: now.Add(-3 * time.Second), Epoch: 9} + if ok, why := fresh.HeldBySince("control", started, now); !ok { + t.Fatalf("a new controller holding the lease was not healthy: %s", why) + } + old := fresh + old.Taken = started.Add(-time.Minute) + if ok, _ := old.HeldBySince("control", started, now); ok { + t.Error("the controller from before the build counted as the new one") + } + elsewhere := fresh + elsewhere.Host = "other" + if ok, _ := elsewhere.HeldBySince("control", started, now); ok { + t.Error("a controller on another machine counted") + } + stale := fresh + stale.Renewed = now.Add(-20 * time.Second) + stale.Taken = stale.Renewed + if ok, _ := stale.HeldBySince("control", started.Add(-time.Minute), now); ok { + t.Error("a lease not renewed within its age counted as held") + } + if ok, _ := (Holder{}).HeldBySince("control", started, now); ok { + t.Error("nobody counted as the holder") + } +} + +// The outcomes the operator is woken for, and the subjects the host reads. +func TestTheWitnessContractsWords(t *testing.T) { + for outcome, urgent := range map[string]bool{OutcomeRolledBack: true, OutcomeNotReversible: true, + OutcomeRestoreFailed: true, OutcomeHalted: true, OutcomeNothingToRestore: false, OutcomeUnwitnessed: false} { + if Urgent(outcome) != urgent { + t.Errorf("%s urgent: %v", outcome, Urgent(outcome)) + } + } + if got := LeaseReadSubject("mesh-controller_lease"); got != "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder" { + t.Errorf("the lease is read on %s", got) + } + if got := PingSubject("ace"); got != "$SRV.PING.node-tools.ace" { + t.Errorf("the node tools are asked on %s", got) + } + for c, m := range map[string]string{ComponentController: "mesh-controller", ComponentEngine: "mesh-host", + ComponentNodeTools: "node-tools"} { + if ModuleOf(c) != m { + t.Errorf("%s is delivered as %s", c, ModuleOf(c)) + } + } +} diff --git a/internal/link/events.go b/internal/link/events.go index cf51d9f..c3c6bad 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -70,6 +70,9 @@ const ( // is spent and the condition is the operator's (novox/hq to-be 45 §7). Never a person's act: those // are the hand-act log's. KeyHealerActed = "healer-acted" + // KeyRolledBack: a build failed its gate on its first machine and was put back there, or could not + // be (novox/hq ADR 0235, to-be 45 §8); or a witness on a machine put a core component back. + KeyRolledBack = "rolled-back" ) // Applied is what a machine now runs, as the mesh states it. @@ -180,6 +183,11 @@ type SourceMoved struct { // rather than inferred from a round number, because "this is all of it" and "this is as much as // I asked for" are the difference between rebuilding a module and leaving it stale. PathsTruncated bool `json:"paths_truncated,omitempty"` + + // Removed are the files among Paths the merge deleted. A module whose manifest is among them was + // deleted at its source: it is forgotten, or said, and never built (novox/hq ADR 0235). Empty from an + // announcer that does not say which files went, and then a build that finds no manifest says it. + Removed []string `json:"removed,omitempty"` } type Upgraded struct { diff --git a/internal/link/protocol.go b/internal/link/protocol.go index ec1a9c8..e289ada 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -11,6 +11,8 @@ import ( "strconv" "strings" "time" + + "github.com/novox/mesh-controller/internal/lease" ) // Exchange is where nodes publish everything they have to say. @@ -251,6 +253,15 @@ type Report struct { // and the mesh's up in its place, and where the found configuration's original was kept. Tunnel *CarriedTunnel `json:"tunnel,omitempty"` + // Rollbacks is what this machine's witnesses decided about a core build that was not healthy in + // bound (novox/hq to-be 45 §8, ADR 0236; the contract is lease/witness.go and mesh-host's + // internal/witness): the node-engine's launcher about the engine, the engine about the controller and + // the node tools. Said on every report while it stands; absent from a host that witnesses nothing. + Rollbacks []lease.Rollback `json:"rollbacks,omitempty"` + // Witness is the witness contract version the host keeps (lease.WitnessContract): a process's + // `witness` and `not-reversible` are sent only to a machine whose report carries it. + Witness int `json:"witness,omitempty"` + // Rekey is a node taking a found tunnel's key as its overlay key after enrolment (novox/hq // ADR 0105). A report carrying one is not an account of the machine: it moves the node's // overlay key and tunnel and nothing else. From c6f3d8cdfad154252f4fec4144b205b7a342d74b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 6 Oct 2026 18:31:54 +0200 Subject: [PATCH 2/7] Take the bus's snapshot through its machine's backup holder before the planned step (hq ADR 0235, 0236) --- cmd/mesh-controller/bus_step.go | 76 +++++++++++++++++++++----- cmd/mesh-controller/gate_test.go | 18 ++++-- internal/broker/nats.go | 10 ++++ internal/broker/testdata/composed.conf | 2 +- 4 files changed, 85 insertions(+), 21 deletions(-) diff --git a/cmd/mesh-controller/bus_step.go b/cmd/mesh-controller/bus_step.go index 415230a..c45ee8f 100644 --- a/cmd/mesh-controller/bus_step.go +++ b/cmd/mesh-controller/bus_step.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" @@ -38,10 +40,11 @@ const ( // busStepBound is how long after its start a bus upgrade must be followed by a healthy bus. var busStepBound = 15 * time.Minute -// takeBusSnapshot snapshots every stream before a bus upgrade and answers where the snapshot is. Nil -// until the controller's JetStream snapshot is built (to-be 45 §8: "the streams are snapshotted"); until -// then a person takes it by hand and says where with --snapshot-taken. -var takeBusSnapshot func(ctx context.Context) (string, error) +// takeBusSnapshot snapshots every stream before a bus upgrade and answers where the snapshot is: the bus +// machine's backup holder backs the bus module up now, whose dump is the streams' snapshot (novox/hq ADR +// 0235). A variable so a test takes none. A person who took one by hand says where with --snapshot-taken, +// and then none is taken — for a bus whose module does not yet carry the snapshot program. +var takeBusSnapshot = snapshotTheBusNow // busPending is what a bus upgrade would do: the bus's module, the machines running it, and, per // machine, the build it was last sent against the build the mesh holds. Empty machines: the mesh holds @@ -174,15 +177,14 @@ func busCommand(ctx context.Context, args []string) error { return nil } where := strings.TrimSpace(*snapshot) - switch { - case takeBusSnapshot != nil: - if where, err = takeBusSnapshot(ctx); err != nil { - return fmt.Errorf("the streams could not be snapshotted, so the bus is not replaced: %w", err) + if where == "" { + for _, n := range moving { + fmt.Printf("snapshotting the bus's streams on %s first (its backup holder, ADR 0235)…\n", n) + if where, err = takeBusSnapshot(ctx, b.module, n); err != nil { + return fmt.Errorf("the streams could not be snapshotted, so the bus is not replaced: %w — a snapshot "+ + "taken by hand is said with --snapshot-taken ", err) + } } - case where == "": - return errors.New("the bus is replaced only after its streams are snapshotted. The mesh does not take the " + - "snapshot itself yet (to-be 45 §8: the controller's JetStream snapshot); take one by hand and say where " + - "with --snapshot-taken . Nothing was done") } from := map[string]bool{} for _, n := range moving { @@ -237,9 +239,7 @@ func busStatus(ctx context.Context) error { fmt.Printf(" %-10s %s\n", n, state) } } - if takeBusSnapshot == nil { - fmt.Println(" the mesh takes no snapshot of the streams itself yet: `bus upgrade` asks where yours is (--snapshot-taken)") - } + fmt.Println(" `bus upgrade` has the bus machine's backup holder snapshot the streams first (ADR 0235)") s, found, err := open.inventory.LatestBusStep(ctx) if err != nil || !found { return err @@ -325,3 +325,49 @@ func sortedKeys(set map[string]bool) []string { sort.Strings(out) return out } + +// busSnapshotWithin is how long the bus machine's backup holder is given to take the bus's snapshot. +var busSnapshotWithin = 15 * time.Minute + +// snapshotTheBusNow asks the bus machine's backup holder to back the bus module up now — its dump is +// the streams' snapshot (novox/hq ADR 0235) — and waits until it says a backup newer than the ask: +// where the snapshot is, as a person reads it. The serving controller's connection, or one of its own. +func snapshotTheBusNow(ctx context.Context, module, node string) (string, error) { + var where string + err := onTheBus(func(conn *nats.Conn) error { + asked := time.Now() + answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "now", node, + map[string]any{"module": module}, 30*time.Second) + if err != nil { + return fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err) + } + if answer.Error != "" { + return fmt.Errorf("%s's backup holder would not take the bus's snapshot: %s", node, answer.Error) + } + deadline := time.Now().Add(busSnapshotWithin) + for { + answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "backed-up", node, map[string]any{}, 10*time.Second) + if err == nil && answer.Error == "" { + if measured, err := readHolder(answer.Result); err == nil { + for item, m := range measured[module] { + if m.LastBackup != nil && m.LastBackup.After(asked) && m.Error == "" { + where = fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, item, + m.LastBackup.UTC().Format(time.RFC3339)) + return nil + } + } + } + } + if time.Now().After(deadline) { + return fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s; the bus is "+ + "not replaced", node, busSnapshotWithin) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Second): + } + } + }) + return where, err +} diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index ff2135b..4c8bcb9 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -403,22 +403,30 @@ func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) { if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" { t.Fatalf("a push may send the bus's machine: %v %v", held, err) } - // The planned step refuses to start without its word on reversibility, and without a snapshot. + // The planned step refuses to start without its word on reversibility, and without a snapshot taken + // first by the bus machine's backup holder. if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") { t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err) } + wasSnapshot := takeBusSnapshot + t.Cleanup(func() { takeBusSnapshot = wasSnapshot }) + takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") } if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil || - !strings.Contains(err.Error(), "snapshot") { - t.Fatalf("a bus upgrade started without a snapshot: %v", err) + !strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 { + t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent) } - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible", "--snapshot-taken", "nightly"}); err != nil { + takeBusSnapshot = func(_ context.Context, module, node string) (string, error) { + return node + "'s restore point of " + module, nil + } + if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil { t.Fatal(err) } if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) { t.Fatalf("the step sent %v, not the bus's machine", sent) } s, found, err := inv.LatestBusStep(ctx) - if err != nil || !found || s.Snapshot != "nightly" || s.Ended != nil || !reflect.DeepEqual(s.Machines, []string{"anchor"}) { + if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil || + !reflect.DeepEqual(s.Machines, []string{"anchor"}) { t.Fatalf("the step is %+v %v %v", s, found, err) } } diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 84284ab..5b88417 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -152,6 +152,12 @@ type SeatVerb struct{ Seat, Verb string } var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: "banned"}, {Seat: "node-backup", Verb: "backed-up"}} +// VerbsTheBusStepAsks are the seat verbs the bus's planned step calls (novox/hq to-be 45 §8, ADR 0236): +// the bus machine's backup holder takes the bus's snapshot now, before the bus is replaced (ADR 0235's +// dump), and says when it is taken (`backed-up`, granted with the self-check's). The one verb of +// the controller's grant that acts, and only through the step a person starts. +var VerbsTheBusStepAsks = []SeatVerb{{Seat: "node-backup", Verb: "now"}} + // perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the // holder's machine (novox/hq ADR 0219): `paused.`, the build agent saying whether it takes work. var perMachineEvents = map[string]bool{"paused.*": true} @@ -342,6 +348,10 @@ func PermissionsFor(p Principal) (Permissions, error) { for _, v := range VerbsTheSelfCheckAsks { pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*") } + // And the bus's planned step: a snapshot taken now, before the bus is replaced (ADR 0236). + for _, v := range VerbsTheBusStepAsks { + pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*") + } // And asks who answers (novox/hq to-be 45 §4, D3): the self-check finds every seat's holder by // the same discovery the console reads. The question only; the answers come to its own inbox. pub = append(pub, "$SRV.INFO") diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index e53cc8a..c4cdabe 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] } allow_responses: { max: 1, ttl: "1m" } } } From d7bf1bae83da2a173ee166529b5a1e681e654e6b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 6 Oct 2026 18:38:51 +0200 Subject: [PATCH 3/7] Name the decision this builds: hq ADR 0236 (0235 is the bus's snapshot) --- cmd/mesh-controller/build.go | 2 +- cmd/mesh-controller/bus_step.go | 6 +++--- cmd/mesh-controller/doctor.go | 18 +++++++++--------- cmd/mesh-controller/gate.go | 6 +++--- cmd/mesh-controller/gate_test.go | 2 +- cmd/mesh-controller/held_back_test.go | 2 +- cmd/mesh-controller/main.go | 2 +- cmd/mesh-controller/plan_retry.go | 2 +- cmd/mesh-controller/push.go | 2 +- cmd/mesh-controller/queue_test.go | 2 +- cmd/mesh-controller/release_plan.go | 10 +++++----- cmd/mesh-controller/status_summary.go | 2 +- cmd/mesh-controller/upgrades.go | 8 ++++---- cmd/mesh-controller/witness.go | 2 +- internal/broker/states_agreement_test.go | 2 +- internal/broker/streams.go | 2 +- internal/catalogue/manifest.go | 2 +- internal/catalogue/seats.go | 2 +- internal/catalogue/upgrade.go | 4 ++-- internal/catalogue/verbs.go | 6 +++--- internal/inventory/busstep.go | 2 +- internal/inventory/catalogue.go | 8 ++++---- internal/inventory/gate.go | 2 +- ...-a-build-rolls-out-gated-and-rolls-back.sql | 4 ++-- internal/inventory/plans.go | 4 ++-- internal/inventory/sent_builds_test.go | 2 +- internal/link/events.go | 4 ++-- 27 files changed, 55 insertions(+), 55 deletions(-) diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index d7fb7da..f860b39 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -570,7 +570,7 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w", result.On, result.Repository, short(result.Commit), err) } - // **A build that failed its gate is never registered again** (novox/hq ADR 0235): an outcome heard + // **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard // twice, or replayed, would otherwise make the build a rollback put back what the module is again, // and the next push would send it. if failed, err := inv.GateFailed(ctx, kept.ID); err != nil { diff --git a/cmd/mesh-controller/bus_step.go b/cmd/mesh-controller/bus_step.go index c45ee8f..4ef2149 100644 --- a/cmd/mesh-controller/bus_step.go +++ b/cmd/mesh-controller/bus_step.go @@ -17,7 +17,7 @@ import ( "github.com/novox/mesh-controller/internal/link" ) -// The bus as a planned step (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0235). +// The bus as a planned step (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0236). // // **A bus upgrade is never rolled out.** The bus carries every declaration, every report and the // controller's own lease; a new bus build that does not come up is a mesh nobody can tell anything, @@ -101,7 +101,7 @@ func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, erro } // busHeld names the machines a push may not send because sending them would replace the bus: the -// planned step's, not a push's (ADR 0235). Said with the remedy. +// planned step's, not a push's (ADR 0236). Said with the remedy. func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) { b, err := pendingBus(ctx, inv) if err != nil { @@ -112,7 +112,7 @@ func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) ( for _, holder := range b.machines { if n == holder && b.moves(n) { out[n] = fmt.Sprintf("sending %s would replace the bus (%s %s → %s), which is a planned step: "+ - "`bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0235)", + "`bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)", n, b.module, short(orNotKnown(b.from[n])), short(b.to)) } } diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index ad98778..fdb9cd3 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -113,26 +113,26 @@ var probeRegistry = []probe{ Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}}, {ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals", From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs}, - // The core's health definitions (novox/hq to-be 45 §8, ADR 0235): what a core component's new build is + // The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is // judged by on its first machine, run against every machine between upgrades too. {ID: "H-controller", Asserts: "the controller lease is held, renewed in time, by a controller that says it is " + - "ready: its self-check ran and status answered in full within ten seconds", From: "ADR 0235, to-be 45 §8", + "ready: its self-check ran and status answered in full within ten seconds", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeControllerHealth}, {ID: "H-engine", Asserts: "every machine heard from has reported its current declaration, under a node-engine " + - "build it names", From: "ADR 0235, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeEngineHealth}, + "build it names", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeEngineHealth}, {ID: "H-tools", Asserts: "every machine heard from that runs the node tools has them answering the bus", - From: "ADR 0235, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeToolsHealth}, + From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeToolsHealth}, {ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses " + - "it to the machines' node tools and back", From: "ADR 0235, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, + "it to the machines' node tools and back", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeBusHealth}, - // The gate's verdicts and the witnesses' rollbacks (ADR 0235): each build that failed its gate keeps its + // The gate's verdicts and the witnesses' rollbacks (ADR 0236): each build that failed its gate keeps its // condition until a newer build passes; each rollback a witness stands by is said. {ID: gateProbe, Asserts: "no build that failed its gate, and no core component a witness put back, goes unsaid; " + - "a newer build that passes its gate clears it", From: "ADR 0235, to-be 45 §8", Kind: kindRolledBack, + "a newer build that passes its gate clears it", From: "ADR 0236, to-be 45 §8", Kind: kindRolledBack, Raises: []string{kindRollbackFailed}, Phase: 4, run: probeGates}, - // The bus's planned step (ADR 0235): open while a person's bus upgrade runs, then checked by H-bus. + // The bus's planned step (ADR 0236): open while a person's bus upgrade runs, then checked by H-bus. {ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " + - "health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0235, to-be 45 §8", + "health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8", Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep}, } diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 6a9fdbe..6fa4936 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -20,7 +20,7 @@ import ( "github.com/novox/mesh-controller/internal/link" ) -// The gate on a release plan's first machine, and the rollback after it (novox/hq ADR 0235, to-be 45 +// The gate on a release plan's first machine, and the rollback after it (novox/hq ADR 0236, to-be 45 // §8, ADR 0227 rule 8). // // **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once @@ -328,7 +328,7 @@ func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module str } // carryUserList sends the machine holding the bus the user list a new controller composes, once that -// controller passed its gate (ADR 0235). The controller's own grants travel in that list, and the old +// controller passed its gate (ADR 0236). The controller's own grants travel in that list, and the old // controller composed the list the plan sent; on 2026-10-06 eight pushes by hand carried a new // controller's grant into it. Not when a build its policy or a plan holds back would go with it (ADR // 0221): then it is said, as a push would say it. @@ -457,7 +457,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str sayRollback(ctx, open, module, g, "") } -// rolledBackEvent is the body of `rolled-back` (ADR 0235): a contract, like a condition's events. +// rolledBackEvent is the body of `rolled-back` (ADR 0236): a contract, like a condition's events. type rolledBackEvent struct { Event string `json:"event"` At time.Time `json:"at"` diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index 4c8bcb9..6c6dc55 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -17,7 +17,7 @@ import ( "github.com/novox/mesh-controller/internal/link" ) -// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0235, to-be 45 §8). +// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8). // gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered, // a plan whose tier built c2, and every send recorded and answered — the machine applies what it is diff --git a/cmd/mesh-controller/held_back_test.go b/cmd/mesh-controller/held_back_test.go index 28fb555..bccf4ec 100644 --- a/cmd/mesh-controller/held_back_test.go +++ b/cmd/mesh-controller/held_back_test.go @@ -193,7 +193,7 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { t.Fatal(err) } } - // Recorded by a person's choice: the default rolls out since novox/hq ADR 0235. + // Recorded by a person's choice: the default rolls out since novox/hq ADR 0236. if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil { t.Fatal(err) } diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 7c43803..d718d00 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -116,7 +116,7 @@ func run() error { return serve(ctx) case "upgrade": return upgradeCommand(ctx, args[1:]) - // The bus as a planned step (novox/hq to-be 45 §8, ADR 0235). + // The bus as a planned step (novox/hq to-be 45 §8, ADR 0236). case "bus": return busCommand(ctx, args[1:]) case "declare": diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index f4e7751..b5c0002 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -178,7 +178,7 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error { return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s", p.Tier, p.ID, p.Note) } - // **A build that failed its gate is not sent again** (novox/hq ADR 0235): it was put back on its first + // **A build that failed its gate is not sent again** (novox/hq ADR 0236): it was put back on its first // machine, and retrying would judge the build the mesh put back, or send the failed one by hand. for _, m := range stopped { if g := p.Modules[m].Gate; g != nil && g.Verdict == inventory.GateFailed { diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 0dce90c..882cc39 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -466,7 +466,7 @@ func pushCommand(ctx context.Context, args []string) error { which, len(asked), strings.Join(asked, ", ")) } - // **The bus is replaced only as a planned step** (novox/hq ADR 0235, to-be 45 §8): a machine whose bus + // **The bus is replaced only as a planned step** (novox/hq ADR 0236, to-be 45 §8): a machine whose bus // would move is not sent by a push — named, it is refused; otherwise it is left and said. busKept, err := busHeld(ctx, inv, asked) if err != nil { diff --git a/cmd/mesh-controller/queue_test.go b/cmd/mesh-controller/queue_test.go index ab97aac..8ac29af 100644 --- a/cmd/mesh-controller/queue_test.go +++ b/cmd/mesh-controller/queue_test.go @@ -625,7 +625,7 @@ func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) { } t.Cleanup(func() { sendRollout = was }) - // a records: a person's choice, since the default rolls out (novox/hq ADR 0235). + // a records: a person's choice, since the default rolls out (novox/hq ADR 0236). if err := open.inventory.SetUpgradeOf(ctx, "a", inventory.Upgrade{Why: "test"}); err != nil { t.Fatal(err) } diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 06d2ae2..2a59c22 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -432,7 +432,7 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string, continue } if failed != "" && deletedAtSource(failed) { - // Deleted at its source by the merge, not broken (novox/hq ADR 0235): the plan goes on. + // Deleted at its source by the merge, not broken (novox/hq ADR 0236): the plan goes on. state.State, state.Why = planDeleted, "deleted at its source: "+firstLine(failed) forgetDeleted(ctx, inv, module, p) } else if failed != "" { @@ -629,7 +629,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, switch { case step.failed != "": // The first machine refused or failed what it was sent, or never said: the gate failed, and - // the build is put back there (novox/hq ADR 0235); the rest are left as they were. + // the build is put back there (novox/hq ADR 0236); the rest are left as they were. gateFailed(ctx, open, p, m, state, firstRunning(state.First, running), step.failed) p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", "))) fmt.Printf("%s: %s\n", p.ID, p.Note) @@ -638,7 +638,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, state.FirstAt.Local().Format("15:04"))) continue } - // **The gate** (novox/hq ADR 0235, to-be 45 §8): the first machine reported the build applied; + // **The gate** (novox/hq ADR 0236, to-be 45 §8): the first machine reported the build applied; // it is judged by its health before anything else is sent — the rest, or, where it is the only // machine, the plan's next step. if !policy.Together && state.FirstAt != nil && len(state.First) > 0 { @@ -684,7 +684,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(step.send, ", "), p.Tier, err) } if step.first { - // What the first machine ran before: what a failed gate puts back (ADR 0235). + // What the first machine ran before: what a failed gate puts back (ADR 0236). if beforeKnown { state.Previous = before[m] } @@ -1328,7 +1328,7 @@ func firstRunning(first, running []string) []string { } // planDeleted is a plan's module that the merge deleted at its source: not built, not sent, and no -// failure of the plan (novox/hq ADR 0235). +// failure of the plan (novox/hq ADR 0236). const planDeleted = "deleted" // forgetDeleted forgets a module a plan found deleted at its source, where nothing holds it, and says diff --git a/cmd/mesh-controller/status_summary.go b/cmd/mesh-controller/status_summary.go index 74bc280..053f8b6 100644 --- a/cmd/mesh-controller/status_summary.go +++ b/cmd/mesh-controller/status_summary.go @@ -195,7 +195,7 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e if report.Ordered() { link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now) } - // What the machine's witnesses put back and stand by (novox/hq ADR 0235): read by the gate and its + // What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its // probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey. if report.Superseded == "" && report.Rekey == nil && report.Node != "" { witnessed.heard(report.Node, report.Rollbacks, now) diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index d6707ab..33cce27 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -205,7 +205,7 @@ func upgradeCommand(ctx context.Context, args []string) error { } if decision.Why == "" { return fmt.Errorf("holding %s back from every merge is a choice a person reads later: --why "+ - "(novox/hq ADR 0235). Nothing was changed", module) + "(novox/hq ADR 0236). Nothing was changed", module) } case "default": if err := inv.ClearUpgradeOf(ctx, module); err != nil { @@ -354,7 +354,7 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error { } } touched := whatTheMergeTouched(from, entries, m) - // **A module the merge deleted is not built** (novox/hq ADR 0235): its manifest is gone, so the build + // **A module the merge deleted is not built** (novox/hq ADR 0236): its manifest is gone, so the build // seat finds nothing saying what it is, and the plan failed on it (`has no module.json at …`) with // every other module of its tier left unsent. It is forgotten where nothing holds it, said otherwise. touched, deleted := splitDeleted(touched, m) @@ -521,7 +521,7 @@ func wouldMove(m link.SourceMoved, entries []inventory.Entry, } // splitDeleted parts the modules a merge touched into those it changed and those whose manifest it -// removed — deleted at their source (ADR 0235). +// removed — deleted at their source (ADR 0236). func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted []inventory.Entry) { removed := map[string]bool{} for _, p := range m.Removed { @@ -852,7 +852,7 @@ const moduleManifestFile = "module.json" // deletedAtSource is whether a build failed because its module's manifest is not at its source any // more — the build seat's own words (internal/builder) — which a merge that deleted the module causes -// when its announcer did not say which files went (ADR 0235). Such a module is not a failure of the plan. +// when its announcer did not say which files went (ADR 0236). Such a module is not a failure of the plan. func deletedAtSource(failed string) bool { return strings.Contains(failed, "has no "+moduleManifestFile+" at ") && strings.Contains(failed, "so there is nothing saying what it is") diff --git a/cmd/mesh-controller/witness.go b/cmd/mesh-controller/witness.go index 544956b..3bf6374 100644 --- a/cmd/mesh-controller/witness.go +++ b/cmd/mesh-controller/witness.go @@ -9,7 +9,7 @@ import ( "github.com/novox/mesh-controller/internal/lease" ) -// The controller's side of its own rollback witness (novox/hq to-be 45 §8, ADR 0235; the contract is +// The controller's side of its own rollback witness (novox/hq to-be 45 §8, ADR 0236; the contract is // internal/lease/witness.go): what the serving controller says of itself in every write of the lease's // key, for the node-engine on the control node to judge a new controller build by. diff --git a/internal/broker/states_agreement_test.go b/internal/broker/states_agreement_test.go index ce86809..b838527 100644 --- a/internal/broker/states_agreement_test.go +++ b/internal/broker/states_agreement_test.go @@ -28,7 +28,7 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) { states = append(states, link.KeySecretReplaced) // And every act a healer takes (novox/hq to-be 45 §7). states = append(states, link.KeyHealerActed) - // And a build put back after its gate failed (novox/hq ADR 0235). + // And a build put back after its gate failed (novox/hq ADR 0236). states = append(states, link.KeyRolledBack) for _, event := range states { if !slices.Contains(broker.ControllerStates, event) { diff --git a/internal/broker/streams.go b/internal/broker/streams.go index 7030e77..0da2e4b 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -214,7 +214,7 @@ var ControllerStates = []string{"applied", "refused", "built-before", // And every act a healer takes on a condition (novox/hq to-be 45 §7, Phase 3): a repair the mesh // made by itself is said like one a person made, never quietly. "healer-acted", - // And a build put back after its gate failed on its first machine (novox/hq ADR 0235, to-be 45 §8). + // And a build put back after its gate failed on its first machine (novox/hq ADR 0236, to-be 45 §8). "rolled-back"} // BusAdvisories are what the bus server says about the mesh's own account that the controller diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 3c214dc..be03268 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -453,7 +453,7 @@ type Manifest struct { // unassignment retires and what the self-check measures are all derived from it. Data *Data `json:"data,omitempty"` - // Upgrade is how this module's new builds reach its machines (novox/hq ADR 0235): rolled out one + // Upgrade is how this module's new builds reach its machines (novox/hq ADR 0236): rolled out one // machine first and gated when unsaid; `together`, or `record` — wait for a person's push — with // why. A person's choice through the `upgrade` verb stands over it; the bus records whatever it says. Upgrade *UpgradePolicy `json:"upgrade,omitempty"` diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index 14b1926..6e12389 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -90,7 +90,7 @@ var defaultSeats = append([]Seat{ "secret-replaced", // Every act a healer takes (novox/hq to-be 45 §7). "healer-acted", - // A build put back after its gate failed (novox/hq ADR 0235, to-be 45 §8). + // A build put back after its gate failed (novox/hq ADR 0236, to-be 45 §8). "rolled-back"}, Serves: ControllerVerbs}, // The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable, diff --git a/internal/catalogue/upgrade.go b/internal/catalogue/upgrade.go index 5580a7f..c8c2857 100644 --- a/internal/catalogue/upgrade.go +++ b/internal/catalogue/upgrade.go @@ -5,7 +5,7 @@ import ( "strings" ) -// What the mesh does when a module's build moves (novox/hq ADR 0235, extending ADR 0162 §3 and ADR +// What the mesh does when a module's build moves (novox/hq ADR 0236, extending ADR 0162 §3 and ADR // 0218 §2). // // **Rolled out by default, one machine first and gated.** With the gate on the first machine and the @@ -62,7 +62,7 @@ const ( ) // DerivedUpgrade is the policy a module's manifest gives it when no person has chosen one, with where -// it came from and why (ADR 0235): +// it came from and why (ADR 0236): // // - **the bus is never rolled**: a module that provides the mesh's bus records, whatever it says — its // upgrade is a planned step a person starts (to-be 45 §8); diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 7781eb7..0e9b535 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -267,8 +267,8 @@ var ControllerVerbs = []Verb{ "probes": "\"true\": the registry — what each probe asserts, and the condition it raises", "signals": "\"true\": the signals table, each row with the age of its newest signal", }, nil, "run", "probes", "signals")}, - // How a module's new builds reach its machines, and the bus's planned step (novox/hq ADR 0235). - {Name: "upgrade", Description: "How each module's new builds reach its machines (novox/hq ADR 0235): rolled " + + // How a module's new builds reach its machines, and the bus's planned step (novox/hq ADR 0236). + {Name: "upgrade", Description: "How each module's new builds reach its machines (novox/hq ADR 0236): rolled " + "out one machine first and judged there at the gate, then the rest — or recorded, waiting for a person's " + "push — with where that comes from (a person, the module, the bus, its irreplaceable data, the default) and " + "why. With module, that one; with policy, a person's choice for it — roll-out, record (with why) or default " + @@ -279,7 +279,7 @@ var ControllerVerbs = []Verb{ "together": "\"true\": with roll-out, every machine at once instead of one machine first", "why": "with policy: why — required for record, kept and said with the policy", }, nil, "together")}, - {Name: "bus", Description: "The bus as a planned step (novox/hq to-be 45 §8, ADR 0235): what a bus upgrade " + + {Name: "bus", Description: "The bus as a planned step (novox/hq to-be 45 §8, ADR 0236): what a bus upgrade " + "would do — the bus's build on each machine against the one the mesh holds — and how the last step went. " + "With upgrade, start one: a person's act with why, after the streams are snapshotted (snapshot-taken says " + "where, while the mesh takes none itself), saying first whether it can be reverted; bus-maintenance is open " + diff --git a/internal/inventory/busstep.go b/internal/inventory/busstep.go index 4659d5e..96a5716 100644 --- a/internal/inventory/busstep.go +++ b/internal/inventory/busstep.go @@ -8,7 +8,7 @@ import ( "github.com/jackc/pgx/v5" ) -// BusStep is one planned bus upgrade (novox/hq to-be 45 §8, ADR 0235). +// BusStep is one planned bus upgrade (novox/hq to-be 45 §8, ADR 0236). type BusStep struct { ID int64 Module string diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 58bd52b..274201c 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -1194,10 +1194,10 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) { // providedBy is what the source column says for a module the control plane ships. const providedBy = "the control plane" -// Upgrade is what the mesh does when a module's current version moves (ADR 0162 §3, ADR 0235). +// Upgrade is what the mesh does when a module's current version moves (ADR 0162 §3, ADR 0236). type Upgrade struct { // RollOut is true when the machines running it are sent the new version: one machine first, judged - // at the gate, then the rest (ADR 0218, ADR 0235). False means record it and stop — the machines + // at the gate, then the rest (ADR 0218, ADR 0236). False means record it and stop — the machines // running it are behind until a person pushes, which the mesh already reports. RollOut bool // Together is true when every machine running it is sent the new version at once. Only meaningful @@ -1223,7 +1223,7 @@ func (u Upgrade) Policy() string { } // upgradeFrom is a module's policy from what the store holds of it: a person's choice, over the module's -// own word, over the default — except that the bus is never rolled out, whoever says so (ADR 0235). +// own word, over the default — except that the bus is never rolled out, whoever says so (ADR 0236). func upgradeFrom(chosen *string, together bool, why, by string, manifest []byte) Upgrade { var m catalogue.Manifest // Leniently: a policy is read from what was registered, and a manifest registered before a field it @@ -1296,7 +1296,7 @@ func (i *Inventory) Upgrades(ctx context.Context) (map[string]Upgrade, error) { return out, rows.Err() } -// ErrBusIsPlanned is a person asking the bus to be rolled out: its upgrade is a planned step (ADR 0235). +// ErrBusIsPlanned is a person asking the bus to be rolled out: its upgrade is a planned step (ADR 0236). var ErrBusIsPlanned = errors.New("the bus is never rolled out: its upgrade is a planned step a person starts " + "with `bus upgrade`, which snapshots its streams first and checks them after") diff --git a/internal/inventory/gate.go b/internal/inventory/gate.go index 729ac29..0a0c5eb 100644 --- a/internal/inventory/gate.go +++ b/internal/inventory/gate.go @@ -12,7 +12,7 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" ) -// The gate's verdicts (novox/hq ADR 0235, to-be 45 §8): what a build did on its first machine, and, +// The gate's verdicts (novox/hq ADR 0236, to-be 45 §8): what a build did on its first machine, and, // for one that failed there, how it was put back. One row per build, written by the plan that rolled it // out, under the lease. diff --git a/internal/inventory/migrations/0073-a-build-rolls-out-gated-and-rolls-back.sql b/internal/inventory/migrations/0073-a-build-rolls-out-gated-and-rolls-back.sql index 6deb22c..e6b312d 100644 --- a/internal/inventory/migrations/0073-a-build-rolls-out-gated-and-rolls-back.sql +++ b/internal/inventory/migrations/0073-a-build-rolls-out-gated-and-rolls-back.sql @@ -1,12 +1,12 @@ -- A module's build rolls out one machine first, judged at a gate, and rolls back there when the gate --- fails (novox/hq ADR 0235, to-be 45 §8, Phase 4). +-- fails (novox/hq ADR 0236, to-be 45 §8, Phase 4). -- -- 1. The upgrade policy becomes a person's choice over the module's own word. Until now every module -- held a policy here, 'record' unless a person had said 'roll-out', and nothing distinguished a -- 'record' somebody chose from the default it always was. From here a null policy is no choice: the -- module's manifest decides (its `upgrade`, its data, whether it is the bus), and its default is to -- roll out. A 'roll-out' a person chose is kept as their choice. A 'record' is the old default and --- becomes no choice — ADR 0235 decides it, and lists every module's resulting policy; a person who +-- becomes no choice — ADR 0236 decides it, and lists every module's resulting policy; a person who -- wants one held again says so with `upgrade record --why`, which is kept with its why. alter table module alter column upgrade drop not null; alter table module alter column upgrade drop default; diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index 791f0d0..7508b60 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -69,10 +69,10 @@ type PlanModule struct { Build string `json:"build,omitempty"` // Previous is the build the first machine ran of this module before the plan sent it the new one — // the commit its last send carried (ADR 0221) — kept at the first send: what a rollback puts back - // (novox/hq ADR 0235). Empty when the machine had never been sent the module, or what it was sent + // (novox/hq ADR 0236). Empty when the machine had never been sent the module, or what it was sent // is not known. Previous string `json:"previous,omitempty"` - // Gate is the new build's judging on its first machine (novox/hq ADR 0235, to-be 45 §8), kept so a + // Gate is the new build's judging on its first machine (novox/hq ADR 0236, to-be 45 §8), kept so a // controller replaced mid-judging resumes it, and read back through `plans` as the rollout's record. Gate *PlanGate `json:"gate,omitempty"` } diff --git a/internal/inventory/sent_builds_test.go b/internal/inventory/sent_builds_test.go index eb9ae85..f3888e1 100644 --- a/internal/inventory/sent_builds_test.go +++ b/internal/inventory/sent_builds_test.go @@ -72,7 +72,7 @@ func TestTheCurrentBuildsAreTheCatalogues(t *testing.T) { if err != nil { t.Fatal(err) } - // Rolled out by default, one machine first and gated (novox/hq ADR 0235). + // Rolled out by default, one machine first and gated (novox/hq ADR 0236). if got := current["resolver"]; got != (CurrentBuild{Commit: "c1", RollOut: true}) { t.Errorf("resolver is at %+v", got) } diff --git a/internal/link/events.go b/internal/link/events.go index c3c6bad..84bb5b5 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -71,7 +71,7 @@ const ( // are the hand-act log's. KeyHealerActed = "healer-acted" // KeyRolledBack: a build failed its gate on its first machine and was put back there, or could not - // be (novox/hq ADR 0235, to-be 45 §8); or a witness on a machine put a core component back. + // be (novox/hq ADR 0236, to-be 45 §8); or a witness on a machine put a core component back. KeyRolledBack = "rolled-back" ) @@ -185,7 +185,7 @@ type SourceMoved struct { PathsTruncated bool `json:"paths_truncated,omitempty"` // Removed are the files among Paths the merge deleted. A module whose manifest is among them was - // deleted at its source: it is forgotten, or said, and never built (novox/hq ADR 0235). Empty from an + // deleted at its source: it is forgotten, or said, and never built (novox/hq ADR 0236). Empty from an // announcer that does not say which files went, and then a build that finds no manifest says it. Removed []string `json:"removed,omitempty"` } From 37229b4db54a20010e5210f214c560c9a865547b Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 6 Oct 2026 18:53:38 +0200 Subject: [PATCH 4/7] Refuse every send that would replace the bus outside its planned step (hq ADR 0236) A plan's send to the bus's machine for another module carried the bus's new build and restarted it under every machine with nobody asking (2026-10-06). The guard is in the one send everything uses; only the bus step passes it. A rebuild that made the same artifacts is no move. --- cmd/mesh-controller/bus_step.go | 59 +++++++++++++++++++++++++++++--- cmd/mesh-controller/gate_test.go | 20 +++++++++++ cmd/mesh-controller/push.go | 15 ++++++++ 3 files changed, 90 insertions(+), 4 deletions(-) diff --git a/cmd/mesh-controller/bus_step.go b/cmd/mesh-controller/bus_step.go index 4ef2149..95a4880 100644 --- a/cmd/mesh-controller/bus_step.go +++ b/cmd/mesh-controller/bus_step.go @@ -54,12 +54,19 @@ type busPending struct { machines []string from map[string]string to string + // same are the commits whose build made the same artifacts as the build the mesh holds. + same map[string]bool } -// moves is whether sending the machine would replace its bus. +// moves is whether sending the machine would replace its bus: a build it was not last sent, unless the +// two builds made the same artifacts — a rebuild of the same source for another module's merge changes +// nothing the machine runs. func (b busPending) moves(machine string) bool { from, known := b.from[machine] - return b.module != "" && b.to != "" && (!known || !sameCommit(from, b.to)) + if b.module == "" || b.to == "" || (known && sameCommit(from, b.to)) { + return false + } + return !known || !b.same[from] } // pendingBus reads what a bus upgrade would do. @@ -85,7 +92,14 @@ func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, erro if b.machines, err = inv.Running(ctx, b.module); err != nil { return b, err } - b.from = map[string]string{} + b.from, b.same = map[string]string{}, map[string]bool{} + made, err := madeBy(ctx, inv, b.module) + if err != nil { + return b, err + } + for commit, refs := range made { + b.same[commit] = refs != "" && refs == made[b.to] + } for _, n := range b.machines { sent, known, err := inv.SentBuilds(ctx, n) if err != nil { @@ -204,7 +218,7 @@ func busCommand(ctx context.Context, args []string) error { fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From, short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it", false: "NOT reversible: the snapshot is the only way back"}[*reversible]) - sent, err := sendRollout(ctx, open, moving) + sent, err := sendRollout(withBusStep(ctx), open, moving) if err != nil { _ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error()) return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err) @@ -371,3 +385,40 @@ func snapshotTheBusNow(ctx context.Context, module, node string) (string, error) }) return where, err } + +// errBusWaits is a send refused because it would replace the bus outside its planned step. +var errBusWaits = errors.New("a new bus build waits for its planned step") + +type busStepKey struct{} + +// withBusStep marks a send as the bus's planned step: the one send that may replace the bus. +func withBusStep(ctx context.Context) context.Context { + return context.WithValue(ctx, busStepKey{}, true) +} + +func busStepSending(ctx context.Context) bool { on, _ := ctx.Value(busStepKey{}).(bool); return on } + +// madeBy is, per commit, the artifacts the newest worked build of a module from it made, as one sorted +// string: what tells a rebuild that changes nothing from one that does. +func madeBy(ctx context.Context, inv *inventory.Inventory, module string) (map[string]string, error) { + builds, err := inv.Builds(ctx, module, 50) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, b := range builds { + if !b.Worked() || b.Commit == "" { + continue + } + if _, seen := out[b.Commit]; seen { + continue + } + var refs []string + for _, a := range b.Made { + refs = append(refs, a.Name+"="+a.Reference) + } + sort.Strings(refs) + out[b.Commit] = strings.Join(refs, " ") + } + return out, nil +} diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index 6c6dc55..0ad1bd4 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -403,6 +403,26 @@ func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) { if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" { t.Fatalf("a push may send the bus's machine: %v %v", held, err) } + // Nor may any other send — a plan's for another module on that machine carried the bus with it. + if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) { + t.Fatalf("a send to the bus's machine was not refused: %v", err) + } + // A rebuild that made the same artifacts is no move. + for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} { + b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}} + b.Asked, b.At = time.Now(), time.Now() + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 { + t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second), + At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image", + Reference: "registry/nats@sha256:new"}}}); err != nil { + t.Fatal(err) + } // The planned step refuses to start without its word on reversibility, and without a snapshot taken // first by the bus machine's backup holder. if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") { diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 882cc39..f8aa3f4 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -1001,6 +1001,21 @@ func sendTo(ctx context.Context, open *stores, names []string) error { // the machines it sent waits for that one too. func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) { inv := open.inventory + // **No send replaces the bus but its planned step** (novox/hq ADR 0236). A plan's send to the bus's + // machine for some other module carried the bus's new build with it on 2026-10-06, and the bus + // restarted under every machine with nobody having asked. Refused whole — the send cannot leave the + // bus behind and carry the rest (ADR 0221 option 2) — and said with the remedy; the plan tries again. + if !busStepSending(ctx) { + held, err := busHeld(ctx, inv, names) + if err != nil { + return nil, err + } + for _, n := range names { + if why, h := held[n]; h { + return nil, fmt.Errorf("%w: %s", errBusWaits, why) + } + } + } ident, err := openIdentity(ctx) if err != nil { return nil, err From 41f7b2c152ee8dc50ed13f53b564eb45c958ae91 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 6 Oct 2026 18:54:12 +0200 Subject: [PATCH 5/7] Tell a rebuild that changes nothing by its artifacts and its manifest (hq ADR 0236) --- cmd/mesh-controller/bus_step.go | 33 ++++----------------------------- internal/inventory/gate.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/cmd/mesh-controller/bus_step.go b/cmd/mesh-controller/bus_step.go index 95a4880..e13854d 100644 --- a/cmd/mesh-controller/bus_step.go +++ b/cmd/mesh-controller/bus_step.go @@ -54,13 +54,13 @@ type busPending struct { machines []string from map[string]string to string - // same are the commits whose build made the same artifacts as the build the mesh holds. + // same are the commits whose build made the same artifacts and manifest as the build the mesh holds. same map[string]bool } // moves is whether sending the machine would replace its bus: a build it was not last sent, unless the -// two builds made the same artifacts — a rebuild of the same source for another module's merge changes -// nothing the machine runs. +// two builds made the same artifacts from the same manifest — a rebuild of the same source for another +// module's merge changes nothing the machine runs. func (b busPending) moves(machine string) bool { from, known := b.from[machine] if b.module == "" || b.to == "" || (known && sameCommit(from, b.to)) { @@ -93,7 +93,7 @@ func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, erro return b, err } b.from, b.same = map[string]string{}, map[string]bool{} - made, err := madeBy(ctx, inv, b.module) + made, err := inv.BuildFingerprints(ctx, b.module) if err != nil { return b, err } @@ -397,28 +397,3 @@ func withBusStep(ctx context.Context) context.Context { } func busStepSending(ctx context.Context) bool { on, _ := ctx.Value(busStepKey{}).(bool); return on } - -// madeBy is, per commit, the artifacts the newest worked build of a module from it made, as one sorted -// string: what tells a rebuild that changes nothing from one that does. -func madeBy(ctx context.Context, inv *inventory.Inventory, module string) (map[string]string, error) { - builds, err := inv.Builds(ctx, module, 50) - if err != nil { - return nil, err - } - out := map[string]string{} - for _, b := range builds { - if !b.Worked() || b.Commit == "" { - continue - } - if _, seen := out[b.Commit]; seen { - continue - } - var refs []string - for _, a := range b.Made { - refs = append(refs, a.Name+"="+a.Reference) - } - sort.Strings(refs) - out[b.Commit] = strings.Join(refs, " ") - } - return out, nil -} diff --git a/internal/inventory/gate.go b/internal/inventory/gate.go index 0a0c5eb..a07639e 100644 --- a/internal/inventory/gate.go +++ b/internal/inventory/gate.go @@ -2,6 +2,8 @@ package inventory import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -216,3 +218,30 @@ func (i *Inventory) RestoreModule(ctx context.Context, b Build) error { } return nil } + +// BuildFingerprints is, per commit, what the newest successful build of a module from it would put on a +// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told +// from one that does (the bus's planned step, ADR 0236). +func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) { + rows, err := i.store.Pool().Query(ctx, + `select commit_hash, made, coalesce(manifest::text, '') from build + where module = $1 and failed = '' and commit_hash <> '' order by at desc limit 50`, module) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]string{} + for rows.Next() { + var commit, manifest string + var made []byte + if err := rows.Scan(&commit, &made, &manifest); err != nil { + return nil, err + } + if _, seen := out[commit]; seen { + continue + } + sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...)) + out[commit] = hex.EncodeToString(sum[:]) + } + return out, rows.Err() +} From 2bfa6ae4a0f76a19512ed5279110fb9ef1844295 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 6 Oct 2026 19:14:25 +0200 Subject: [PATCH 6/7] No build reaches a machine without a gate; a release plan walks what waits (hq ADR 0236) A send carries the machine's whole declaration, so at the switch to roll the next send of anything would have carried the old default's backlog, unjudged, to every machine. A gated send now carries and judges everything waiting on its machine; every other send is refused or leaves the machine; a release plan walks what waits one machine at a time, the control node last, and one that fails holds the next until a person releases it. --- cmd/mesh-controller/bus_step.go | 2 +- cmd/mesh-controller/gate.go | 47 +- cmd/mesh-controller/healers.go | 3 + cmd/mesh-controller/held_back.go | 24 +- cmd/mesh-controller/held_back_test.go | 16 +- cmd/mesh-controller/push.go | 32 + cmd/mesh-controller/release.go | 603 ++++++++++++++++++ cmd/mesh-controller/release_plan.go | 42 +- cmd/mesh-controller/release_test.go | 257 ++++++++ cmd/mesh-controller/seatverbs.go | 10 + cmd/mesh-controller/upgrades.go | 28 +- internal/catalogue/verbs.go | 8 +- internal/inventory/gate.go | 68 +- ...a-build-rolls-out-gated-and-rolls-back.sql | 5 + internal/inventory/plans.go | 58 +- 15 files changed, 1163 insertions(+), 40 deletions(-) create mode 100644 cmd/mesh-controller/release.go create mode 100644 cmd/mesh-controller/release_test.go diff --git a/cmd/mesh-controller/bus_step.go b/cmd/mesh-controller/bus_step.go index e13854d..94ee7c2 100644 --- a/cmd/mesh-controller/bus_step.go +++ b/cmd/mesh-controller/bus_step.go @@ -218,7 +218,7 @@ func busCommand(ctx context.Context, args []string) error { fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From, short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it", false: "NOT reversible: the snapshot is the only way back"}[*reversible]) - sent, err := sendRollout(withBusStep(ctx), open, moving) + sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving) if err != nil { _ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error()) return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err) diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 6fa4936..4316ac8 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -139,7 +139,7 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string) } else { f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what") } - if component == lease.ComponentController { + if theLease != nil { h, found, err := theLease.holder(ctx) switch { case err != nil: @@ -147,6 +147,9 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string) case found: f.holder = &h } + if component != lease.ComponentController && f.holderErr != nil { + f.holderErr = nil // read only for the controller's own judging + } } return f, nil } @@ -263,12 +266,31 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri To: state.Commit, Since: &start} state.Gate = g } + pairs := []judged{} + for _, n := range g.Machines { + pairs = append(pairs, judged{module: module, node: n}) + } + return judgeMoves(ctx, open, g, pairs, now) +} + +// judged is one module on one machine, as a gate judges it. +type judged struct{ module, node string } + +// judgeMoves takes one judging of a gate over the modules it judges on their machines — its own, and +// everything the send carried (Carried) — and records it: a pass counted, a pass missed (what is +// wanting, and which modules), or the verdict. Answers the verdict once there is one. +func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs []judged, now time.Time) (string, error) { if g.Verdict != "" { return g.Verdict, nil } if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery { return "", nil } + for _, c := range g.Carried { + if !slices.Contains(pairs, judged{module: c.Module, node: c.Node}) { + pairs = append(pairs, judged{module: c.Module, node: c.Node}) + } + } shelf, err := open.inventory.Catalogue(ctx) if err != nil { return "", err @@ -278,8 +300,16 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri return "", err } worst, why := healthGood, "" - for _, n := range g.Machines { - h, said := judgeHealth(module, g.Component, shelf[module], n, *g.Since, facts) + var failing []string + broken := map[string]bool{} + for _, j := range pairs { + h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, facts) + if h != healthGood && !slices.Contains(failing, j.module) { + failing = append(failing, j.module) + } + if h == healthBroken { + broken[j.module] = true + } if h > worst { worst, why = h, said } else if h == worst && h != healthGood && why == "" { @@ -288,15 +318,17 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri } switch { case worst == healthBroken: + // What broke is put back; what was only not yet healthy beside it is too — they moved together. + g.Failing = failing decide(g, inventory.GateFailed, why, now) case worst == healthNotYet: - g.Passes, g.LastPass, g.Last = 0, nil, why + g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing if now.Sub(*g.Since) > gateBound { decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now) } default: g.Passes++ - g.LastPass, g.Last = &now, "" + g.LastPass, g.Last, g.Failing = &now, "", nil if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle { decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes, now.Sub(*g.Since).Round(time.Second)), now) @@ -320,6 +352,7 @@ func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module str if err != nil && state.Build != "" { fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err) } + passCarried(ctx, open, p, g, module) fmt.Printf("%s: %s passed its gate on %s (%s); the rest are sent\n", p.ID, module, strings.Join(g.Machines, ", "), g.Why) if module == catalogue.ControllerSeatName { @@ -443,7 +476,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str if err := inv.SavePlan(ctx, p); err != nil { fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err) } - sent, err := sendRollout(ctx, open, g.Machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines) if err != nil { notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ "sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0])) @@ -553,6 +586,8 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error out = append(out, witnessObservation(node, r)) } } + // A release held after a failed one, while builds still wait for a gate (ADR 0236). + out = append(out, backlogObservation()...) return sortedFound(dedupeObservations(out)), nil } diff --git a/cmd/mesh-controller/healers.go b/cmd/mesh-controller/healers.go index e77eb2d..a7471a7 100644 --- a/cmd/mesh-controller/healers.go +++ b/cmd/mesh-controller/healers.go @@ -614,6 +614,9 @@ func lastReportOf(ctx context.Context, inv *inventory.Inventory, node string) (i // planStale is why a plan's wait is superseded or finished, and the state closing it leaves it in; empty // when it is neither, which is not H2's to repair. func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (state, why string, err error) { + if p.Release != nil { + return "", "", nil // a release plan walks machines, and its own gate says when it is done (ADR 0236) + } recent, err := inv.RecentPlans(ctx, 50) if err != nil { return "", "", err diff --git a/cmd/mesh-controller/held_back.go b/cmd/mesh-controller/held_back.go index 22da1e3..fc22083 100644 --- a/cmd/mesh-controller/held_back.go +++ b/cmd/mesh-controller/held_back.go @@ -120,6 +120,10 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string if err != nil { return nil, err } + f, err := readMoveFacts(ctx, inv) + if err != nil { + return nil, err + } for _, node := range names { plan, _, err := planFor(ctx, open, node) if err != nil { @@ -133,7 +137,25 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string if err != nil { return nil, err } - if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 { + // A rebuild that put the same thing on the machine is no move (ADR 0236): read as the build it + // runs. + same := map[string]string{} + for m, was := range sent { + same[m] = was + if f.identical(m, was, current[m].Commit) { + same[m] = current[m].Commit + } + } + why := heldBack(node, modules, same, known, current, plans) + // And a build no gate has seen is not carried by a send that does not judge it (ADR 0236). + for _, mv := range f.moves(node, modules, same, known, false) { + if planStillToSend(plans, mv.Module, node) == "" { + why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+ + "plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To))) + } + } + if len(why) > 0 { + sort.Strings(why) out[node] = why } } diff --git a/cmd/mesh-controller/held_back_test.go b/cmd/mesh-controller/held_back_test.go index bccf4ec..0507b64 100644 --- a/cmd/mesh-controller/held_back_test.go +++ b/cmd/mesh-controller/held_back_test.go @@ -264,7 +264,8 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { t.Fatalf("the push did not say both:\n%s", said.String()) } - // A policy that rolls out: the laptop is a consequence like any other, and sent. + // A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it + // (novox/hq ADR 0236). if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil { t.Fatal(err) } @@ -273,6 +274,19 @@ func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { compose, d, "", &said); err != nil { t.Fatal(err) } + if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") { + t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String()) + } + // Once it passed a gate on some machine, the laptop is a consequence like any other, and sent. + if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2", + Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { + t.Fatal(err) + } + said.Reset() + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, + compose, d, "", &said); err != nil { + t.Fatal(err) + } if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before { t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String()) } diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index f8aa3f4..fba0f7b 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -1,6 +1,7 @@ package main import ( + "slices" "context" "crypto/sha256" "encoding/hex" @@ -466,6 +467,28 @@ func pushCommand(ctx context.Context, args []string) error { which, len(asked), strings.Join(asked, ", ")) } + // **A whole-mesh push sends no build a gate has not seen** (novox/hq ADR 0236): a machine where one + // waits is left, named, for the release plan — `push ` still sends one machine by a person's word. + if len(args) == 0 { + f, err := readMoveFacts(ctx, inv) + if err != nil { + return err + } + var kept []string + for _, n := range asked { + moves, err := machineMoves(ctx, open, f, n, false) + if err != nil { + return err + } + if len(moves) > 0 { + fmt.Printf(" %s is left: %d build(s) wait there for a gate, which a release plan sends one machine "+ + "at a time (`upgrade backlog` lists them; `push %s` sends it by name)\n", n, len(moves), n) + continue + } + kept = append(kept, n) + } + asked = kept + } // **The bus is replaced only as a planned step** (novox/hq ADR 0236, to-be 45 §8): a machine whose bus // would move is not sent by a push — named, it is refused; otherwise it is left and said. busKept, err := busHeld(ctx, inv, asked) @@ -1030,7 +1053,16 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er if err != nil { return nil, err } + added := "" + if behind && !slices.Contains(names, holder) { + added = holder + } names = brokerFirst(names, holder, behind) + // **No build moves on a machine without a gate** (novox/hq ADR 0236): a send outside its scope that + // would carry one is refused, said with what waits; the bus's machine added for its user list is left. + if names, err = ungatedIn(ctx, open, names, added); err != nil { + return nil, err + } // Held from composing to sending (novox/hq ADR 0100); a caller that holds them already — // converge, which flips the node and then sends it — is not made to wait on itself. diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go new file mode 100644 index 0000000..226bad1 --- /dev/null +++ b/cmd/mesh-controller/release.go @@ -0,0 +1,603 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "slices" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// No build reaches a machine without a gate (novox/hq ADR 0236). +// +// **A send carries the machine's whole declaration.** A plan sending one module to its first machine +// sends every other module whose build moved there too; a cascade, a healer's resend and a whole-mesh +// push do the same. Under the old default (`record`) builds were registered and sent nowhere, so on the +// day the default became `roll` every machine was behind on dozens of builds no gate had seen — and the +// next send of anything would have restarted all of them at once, on every machine. +// +// So **a build moves on a machine only through a send that judges it there**, or a person's: +// +// - a gated send — a plan's first machine, a release plan's machine — carries every move waiting on that +// machine, and its gate judges each of them; a pass is each build's verdict, a failure puts back what +// failed; +// - a module a send exists for may move anywhere it is sent: a policy of *together*, a rollback, a build +// whose gate already passed; +// - a send naming a machine by a person (`push `, the bus step) carries what it carries; +// - every other send — a plan's "rest", a cascade, a healer's, the bus's user list carried — is refused, +// or leaves the machine, while a move there waits for a gate. A rebuild that made the same artifacts +// from the same manifest is no move. +// +// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no +// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last, +// each sent and judged before the next. A release plan that fails stops the next one opening on its own: +// a person releases it again (`upgrade release-backlog --why`), after the condition says why. + +// sendScope is what a send may carry that no gate has seen. +type sendScope struct { + // judged are the machines whose every move this send's gate judges. + judged map[string]bool + // modules are those a send exists for, which may move wherever it goes. + modules map[string]bool + // person is a person's act: it carries what it carries. + person bool +} + +type sendScopeKey struct{} + +func withScope(ctx context.Context, s sendScope) context.Context { + return context.WithValue(ctx, sendScopeKey{}, s) +} + +func scopeOf(ctx context.Context) sendScope { + s, _ := ctx.Value(sendScopeKey{}).(sendScope) + return s +} + +// errUngated is a send refused because it would carry a build no gate has seen. +var errUngated = errors.New("a build waits there for its gate") + +// errWalkedElsewhere is a gated send refused because a plan that has started walks a move there. +var errWalkedElsewhere = errors.New("a plan already walking a build there sends it") + +// moveFacts is what tells a move from a rebuild, and a gated build from one no gate has seen. +type moveFacts struct { + current map[string]inventory.CurrentBuild + fps map[string]map[string]string + passed map[string]map[string]bool + plans []inventory.Plan +} + +func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) { + var f moveFacts + var err error + if f.current, err = inv.CurrentBuilds(ctx); err != nil { + return f, err + } + if f.fps, err = inv.Fingerprints(ctx); err != nil { + return f, err + } + if f.passed, err = inv.PassedCommits(ctx); err != nil { + return f, err + } + f.plans, err = inv.OpenPlans(ctx) + return f, err +} + +// identical is whether two builds of a module put the same thing on a machine: the same commit, or +// builds that made the same artifacts from the same manifest. +func (f moveFacts) identical(module, a, b string) bool { + if a == b || sameCommit(a, b) { + return true + } + fa := f.fps[module][a] + return fa != "" && fa == f.fps[module][b] +} + +// gated is whether a build of a module from this commit — or one identical to it — passed a gate. +func (f moveFacts) gated(module, commit string) bool { + for c := range f.passed[module] { + if f.identical(module, c, commit) { + return true + } + } + return false +} + +// moves is what a machine's next send would move that no gate has seen: modules whose policy rolls out +// (a recorded one is a person's push), that the machine was sent before, moving to a build not identical +// to the one it runs and that has passed no gate. A machine whose last send's builds are not known names +// none: the cascade holds it whole (ADR 0221). +// +// With all, a move to a build that passed a gate elsewhere counts too: what a gated send carries. +func (f moveFacts) moves(node string, modules []string, sent map[string]string, known, all bool) []inventory.CarriedMove { + if !known { + return nil + } + var out []inventory.CarriedMove + for _, m := range modules { + now := f.current[m] + was, carried := sent[m] + if !now.RollOut || !carried || f.identical(m, was, now.Commit) || (!all && f.gated(m, now.Commit)) { + continue + } + out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: now.Commit}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) + return out +} + +// walkedBy is the open plan that has started walking a module's build — sent it to a first machine, +// not yet passed — other than to this machine; empty when none does. +func (f moveFacts) walkedBy(module, node string) string { + for _, p := range f.plans { + s, holds := p.Modules[module] + if !p.Open() || !holds || s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(s.First, node) { + continue + } + if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed { + continue + } + return p.ID + } + return "" +} + +// machineMoves is the moves no gate has seen on one machine, read from what it would be sent now. +var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + plan, _, err := planFor(ctx, open, node) + if err != nil { + return nil, nil // it cannot be worked out: the send says why + } + modules := make([]string, 0, len(plan.Modules)) + for _, m := range plan.Modules { + modules = append(modules, m.Module) + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil +} + +// ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the +// machines named, and why; the machine holding the bus, added only for its user list, is left instead. +func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) { + scope := scopeOf(ctx) + if scope.person { + return names, nil + } + f, err := readMoveFacts(ctx, open.inventory) + if err != nil { + return nil, err + } + var kept, refused []string + for _, n := range names { + moves, err := machineMoves(ctx, open, f, n, false) + if err != nil { + return nil, err + } + var waiting []string + for _, mv := range moves { + if !scope.judged[n] && !scope.modules[mv.Module] { + waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To))) + } + } + switch { + case len(waiting) == 0: + kept = append(kept, n) + case n == addedHolder: + fmt.Printf("%s holds the bus and its user list changed, and it is not sent now: %s wait there for a gate "+ + "— the bus may refuse what was newly granted until it is\n", n, strings.Join(waiting, ", ")) + default: + refused = append(refused, fmt.Sprintf("%s (%s)", n, strings.Join(waiting, ", "))) + } + } + if len(refused) > 0 { + return nil, fmt.Errorf("%w: %s — a release plan sends them, one machine at a time, each judged (novox/hq ADR "+ + "0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; ")) + } + return kept, nil +} + +// gatedSend sends one machine everything waiting there, under a gate that judges it all: what moved is +// answered, with the build each moved to, for the gate to judge and to put back. Refused while a plan that +// has started walks one of those builds elsewhere: that plan sends it here once its gate passed. +func gatedSend(ctx context.Context, open *stores, node string, own *inventory.CarriedMove) ([]inventory.CarriedMove, []string, error) { + inv := open.inventory + f, err := readMoveFacts(ctx, inv) + if err != nil { + return nil, nil, err + } + moves, err := machineMoves(ctx, open, f, node, true) + if err != nil { + return nil, nil, err + } + for _, mv := range moves { + if own != nil && mv.Module == own.Module { + continue + } + if id := f.walkedBy(mv.Module, node); id != "" { + return nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, + mv.Module, short(mv.To), node, id) + } + } + if own != nil && !slices.ContainsFunc(moves, func(mv inventory.CarriedMove) bool { return mv.Module == own.Module }) { + moves = append(moves, *own) + } + if len(moves) == 0 && own == nil { + return nil, nil, nil + } + for i := range moves { + if moves[i].Build == "" { + if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil { + return nil, nil, err + } + } + } + sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node}) + if err != nil { + return nil, nil, err + } + return moves, sent, nil +} + +// passCarried keeps a pass as the verdict of every build the gate judged beside its own module. +func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) { + seen := map[string]bool{} + for _, c := range g.Carried { + if c.Module == except || c.Build == "" || seen[c.Build] { + continue + } + seen[c.Build] = true + if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To, + Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: g.Why, + Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil { + fmt.Printf("%s: %s passed its gate on %s, and the verdict could not be kept: %v\n", p.ID, c.Module, c.Node, err) + } + } +} + +// failCarried puts back every build the gate carried that it found wanting, on the machines that were +// sent it, once each. +func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) { + var notes []string + if p.Note != "" { + notes = append(notes, p.Note) + } + done := map[string]bool{except: true} + for _, c := range g.Carried { + if done[c.Module] || (len(g.Failing) > 0 && !slices.Contains(g.Failing, c.Module)) { + continue + } + done[c.Module] = true + machines, err := sentTheBuild(ctx, open, c.Module, c.To) + if err != nil || len(machines) == 0 { + machines = []string{c.Node} + } + state := &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To} + p.Note = "" + gateFailed(ctx, open, p, c.Module, state, machines, g.Why) + notes = append(notes, p.Note) + } + p.State = inventory.PlanFailed + p.Note = strings.Join(notes, "; ") +} + +// sentTheBuild is every machine running a module that was last sent this build of it. +func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) { + running, err := open.inventory.Running(ctx, module) + if err != nil { + return nil, err + } + var out []string + for _, n := range running { + sent, known, err := open.inventory.SentBuilds(ctx, n) + if err != nil { + return nil, err + } + if known && sameCommit(sent[module], commit) { + out = append(out, n) + } + } + return out, nil +} + +// releaseRepository is what a release plan says it is for, where a merge's says its repository. +const releaseRepository = "the builds waiting for a gate" + +// releaseHeard is the machines a release plan may send: heard within their heartbeat's bound. A machine +// away is left, not judged against a bound it cannot meet. A variable so a test says who is heard. +var releaseHeard = func(ctx context.Context, open *stores) (map[string]bool, error) { + if d := doctorFrom; d != nil && d.watchdogs != nil { + return heardMachines(d), nil + } + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return nil, err + } + out := map[string]bool{} + for _, r := range reports { + if r.At != nil && time.Since(*r.At) < 15*time.Minute { + out[r.Node] = true + } + } + return out, nil +} + +// backlogNow is what the newest look found waiting, for `upgrade backlog` and the gate's probe. +var backlogNow struct { + held string + waiting map[string][]inventory.CarriedMove +} + +// waitingMoves is every machine's moves no gate has seen and no started plan walks. +func waitingMoves(ctx context.Context, open *stores, all bool) (map[string][]inventory.CarriedMove, error) { + f, err := readMoveFacts(ctx, open.inventory) + if err != nil { + return nil, err + } + nodes, err := open.inventory.Nodes(ctx) + if err != nil { + return nil, err + } + out := map[string][]inventory.CarriedMove{} + for _, n := range nodes { + moves, err := machineMoves(ctx, open, f, n.Name, all) + if err != nil { + return nil, err + } + for _, mv := range moves { + if f.walkedBy(mv.Module, n.Name) == "" { + out[n.Name] = append(out[n.Name], mv) + } + } + } + return out, nil +} + +// releaseBacklog opens a release plan when builds wait for a gate and none is open; not after a release +// plan failed, until a person releases one (by). Called with the plans held. +func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Plan, error) { + inv := open.inventory + plans, err := inv.OpenPlans(ctx) + if err != nil { + return nil, err + } + for _, p := range plans { + if p.Release != nil { + if by != "" { + return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID) + } + return nil, nil + } + } + waiting, err := waitingMoves(ctx, open, false) + if err != nil { + return nil, err + } + backlogNow.waiting, backlogNow.held = waiting, "" + if len(waiting) == 0 { + return nil, nil + } + // Opened by what no gate has seen; it walks every machine where anything of the release waits — a + // build that passed on the first machine still goes to the next one by this plan, judged there too. + if waiting, err = waitingMoves(ctx, open, true); err != nil { + return nil, err + } + if by == "" { + recent, err := inv.RecentPlans(ctx, 50) + if err != nil { + return nil, err + } + for _, p := range recent { + if p.Release == nil { + continue + } + if p.State == inventory.PlanFailed { + backlogNow.held = fmt.Sprintf("%s failed (%s); what waits is released again by a person", p.ID, p.Note) + return nil, nil + } + break + } + } + heard, err := releaseHeard(ctx, open) + if err != nil { + return nil, err + } + controllers, err := inv.Running(ctx, catalogue.ControllerSeatName) + if err != nil { + return nil, err + } + var order, last []string + modules := map[string]bool{} + for node, moves := range waiting { + if !heard[node] { + continue + } + for _, mv := range moves { + modules[mv.Module] = true + } + if slices.Contains(controllers, node) { + last = append(last, node) + } else { + order = append(order, node) + } + } + if len(order)+len(last) == 0 { + return nil, nil + } + sort.Strings(order) + sort.Strings(last) + order = append(order, last...) + names := make([]string, 0, len(modules)) + for m := range modules { + names = append(names, m) + } + sort.Strings(names) + now := time.Now().UTC() + p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository, + Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{}, + Release: &inventory.PlanRelease{Order: order, By: by}, + Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names), + strings.Join(order, ", "))} + if err := inv.SavePlan(ctx, &p); err != nil { + return nil, err + } + fmt.Printf("%s: %s\n", p.ID, p.Note) + return &p, nil +} + +// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a +// gate, or the machine being judged judged once more; the plan done when every machine is. +func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) { + r := p.Release + now := time.Now().UTC() + if r.Gate == nil { + heard, err := releaseHeard(ctx, open) + if err != nil { + return false, err + } + for r.Next < len(r.Order) { + node := r.Order[r.Next] + if !heard[node] { + r.Skipped = append(r.Skipped, node) + r.Next++ + continue + } + moves, sent, err := gatedSend(ctx, open, node, nil) + if errors.Is(err, errWalkedElsewhere) { + note := fmt.Sprintf("waiting before %s: %v", node, err) + changed := p.Note != note + p.Note = note + return changed, nil + } + if err != nil { + return false, fmt.Errorf("sending %s what waits there: %w", node, err) + } + if len(moves) == 0 { + r.Done = append(r.Done, node) + r.Next++ + continue + } + r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves} + p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves)) + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil + } + p.State, p.Tier = inventory.PlanDone, len(p.Tiers) + p.Note = fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", "))) + if len(r.Skipped) > 0 { + p.Note += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") + } + return true, nil + } + g := r.Gate + verdict, err := judgeMoves(ctx, open, g, nil, now) + if err != nil { + return false, err + } + switch verdict { + case "": + note := fmt.Sprintf("judging %s: %s", strings.Join(g.Machines, ", "), gateLine(g)) + changed := p.Note != note + p.Note = note + return changed, nil + case inventory.GatePassed: + passCarried(ctx, open, p, g, "") + r.Done = append(r.Done, firstOf(g.Machines)) + r.Next++ + r.Gate = nil + return true, nil + } + p.Note = "" + failCarried(ctx, open, p, g, "") + p.Note = fmt.Sprintf("failed its gate on %s: %s — %s", strings.Join(g.Machines, ", "), g.Why, p.Note) + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil +} + +// backlogObservation is what the gate's probe says of a release held after a failure. +func backlogObservation() []conditions.Observation { + if backlogNow.held == "" || len(backlogNow.waiting) == 0 { + return nil + } + n := 0 + var machines []string + for node, moves := range backlogNow.waiting { + n += len(moves) + machines = append(machines, node) + } + sort.Strings(machines) + return []conditions.Observation{{Scope: conditions.ScopeMesh, ID: "release", Token: "held", Kind: "release-held", + Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%d build move(s) on %s wait for a gate and are not released: %s — `upgrade backlog` lists "+ + "them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)}} +} + +// backlogCommand is `upgrade backlog`, read-only, and `upgrade release-backlog --why`. +func backlogCommand(ctx context.Context, sub string, args []string) error { + set := flag.NewFlagSet("upgrade "+sub, flag.ContinueOnError) + why := addHandActFlags(set) + if rest, err := parseAround(set, args); err != nil { + return err + } else if len(rest) > 0 { + return errors.New("upgrade backlog | upgrade release-backlog --why ") + } + if sub == "release-backlog" { + if err := why.require("upgrade release-backlog"); err != nil { + return err + } + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + if sub == "release-backlog" { + release, err := open.inventory.HoldPlans(ctx, true) + if err != nil { + return err + } + defer release() + why.record(ctx, "upgrade release-backlog", nil) + p, err := releaseBacklog(ctx, open, link.Caller()) + if err != nil { + return err + } + if p == nil { + fmt.Println("nothing waits for a gate on any machine heard from: nothing to release") + return nil + } + fmt.Printf("%s releases it; `plans %s` says where it is\n", p.ID, p.ID) + return nil + } + waiting, err := waitingMoves(ctx, open, false) + if err != nil { + return err + } + if len(waiting) == 0 { + fmt.Println("no build waits for a gate on any machine") + return nil + } + nodes := make([]string, 0, len(waiting)) + for n := range waiting { + nodes = append(nodes, n) + } + sort.Strings(nodes) + for _, n := range nodes { + fmt.Printf("%s: %d build(s) wait for a gate\n", n, len(waiting[n])) + for _, mv := range waiting[n] { + fmt.Printf(" %-28s %s → %s\n", mv.Module, short(mv.From), short(mv.To)) + } + } + return nil +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 2a59c22..1adb2fa 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -476,6 +476,10 @@ func advancePlans(ctx context.Context, open *stores) { return } defer release() + // What waits for a gate, released one machine at a time (ADR 0236). + if _, err := releaseBacklog(ctx, open, ""); err != nil { + fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err) + } advanceHeld(ctx, open) } @@ -531,6 +535,9 @@ func advanceHeld(ctx context.Context, open *stores) { func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) (bool, error) { inv := open.inventory + if p.Release != nil { + return advanceRelease(ctx, open, p) + } if p.Tier >= len(p.Tiers) { p.State = inventory.PlanDone fmt.Printf("%s: done — %s at %s, %d tier(s)\n", p.ID, p.Repository, short(p.Commit), len(p.Tiers)) @@ -631,6 +638,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, // The first machine refused or failed what it was sent, or never said: the gate failed, and // the build is put back there (novox/hq ADR 0236); the rest are left as they were. gateFailed(ctx, open, p, m, state, firstRunning(state.First, running), step.failed) + if state.Gate != nil && len(state.Gate.Carried) > 0 { + failCarried(ctx, open, p, state.Gate, m) + } p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", "))) fmt.Printf("%s: %s\n", p.ID, p.Note) return true, nil @@ -653,6 +663,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, continue case inventory.GateFailed: gateFailed(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why) + if len(state.Gate.Carried) > 0 { + failCarried(ctx, open, p, state.Gate, m) + } p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", "))) fmt.Printf("%s: %s\n", p.ID, p.Note) return true, nil @@ -677,7 +690,20 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, } // What sendToEach answers, not what was asked: the machine holding the bus is sent before // the first when its user list must change (issue 249), and the plan waits for it too. - sent, err := sendRollout(ctx, open, step.send) + var sent []string + var carried []inventory.CarriedMove + switch { + case step.first: + // **A gated send** (ADR 0236): everything waiting on the first machine goes with the build, + // and the gate judges all of it there. + own := inventory.CarriedMove{Module: m, Node: step.send[0], From: before[m], To: state.Commit, Build: state.Build} + carried, sent, err = gatedSend(ctx, open, step.send[0], &own) + case policy.Together: + sent, err = sendRollout(withScope(ctx, sendScope{modules: map[string]bool{m: true}}), open, step.send) + default: + // The rest, after the gate passed: nothing else may move with it that no gate has seen. + sent, err = sendRollout(ctx, open, step.send) + } if err != nil { // Not marked sent, so the next step tries again (issue 249): a grant that could not be // issued is a send that did not happen. @@ -690,6 +716,8 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, } state.First = sent state.FirstAt = &now + state.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, running), + From: state.Previous, To: state.Commit, Since: &now, Carried: carried} p.State = inventory.PlanRolling p.Note = fmt.Sprintf("tier %d built; sent %s to %s first", p.Tier, m, strings.Join(sent, ", ")) fmt.Printf("%s: tier %d built; sent %s to %s first, the rest once it reports it applied\n", @@ -1059,6 +1087,18 @@ func plansCommand(ctx context.Context, args []string) error { return err } fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}))) + if r := p.Release; r != nil { + // A release plan's walk (ADR 0236): machines done, the one judged, those to come. + fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "), + orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", "))) + if r.Gate != nil { + fmt.Printf(" %s\n", gateLine(r.Gate)) + for _, c := range r.Gate.Carried { + fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To)) + } + } + return nil + } for i, tier := range p.Tiers { marker := " " if i == p.Tier && p.Open() { diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go new file mode 100644 index 0000000..11c4c82 --- /dev/null +++ b/cmd/mesh-controller/release_test.go @@ -0,0 +1,257 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// The backlog the old default left — builds registered and sent nowhere — is released by a plan, one +// machine at a time, each judged, never by the next send of something else (novox/hq ADR 0236). + +type backlogMesh struct { + open *stores + sent [][]string + broken map[string]bool +} + +// aBacklog is a mesh where `app` moved c1 → c2 on anchor and laptop, `late` moved on laptop only, and +// `same` was rebuilt with the very artifacts it had: two machines heard, every send applied at once. +func aBacklog(t *testing.T) *backlogMesh { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + b := &backlogMesh{open: open, broken: map[string]bool{}} + withConditionsInMemory(t) + was := doctorFrom + doctorFrom = nil + t.Cleanup(func() { doctorFrom = was }) + + build := func(module, commit, made string, asked time.Time) { + manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit, + Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: made}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit, + Asked: asked}); err != nil { + t.Fatal(err) + } + } + old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) + on := map[string][]string{"app": {"anchor", "laptop"}, "late": {"laptop"}, "same": {"anchor", "laptop"}} + for _, m := range []string{"app", "late", "same"} { + build(m, "c1", "sha256:"+m+"-1", old) + for _, n := range on[m] { + if _, err := inv.Assign(ctx, n, m); err != nil { + t.Fatal(err) + } + } + } + for _, n := range []string{"anchor", "laptop"} { + sent := map[string]string{} + for m, nodes := range on { + for _, x := range nodes { + if x == n { + sent[m] = "c1" + } + } + } + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil { + t.Fatal(err) + } + } + build("app", "c2", "sha256:app-2", now) + build("late", "c2", "sha256:late-2", now) + build("same", "c2", "sha256:same-1", now) // rebuilt, the same bytes + + assigned := func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + modules, err := open.inventory.Assigned(ctx, node) + if err != nil { + return nil, err + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil + } + wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts + machineMoves = assigned + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"anchor": true, "laptop": true}, nil + } + n := 0 + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + b.sent = append(b.sent, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + modules, _ := open.inventory.Assigned(ctx, node) + carried := map[string]string{} + for _, m := range modules { + carried[m] = current[m].Commit + } + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { + return nil, err + } + if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + if b.broken[r.Node] { + r.Outcome = inventory.OutcomeFailed + } + f.reports[r.Node] = r + } + return f, nil + } + wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound + gateSettle, gateEvery = 0, time.Hour + t.Cleanup(func() { + machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather + gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound + }) + return b +} + +func (b *backlogMesh) release(t *testing.T) inventory.Plan { + t.Helper() + plans, err := b.open.inventory.RecentPlans(t.Context(), 10) + if err != nil { + t.Fatal(err) + } + for _, p := range plans { + if p.Release != nil { + return p + } + } + t.Fatal("no release plan") + return inventory.Plan{} +} + +// The backlog goes out one machine at a time: the first judged before the second is sent, each build's +// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused. +func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) { + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + + f, err := readMoveFacts(ctx, inv) + if err != nil { + t.Fatal(err) + } + moves, _ := machineMoves(ctx, b.open, f, "laptop", false) + var names []string + for _, mv := range moves { + names = append(names, mv.Module) + } + if !reflect.DeepEqual(names, []string{"app", "late"}) { + t.Fatalf("laptop waits for %v; a rebuild with the same bytes is no move", names) + } + // Nothing else may carry them: a send that judges nothing is refused. + if _, err := ungatedIn(ctx, b.open, []string{"laptop"}, ""); !errors.Is(err, errUngated) { + t.Fatalf("a send that judges nothing would carry them: %v", err) + } + + advancePlans(ctx, b.open) + if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: the first machine alone, before it is judged", b.sent) + } + p := b.release(t) + if !reflect.DeepEqual(p.Release.Order, []string{"anchor", "laptop"}) || p.Release.Gate == nil { + t.Fatalf("the release plan is %+v", p.Release) + } + gateEvery = 0 + for i := 0; i < 4; i++ { + advancePlans(ctx, b.open) + } + if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("sent %v", b.sent) + } + p = b.release(t) + if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) { + t.Fatalf("the release plan is %s: %s %+v", p.State, p.Note, p.Release) + } + for _, build := range []string{"build-app-c2", "build-late-c2"} { + if v, found, err := inv.GateOf(ctx, build); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("%s's pass was not kept: %+v %v %v", build, v, found, err) + } + } + // Nothing waits now, and nothing opens again. + if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil { + t.Fatalf("a release opened with nothing waiting: %+v %v", p, err) + } +} + +// A release that fails on its first machine puts back what it carried there, goes no further, and the +// next one waits for a person, said as a condition; a person releases it with why. +func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) { + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + gateEvery = 0 + b.broken["anchor"] = true + advancePlans(ctx, b.open) + + p := b.release(t) + if p.State != inventory.PlanFailed { + t.Fatalf("the release is %s: %s", p.State, p.Note) + } + if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"anchor"}}) { + t.Fatalf("sent %v: the first machine, then the put-back to it, and nothing to laptop", b.sent) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { + t.Fatalf("app is at %s, not put back", current["app"].Commit) + } + if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { + t.Fatal("app's build is not marked failed at its gate") + } + // late still waits on laptop, and is not released on its own after a failure. + if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil { + t.Fatalf("a release opened after a failed one: %+v %v", p, err) + } + if obs := backlogObservation(); len(obs) != 1 || !strings.Contains(obs[0].Summary, "release-backlog") { + t.Fatalf("the held release is not said: %+v", obs) + } + b.broken["anchor"] = false + if err := backlogCommand(ctx, "release-backlog", []string{"--why", "anchor is fixed"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, b.open) + } + if p := b.release(t); p.State != inventory.PlanDone || p.Release.By == "" { + t.Fatalf("the person's release is %s (%+v)", p.State, p.Release) + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { + t.Fatalf("late was not released to laptop: %v", sent) + } +} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index d4a04f0..3517e34 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -509,6 +509,16 @@ func (a *verbArguments) commandLine() ([]string, error) { } return argv, nil case "upgrade": + if on("backlog") { + return []string{"upgrade", "backlog"}, nil + } + if on("release-backlog") { + argv := []string{"upgrade", "release-backlog", "--why", str("why")} + if c := str("cause"); c != "" { + argv = append(argv, "--cause", c) + } + return argv, nil + } m, policy := str("module"), str("policy") if m == "" { if policy != "" { diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index 33cce27..515d311 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -74,24 +74,12 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error { u.Module, shortCommit(u.Commit), id, readableList(on)) return nil } - if decision.Together { - fmt.Printf("%s moved to %s; sending %s together\n", - u.Module, shortCommit(u.Commit), readableList(on)) - return askAgainOnGrants(sendTo(ctx, f.open, on)) - } - // One at a time, and stopping at the first that fails. - // - // **Stopping is the point.** The machines are done one after another precisely so that a - // version that breaks the first one does not reach the rest; carrying on past a failure would - // make this the same as sending them together, only slower. - fmt.Printf("%s moved to %s; sending %s one at a time\n", - u.Module, shortCommit(u.Commit), readableList(on)) - for _, node := range on { - if err := sendTo(ctx, f.open, []string{node}); err != nil { - return askAgainOnGrants(fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w", - node, u.Module, err)) - } - } + // **No plan holds it: a release plan sends it** (novox/hq ADR 0236). A build asked outside a plan — a + // `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it + // now waits for a gate like any other build, and the release plan walks it through the machines, one + // at a time, each judged. + fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine at a time, each judged "+ + "(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on)) return nil } @@ -147,6 +135,10 @@ func isAre(n int) string { // upgradeCommand says what should happen when a module's current version moves (ADR 0162 §3, ADR // 0235): every module's policy and where it comes from; one module's; or a person's choice for one. func upgradeCommand(ctx context.Context, args []string) error { + // What waits for a gate, and releasing it by a person's word (ADR 0236). + if len(args) > 0 && (args[0] == "backlog" || args[0] == "release-backlog") { + return backlogCommand(ctx, args[0], args[1:]) + } set := flag.NewFlagSet("upgrade", flag.ContinueOnError) together := set.Bool("together", false, "send every machine running it at once, instead of one machine first") diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 0e9b535..d6020d2 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -277,8 +277,12 @@ var ControllerVerbs = []Verb{ "module": "one module", "policy": "with module: roll-out, record or default", "together": "\"true\": with roll-out, every machine at once instead of one machine first", - "why": "with policy: why — required for record, kept and said with the policy", - }, nil, "together")}, + "why": "with policy or release-backlog: why — required for record and for a release, kept", + "backlog": "\"true\": every build waiting for a gate, per machine — what a release plan would send", + "release-backlog": "\"true\": release what waits now, one machine at a time, each judged — after a release " + + "plan failed, the mesh waits for this; with why", + "cause": "with release-backlog: the cause in a word (optional)", + }, nil, "together", "backlog", "release-backlog")}, {Name: "bus", Description: "The bus as a planned step (novox/hq to-be 45 §8, ADR 0236): what a bus upgrade " + "would do — the bus's build on each machine against the one the mesh holds — and how the last step went. " + "With upgrade, start one: a person's act with why, after the streams are snapshotted (snapshot-taken says " + diff --git a/internal/inventory/gate.go b/internal/inventory/gate.go index a07639e..9d32b07 100644 --- a/internal/inventory/gate.go +++ b/internal/inventory/gate.go @@ -46,8 +46,8 @@ type GateVerdict struct { Epoch uint64 } -// RecordGate writes a build's verdict. **A failed build's row is written once**: a second failure for -// the same build is refused with ErrGateKept, which is what keeps a rollback to one per build — the row +// RecordGate writes a build's verdict. A build that passed on one machine may still fail on the next one +// judged; **a failed build's row is written once**: any later verdict for it is refused with ErrGateKept, which is what keeps a rollback to one per build — the row // is written before the rollback's send, and a controller replaced in between finds it. func (i *Inventory) RecordGate(ctx context.Context, v GateVerdict) error { epoch, err := i.actingEpoch(ctx) @@ -64,7 +64,7 @@ func (i *Inventory) RecordGate(ctx context.Context, v GateVerdict) error { on conflict (build) do update set verdict = excluded.verdict, rollback = excluded.rollback, why = excluded.why, previous = excluded.previous, machines = excluded.machines, judged_at = now(), epoch = excluded.epoch - where build_gate.verdict = 'passed' and excluded.verdict = 'passed'`, + where build_gate.verdict = 'passed'`, v.Build, v.Module, v.Commit, v.Previous, v.Plan, v.Machines, v.Verdict, v.Rollback, v.Why, v.Component, v.JudgingFrom, epoch) if err != nil { @@ -245,3 +245,65 @@ func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[s } return out, rows.Err() } + +// Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint. +func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) { + rows, err := i.store.Pool().Query(ctx, + `select module, commit_hash, made, coalesce(manifest::text, '') from build + where module is not null and failed = '' and commit_hash <> '' order by at desc`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]map[string]string{} + for rows.Next() { + var module, commit, manifest string + var made []byte + if err := rows.Scan(&module, &commit, &made, &manifest); err != nil { + return nil, err + } + if out[module] == nil { + out[module] = map[string]string{} + } + if _, seen := out[module][commit]; seen { + continue + } + sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...)) + out[module][commit] = hex.EncodeToString(sum[:]) + } + return out, rows.Err() +} + +// PassedCommits is, per module, the commits a build of which passed its gate on some machine. +func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bool, error) { + rows, err := i.store.Pool().Query(ctx, `select module, commit_hash from build_gate where verdict = 'passed'`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]map[string]bool{} + for rows.Next() { + var module, commit string + if err := rows.Scan(&module, &commit); err != nil { + return nil, err + } + if out[module] == nil { + out[module] = map[string]bool{} + } + out[module][commit] = true + } + return out, rows.Err() +} + +// BuildOf is the id of the newest successful build of a module from a commit; empty when none is +// recorded (a manifest handed over by hand). +func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) { + var id string + err := i.store.Pool().QueryRow(ctx, + `select id from build where module = $1 and commit_hash = $2 and failed = '' order by at desc limit 1`, + module, commit).Scan(&id) + if errors.Is(err, pgx.ErrNoRows) { + return "", nil + } + return id, err +} diff --git a/internal/inventory/migrations/0073-a-build-rolls-out-gated-and-rolls-back.sql b/internal/inventory/migrations/0073-a-build-rolls-out-gated-and-rolls-back.sql index e6b312d..9d57766 100644 --- a/internal/inventory/migrations/0073-a-build-rolls-out-gated-and-rolls-back.sql +++ b/internal/inventory/migrations/0073-a-build-rolls-out-gated-and-rolls-back.sql @@ -65,3 +65,8 @@ create table bus_step ( outcome text not null default '' check (outcome in ('', 'done', 'failed')), found text not null default '' ); + +-- 4. A release plan (ADR 0236): the builds that wait for a gate — the backlog the old default left, and +-- whatever a plan built and did not send — walked through the machines one at a time, each judged +-- before the next. Its walk is kept with the plan. +alter table release_plan add column release jsonb; diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index 7508b60..cbb37f4 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -38,6 +38,9 @@ type Plan struct { Revision int64 `json:"revision"` // Epoch is the controller lease epoch that wrote it last; zero for a write that claimed none. Epoch uint64 `json:"epoch,omitempty"` + // Release is set on a release plan (novox/hq ADR 0236): not a merge's, but the builds waiting for a + // gate, walked through the machines one at a time. + Release *PlanRelease `json:"release,omitempty"` } // ErrPlanMoved is a save against a plan written by somebody else since it was read. @@ -103,6 +106,35 @@ type PlanGate struct { Rollback string `json:"rollback,omitempty"` // Kept says a passing verdict was written to the gate's records. Kept bool `json:"kept,omitempty"` + // Carried is every module whose build moved on the judged machines with the send — the plan's own + // module and whatever else was waiting there for a gate (novox/hq ADR 0236): each is judged here, a + // pass is its verdict too, and one that fails is put back. + Carried []CarriedMove `json:"carried,omitempty"` + // Failing names the modules the last judging found wanting. + Failing []string `json:"failing,omitempty"` +} + +// CarriedMove is one module's build moving on a machine with a gated send. +type CarriedMove struct { + Module string `json:"module"` + Node string `json:"node"` + From string `json:"from,omitempty"` + To string `json:"to"` + Build string `json:"build,omitempty"` +} + +// PlanRelease is a release plan's walk through the machines (novox/hq ADR 0236): every module build +// that waits for a gate, sent one machine at a time, each judged before the next. +type PlanRelease struct { + Order []string `json:"order"` + Next int `json:"next"` + // Gate is the machine being judged; nil between machines. + Gate *PlanGate `json:"gate,omitempty"` + Done []string `json:"done,omitempty"` + // Skipped are the machines not heard from when their turn came, left as they were. + Skipped []string `json:"skipped,omitempty"` + // By is the person who released it, empty when the mesh did. + By string `json:"by,omitempty"` } // The states a plan passes through. @@ -138,6 +170,12 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error { if err != nil { return err } + var release []byte + if p.Release != nil { + if release, err = json.Marshal(p.Release); err != nil { + return err + } + } // **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the // plan moves, in the same transaction as the move, so no save can move a tier unmeasured or // measure one twice. @@ -153,15 +191,16 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error { var revision int64 err = tx.QueryRow(ctx, `insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, - branch, tier_entered, revision, epoch) - values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13) + branch, tier_entered, revision, epoch, release) + values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14) on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier, tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch, - tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch + tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch, + release = excluded.release where release_plan.revision = $12 returning revision`, p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered, - p.Revision, epoch).Scan(&revision) + p.Revision, epoch, release).Scan(&revision) if errors.Is(err, pgx.ErrNoRows) { // The row is there and at another revision — moved since this was read, or there already // when this one is new: either way not this writer's to overwrite. (A plan saved before plans @@ -216,7 +255,7 @@ func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) { func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) { rows, err := i.store.Pool().Query(ctx, `select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch, - coalesce(tier_entered, created), revision, coalesce(epoch, 0) + coalesce(tier_entered, created), revision, coalesce(epoch, 0), release from release_plan `+tail) if err != nil { return nil, err @@ -225,12 +264,17 @@ func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) { var out []Plan for rows.Next() { var p Plan - var tiers, modules []byte + var tiers, modules, release []byte var epoch int64 if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State, - &p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch); err != nil { + &p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch, &release); err != nil { return nil, err } + if len(release) > 0 { + if err := json.Unmarshal(release, &p.Release); err != nil { + return nil, err + } + } p.Epoch = uint64(epoch) if err := json.Unmarshal(tiers, &p.Tiers); err != nil { return nil, err From dcec6a4db30bfca82b230d1baba4381f61c7dfbe Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 6 Oct 2026 19:14:35 +0200 Subject: [PATCH 7/7] gofmt --- cmd/mesh-controller/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index fba0f7b..4a66640 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -1,7 +1,6 @@ package main import ( - "slices" "context" "crypto/sha256" "encoding/hex" @@ -13,6 +12,7 @@ import ( "io" "log" "os" + "slices" "sort" "strings" "time"