diff --git a/cmd/mesh-controller/batches.go b/cmd/mesh-controller/batches.go index b3c09aff..1acbe182 100644 --- a/cmd/mesh-controller/batches.go +++ b/cmd/mesh-controller/batches.go @@ -219,10 +219,10 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T } if p.Batch() { if err := keepBatch(ctx, inv, &p, now, ""); err != nil { - return notNow(err) + return unsaved(err) } } else if err := nameMerges(ctx, inv, &p); err != nil { - return notNow(err) + return unsaved(err) } fmt.Printf("%s merged into %s (%.8s) after a later merge of its branch was put in %s (%s): that %s "+ "answers it\n", repository, m.Base, m.Commit, p.ID, p.State, walkOrBatch(p)) @@ -239,7 +239,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T } holdFor(&batch, m.Head, waits, holding, own, now) if err := keepBatch(ctx, inv, &batch, now, ""); err != nil { - return notNow(err) + return unsaved(err) } fmt.Printf("%s merged into %s (%.8s); %s, it joins %s — %s\n", repository, m.Base, m.Commit, why, batch.ID, batchWords(batch, now)) @@ -820,6 +820,12 @@ func nameMerges(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan // walk open, behind which it is said queued (cutUnrelated). While the search runs everything waits. func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error { inv := open.inventory + // A merge found in two records is the half-done move of a crash: put right before anything is decided from it. + if repaired, err := inv.RepairMerges(ctx); err != nil { + return err + } else if repaired > 0 { + fmt.Printf("%d merge(s) found in two plans, the half of a move a restart interrupted, are kept by the plan they were moved to\n", repaired) + } if err := searchAfterFailures(ctx, inv); err != nil { return err } diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 5a6802e1..b89e30b2 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -152,6 +152,8 @@ func buildFrom(result link.BuildResult) inventory.Build { Path: result.Path, // What it was made from (novox/hq issue 280): two builds with one are one build. SourceFingerprint: result.SourceFingerprint, + // And its reuse key (novox/hq ADR 0350 §1), which the chain names (ADR 0408 §1). + ReuseKey: result.ReuseKey, } // When it was asked, which is what orders it against another build of the same module // (novox/hq 04-ISSUES/219) — not when it was heard. @@ -254,6 +256,18 @@ func buildsCommand(ctx context.Context, args []string) error { for _, made := range b.Made { fmt.Printf(" %-10s %s\n", made.Kind, made.Reference) } + // What it published and under what reuse key (novox/hq ADR 0408 §1). + if p := b.Published; p != nil { + if p.Artifact != "" { + fmt.Printf(" published %s\n", p.Artifact) + } + if p.Package != nil { + fmt.Printf(" published %s@%s\n", p.Package.Name, p.Package.Version) + } + } + if b.ReuseKey != "" { + fmt.Printf(" reuse key %s\n", b.ReuseKey) + } if !b.Worked() { // The builder's own first line. The whole failure is often a build log, and printing // it here would bury every other row. diff --git a/cmd/mesh-controller/chain.go b/cmd/mesh-controller/chain.go new file mode 100644 index 00000000..18436e8b --- /dev/null +++ b/cmd/mesh-controller/chain.go @@ -0,0 +1,181 @@ +package main + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/novox/mesh-controller/internal/chain" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the +// bus's key-value store, written by the controller that holds the lease. The tables of the controller's store are +// copied into them once, by the serving controller at its start, and a kind is read from its bucket only once the move +// has recorded it (inventory.ChainReads). + +// chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes. +func chainHolds(ctx context.Context) error { + _, err := theLease.epoch(ctx) + return err +} + +// openTheChain opens the chain's buckets on the bus: the serving controller's connection, or one dialled for a command. +func openTheChain(ctx context.Context) (*chain.Chain, error) { + js, err := aBus() + if err != nil { + return nil, err + } + // Not closed: a borrowed connection closes nothing, and a dialled one lasts as long as the command. + return chain.OnTheBus(ctx, js.Conn(), chainHolds) +} + +// kindChainMoveRefused is the condition of a move that stopped without switching a kind. +const kindChainMoveRefused = "chain-move-refused" + +// chainMoveRetry is how long a move that failed for want of the bus waits before it tries again. +const chainMoveRetry = 30 * time.Second + +// movingTheChain copies the tables into the buckets and records it, at the serving controller's start: after the buckets +// exist and the lease is held, before anything reads a kind that has not moved. **The first try is made here, before the +// controller serves**, so what it serves is read from one place. A move that fails for want of the bus is tried again +// every half minute while the controller serves from the tables; one that is refused — a count that differs, a value +// over the bus's largest — is raised as an urgent condition, and the tables stay what is read: nothing is half-switched. +func movingTheChain(ctx context.Context, inv *inventory.Inventory) { + say := func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) } + // The first try is made here, before the controller serves; the tries after it, while it serves. + if movedOrStopped(ctx, inv, say) { + return + } + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(chainMoveRetry): + } + if movedOrStopped(ctx, inv, say) { + return + } + } + }() +} + +// movedOrStopped makes one try at the move and says whether there is nothing more to try: moved, refused, or the controller +// ending. False for a failure of the kind that is tried again (the bus away, a write not granted yet, the lease not held). +func movedOrStopped(ctx context.Context, inv *inventory.Inventory, say func(string, ...any)) bool { + _, err := inv.MoveChain(ctx, say) + var refusal *inventory.Refusal + switch { + case err == nil: + go clearChainMove(ctx) + go askAgainSoon(ctx, inv) + return true + case errors.As(err, &refusal): + say("%v", err) + go raiseChainMove(ctx, refusal) + return true + case ctx.Err() != nil: + return true + } + say("the move to the bus did not finish, and the tables are read until it does: %v", err) + return false +} + +// chainMoveObservation is the urgent condition for a refused move. +func chainMoveObservation(r *inventory.Refusal) conditions.Observation { + return conditions.Observation{Scope: conditions.ScopeBus, ID: "chain." + r.Kind, Token: kindChainMoveRefused, + Kind: kindChainMoveRefused, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, + Summary: "the controller's " + r.Kind + " were not moved to the bus: " + r.Why, Source: "chain move"} +} + +// whenKept runs f once the serving controller keeps conditions: the move runs before they are kept. +func whenKept(ctx context.Context, f func(k *conditions.Keeper)) { + for conditionsFrom == nil { + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } + f(conditionsFrom) +} + +func raiseChainMove(ctx context.Context, r *inventory.Refusal) { + whenKept(ctx, func(k *conditions.Keeper) { + if _, err := k.Observe(ctx, chainMoveObservation(r)); err != nil { + fmt.Printf("chain: the condition for the refused move could not be raised: %v\n", err) + } + }) +} + +func clearChainMove(ctx context.Context) { + whenKept(ctx, func(k *conditions.Keeper) { + for _, kind := range []string{inventory.KindBuilds, inventory.KindRollouts, inventory.KindAssignments} { + key := chainMoveObservation(&inventory.Refusal{Kind: kind}).Key() + if _, err := k.Clear(ctx, key, "the "+kind+" are on the bus"); err != nil { + fmt.Printf("chain: the condition for the move of the %s could not be cleared: %v\n", kind, err) + } + } + }) +} + +// attachTheChain gives the inventory of the serving controller the chain on its connection, then moves what has not moved. +func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error { + bus, ok := server.Bus().(link.OverNATS) + if !ok { + return errors.New("the chain lives on the bus, and this control plane is not on it") + } + c, err := chain.OnTheBus(ctx, bus.Conn, chainHolds) + if err != nil { + return err + } + inv.UseChain(c) + movingTheChain(ctx, inv) + return nil +} + +// askAgain asks once more for each build request the move found never handed over (novox/hq issue 981 note 7 Q5): the +// requests a merge could not ask for, which a person's hand `build` would otherwise have to repeat. **At most once**: a +// request is claimed before it is asked and settled after, so a crash between loses the ask, which is where it stood. +func askAgain(ctx context.Context, inv *inventory.Inventory) { + pending, err := inv.RequestsToAskAgain(ctx) + if err != nil { + fmt.Printf("chain: the requests never handed over cannot be read: %v\n", err) + return + } + for _, r := range pending { + if claimed, err := inv.ClaimToAskAgain(ctx, r.ID); err != nil || !claimed { + if err != nil { + fmt.Printf("chain: %s cannot be claimed to be asked again: %v\n", r.ID, err) + } + continue + } + id, err := askABuild(ctx, buildSource{Repository: r.Repository, Seat: r.Seat}, r.Path, r.Ref) + if err != nil { + if serr := inv.AskedAgainAs(ctx, r.ID, "not asked again: "+err.Error()); serr != nil { + fmt.Printf("chain: %s was not asked again, and that could not be kept: %v\n", r.ID, serr) + } + fmt.Printf("chain: %s was never handed over and cannot be asked again: %v\n", r.ID, err) + continue + } + recordBuildRequest(ctx, inv, inventory.BuildRequest{ID: id, Repository: r.Repository, Seat: r.Seat, Path: r.Path, + Ref: r.Ref, Commit: r.Commit, For: r.For}) + if err := inv.AskedAgainAs(ctx, r.ID, id); err != nil { + fmt.Printf("chain: %s was asked again as %s, and that could not be kept: %v\n", r.ID, id, err) + } + fmt.Printf("chain: %s, never handed over, is asked again as %s\n", r.ID, id) + } +} + +// askAgainSoon waits for the controller to be serving, which an ask needs, and then asks again. +func askAgainSoon(ctx context.Context, inv *inventory.Inventory) { + select { + case <-ctx.Done(): + case <-time.After(time.Minute): + askAgain(ctx, inv) + } +} diff --git a/cmd/mesh-controller/chain_integration_test.go b/cmd/mesh-controller/chain_integration_test.go new file mode 100644 index 00000000..c6a98e1c --- /dev/null +++ b/cmd/mesh-controller/chain_integration_test.go @@ -0,0 +1,209 @@ +//go:build integration + +package main + +import ( + "context" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/chain" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" + "github.com/novox/mesh-controller/internal/link" +) + +// **Acceptance 10 of design 47: one read of an assignment names the commit, the build and the rollout that put it there, +// and one read of the rollout lists the assignments it changed** (novox/hq ADR 0408, issue 1095). It enters by the live +// trigger — a merge heard, its batch cut into a plan when the window closed, its build answered through the build seat's +// path, the plan advanced — and writes no key and no row by hand. Only the machine's end of the send is the test's: the send +// is recorded as the controller records one it made, and the gate's facts are the store's. +func TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asked := asksRecorded(t) + ctx := t.Context() + inv := open.inventory + if _, err := assign(ctx, open, "anchor", "notes"); err != nil { + t.Fatal(err) + } + + var sent [][]string + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + sent = append(sent, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + digest := fmt.Sprintf("d-%s-%d", node, len(sent)) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, + map[string]string{"notes": current["notes"].Commit}); err != nil { + return nil, err + } + } + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + wasGather := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + f.reports[r.Node] = r + } + return f, nil + } + t.Cleanup(func() { gatherGateFacts = wasGather }) + + // A merge of notes is heard, and its window closes: the batch is cut into the walk. + hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second)) + cutAt(t, open, t0.Add(time.Second+90*time.Second)) + ws, _ := walks(t, open) + if len(ws) != 1 { + t.Fatalf("%d walk(s) after the cut", len(ws)) + } + rollout := ws[0].ID + + // The walk asks its first tier (at the cut, or at its next step); the build seat answers, and the controller takes the + // outcome in as it does. + asking, err := inv.PlanByID(ctx, rollout) + if err != nil { + t.Fatal(err) + } + for try := 0; try < 3 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == ""); try++ { + advancePlans(ctx, open) + if asking, err = inv.PlanByID(ctx, rollout); err != nil { + t.Fatal(err) + } + } + buildID := asking.Modules["notes"].Build + if buildID == "" || len(*asked) != 1 || !strings.HasSuffix((*asked)[0], "#"+buildID) { + t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID) + } + digest := "sha256:" + strings.Repeat("a", 64) + result := outcome(buildID, "modules/notes", "c-notes", "notes", "") + result.ReuseKey = "rk-notes" + result.Made = []link.MadeArtifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes/notes@" + digest}} + if err := (builds{inv: inv, open: open}).Built(ctx, result); err != nil { + t.Fatal(err) + } + + // The walk sends the tier to its first machine. + for try := 0; try < 3 && len(sent) == 0; try++ { + advancePlans(ctx, open) + } + if !slices.Equal(slices.Concat(sent...), []string{"anchor"}) { + t.Fatalf("sent %v", sent) + } + + // One read of the assignment. + on, err := inv.AssignmentsOn(ctx, "anchor") + a, ok := assignmentOf(on, "notes") + if err != nil || !ok { + t.Fatalf("the assignments of anchor: %+v %v", on, err) + } + // Beside it a module pulled in by the assignment, which no send of this walk put anywhere. + if wireguard, pulled := assignmentOf(on, "mesh-wireguard"); pulled && (wireguard.Build != "" || wireguard.Rollout != "") { + t.Fatalf("a module this rollout did not send names a build or a rollout: %+v", wireguard) + } + if a.Module != "notes" || a.Commit != "c-notes" || a.Build != buildID || a.Rollout != rollout || a.Artifact != digest || + a.Since.IsZero() || a.Assigned.IsZero() || !a.Stands() { + t.Fatalf("the assignment names %+v, not commit c-notes, build %s, artifact %s, rollout %s", a, buildID, digest, rollout) + } + // One read of the rollout. + p, err := inv.PlanByID(ctx, rollout) + if err != nil { + t.Fatal(err) + } + wantChange := chain.Change{Node: "anchor", Module: "notes", After: chain.Ref{Build: buildID, Commit: "c-notes"}} + if !slices.Contains(p.Changed, wantChange) || !slices.Contains(p.Builds, buildID) || len(p.Steps) != 1 || + p.Steps[0].Node != "anchor" || p.Steps[0].Sent == nil { + t.Fatalf("the rollout lists changed %+v, builds %v, steps %+v; want %+v", p.Changed, p.Builds, p.Steps, wantChange) + } + // And one read of the build names the same commit, with what it published and its reuse key. + built, found, err := inv.BuildByID(ctx, buildID) + if err != nil || !found || built.Commit != "c-notes" || built.Module != "notes" { + t.Fatalf("the build: %+v %v %v", built, found, err) + } + lately, err := inv.Builds(ctx, "notes", 1) + if err != nil || len(lately) != 1 || lately[0].ReuseKey != "rk-notes" || lately[0].Published == nil || + lately[0].Published.Artifact != digest { + t.Fatalf("what the build published: %+v %v", lately, err) + } + // The verbs say the links. + if lines := changedLines(p); len(lines) != 2 || !strings.Contains(lines[1], "anchor/notes") || + !strings.Contains(lines[1], buildID) { + t.Fatalf("`plans` says %v", lines) + } + if lines := runLines(on); len(lines) != 2 || !strings.Contains(lines[1], buildID) || !strings.Contains(lines[1], rollout) { + t.Fatalf("`node show` says %v", lines) + } +} + +// A machine sent again what it runs says nothing new, and a push by hand with no rollout fills what is missing and blanks +// nothing (issue 1095). +func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + register(t, open, catalogue.Manifest{Module: "notes", Version: "1"}) + if _, err := assign(ctx, open, "anchor", "notes"); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-1", Module: "notes", Commit: "c1", Repository: "novox/notes"}); err != nil { + t.Fatal(err) + } + node := nodeID(t, open, "anchor") + inRollout := inventory.WithRollout(ctx, "plan-1", map[string]inventory.RolloutModule{"notes": {Commit: "c1", Build: "build-1"}}) + if err := inv.RecordSent(inRollout, node, "d1", map[string]string{"notes": "c1"}); err != nil { + t.Fatal(err) + } + // Sent again by hand, with no rollout: the assignment still names the rollout that put the build there. + if err := inv.RecordSent(ctx, node, "d2", map[string]string{"notes": "c1"}); err != nil { + t.Fatal(err) + } + on, _ := inv.AssignmentsOn(ctx, "anchor") + if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" { + t.Fatalf("the assignment names %+v", on) + } + // A declaration sent by hand with no builds known says nothing of them. + if err := inv.RecordSent(ctx, node, "d3", nil); err != nil { + t.Fatal(err) + } + if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c1" { + t.Fatalf("a send of unknown builds changed %+v", on) + } + // A module sent a commit no build is kept of is named by its commit alone. + if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil { + t.Fatal(err) + } + if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c2" || runOf(on, "notes").Build != "" || + runOf(on, "notes").Rollout != "" { + t.Fatalf("the assignment after a push of c2 by hand names %+v", on) + } +} + +// assignmentOf is one module's assignment among those of a machine: assigning a module assigns what it requires beside it. +func assignmentOf(on []chain.Assignment, module string) (chain.Assignment, bool) { + for _, a := range on { + if a.Module == module { + return a, true + } + } + return chain.Assignment{}, false +} + +func runOf(on []chain.Assignment, module string) chain.Assignment { + a, _ := assignmentOf(on, module) + return a +} diff --git a/cmd/mesh-controller/chain_links.go b/cmd/mesh-controller/chain_links.go new file mode 100644 index 00000000..b6d01adb --- /dev/null +++ b/cmd/mesh-controller/chain_links.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "fmt" + "slices" + "time" + + "github.com/novox/mesh-controller/internal/chain" + "github.com/novox/mesh-controller/internal/inventory" +) + +// The links a rollout makes (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): the send of a plan's tier puts a +// build on a machine; the assignment of the module there names the build, its commit and the rollout (written by the record of +// the send, inventory.RecordSent, from the context a send is made in), and the rollout lists the assignments it changed with the +// build before and after. + +// rolloutModules is what a plan sends of each module: the commit and the build of it. +func rolloutModules(p *inventory.Plan) map[string]inventory.RolloutModule { + out := make(map[string]inventory.RolloutModule, len(p.Modules)) + for m, s := range p.Modules { + if s != nil && s.Commit != "" { + out[m] = inventory.RolloutModule{Commit: s.Commit, Build: s.Build} + } + } + return out +} + +// sendingAs is the context a plan's send is made in: what is recorded of the send names this rollout. +func sendingAs(ctx context.Context, p *inventory.Plan) context.Context { + return inventory.WithRollout(ctx, p.ID, rolloutModules(p)) +} + +// ranBefore is what each module on a machine ran before a send, as the chain names it: its assignment's build and commit, +// else the commit the machine was last sent for it. Read before the send, which writes the new ones. +func ranBefore(ctx context.Context, inv *inventory.Inventory, node string) map[string]chain.Ref { + was := map[string]chain.Ref{} + if sent, known, err := inv.SentBuilds(ctx, node); err == nil && known { + for m, commit := range sent { + was[m] = chain.Ref{Commit: commit} + } + } + if ran, err := inv.AssignmentRuns(ctx, node); err == nil { + for m, r := range ran { + was[m] = r + } + } + return was +} + +// noteChanges writes into a plan that a send to a machine changed the assignments of the modules it carried: the build and +// commit before and after, the builds the rollout carried, and the step taken. A change noted already for the machine and the +// module is replaced, so a send made again says it once. +func noteChanges(p *inventory.Plan, node string, modules []string, was map[string]chain.Ref, at time.Time) { + for _, m := range modules { + s := p.Modules[m] + if s == nil { + continue + } + change := chain.Change{Node: node, Module: m, Before: was[m], After: chain.Ref{Build: s.Build, Commit: s.Commit}} + if at := slices.IndexFunc(p.Changed, func(c chain.Change) bool { return c.Node == node && c.Module == m }); at >= 0 { + change.Before = p.Changed[at].Before + p.Changed[at] = change + } else { + p.Changed = append(p.Changed, change) + } + if s.Build != "" && !slices.Contains(p.Builds, s.Build) { + p.Builds = append(p.Builds, s.Build) + } + } + if !slices.ContainsFunc(p.Steps, func(s chain.Step) bool { return s.Tier == p.Tier && s.Node == node }) { + sent := at.UTC() + p.Steps = append(p.Steps, chain.Step{Tier: p.Tier, Node: node, Sent: &sent}) + } +} + +// refSaid is a build and the commit it was made from as a person reads them; "nothing" for a module that ran none. +func refSaid(r chain.Ref) string { + switch { + case r.Build == "" && r.Commit == "": + return "nothing" + case r.Build == "": + return "commit " + short(r.Commit) + } + return r.Build + " (" + short(r.Commit) + ")" +} + +// changedLines say the assignments a rollout changed: the machine, the module, and the build before and after. +func changedLines(p inventory.Plan) []string { + if len(p.Changed) == 0 { + return nil + } + out := []string{" changed:"} + for _, c := range p.Changed { + out = append(out, fmt.Sprintf(" %s/%s %s → %s", c.Node, c.Module, refSaid(c.Before), refSaid(c.After))) + } + return out +} + +// runLines say what each module assigned to a machine runs there, as its assignment names it: the build, the commit, the +// rollout that put it there and since when. +func runLines(assigned []chain.Assignment) []string { + var out []string + for _, a := range assigned { + if a.Build == "" && a.Commit == "" { + continue + } + line := fmt.Sprintf(" %-22s %s", a.Module, refSaid(chain.Ref{Build: a.Build, Commit: a.Commit})) + if a.Rollout != "" { + line += ", sent by " + a.Rollout + } + if !a.Since.IsZero() { + line += " at " + a.Since.Local().Format("2006-01-02 15:04") + } + out = append(out, line) + } + if len(out) == 0 { + return nil + } + return append([]string{" what they run, as the chain names it:"}, out...) +} diff --git a/cmd/mesh-controller/chain_links_test.go b/cmd/mesh-controller/chain_links_test.go new file mode 100644 index 00000000..8652bd60 --- /dev/null +++ b/cmd/mesh-controller/chain_links_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/chain" + "github.com/novox/mesh-controller/internal/inventory" +) + +// **A rollout lists the assignments it changed, each with the build before and after** (ADR 0408 §1, issue 1095): one +// change for a machine and a module however often the send is noted, the build before kept from the first, the builds the +// rollout carried once each, and one step for a machine and a tier. +func TestARolloutListsTheAssignmentsItChangedWithTheBuildBeforeAndAfter(t *testing.T) { + at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) + p := inventory.Plan{ID: "plan-1", Tier: 1, Modules: map[string]*inventory.PlanModule{ + "notes": {Commit: "c2", Build: "b2"}, "wiki": {Commit: "c2w", Build: "b2w"}, "nothing": nil}} + was := map[string]chain.Ref{"notes": {Build: "b1", Commit: "c1"}, "wiki": {Commit: "c1w"}} + + noteChanges(&p, "anchor", []string{"notes", "wiki", "nothing"}, was, at) + noteChanges(&p, "laptop", []string{"notes"}, nil, at) + // The send noted again, after what was sent had become what the machine runs: the first before stands. + noteChanges(&p, "anchor", []string{"notes"}, map[string]chain.Ref{"notes": {Build: "b2", Commit: "c2"}}, at.Add(time.Minute)) + + want := []chain.Change{ + {Node: "anchor", Module: "notes", Before: chain.Ref{Build: "b1", Commit: "c1"}, After: chain.Ref{Build: "b2", Commit: "c2"}}, + {Node: "anchor", Module: "wiki", Before: chain.Ref{Commit: "c1w"}, After: chain.Ref{Build: "b2w", Commit: "c2w"}}, + {Node: "laptop", Module: "notes", After: chain.Ref{Build: "b2", Commit: "c2"}}, + } + if len(p.Changed) != len(want) { + t.Fatalf("changed %+v", p.Changed) + } + for k := range want { + if p.Changed[k] != want[k] { + t.Errorf("change %d is %+v, want %+v", k, p.Changed[k], want[k]) + } + } + if len(p.Builds) != 2 || p.Builds[0] != "b2" || p.Builds[1] != "b2w" { + t.Errorf("builds %v", p.Builds) + } + if len(p.Steps) != 2 || p.Steps[0].Node != "anchor" || p.Steps[0].Tier != 1 || p.Steps[0].Sent == nil || p.Steps[1].Node != "laptop" { + t.Errorf("steps %+v", p.Steps) + } + lines := changedLines(p) + if len(lines) != 4 || !strings.Contains(lines[1], "anchor/notes b1 (c1) → b2 (c2)") || + !strings.Contains(lines[3], "laptop/notes nothing → b2 (c2)") { + t.Errorf("changed lines %q", lines) + } + if got := changedLines(inventory.Plan{}); got != nil { + t.Errorf("a plan that changed nothing says %q", got) + } +} + +func TestWhatAMachineRunsIsSaidAsItsAssignmentNamesIt(t *testing.T) { + at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) + lines := runLines([]chain.Assignment{ + {Node: "anchor", Module: "notes", Build: "b2", Commit: "c2c2c2c2c2c2", Rollout: "plan-1", Since: at}, + {Node: "anchor", Module: "web", Commit: "c3"}, + {Node: "anchor", Module: "never-sent"}, + }) + if len(lines) != 3 || !strings.Contains(lines[1], "notes") || !strings.Contains(lines[1], "b2 (c2c2c2c2)") || + !strings.Contains(lines[1], "sent by plan-1") || !strings.Contains(lines[2], "web") || !strings.Contains(lines[2], "commit c3") { + t.Fatalf("lines %q", lines) + } + if runLines([]chain.Assignment{{Node: "a", Module: "m"}}) != nil { + t.Fatal("a module never sent is said to run something") + } +} diff --git a/cmd/mesh-controller/merge_unsaved_integration_test.go b/cmd/mesh-controller/merge_unsaved_integration_test.go new file mode 100644 index 00000000..095930b9 --- /dev/null +++ b/cmd/mesh-controller/merge_unsaved_integration_test.go @@ -0,0 +1,71 @@ +//go:build integration + +package main + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/chain" + "github.com/novox/mesh-controller/internal/link" +) + +// refusing is a bucket that refuses to write while it is told to. +type refusing struct { + chain.Backend + refuse atomic.Bool +} + +func (r *refusing) Create(ctx context.Context, key string, value []byte) (uint64, error) { + if r.refuse.Load() { + return 0, nats.ErrTimeout + } + return r.Backend.Create(ctx, key, value) +} + +func (r *refusing) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { + if r.refuse.Load() { + return 0, nats.ErrTimeout + } + return r.Backend.Update(ctx, key, value, revision) +} + +// **A merge whose plan cannot be saved is held, recorded nowhere, and recorded when it is handed over again** (ADR 0407 +// decision 3, issue 1092): hearMerge answers ErrTryAgain — the bus leaves the announcement unacknowledged and hands it +// over again — and the merge is in no record until the plan holding it is written. +func TestAMergeWhosePlanCannotBeSavedIsHeldAndRecordedWhenHandedOverAgain(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksWithPaths(t) + ctx := t.Context() + rollouts := &refusing{Backend: chain.NewMemory()} + open.inventory.UseChain(chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil })) + merge := catalogueMerge("cat1aaaaaaaa", "app", t0) + + rollouts.refuse.Store(true) + err := (following{open: open}).hearMerge(ctx, merge, t0.Add(time.Second)) + if !errors.Is(err, link.ErrTryAgain) { + t.Fatalf("a merge whose plan could not be saved answered %v, which the bus takes as done", err) + } + if _, known, err := open.inventory.MergeOf(ctx, "novox/mesh-catalog", merge.Commit); err != nil || known { + t.Fatalf("the merge is kept though its plan was not: %v %v", known, err) + } + if _, bs := walks(t, open); len(bs) != 0 { + t.Fatalf("a batch was kept though its write failed: %+v", bs) + } + + rollouts.refuse.Store(false) + hear(t, open, merge, t0.Add(10*time.Second)) + _, bs := walks(t, open) + if len(bs) != 1 { + t.Fatalf("the merge handed over again made %d batch(es)", len(bs)) + } + if m, known, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", merge.Commit); !known || m.Plan != bs[0].ID { + t.Fatalf("the merge is %+v %v, and the batch %s", m, known, bs[0].ID) + } +} diff --git a/cmd/mesh-controller/merge_unsaved_test.go b/cmd/mesh-controller/merge_unsaved_test.go new file mode 100644 index 00000000..c967060f --- /dev/null +++ b/cmd/mesh-controller/merge_unsaved_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "errors" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +// **A merge whose plan could not be saved is held, never taken** (novox/hq ADR 0407 decision 3, issue 1092): the merge is +// carried until its plan is saved, and the bus acknowledges the announcement only when the controller answers without +// ErrTryAgain (link.Server.sourceMoved), so a failed save of any kind must answer it. +func TestAFailureToSaveAMergesPlanIsHeldAndNotTaken(t *testing.T) { + cause := errors.New("the plan for novox/mesh-catalog is not written: the plan was written by somebody else since it was read") + held := unsaved(cause) + if !errors.Is(held, link.ErrTryAgain) || !errors.Is(held, cause) { + t.Fatalf("a failed save answers %v", held) + } + if unsaved(nil) != nil { + t.Fatal("a save that worked answers an error") + } + // Not the same as notNow, which holds a merge only for a store that is away. + if errors.Is(notNow(cause), link.ErrTryAgain) { + t.Fatal("notNow holds every failure; the premise of this test is gone") + } +} diff --git a/cmd/mesh-controller/nodes.go b/cmd/mesh-controller/nodes.go index c446da3d..a297e7b3 100644 --- a/cmd/mesh-controller/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -929,6 +929,12 @@ func showNode(ctx context.Context, stored *stores, name string) error { } if len(assigned) > 0 { fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", ")) + // And what each runs, as its assignment names it (novox/hq ADR 0408, issue 1095). + if on, err := inv.AssignmentsOn(ctx, name); err == nil { + for _, line := range runLines(on) { + fmt.Println(line) + } + } } // And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same // reason its condition says. Not computable is said, never read as "all applied". diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 3008cc42..2ae089e6 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -172,6 +172,11 @@ func serve(ctx context.Context) (err error) { return err } defer server.Close() + // The chain on the bus, and the move of what the tables still hold into it (novox/hq ADR 0408 §8): before anything + // reads builds, rollouts or assignments. + if err := attachTheChain(ctx, inv, server); err != nil { + return err + } // The bus's own objects, asserted on every start. **Not created once at genesis**: a stream // somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was diff --git a/cmd/mesh-controller/record_mirrors.go b/cmd/mesh-controller/record_mirrors.go index 6889d79e..970956e8 100644 --- a/cmd/mesh-controller/record_mirrors.go +++ b/cmd/mesh-controller/record_mirrors.go @@ -2,12 +2,15 @@ package main import ( "context" + "encoding/json" "fmt" + "sort" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/recordmirror" ) @@ -41,7 +44,7 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats. return e } sources := map[string]func(context.Context) (map[string][]byte, error){ - recordmirror.Assignments.Name: inv.AssignmentsMirror, + recordmirror.Assignments.Name: inv.AssignmentsMirrorToWrite, recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror, recordmirror.Predefined.Name: inv.PredefinedMirror, } @@ -49,10 +52,18 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats. failing := map[string]string{} tick := time.NewTicker(mirrorEvery) defer tick.Stop() + events := &assignmentEvents{} for { if holds() { + // The assignments, once they are keys of the bus, are written by the verbs that change them (novox/hq ADR + // 0408): this tick keeps the provisioner's plan in their values and says what changed; the mirror of the store + // writes them only until the move. + onTheBus := keepingAssignments(ctx, inv, announce, epoch, events, failing) for _, m := range recordmirror.Mirrors() { - why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds) + if m.Name == recordmirror.Assignments.Name && onTheBus { + continue + } + why := mirrorOnce(ctx, inv, api, m, writers, sources[m.Name], announce, epoch, holds) if why != failing[m.Name] { if why != "" { fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why) @@ -72,7 +83,7 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats. } // mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could. -func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer, +func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer, source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64, holds func() bool) string { reading, cancel := context.WithTimeout(ctx, 30*time.Second) @@ -84,6 +95,9 @@ func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mir return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err) } w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds} + if m.Name == recordmirror.Assignments.Name { + w.Holds = assignmentsHolds(inv, holds) + } writers[m.Name] = w } want, err := source(reading) @@ -104,3 +118,88 @@ func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mir } return "" } + +// assignmentEvents is the assignments as the bucket held them at the last look, to say what changed since. +type assignmentEvents struct { + known map[string]inventory.KeptValue +} + +// keepingAssignments is the tick's work for the assignments once they are on the bus: the provisioner's plan written into +// the values it belongs to, and one `assignment-changed` event for each key that is new, changed or ended since the last +// look — whoever wrote it, a verb in this process or a command in another. False when they are not on the bus yet. +func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce recordmirror.Announcer, epoch func() uint64, + events *assignmentEvents, failing map[string]string) bool { + reading, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + const name = "assignments" + if _, on, err := inv.KeptAssignments(reading); err != nil || !on { + if err != nil { + // Not known to be off the bus: the store's mirror is not written over what the bus may hold. + if failing[name] == "" { + failing[name] = err.Error() + fmt.Printf("the assignments cannot be read from the bus: %v\n", err) + } + return true + } + return false + } + if _, _, err := inv.KeepAssignmentPlans(reading); err != nil { + if failing[name] != err.Error() { + failing[name] = err.Error() + fmt.Printf("the provisioner's plans are not kept in the assignments: %v\n", err) + } + } else if failing[name] != "" { + fmt.Printf("the provisioner's plans are kept in the assignments again\n") + failing[name] = "" + } + now, _, err := inv.KeptAssignments(reading) + if err != nil { + return true + } + if events.known != nil { + keys := make([]string, 0, len(now)+len(events.known)) + for key := range now { + keys = append(keys, key) + } + for key := range events.known { + if _, still := now[key]; !still { + keys = append(keys, key) + } + } + sort.Strings(keys) + for _, key := range keys { + was, had := events.known[key] + is, has := now[key] + e := recordmirror.Event{Mirror: recordmirror.Assignments.Name, Key: key, Revision: is.Revision, Epoch: epoch(), + At: time.Now().UTC()} + switch { + case has && !had: + e.Change = recordmirror.Added + case has && had && was.Revision != is.Revision: + e.Change, e.Fields = recordmirror.Changed, recordmirror.ChangedFields(was.Value, is.Value) + case !has && had: + e.Change, e.Revision = recordmirror.Removed, 0 + default: + continue + } + body, err := json.Marshal(e) + if err != nil { + continue + } + for try := 0; try < 2; try++ { + if err = announce.PublishSeatEvent(reading, broker.ControllerSeat, recordmirror.Assignments.Event, body); err == nil { + break + } + } + } + } + events.known = now + return true +} + +// assignmentsHolds is the assignments mirror's leave to write: the lease's, and the assignments neither being switched to +// the bus nor on it. Asked by the writer before every write and purge, so a pass that read its source before the switch +// writes nothing after it (ADR 0408 §4). +func assignmentsHolds(inv *inventory.Inventory, holds func() bool) func() bool { + return func() bool { return holds() && inv.AssignmentsMirrorMayWrite() } +} diff --git a/cmd/mesh-controller/record_mirrors_switch_test.go b/cmd/mesh-controller/record_mirrors_switch_test.go new file mode 100644 index 00000000..88ed29a1 --- /dev/null +++ b/cmd/mesh-controller/record_mirrors_switch_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "errors" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/recordmirror" +) + +// **A writer of the store's mirror of the assignments, made before the assignments are switched to the bus, writes and +// purges nothing after** (ADR 0408 §4, issue 1093): its leave to write is asked before every write, and false once they are +// switching or moved. +func TestAMirrorWriterMadeBeforeTheSwitchWritesAndPurgesNothingAfter(t *testing.T) { + inv := inventory.NewForTest() + w := &recordmirror.Writer{Mirror: recordmirror.Assignments, Holds: assignmentsHolds(inv, func() bool { return true })} + if !w.Holds() { + t.Fatal("the writer may not write before the switch") + } + inv.BeginSwitchForTest(inventory.KindAssignments) + if w.Holds() { + t.Fatal("the writer may write while the assignments are switching") + } + // Refused before it reads the bucket (it has none here) or lays or ends a key. + r, err := w.Sync(t.Context(), map[string][]byte{"anchor.notes": []byte(`{}`)}) + if !errors.Is(err, recordmirror.ErrNotHeld) || r.Written != 0 || r.Removed != 0 { + t.Fatalf("the writer wrote after the switch: %+v %v", r, err) + } +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 5b5d1590..a2a28ce1 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -16,6 +16,7 @@ import ( "github.com/novox/mesh-controller/internal/builder" "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/chain" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -1054,7 +1055,11 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, } } sort.Strings(machines) - sent, err := sendRollout(withScope(ctx, scope), open, machines) + was := map[string]map[string]chain.Ref{} + for _, n := range machines { + was[n] = ranBefore(ctx, inv, n) + } + sent, err := sendRollout(withScope(sendingAs(ctx, p), scope), open, machines) if err != nil { return false, fmt.Errorf("sending %s to %s after tier %d: %w", strings.Join(rest, ", "), strings.Join(machines, ", "), p.Tier, err) @@ -1076,6 +1081,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, } } noteSends(ctx, p.Tier, []string{n}, carried, now) + noteChanges(p, n, carried, was[n], now) } fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "), strings.Join(sent, ", ")) @@ -1324,6 +1330,10 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string s := p.Modules[m] owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build}) } + // What the assignments name before the send, for the rollout to say what it changed; and the send is made as this + // rollout, so what is recorded of it names it (novox/hq ADR 0408, issue 1095). + was := ranBefore(ctx, inv, node) + ctx = sendingAs(ctx, p) // A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges // all of it there. carried, sent, grants, err := gatedSend(ctx, open, node, owns) @@ -1350,6 +1360,9 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string } } noteSends(ctx, p.Tier, sent, modules, now) + for _, n := range sent { + noteChanges(p, n, modules, was, now) + } p.State = inventory.PlanRolling p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "), strings.Join(sent, ", ")) @@ -2016,6 +2029,10 @@ func plansCommand(ctx context.Context, args []string) error { } } } + // The assignments it changed, each with the build and commit before and after (novox/hq ADR 0408, issue 1095). + for _, line := range changedLines(p) { + fmt.Println(line) + } return nil } if *whatIf != "" { diff --git a/cmd/mesh-controller/stores.go b/cmd/mesh-controller/stores.go index 5fef54d7..ff727dbc 100644 --- a/cmd/mesh-controller/stores.go +++ b/cmd/mesh-controller/stores.go @@ -132,6 +132,9 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) { } // And where config values are read once the move of config has run (issue 664): mesh-config_resolved. inv.ReadsConfigFrom(readConfigValues) + // And the chain, opened when a kind that moved to the bus is first read (novox/hq ADR 0408): a command that reads none + // has no use of a connection. + inv.UseChainOpener(openTheChain) return inv, nil } diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index ea9ec498..94d25d4d 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -303,6 +303,16 @@ func notNow(err error) error { return err } +// unsaved marks the failure to save the plan a merge was added to, whatever it was: the merge is carried until that save +// writes it (inventory.AddMerge), so the announcement is held and handed over again, never taken (novox/hq ADR 0407 +// decision 3, ADR 0408, issue 1092). The bus acknowledges it only after the save succeeded. +func unsaved(err error) error { + if err == nil { + return nil + } + return fmt.Errorf("%w: the merge is not kept until its plan is saved: %w", link.ErrTryAgain, err) +} + // SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which // becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go. func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error { diff --git a/internal/broker/controller_buckets.go b/internal/broker/controller_buckets.go index 088f1edf..1535ecb3 100644 --- a/internal/broker/controller_buckets.go +++ b/internal/broker/controller_buckets.go @@ -92,9 +92,47 @@ const ( MirrorPredefined = "predefined" ) -// ControllerMirrors are the local names of the controller's record mirrors, which a module may read. +// The chain's buckets (novox/hq ADR 0408, issue 1089): builds, rollouts and assignments are one chain kept on the bus +// alone, one bucket each, one key per record, written by the lease holder by compare-and-set. The assignments bucket is +// AssignmentsBucket above, which keeps its name and its readers (ADR 0408 §3). Nothing is removed from them (§4), they keep +// ChainHistory earlier revisions of every key (§5), and a value is never cut (§6). +var ( + BuildsBucket = BucketName(ControllerSeat, ChainBuilds) + RolloutsBucket = BucketName(ControllerSeat, ChainRollouts) +) + +// **The assignments bucket keeps the marker age (MirrorMarkerTTL) it was made with as a mirror.** A stream that allows +// messages an age cannot be made to disallow them (the server refuses the update: "message TTL status can not be +// disabled", nats-server stream.go), so asserting the bucket without the age would fail the controller's start on a +// mesh that has the bucket. The chain never purges or removes a key (ADR 0408 §4), so no marker is ever written and the +// age ends nothing; it is kept only so the stream's configuration stays one the server accepts. +// +// The chain buckets' local names, as a reader names them in `reads` after `mesh-controller.`. +const ( + ChainBuilds = "builds" + ChainRollouts = "rollouts" +) + +// The chain buckets' bounds. ChainHistory is the bus's largest history (ADR 0408 §5). A value is at most ChainValueBytes, +// just under the server's largest message of a mebibyte, which also carries the subject and the headers (ADR 0408 §6): +// the controller writes none larger and the move refuses to. The buckets' size is the arithmetic of the survey of +// issue 981: 8585 builds of 1 to 40 KB, 1049 rollouts of about 2 KB a revision and 64 revisions, so 512 MiB for the two +// that grow, and the assignments, 285 of under a kilobyte, 64 MiB. +const ( + ChainHistory = 64 + ChainValueBytes = 1<<20 - 8<<10 + ChainBuildsBytes = 512 << 20 + ChainRolloutsBytes = 512 << 20 + ChainAssignmentBytes = 64 << 20 +) + +// ChainBuckets are the three buckets of the chain: builds, rollouts and assignments. +func ChainBuckets() []string { return []string{BuildsBucket, RolloutsBucket, AssignmentsBucket} } + +// ControllerMirrors are the local names of the controller's buckets a module may read: the record mirrors and the +// chain's buckets. func ControllerMirrors() []string { - return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined} + return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts} } // The mirrors' caps (design 54 §5): one value per key, no age, 64 KiB a value, 64 MiB a bucket. @@ -151,14 +189,14 @@ func IsControllerBucket(bucket string) bool { bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket || bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket || bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket || - bucket == DeliveryMergesBucket + bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket } // ControllerBuckets are the controller's own buckets, in the order they are asserted. func ControllerBuckets() []string { return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket, AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket, - DeliveryMergesBucket} + DeliveryMergesBucket, BuildsBucket, RolloutsBucket} } // ControllerBucketsAsserter is what raising the controller's buckets needs of a connection. @@ -315,11 +353,36 @@ func (j *JetStream) EnsureControllerBuckets() error { }); err != nil { return fmt.Errorf("asserting bucket %s: %w", DeliveryMergesBucket, err) } + // **The chain: no age, no removal, history 64, no cut** (ADR 0408 §4 to §6). A key is never removed, so no purge + // marker needs an age and a key that expired would be a record the bus forgot. + for _, c := range []struct { + bucket, why string + bytes int64 + }{ + {BuildsBucket, "every build, one key each by its id, with its repository, commit, reuse key and what it " + + "published (novox/hq ADR 0408, design 47): written by the controller alone, never removed, read through `reads`", + ChainBuildsBytes}, + {RolloutsBucket, "every rollout, one key each by its id: the commit it fixed per repository, its tiers, the " + + "builds it carried and the assignments it changed (novox/hq ADR 0408, design 47): written by the " + + "controller alone, never removed, read through `reads`", ChainRolloutsBytes}, + } { + if _, err := js.CreateOrUpdateKeyValue(ctx, chainBucket(c.bucket, c.why, c.bytes, 0)); err != nil { + return fmt.Errorf("asserting bucket %s: %w", c.bucket, err) + } + } + // **The assignments keep the marker age they had as a mirror.** A stream that allows messages an age cannot be + // made to disallow them (the server refuses: "message TTL status can not be disabled"), so the bucket that was + // the assignments mirror is asserted with the age it was made with. Nothing is purged from it, so no marker is ever + // written and the age ends nothing. + if _, err := js.CreateOrUpdateKeyValue(ctx, chainBucket(AssignmentsBucket, + "which module is assigned to which node, keyed ., with the build, commit and rollout it runs "+ + "there (novox/hq ADR 0408, design 47): written by the controller alone, never removed, read through `reads`", + ChainAssignmentBytes, MirrorMarkerTTL)); err != nil { + return fmt.Errorf("asserting bucket %s: %w", AssignmentsBucket, err) + } // **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose // marker lives a day; a key that expired would be a record the bus forgot while the store still holds it. for _, m := range []struct{ bucket, why string }{ - {AssignmentsBucket, "which module is assigned to which node, keyed . (novox/hq ADR 0289, " + - "design 54): written by the controller alone from its store when an assignment changes, read through `reads`"}, {DeclaredConfigBucket, "each registered module's config declarations as its manifest holds them, keyed by " + "the module (novox/hq ADR 0348, design 57 F3): written by the controller alone from its store when a " + "registration changes them, read through `reads`"}, @@ -342,6 +405,20 @@ func (j *JetStream) EnsureControllerBuckets() error { return nil } +// chainBucket is the options of a bucket of the chain: sixty-four revisions of each key, no age, a value up to +// ChainValueBytes, on disk. +func chainBucket(bucket, why string, bytes int64, markers time.Duration) jetstream.KeyValueConfig { + return jetstream.KeyValueConfig{ + Bucket: bucket, + Description: why, + History: ChainHistory, + MaxValueSize: ChainValueBytes, + MaxBytes: bytes, + Storage: jetstream.FileStorage, + LimitMarkerTTL: markers, + } +} + // EnsureLeaseBucket creates the lease's bucket if absent and brings its options to match (to-be 45 // §6). **Before the lease is taken, by any candidate**: it is the lease's own precondition, and asserting // a bucket that exists changes nothing. File storage, so its revisions — the epochs — outlive a restart of diff --git a/internal/broker/controller_buckets_integration_test.go b/internal/broker/controller_buckets_integration_test.go index 106bb3b9..9f6e8d5d 100644 --- a/internal/broker/controller_buckets_integration_test.go +++ b/internal/broker/controller_buckets_integration_test.go @@ -259,3 +259,52 @@ func TestAMergeTheOldConsumerHadNotAcknowledgedIsTheMergesConsumers(t *testing.T t.Fatalf("the merges consumer holds %v (%v); want the merge the old consumer never acknowledged", merges, err) } } + +// **The chain's buckets keep sixty-four revisions, the assignments' that were a mirror's included** (ADR 0408 §3 to §5, +// issue 1089): a bucket made as a mirror — one revision, a marker age — is brought to the chain's options without the +// server refusing the update, and a key written three times keeps three values. +func TestTheChainBucketsKeepSixtyFourRevisionsOnTheBus(t *testing.T) { + js, err := Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer js.Close() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + // The bucket as the mirror made it. + if _, err := api.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: AssignmentsBucket, History: 1, + MaxValueSize: MirrorValueBytes, MaxBytes: MirrorBytes, Storage: jetstream.FileStorage, + LimitMarkerTTL: MirrorMarkerTTL}); err != nil { + t.Fatal(err) + } + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatalf("a mirror's bucket is not brought to the chain's options: %v", err) + } + for _, name := range ChainBuckets() { + kv, err := api.KeyValue(ctx, name) + if err != nil { + t.Fatal(err) + } + status, err := kv.Status(ctx) + if err != nil { + t.Fatal(err) + } + cfg := status.(*jetstream.KeyValueBucketStatus).StreamInfo().Config + if status.History() != ChainHistory || status.TTL() != 0 || cfg.MaxMsgSize < 0 || cfg.MaxMsgSize != ChainValueBytes { + t.Errorf("%s keeps %d revisions, age %s, a value of %d bytes", name, status.History(), status.TTL(), cfg.MaxMsgSize) + } + for _, value := range []string{"one", "two", "three"} { + if _, err := kv.Put(ctx, "k", []byte(value)); err != nil { + t.Fatal(err) + } + } + history, err := kv.History(ctx, "k") + if err != nil || len(history) != 3 { + t.Errorf("%s keeps %d values of a key written three times: %v", name, len(history), err) + } + } +} diff --git a/internal/broker/controller_buckets_test.go b/internal/broker/controller_buckets_test.go index 7b9d9f9b..1db7465c 100644 --- a/internal/broker/controller_buckets_test.go +++ b/internal/broker/controller_buckets_test.go @@ -4,6 +4,8 @@ import ( "slices" "strings" "testing" + + "github.com/nats-io/nats.go/jetstream" ) // **The controller may write every bucket it writes** (novox/hq to-be 45 §1, issue 269). Writing a @@ -118,7 +120,7 @@ func TestTheMirrorsListAndTheWritersTableAgree(t *testing.T) { module := Principal{Kind: KindModule, Module: ControllerSeat, Node: "one"} rowed := map[string]bool{} for _, row := range WritersTable { - if !strings.HasSuffix(row.State, " mirror") { + if !strings.HasSuffix(row.State, " mirror") && !strings.HasSuffix(row.State, " of the chain") { continue } for _, subject := range row.Subjects { @@ -254,3 +256,52 @@ func TestTheControllerMayAskMeshConfigInItsOwnName(t *testing.T) { // **Delivering this change loses no merge** (review of issue 973): a merge published under the old filter of the // `controller` consumer and never acknowledged by it is delivered by the merges consumer once the assertion has // made it, because it starts where `controller` has acknowledged up to, not from now. + +// **The chain's buckets keep sixty-four revisions and no age** (ADR 0408 §4 to §6, issue 1089): the bus's largest +// history, no age that could forget a key, a value up to just under the server's largest message, on disk. +func TestTheChainBucketsKeepSixtyFourRevisionsAndNoAge(t *testing.T) { + if want := []string{"mesh-controller_builds", "mesh-controller_rollouts", "mesh-controller_assignments"}; !slices.Equal(ChainBuckets(), want) { + t.Fatalf("the chain's buckets are %v, not %v", ChainBuckets(), want) + } + for _, bucket := range ChainBuckets() { + cfg := chainBucket(bucket, "why", ChainBuildsBytes, 0) + if cfg.History != 64 || cfg.TTL != 0 || cfg.Storage != jetstream.FileStorage { + t.Errorf("%s keeps %d revisions, age %v, on %v", bucket, cfg.History, cfg.TTL, cfg.Storage) + } + if cfg.MaxValueSize <= MirrorValueBytes || cfg.MaxValueSize >= 1<<20 { + t.Errorf("%s takes a value of %d bytes, not just under a mebibyte", bucket, cfg.MaxValueSize) + } + if cfg.Replicas > 1 { + t.Errorf("%s is replicated %d times", bucket, cfg.Replicas) + } + } +} + +// **Every bucket of the chain is the controller's, listed for a reader, and has a writer row** (ADR 0408 §3). +func TestEveryChainBucketIsOnTheControllersListAndHasAWriterRow(t *testing.T) { + controller := Principal{Kind: KindController} + module := Principal{Kind: KindModule, Module: ControllerSeat, Node: "one"} + for _, bucket := range ChainBuckets() { + if !IsControllerBucket(bucket) || !slices.Contains(ControllerBuckets(), bucket) { + t.Errorf("%s is not on the controller's list of buckets", bucket) + } + local := strings.TrimPrefix(bucket, ControllerSeat+"_") + if !slices.Contains(ControllerMirrors(), local) { + t.Errorf("a module cannot name %s in `reads`", bucket) + } + rows := 0 + for _, row := range WritersTable { + for _, subject := range row.Subjects { + if subject == "$KV."+bucket+".>" { + rows++ + if row.Writes == nil || !row.Writes(controller, subject) || row.Writes(module, subject) { + t.Errorf("%q is not written by the controller alone", row.State) + } + } + } + } + if rows != 1 { + t.Errorf("%s has %d writer rows", bucket, rows) + } + } +} diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 07d30883..9408ee8a 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -403,6 +403,16 @@ var DegradableGrants = []DegradableGrant{ Reason: "refused: the controller then keeps no merge mesh-delivery announced, so the build of a moved module is " + "refused as it is without the operator's Approve — failing closed (ADR 0420 §7, issue 1025)", }, + { + Subject: "$KV." + BuildsBucket + ".>", + Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " + + "first push, and writes the chain only after that, holding the lease (ADR 0408 §2 and §8, issue 1089)", + }, + { + Subject: "$KV." + RolloutsBucket + ".>", + Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " + + "first push, and writes the chain only after that, holding the lease (ADR 0408 §2 and §8, issue 1089)", + }, { Subject: "$KV." + RolloutSilentBucket + ".>", Reason: "refused: the controller looks at the plans when a wait for a machine's report is up, by a timer of its own " + diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index d9e24d27..fc21c54a 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/broker/writers.go b/internal/broker/writers.go index bf7a5960..2520359d 100644 --- a/internal/broker/writers.go +++ b/internal/broker/writers.go @@ -193,9 +193,18 @@ var WritersTable = []WriterRow{ // The record mirrors (novox/hq ADR 0289 §9, design 54 §2): rows of this table, so a grant that lets // anything else write them is refused at composition. Built so far: the assignments and each registered // module's declared config (design 57 F3), and the list of predefined keys kept (design 57 K3). - {State: "the assignments mirror", Writer: "controller (lease holder), from its store, only on a change", - KeptIn: "key-value " + AssignmentsBucket, Others: "read through `reads` as mesh-controller." + MirrorAssignments, + {State: "the assignments mirror", Writer: "controller (lease holder): the verbs that assign, and the sends that record what runs; " + + "a mirror of its store only until the assignments are moved to the bus (novox/hq ADR 0408)", + KeptIn: "key-value " + AssignmentsBucket + ", the record once moved", Others: "read through `reads` as mesh-controller." + MirrorAssignments, Subjects: kvOf(AssignmentsBucket), Writes: isController}, + // The chain (novox/hq ADR 0408): builds and rollouts beside the assignments above, which keep their bucket. Not + // mirrors of the store but the record, written by the lease holder by compare-and-set and removed by nobody. + {State: "the builds of the chain", Writer: "controller (lease holder), compare-and-set on the key's revision", + KeptIn: "key-value " + BuildsBucket + ", sixty-four revisions of each key, none removed", + Others: "read through `reads` as mesh-controller." + ChainBuilds, Subjects: kvOf(BuildsBucket), Writes: isController}, + {State: "the rollouts of the chain", Writer: "controller (lease holder), compare-and-set on the key's revision", + KeptIn: "key-value " + RolloutsBucket + ", sixty-four revisions of each key, none removed", + Others: "read through `reads` as mesh-controller." + ChainRollouts, Subjects: kvOf(RolloutsBucket), Writes: isController}, {State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change", KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig, Subjects: kvOf(DeclaredConfigBucket), Writes: isController}, diff --git a/internal/broker/writers_test.go b/internal/broker/writers_test.go index 6f929662..4ac55259 100644 --- a/internal/broker/writers_test.go +++ b/internal/broker/writers_test.go @@ -38,6 +38,9 @@ var designRows = []string{ "a message a consumer gave up on", // The record mirrors, rows of to-be 45 §1's table by design 54 §2 (ADR 0289 §9). "the assignments mirror", + // The chain beside the assignments, the record itself and no mirror of one (ADR 0408, design 47, issue 1089). + "the builds of the chain", + "the rollouts of the chain", "the declared-config mirror", "the predefined-keys mirror", // The provisioner client's word on what it placed (novox/hq ADR 0349 §4, design 57 P6). diff --git a/internal/catalogue/state.go b/internal/catalogue/state.go index b36d9564..1d08e229 100644 --- a/internal/catalogue/state.go +++ b/internal/catalogue/state.go @@ -130,7 +130,7 @@ func StateProblems(m Manifest) []string { // controller's account alone (the writers table); declared as a module's state, they would be asserted a // second way and granted to the module's own principal as well. Named here so a read of one is a read of // kept state. The bus's list is broker.ControllerMirrors; a test keeps the two agreeing. -var ControllerMirrors = []string{"assignments", "declared-config", "predefined"} +var ControllerMirrors = []string{"assignments", "declared-config", "predefined", "builds", "rollouts"} // StateReadsNothingDeclares is every read across a catalogue whose owner is present and declares no // such state. An absent owner says nothing — a module may be installed long before the one whose diff --git a/internal/chain/backend.go b/internal/chain/backend.go new file mode 100644 index 00000000..b99f3673 --- /dev/null +++ b/internal/chain/backend.go @@ -0,0 +1,275 @@ +package chain + +import ( + "context" + "errors" + "sort" + "sync" + "time" +) + +// Entry is one key's value at a revision. Caught is set on the entry a watch sends once, after the values the bucket +// held when it began, and carries no key. +type Entry struct { + Key string + // Deleted marks the end of a key (Remove); it holds no value. + Deleted bool + Value []byte + Revision uint64 + At time.Time + Caught bool +} + +// Backend is one bucket of the chain. **It has no delete and no purge**: no key of the chain is ever removed (ADR +// 0408 §4), so nothing that holds a Backend can remove one. +type Backend interface { + // Get is one key's value and revision; false when it holds none. + Get(ctx context.Context, key string) (Entry, bool, error) + // Create writes a key that holds nothing and fails with ErrExists when it holds something. + Create(ctx context.Context, key string, value []byte) (uint64, error) + // Update writes a key at the revision it was read at and fails with ErrMoved when it moved. + Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) + // Keys is every key. An error is an error: never an empty bucket. + Keys(ctx context.Context) ([]string, error) + // Watch sends every key's current value, then a Caught entry, then each later write, until the context ends. + Watch(ctx context.Context) (<-chan Entry, error) + // History is a key's earlier values and its current one, oldest first, at most the bucket keeps. + History(ctx context.Context, key string) ([]Entry, error) +} + +var ( + // ErrMoved is a compare-and-set that lost: somebody wrote the key since it was read. + ErrMoved = errors.New("the key was written by somebody else since it was read") + // ErrExists is a create of a key that holds a value. + ErrExists = errors.New("the key already holds a value") + // ErrNotHeld is a write refused because this controller does not hold the lease. + ErrNotHeld = errors.New("this controller does not hold the lease, so it writes nothing of the chain") + // ErrTooLarge is a value over the largest the bus holds: never cut (ADR 0408 §6), so refused. + ErrTooLarge = errors.New("the value is larger than the bus holds, and a value of the chain is never cut") +) + +// Memory is a Backend held in this process, for tests and for nothing else. It keeps what the bucket keeps: every +// earlier value, up to History of them, and a revision that rises with every write of the bucket. +type Memory struct { + // Keeps is how many values of a key it keeps; zero keeps sixty-four. + Keeps int + + mu sync.Mutex + values map[string][]Entry + revision uint64 + watchers []*watcher + // Now is the clock the entries are dated by; nil is time.Now. + Now func() time.Time + // writes counts the creates and updates taken. + writes int + // fail is what every call answers, when set. + fail error +} + +// NewMemory is an empty bucket. +func NewMemory() *Memory { return &Memory{values: map[string][]Entry{}} } + +// SetFail makes every call answer err from now on, or none when err is nil: a bus that is away. +func (m *Memory) SetFail(err error) { + m.mu.Lock() + defer m.mu.Unlock() + m.fail = err +} + +// Writes is how many writes the bucket took. +func (m *Memory) Writes() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.writes +} + +func (m *Memory) keep() int { + if m.Keeps > 0 { + return m.Keeps + } + return 64 +} + +func (m *Memory) now() time.Time { + if m.Now != nil { + return m.Now() + } + return time.Now() +} + +// last is a key's newest entry, false when it holds none. +func (m *Memory) last(key string) (Entry, bool) { + h := m.values[key] + if len(h) == 0 { + return Entry{}, false + } + return h[len(h)-1], true +} + +func (m *Memory) Get(_ context.Context, key string) (Entry, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return Entry{}, false, m.fail + } + e, ok := m.last(key) + if !ok { + return Entry{}, false, nil + } + return clone(e), true, nil +} + +func (m *Memory) Create(_ context.Context, key string, value []byte) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return 0, m.fail + } + if _, held := m.last(key); held { + return 0, ErrExists + } + return m.write(key, value), nil +} + +func (m *Memory) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return 0, m.fail + } + e, held := m.last(key) + if !held || e.Revision != revision { + return 0, ErrMoved + } + return m.write(key, value), nil +} + +// write is called with the lock held. +func (m *Memory) write(key string, value []byte) uint64 { + m.revision++ + m.writes++ + e := Entry{Key: key, Value: append([]byte(nil), value...), Revision: m.revision, At: m.now()} + h := append(m.values[key], e) + if len(h) > m.keep() { + h = h[len(h)-m.keep():] + } + m.values[key] = h + for _, w := range m.watchers { + w.push(clone(e)) + } + return e.Revision +} + +func (m *Memory) Keys(_ context.Context) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return nil, m.fail + } + out := make([]string, 0, len(m.values)) + for k := range m.values { + if _, held := m.last(k); held { + out = append(out, k) + } + } + sort.Strings(out) + return out, nil +} + +// watcher is one watch of a Memory: an unbounded queue, so a watch that is slow is behind and never misses a write. +type watcher struct { + mu sync.Mutex + q []Entry + wake chan struct{} +} + +func (w *watcher) push(e Entry) { + w.mu.Lock() + w.q = append(w.q, e) + w.mu.Unlock() + select { + case w.wake <- struct{}{}: + default: + } +} + +func (w *watcher) pop() (Entry, bool) { + w.mu.Lock() + defer w.mu.Unlock() + if len(w.q) == 0 { + return Entry{}, false + } + e := w.q[0] + w.q = w.q[1:] + return e, true +} + +func (m *Memory) Watch(ctx context.Context) (<-chan Entry, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return nil, m.fail + } + w := &watcher{wake: make(chan struct{}, 1)} + keys := make([]string, 0, len(m.values)) + for k := range m.values { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if e, held := m.last(k); held { + w.q = append(w.q, clone(e)) + } + } + w.q = append(w.q, Entry{Caught: true}) + m.watchers = append(m.watchers, w) + out := make(chan Entry) + go func() { + defer close(out) + defer func() { + m.mu.Lock() + for i, x := range m.watchers { + if x == w { + m.watchers = append(m.watchers[:i], m.watchers[i+1:]...) + break + } + } + m.mu.Unlock() + }() + for { + e, ok := w.pop() + if !ok { + select { + case <-ctx.Done(): + return + case <-w.wake: + } + continue + } + select { + case out <- e: + case <-ctx.Done(): + return + } + } + }() + return out, nil +} + +func (m *Memory) History(_ context.Context, key string) ([]Entry, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return nil, m.fail + } + out := make([]Entry, 0, len(m.values[key])) + for _, e := range m.values[key] { + out = append(out, clone(e)) + } + return out, nil +} + +func clone(e Entry) Entry { + e.Value = append([]byte(nil), e.Value...) + return e +} diff --git a/internal/chain/bus.go b/internal/chain/bus.go new file mode 100644 index 00000000..c94f6373 --- /dev/null +++ b/internal/chain/bus.go @@ -0,0 +1,135 @@ +package chain + +import ( + "context" + "errors" + "fmt" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" +) + +// OnTheBus opens the chain's three buckets on a connection. The controller asserts them at its start +// (broker.EnsureControllerBuckets), so one older than this has not, and the error says so. +func OnTheBus(ctx context.Context, conn *nats.Conn, holds Holds) (*Chain, error) { + api, err := jetstream.New(conn) + if err != nil { + return nil, err + } + open := func(bucket string) (Backend, error) { + kv, err := api.KeyValue(ctx, bucket) + if err != nil { + return nil, fmt.Errorf("the bucket %s is not on the bus — the controller asserts it at its start, so one "+ + "older than this has not: %w", bucket, err) + } + return busBucket{kv}, nil + } + builds, err := open(broker.BuildsBucket) + if err != nil { + return nil, err + } + rollouts, err := open(broker.RolloutsBucket) + if err != nil { + return nil, err + } + assignments, err := open(broker.AssignmentsBucket) + if err != nil { + return nil, err + } + return New(builds, rollouts, assignments, holds), nil +} + +// busBucket is a Backend on a key-value bucket. It calls no delete and no purge: the type has none to call (ADR 0408 §4). +type busBucket struct{ kv jetstream.KeyValue } + +func entryOf(e jetstream.KeyValueEntry) Entry { + return Entry{Key: e.Key(), Value: e.Value(), Revision: e.Revision(), At: e.Created()} +} + +func (b busBucket) Get(ctx context.Context, key string) (Entry, bool, error) { + e, err := b.kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return Entry{}, false, nil + } + if err != nil { + return Entry{}, false, err + } + return entryOf(e), true, nil +} + +func (b busBucket) Create(ctx context.Context, key string, value []byte) (uint64, error) { + rev, err := b.kv.Create(ctx, key, value) + if errors.Is(err, jetstream.ErrKeyExists) { + return 0, ErrExists + } + return rev, err +} + +func (b busBucket) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { + rev, err := b.kv.Update(ctx, key, value, revision) + var apiErr *jetstream.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence { + return 0, ErrMoved + } + return rev, err +} + +func (b busBucket) Keys(ctx context.Context) ([]string, error) { + keys, err := b.kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +func (b busBucket) Watch(ctx context.Context) (<-chan Entry, error) { + w, err := b.kv.WatchAll(ctx, jetstream.IgnoreDeletes()) + if err != nil { + return nil, err + } + out := make(chan Entry) + go func() { + defer close(out) + defer func() { _ = w.Stop() }() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + return + } + v := Entry{Caught: true} + if e != nil { + v = entryOf(e) + } + select { + case out <- v: + case <-ctx.Done(): + return + } + } + } + }() + return out, nil +} + +func (b busBucket) History(ctx context.Context, key string) ([]Entry, error) { + all, err := b.kv.History(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + out := make([]Entry, 0, len(all)) + for _, e := range all { + if e.Operation() != jetstream.KeyValuePut { + continue // a marker an earlier mirror left; nothing of the chain makes one + } + out = append(out, entryOf(e)) + } + return out, nil +} diff --git a/internal/chain/chain.go b/internal/chain/chain.go new file mode 100644 index 00000000..bb3f6862 --- /dev/null +++ b/internal/chain/chain.go @@ -0,0 +1,210 @@ +package chain + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "strings" + + "github.com/novox/mesh-controller/internal/broker" +) + +// Bucket is one kind of record of the chain, typed. Reads need nothing; **every write asks first whether this +// controller holds the lease** and writes nothing when it does not (ADR 0408 §2). +type Bucket[T any] struct { + b Backend + holds Holds +} + +// Holds says whether this controller may write now: nil when it holds the lease, and why not when it does not. The +// serving controller answers from its lease, and a command from the holder's (the epoch gate). +type Holds func(ctx context.Context) error + +// Kept is a record as the bucket holds it: with its revision, the one a write must find. +type Kept[T any] struct { + Key string + Value T + Revision uint64 +} + +// validKey is what the bus takes as a key: letters, digits and - _ / = . with no dot at either end. +var validKey = regexp.MustCompile(`^[-/_=.a-zA-Z0-9]+$`) + +// ErrBadKey is a key the bus does not take: refused loudly, never rewritten into another. +var ErrBadKey = errors.New("the bus takes no key of that spelling") + +// ValidKey says the bus takes the key: letters, digits and - _ / = . with no dot at either end. +func ValidKey(key string) bool { + return validKey.MatchString(key) && !strings.HasPrefix(key, ".") && !strings.HasSuffix(key, ".") +} + +func (s Bucket[T]) guard(ctx context.Context, key string, value []byte) error { + if s.holds == nil { + return ErrNotHeld + } + if err := s.holds(ctx); err != nil { + return fmt.Errorf("%w: %v", ErrNotHeld, err) + } + if !ValidKey(key) { + return fmt.Errorf("%w: %q", ErrBadKey, key) + } + if len(value) > broker.ChainValueBytes { + return fmt.Errorf("%w: %d bytes, the most is %d", ErrTooLarge, len(value), broker.ChainValueBytes) + } + return nil +} + +func decode[T any](e Entry) (Kept[T], error) { + var v T + if err := json.Unmarshal(e.Value, &v); err != nil { + return Kept[T]{}, fmt.Errorf("the value of %s cannot be read: %w", e.Key, err) + } + return Kept[T]{Key: e.Key, Value: v, Revision: e.Revision}, nil +} + +// Get is one record; false when the key holds none. +func (s Bucket[T]) Get(ctx context.Context, key string) (Kept[T], bool, error) { + e, ok, err := s.b.Get(ctx, key) + if err != nil || !ok { + return Kept[T]{}, false, err + } + e.Key = key + k, err := decode[T](e) + return k, err == nil, err +} + +// Create writes a key that holds nothing: ErrExists when it holds something. It returns the revision. +func (s Bucket[T]) Create(ctx context.Context, key string, v T) (uint64, error) { + body, err := json.Marshal(v) + if err != nil { + return 0, err + } + if err := s.guard(ctx, key, body); err != nil { + return 0, err + } + return s.b.Create(ctx, key, body) +} + +// Update writes a key at the revision it was read at: ErrMoved when it moved, the key unchanged. +func (s Bucket[T]) Update(ctx context.Context, key string, v T, revision uint64) (uint64, error) { + body, err := json.Marshal(v) + if err != nil { + return 0, err + } + if err := s.guard(ctx, key, body); err != nil { + return 0, err + } + return s.b.Update(ctx, key, body, revision) +} + +// All is every record, by key, read in one pass of a watch. An error is an error: never an empty bucket. +func (s Bucket[T]) All(ctx context.Context) (map[string]Kept[T], error) { + ctx, stop := context.WithCancel(ctx) + defer stop() + w, err := s.b.Watch(ctx) + if err != nil { + return nil, err + } + out := map[string]Kept[T]{} + for { + select { + case <-ctx.Done(): + return nil, fmt.Errorf("reading the bucket: %w", ctx.Err()) + case e, open := <-w: + if !open { + return nil, errors.New("the bucket stopped answering before it was read whole") + } + if e.Caught { + return out, nil + } + k, err := decode[T](e) + if err != nil { + return nil, err + } + out[e.Key] = k + } + } +} + +// History is a key's earlier records and its current one, oldest first. +func (s Bucket[T]) History(ctx context.Context, key string) ([]Kept[T], error) { + all, err := s.b.History(ctx, key) + if err != nil { + return nil, err + } + out := make([]Kept[T], 0, len(all)) + for _, e := range all { + e.Key = key + k, err := decode[T](e) + if err != nil { + return nil, err + } + out = append(out, k) + } + return out, nil +} + +// Keys is every key of the bucket. +func (s Bucket[T]) Keys(ctx context.Context) ([]string, error) { return s.b.Keys(ctx) } + +// Watch is the bucket's raw entries: its current values, a Caught mark, then each write. +func (s Bucket[T]) Watch(ctx context.Context) (<-chan Entry, error) { return s.b.Watch(ctx) } + +// GetRaw is one key's value as the bucket holds it, for a writer that changes a field of a record and must keep the rest +// exactly as it is. +func (s Bucket[T]) GetRaw(ctx context.Context, key string) (Entry, bool, error) { + e, ok, err := s.b.Get(ctx, key) + e.Key = key + return e, ok, err +} + +// CreateRaw writes a key's first value as bytes, with the same guards as Create. +func (s Bucket[T]) CreateRaw(ctx context.Context, key string, value []byte) (uint64, error) { + if err := s.guard(ctx, key, value); err != nil { + return 0, err + } + return s.b.Create(ctx, key, value) +} + +// UpdateRaw writes a key's value as bytes, at the revision it was read at, with the same guards as Update. +func (s Bucket[T]) UpdateRaw(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { + if err := s.guard(ctx, key, value); err != nil { + return 0, err + } + return s.b.Update(ctx, key, value, revision) +} + +// Chain is the three buckets of the chain. +type Chain struct { + Builds Bucket[Build] + Rollouts Bucket[Rollout] + Assignments Bucket[Assignment] + + rollouts Backend + holds Holds +} + +// RolloutsOf is the rollouts bucket typed as the controller's plan, which carries more than the chain's Rollout names: +// the same keys and values, read and written as the plan. +func RolloutsOf[T any](c *Chain) Bucket[T] { return Bucket[T]{c.rollouts, c.holds} } + +// New is a chain over three backends, written only while holds says this controller may write. A nil holds writes nothing. +func New(builds, rollouts, assignments Backend, holds Holds) *Chain { + return &Chain{ + Builds: Bucket[Build]{builds, holds}, + Rollouts: Bucket[Rollout]{rollouts, holds}, + Assignments: Bucket[Assignment]{assignments, holds}, + rollouts: rollouts, + holds: holds, + } +} + +// InMemory is a chain held in this process, for tests, written whenever holds says so; nil always holds. +func InMemory(holds Holds) *Chain { + if holds == nil { + holds = func(context.Context) error { return nil } + } + return New(NewMemory(), NewMemory(), NewMemory(), holds) +} diff --git a/internal/chain/chain_test.go b/internal/chain/chain_test.go new file mode 100644 index 00000000..42c216a0 --- /dev/null +++ b/internal/chain/chain_test.go @@ -0,0 +1,391 @@ +package chain + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" +) + +func at(h int) time.Time { return time.Date(2026, 10, 18, h, 0, 0, 0, time.UTC) } + +// **A build is kept whole and read back with its reuse key and what it published** (ADR 0408 §1): its module, +// repository, commit, reuse key, and an artifact by digest or a package release by name and version. +func TestABuildIsKeptWholeAndReadBackWithItsReuseKeyAndWhatItPublished(t *testing.T) { + ctx := context.Background() + c := InMemory(nil) + artifact := Build{ID: "b1", Repository: "mesh-catalog", Module: "notes", Commit: "c1", ReuseKey: "rk-1", + Published: &Published{Artifact: "sha256:aa"}, Asked: at(1), At: at(2), + Made: []Artifact{{Name: "notes", Kind: "image", Reference: "sha256:aa"}}} + pkg := Build{ID: "b2", Repository: "mesh-sdk", Module: "sdk", Commit: "c2", ReuseKey: "rk-2", + Published: &Published{Package: &Package{Name: "mesh-sdk", Version: "1.4.0"}}} + for _, b := range []Build{artifact, pkg} { + if _, err := c.Builds.Create(ctx, b.ID, b); err != nil { + t.Fatal(err) + } + } + got, ok, err := c.Builds.Get(ctx, "b1") + if err != nil || !ok { + t.Fatalf("read b1: %v %v", ok, err) + } + if got.Value.Module != "notes" || got.Value.Repository != "mesh-catalog" || got.Value.Commit != "c1" || + got.Value.ReuseKey != "rk-1" || got.Value.Published.Artifact != "sha256:aa" || len(got.Value.Made) != 1 { + t.Fatalf("the build came back as %+v", got.Value) + } + got, _, _ = c.Builds.Get(ctx, "b2") + if p := got.Value.Published; p == nil || p.Package == nil || p.Package.Name != "mesh-sdk" || p.Package.Version != "1.4.0" { + t.Fatalf("the package came back as %+v", p) + } + if _, ok, _ := c.Builds.Get(ctx, "nope"); ok { + t.Fatal("a key nobody wrote holds a build") + } +} + +// **A write against a revision that moved is refused and nothing is laid over it** (ADR 0408 §2). +func TestAWriteAgainstAMovedRevisionIsRefusedAndNothingIsLaidOver(t *testing.T) { + ctx := context.Background() + c := InMemory(nil) + first, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}) + if err != nil { + t.Fatal(err) + } + if _, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes"}); !errors.Is(err, ErrExists) { + t.Fatalf("a create of a key that holds a value answered %v", err) + } + second, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c2"}, first) + if err != nil { + t.Fatal(err) + } + // A writer that read the first revision writes now. + if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "stale"}, first); !errors.Is(err, ErrMoved) { + t.Fatalf("a write at a moved revision answered %v", err) + } + got, _, _ := c.Assignments.Get(ctx, "anchor.notes") + if got.Value.Commit != "c2" || got.Revision != second { + t.Fatalf("the key that moved was laid over: %+v at %d", got.Value, got.Revision) + } +} + +// **No key of the chain is ever removed** (ADR 0408 §4): the Backend has no delete and no purge, so nothing that holds +// one can remove a key; and an assignment of a module taken off its node is kept, with its earlier values. +func TestNoKeyOfTheChainIsEverRemoved(t *testing.T) { + var backend Backend = NewMemory() + if _, ok := backend.(interface { + Delete(context.Context, string, uint64) error + }); ok { + t.Fatal("the backend can delete") + } + if _, ok := backend.(interface { + Purge(context.Context, string) error + }); ok { + t.Fatal("the backend can purge") + } + if _, ok := any(InMemory(nil).Builds).(interface { + Delete(context.Context, string) error + }); ok { + t.Fatal("a bucket of the chain can delete") + } + ctx := context.Background() + c := InMemory(nil) + rev, _ := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}) + if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}, rev); err != nil { + t.Fatal(err) + } + keys, err := c.Assignments.Keys(ctx) + if err != nil || len(keys) != 1 || keys[0] != "anchor.notes" { + t.Fatalf("keys %v %v", keys, err) + } + history, err := c.Assignments.History(ctx, "anchor.notes") + if err != nil || len(history) != 2 { + t.Fatalf("the key keeps %d values: %v", len(history), err) + } +} + +// **Each key keeps up to sixty-four earlier values** (ADR 0408 §5), oldest first. +func TestAKeyKeepsItsEarlierValuesUpToSixtyFour(t *testing.T) { + ctx := context.Background() + c := InMemory(nil) + rev, _ := c.Rollouts.Create(ctx, "r1", Rollout{ID: "r1", State: "n0"}) + for i := 1; i <= 70; i++ { + var err error + if rev, err = c.Rollouts.Update(ctx, "r1", Rollout{ID: "r1", State: "n" + string(rune('A'+i%26))}, rev); err != nil { + t.Fatal(err) + } + } + history, err := c.Rollouts.History(ctx, "r1") + if err != nil { + t.Fatal(err) + } + if len(history) != broker.ChainHistory { + t.Fatalf("the key keeps %d values and the bus keeps %d", len(history), broker.ChainHistory) + } + for i := 1; i < len(history); i++ { + if history[i].Revision <= history[i-1].Revision { + t.Fatal("the history is not oldest first") + } + } +} + +// **A value over the largest message is refused, not cut, and the key is unchanged** (ADR 0408 §6). +func TestAValueOverTheLargestMessageIsRefusedNotCut(t *testing.T) { + ctx := context.Background() + c := InMemory(nil) + small := Build{ID: "b1", Repository: "r", Commit: "c"} + rev, err := c.Builds.Create(ctx, "b1", small) + if err != nil { + t.Fatal(err) + } + huge := Build{ID: "b1", Repository: "r", Commit: "c", Manifest: json.RawMessage(`"` + strings.Repeat("x", 1<<20) + `"`)} + if _, err := c.Builds.Update(ctx, "b1", huge, rev); !errors.Is(err, ErrTooLarge) { + t.Fatalf("a value of a mebibyte answered %v", err) + } + if _, err := c.Builds.Create(ctx, "b2", huge); !errors.Is(err, ErrTooLarge) { + t.Fatalf("a created value of a mebibyte answered %v", err) + } + got, _, _ := c.Builds.Get(ctx, "b1") + if got.Revision != rev || len(got.Value.Manifest) != 0 { + t.Fatalf("the key was changed: %+v", got) + } + if _, ok, _ := c.Builds.Get(ctx, "b2"); ok { + t.Fatal("a refused create left a key") + } + // The largest that fits is written whole. + fits := Build{ID: "b3", Manifest: json.RawMessage(`"` + strings.Repeat("x", broker.ChainValueBytes-1024) + `"`)} + if _, err := c.Builds.Create(ctx, "b3", fits); err != nil { + t.Fatalf("a value under the largest was refused: %v", err) + } + got, _, _ = c.Builds.Get(ctx, "b3") + if len(got.Value.Manifest) != len(fits.Manifest) { + t.Fatal("a value under the largest was cut") + } +} + +// **A rollout names every assignment it changed, with the build before and after** (ADR 0408 §1). +func TestARolloutNamesEveryAssignmentItChangedWithBuildBeforeAndAfter(t *testing.T) { + ctx := context.Background() + c := InMemory(nil) + r := Rollout{ID: "r1", State: "rolling", Commits: []Commit{{Repository: "mesh-catalog", Commit: "c2"}}, + Tiers: [][]string{{"notes"}}, Links: Links{Builds: []string{"b2"}, Changed: []Change{ + {Node: "anchor", Module: "notes", Before: Ref{Build: "b1", Commit: "c1"}, After: Ref{Build: "b2", Commit: "c2"}}, + {Node: "g14", Module: "notes", After: Ref{Build: "b2", Commit: "c2"}}}}} + if _, err := c.Rollouts.Create(ctx, "r1", r); err != nil { + t.Fatal(err) + } + got, _, _ := c.Rollouts.Get(ctx, "r1") + if len(got.Value.Changed) != 2 || got.Value.Changed[0].Before.Build != "b1" || got.Value.Changed[0].After.Build != "b2" || + got.Value.Changed[1].Before.Build != "" || got.Value.Changed[1].After.Commit != "c2" { + t.Fatalf("the rollout came back as %+v", got.Value) + } + // And in the contract's words: `changed`, with `before` and `after` naming `build` and `commit`. + raw, _ := json.Marshal(got.Value) + for _, want := range []string{`"changed":[`, `"before":{"build":"b1","commit":"c1"}`, `"after":{"build":"b2","commit":"c2"}`, + `"builds":["b2"]`, `"commits":[`, `"tiers":[["notes"]]`} { + if !strings.Contains(string(raw), want) { + t.Errorf("the rollout's value lacks %s: %s", want, raw) + } + } +} + +// **An assignment names its build, artifact or package, commit and rollout**, and keeps the fields its readers read +// under the names they read them by (ADR 0408 §3). +func TestAnAssignmentNamesBuildArtifactCommitAndRollout(t *testing.T) { + a := Assignment{Node: "anchor", Module: "notes", Assigned: at(3), RequiredBy: "wiki", + Provisioning: json.RawMessage(`{"plan":1}`), Build: "b2", Artifact: "sha256:bb", Commit: "c2", Rollout: "r1", Since: at(4)} + raw, _ := json.Marshal(a) + for _, want := range []string{`"node":"anchor"`, `"module":"notes"`, `"assigned":"2026-10-18T03:00:00Z"`, + `"required-by":"wiki"`, `"provisioning":{"plan":1}`, `"build":"b2"`, `"artifact":"sha256:bb"`, `"commit":"c2"`, + `"rollout":"r1"`, `"since":"2026-10-18T04:00:00Z"`} { + if !strings.Contains(string(raw), want) { + t.Errorf("the assignment's value lacks %s: %s", want, raw) + } + } + if a.Key() != "anchor.notes" { + t.Errorf("key %s", a.Key()) + } + // An assignment of today carries none of the new fields, and says none: a reader of today reads the bytes it did. + raw, _ = json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at(3)}) + if string(raw) != `{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z"}` { + t.Errorf("an assignment without a chain is %s", raw) + } + // A package in place of an artifact. + raw, _ = json.Marshal(Assignment{Node: "a", Module: "m", Package: &Package{Name: "n", Version: "1"}}) + if !strings.Contains(string(raw), `"package":{"name":"n","version":"1"}`) { + t.Errorf("the package is %s", raw) + } +} + +// **The writer without the lease writes nothing** (ADR 0408 §2): it reads, and every create and update is refused with +// the bucket untouched. +func TestTheWriterWithoutTheLeaseWritesNothing(t *testing.T) { + ctx := context.Background() + var away error + builds, rollouts, assignments := NewMemory(), NewMemory(), NewMemory() + c := New(builds, rollouts, assignments, func(context.Context) error { return away }) + rev, err := c.Builds.Create(ctx, "b1", Build{ID: "b1"}) + if err != nil { + t.Fatal(err) + } + away = errors.New("another controller holds the lease") + if _, err := c.Builds.Create(ctx, "b2", Build{ID: "b2"}); !errors.Is(err, ErrNotHeld) { + t.Fatalf("a create without the lease answered %v", err) + } + if _, err := c.Builds.Update(ctx, "b1", Build{ID: "b1", Failed: "x"}, rev); !errors.Is(err, ErrNotHeld) { + t.Fatalf("an update without the lease answered %v", err) + } + if _, err := c.Rollouts.Create(ctx, "r", Rollout{ID: "r"}); !errors.Is(err, ErrNotHeld) { + t.Fatalf("a rollout written without the lease: %v", err) + } + if _, err := c.Assignments.Create(ctx, "a.b", Assignment{}); !errors.Is(err, ErrNotHeld) { + t.Fatalf("an assignment written without the lease: %v", err) + } + if builds.Writes() != 1 || rollouts.Writes() != 0 || assignments.Writes() != 0 { + t.Fatalf("writes %d %d %d", builds.Writes(), rollouts.Writes(), assignments.Writes()) + } + // Reads still answer. + if _, ok, err := c.Builds.Get(ctx, "b1"); err != nil || !ok { + t.Fatalf("a read without the lease: %v %v", ok, err) + } + if all, err := c.Builds.All(ctx); err != nil || len(all) != 1 { + t.Fatalf("all: %v %v", all, err) + } + // And a controller with no lease function at all holds nothing. + if _, err := New(NewMemory(), NewMemory(), NewMemory(), nil).Builds.Create(ctx, "b", Build{}); !errors.Is(err, ErrNotHeld) { + t.Fatalf("a chain with no lease wrote: %v", err) + } +} + +// **A bucket that is away is an error, never an empty bucket** (ADR 0227 rule 4). +func TestABucketThatIsAwayIsAnErrorNotAnEmptyOne(t *testing.T) { + ctx := context.Background() + m := NewMemory() + c := New(m, NewMemory(), NewMemory(), func(context.Context) error { return nil }) + if _, err := c.Builds.Create(ctx, "b1", Build{ID: "b1"}); err != nil { + t.Fatal(err) + } + away := errors.New("the bus is away") + m.SetFail(away) + if all, err := c.Builds.All(ctx); !errors.Is(err, away) || all != nil { + t.Fatalf("all answered %v %v", all, err) + } + if _, _, err := c.Builds.Get(ctx, "b1"); !errors.Is(err, away) { + t.Fatalf("get answered %v", err) + } +} + +// A watch hands over every key's value, says it has, and then each write. +func TestAWatchHandsOverTheValuesThenEachWrite(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + c := InMemory(nil) + if _, err := c.Builds.Create(ctx, "b1", Build{ID: "b1"}); err != nil { + t.Fatal(err) + } + w, err := c.Builds.Watch(ctx) + if err != nil { + t.Fatal(err) + } + next := func() Entry { + select { + case e := <-w: + return e + case <-ctx.Done(): + t.Fatal("the watch said nothing") + return Entry{} + } + } + if e := next(); e.Key != "b1" || e.Caught { + t.Fatalf("first %+v", e) + } + if e := next(); !e.Caught { + t.Fatalf("second %+v", e) + } + if _, err := c.Builds.Create(ctx, "b2", Build{ID: "b2"}); err != nil { + t.Fatal(err) + } + if e := next(); e.Key != "b2" { + t.Fatalf("third %+v", e) + } +} + +// **A key the bus does not take is refused, never rewritten into another** (ADR 0408 §2): a write names the key loudly. +func TestAKeyTheBusTakesNoSpellingOfIsRefused(t *testing.T) { + ctx := context.Background() + c := InMemory(nil) + for _, key := range []string{"", "build one", "build*", ".hidden", "trailing.", "a>b"} { + if _, err := c.Builds.Create(ctx, key, Build{ID: key}); !errors.Is(err, ErrBadKey) { + t.Errorf("the key %q answered %v", key, err) + } + } + for _, key := range []string{"build-1700000000000000000", "anchor.notes", "a/b_c=d"} { + if _, err := c.Builds.Create(ctx, key, Build{ID: key}); err != nil { + t.Errorf("the key %q answered %v", key, err) + } + } +} + +// An empty bucket is read as empty, and the read ends. +func TestAnEmptyBucketIsReadAsEmpty(t *testing.T) { + all, err := InMemory(nil).Assignments.All(context.Background()) + if err != nil || len(all) != 0 { + t.Fatalf("%v %v", all, err) + } +} + +// **An assignment that has ended still stands in the bucket as a value that says so, and nothing in the chain can remove a +// key** (ADR 0408 §4, issue 1093): the value with `ended` is no assignment to a reader, and the earlier value stays in +// history. +func TestAnEndedAssignmentIsNoAssignmentAndStaysInTheBucket(t *testing.T) { + ctx := context.Background() + c := InMemory(nil) + rev, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Assigned: at(1), Commit: "c1"}) + if err != nil { + t.Fatal(err) + } + standing, _, _ := c.Assignments.Get(ctx, "anchor.notes") + if !standing.Value.Stands() { + t.Fatal("a value that has not ended does not stand") + } + ended := standing.Value + ended.Ended = at(2) + if _, err := c.Assignments.Update(ctx, "anchor.notes", ended, rev); err != nil { + t.Fatal(err) + } + got, found, _ := c.Assignments.Get(ctx, "anchor.notes") + if !found || got.Value.Stands() || got.Value.Commit != "c1" || got.Value.Ended.IsZero() { + t.Fatalf("the ended value is %+v (found %v)", got.Value, found) + } + if history, _ := c.Assignments.History(ctx, "anchor.notes"); len(history) != 2 || !history[0].Value.Stands() { + t.Fatalf("the earlier values are %+v", history) + } + if (Assignment{Node: "a", Module: "m", Assigned: at(1), State: "pending"}).Stands() { + t.Fatal("a pending value stands") + } + raw, _ := json.Marshal(ended) + if !strings.Contains(string(raw), `"ended":"2026-10-18T02:00:00Z"`) { + t.Fatalf("the ended value is %s", raw) + } + // And a value that stands says nothing of either, so a reader of today reads the bytes it did. + raw, _ = json.Marshal(Assignment{Node: "a", Module: "m", Assigned: at(1)}) + if strings.Contains(string(raw), "ended") || strings.Contains(string(raw), "state") { + t.Fatalf("a standing value says %s", raw) + } + // No bucket, and no chain, has a removal. + type remover interface { + Remove(context.Context, string, uint64) error + } + for name, x := range map[string]any{"builds": c.Builds, "rollouts": c.Rollouts, "assignments": c.Assignments, "chain": c, + "a backend": NewMemory()} { + if _, has := x.(remover); has { + t.Errorf("%s can remove a key", name) + } + if _, has := x.(interface { + RemoveAssignment(context.Context, string, uint64) error + }); has { + t.Errorf("%s can remove an assignment", name) + } + } +} diff --git a/internal/chain/records.go b/internal/chain/records.go new file mode 100644 index 00000000..709aa833 --- /dev/null +++ b/internal/chain/records.go @@ -0,0 +1,201 @@ +// Package chain keeps the controller's chain on the bus alone (novox/hq ADR 0408, design 47): module, commit, build, +// published artifact or package, rollout and assignment are one chain, a bucket each of the bus's key-value store, +// each the only source of its kind, written by the controller that holds the lease by compare-and-set. +// +// **No key is removed.** A Backend has no delete and no purge (ADR 0408 §4): an assignment of a module taken off its +// node is kept, saying so. **A value is never cut** (§6): one over the largest the bus holds is refused, and the key +// stays as it was. **Each key keeps its earlier values** (§5), read back with History. +package chain + +import ( + "encoding/json" + "time" +) + +// Ref names a build and the commit it was made from: one end of a change. +type Ref struct { + Build string `json:"build,omitempty"` + Commit string `json:"commit,omitempty"` +} + +// Package is a release a build published to the package registry. +type Package struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// Published is what a build published: an artifact by its digest, or a package release by name and version. +type Published struct { + Artifact string `json:"artifact,omitempty"` + Package *Package `json:"package,omitempty"` +} + +// Artifact is one thing a build made. +type Artifact struct { + Name string `json:"name"` + Kind string `json:"kind"` + Reference string `json:"reference"` +} + +// ReadRepository is a repository a build read source from besides the module's own. +type ReadRepository struct { + Repository string `json:"repository"` + Ref string `json:"ref,omitempty"` + Paths []string `json:"paths,omitempty"` + Own bool `json:"own,omitempty"` +} + +// Source is the build source a build read in one repository. +type Source struct { + Repository string `json:"repository,omitempty"` + Ref string `json:"ref,omitempty"` + Paths []string `json:"paths"` +} + +// The states of a build record. A record in state asked is a request that no outcome has answered (issue 1091); one +// without a state is an outcome, and keeps the Request it was asked with. +const BuildAsked = "asked" + +// The states of a request's Reask. +const ( + ReaskPending = "pending" + ReaskClaimed = "claimed" +) + +// Request is a build as it was asked for (novox/hq issue 325, ADR 0408 §1): kept in the build's own record under its id, +// first alone as state asked and then beside the outcome that answered it. +type Request struct { + // Repository and Seat are the source as the mesh spells it; Path is the module's directory in it, Ref the branch or + // commit asked, and Commit the commit that made the ask, where one did. + Repository string `json:"repository,omitempty"` + Seat string `json:"seat,omitempty"` + Path string `json:"path,omitempty"` + Ref string `json:"ref,omitempty"` + Commit string `json:"commit,omitempty"` + // For says who asked: merge, plan, build (a person), assign. + For string `json:"for,omitempty"` + // NotAsked is why the ask could not be handed over; OutcomeUnknown why an asker that handed it over stopped waiting. + NotAsked string `json:"not-asked,omitempty"` + OutcomeUnknown string `json:"outcome-unknown,omitempty"` + At time.Time `json:"at,omitzero"` + AtTerminal bool `json:"at-terminal,omitempty"` + // Reask is, on a request the move found never handed over, "pending" until the controller asks it again, "claimed" + // while it does, and the build id it was asked as once it has: published once, never twice (issue 981 note 7 Q5). + Reask string `json:"reask,omitempty"` +} + +// Build is one attempt, whichever way it went, kept under its id. It names its module, its repository and commit, its +// reuse key (novox/hq ADR 0350) and what it published (ADR 0408 §1). +type Build struct { + ID string `json:"id"` + State string `json:"state,omitempty"` + Repository string `json:"repository"` + Ref string `json:"ref,omitempty"` + Module string `json:"module,omitempty"` + Commit string `json:"commit,omitempty"` + On string `json:"on,omitempty"` + Path string `json:"path,omitempty"` + Manifest json.RawMessage `json:"manifest,omitempty"` + Against []string `json:"against,omitempty"` + Mirrored []string `json:"mirrored,omitempty"` + Read []ReadRepository `json:"read,omitempty"` + Sources []Source `json:"sources,omitempty"` + SourceFingerprint string `json:"source-fingerprint,omitempty"` + Failed string `json:"failed,omitempty"` + Made []Artifact `json:"made,omitempty"` + Asked time.Time `json:"asked,omitzero"` + At time.Time `json:"at,omitzero"` + ReuseKey string `json:"reuse-key,omitempty"` + Published *Published `json:"published,omitempty"` + Request *Request `json:"request,omitempty"` +} + +// Commit is one repository's commit a rollout carries. +type Commit struct { + Repository string `json:"repository"` + Branch string `json:"branch,omitempty"` + Commit string `json:"commit"` + Merged time.Time `json:"merged,omitzero"` +} + +// Change is one assignment a rollout changed: the build and commit it ran before and the ones it runs after. +type Change struct { + Node string `json:"node"` + Module string `json:"module"` + Before Ref `json:"before"` + After Ref `json:"after"` +} + +// Step is one tier sent to one machine. +type Step struct { + Tier int `json:"tier"` + Node string `json:"node"` + Sent *time.Time `json:"sent,omitempty"` + Reported *time.Time `json:"reported,omitempty"` +} + +// Links are the fields a rollout gains on the chain: the builds it carried, the machine steps it has taken and the +// assignments it changed, each with the build before and after. The controller's plan embeds them, so the plan's own +// fields and these are one value (ADR 0408 §1). +type Links struct { + Builds []string `json:"builds,omitempty"` + Steps []Step `json:"steps,omitempty"` + Changed []Change `json:"changed,omitempty"` +} + +// HeldMerge is a merge a rollout holds: the columns the controller's table of batched merges had, kept in the record of the +// batch or walk that answers it (novox/hq ADR 0276, issue 1092). A merge no plan answers (Waiting) stays in the record +// that held it last. Placed is when it entered this record, so one found in two records after a crash between the two +// writes is read from the one it entered last. +type HeldMerge struct { + Repository string `json:"repository"` + Branch string `json:"branch,omitempty"` + Commit string `json:"commit"` + Merged time.Time `json:"merged,omitzero"` + Heard time.Time `json:"heard,omitzero"` + Event json.RawMessage `json:"event,omitempty"` + Alone bool `json:"alone,omitempty"` + Waiting bool `json:"waiting,omitempty"` + Placed time.Time `json:"placed,omitzero"` +} + +// Rollout is the controller's record of one batch's delivery, kept under the batch's id: the commit it fixed per +// repository, its tiers, the builds it carried and the assignments it changed (ADR 0408 §1). The controller's plan +// carries more (its modules, its times, its backups) beside these, under the same key. +type Rollout struct { + ID string `json:"id"` + State string `json:"state,omitempty"` + Commits []Commit `json:"commits,omitempty"` + Tiers [][]string `json:"tiers,omitempty"` + Links +} + +// Assignment is one module on one node, kept under `.`. The first five fields are the ones its readers +// have always read, with the names they have always had (novox/hq ADR 0408 §3); the rest name the chain: the build, the +// artifact (by digest) or the package it runs, its commit, the rollout that put it there and since when. +type Assignment struct { + Node string `json:"node"` + Module string `json:"module"` + Assigned time.Time `json:"assigned,omitzero"` + RequiredBy string `json:"required-by,omitempty"` + Provisioning json.RawMessage `json:"provisioning,omitempty"` + + Build string `json:"build,omitempty"` + Artifact string `json:"artifact,omitempty"` + Package *Package `json:"package,omitempty"` + Commit string `json:"commit,omitempty"` + Rollout string `json:"rollout,omitempty"` + Since time.Time `json:"since,omitzero"` + + // Ended is when the assignment ended (an unassignment): absent while it stands. **No key of the chain is removed** + // (ADR 0408 §4), so an unassigned module keeps its key, saying so; every reader treats a value with Ended, or with a + // State, as no assignment. State is, in a later delivery, the pending assignment waiting for its build (issue 1094). + Ended time.Time `json:"ended,omitzero"` + State string `json:"state,omitempty"` +} + +// Stands says the value is an assignment that holds: it has not ended and is not waiting in a state of its own. +func (a Assignment) Stands() bool { return a.Ended.IsZero() && a.State == "" } + +// Key is an assignment's key. +func (a Assignment) Key() string { return a.Node + "." + a.Module } diff --git a/internal/inventory/adoption.go b/internal/inventory/adoption.go index 02f3da4e..7d01b062 100644 --- a/internal/inventory/adoption.go +++ b/internal/inventory/adoption.go @@ -66,16 +66,14 @@ func (i *Inventory) Take(ctx context.Context, nodeName, module string) error { // take is Take without the adopted check, for converging, which takes every assigned module in // the same act that makes the node converged. func (i *Inventory) take(ctx context.Context, node Node, module string) error { - var assigned bool - if err := i.store.Pool().QueryRow(ctx, - `select exists (select 1 from assignment where node = $1 and module = $2)`, - node.ID, module).Scan(&assigned); err != nil { + assigned, err := i.isAssigned(ctx, node.Name, module) + if err != nil { return err } if !assigned { return fmt.Errorf("%w: %s is not on %s; assign it first", ErrNotAssigned, module, node.Name) } - _, err := i.store.Pool().Exec(ctx, + _, err = i.store.Pool().Exec(ctx, `insert into taken (node, module) values ($1, $2) on conflict do nothing`, node.ID, module) return err } @@ -95,27 +93,24 @@ func (i *Inventory) Converge(ctx context.Context, nodeName string) ([]string, er return nil, err } defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() - rows, err := tx.Query(ctx, - `insert into taken (node, module) - select node, module from assignment where node = $1 - on conflict do nothing - returning module`, node.ID) + assigned, err := i.assignedModules(ctx, node.Name) if err != nil { return nil, err } var took []string - for rows.Next() { + for _, module := range assigned { var m string - if err := rows.Scan(&m); err != nil { - rows.Close() + err := tx.QueryRow(ctx, + `insert into taken (node, module) values ($1, $2) on conflict do nothing returning module`, + node.ID, module).Scan(&m) + if errors.Is(err, pgx.ErrNoRows) { + continue // taken already + } + if err != nil { return nil, err } took = append(took, m) } - rows.Close() - if err := rows.Err(); err != nil { - return nil, err - } if _, err := tx.Exec(ctx, `update node set adopted = false, adopted_since = null, converged_at = now(), held = null, reachable = null, firewall = null, adoption_reported = null diff --git a/internal/inventory/batches.go b/internal/inventory/batches.go index 5e3c5e22..c0cd6ab2 100644 --- a/internal/inventory/batches.go +++ b/internal/inventory/batches.go @@ -33,6 +33,11 @@ type BatchedMerge struct { // AddMerge keeps a merge for a plan; false when the merge was kept already, for whatever plan. func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error) { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return false, err + } else if on { + return i.chainAddMerge(ctx, c, m) + } tag, err := i.store.Pool().Exec(ctx, `insert into batched_merge (repository, branch, commit_hash, merged_at, heard_at, event, plan_id, alone) values (lower($1), $2, $3, $4, $5, $6, nullif($7, ''), $8) on conflict do nothing`, @@ -45,6 +50,11 @@ func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error) // MergeOf is one kept merge; false when it was never heard. func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (BatchedMerge, bool, error) { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return BatchedMerge{}, false, err + } else if on { + return i.chainMergeOf(ctx, c, repository, commit) + } ms, err := i.merges(ctx, `where repository = lower($1) and commit_hash = $2`, repository, commit) if err != nil || len(ms) == 0 { return BatchedMerge{}, false, err @@ -54,12 +64,22 @@ func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (Bat // MergesOf is every merge a plan answers, oldest merge first. func (i *Inventory) MergesOf(ctx context.Context, plan string) ([]BatchedMerge, error) { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return nil, err + } else if on { + return i.chainMergesOf(ctx, c, plan) + } return i.merges(ctx, `where plan_id = $1 order by merged_at nulls last, heard_at`, plan) } // LaterMergesOf is every merge of a repository's branch made after a moment and answered by a plan, the // newest first. func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string, after time.Time) ([]BatchedMerge, error) { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return nil, err + } else if on { + return i.chainLaterMergesOf(ctx, c, repository, branch, after) + } return i.merges(ctx, `where repository = lower($1) and branch = $2 and merged_at > $3 and plan_id is not null order by merged_at desc`, repository, branch, after) } @@ -67,6 +87,11 @@ func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string // KeptOnBranch says the controller keeps a merge of a repository's branch, and when it heard the branch's first: a merge of a branch it keeps merges of, made since then, is the record's to answer, whatever a // module's last look says (novox/hq ADR 0276). func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string) (bool, time.Time, error) { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return false, time.Time{}, err + } else if on { + return i.chainKeptOnBranch(ctx, c, repository, branch) + } var first *time.Time var kept bool err := i.store.Pool().QueryRow(ctx, @@ -80,12 +105,22 @@ func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string) // AloneMerges is every merge waiting to be walked on its own commit, the newest first (ADR 0276 decision 3). func (i *Inventory) AloneMerges(ctx context.Context) ([]BatchedMerge, error) { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return nil, err + } else if on { + return i.chainAloneMerges(ctx, c) + } return i.merges(ctx, `where plan_id is null and alone order by merged_at desc nulls last, heard_at desc`) } // AnswerMerge moves a merge to the plan that answers it now; an empty plan with alone leaves it waiting to // be walked on its own commit. func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan string, alone bool) error { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return err + } else if on { + return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}}) + } _, err := i.store.Pool().Exec(ctx, `update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`, repository, commit, plan, alone) @@ -95,6 +130,11 @@ func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan st // Batches is every batch not yet cut: assembling or queued, oldest first. One at a time is the rule; more // are read so a fault is seen. func (i *Inventory) Batches(ctx context.Context) ([]Plan, error) { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return nil, err + } else if on { + return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated) + } return i.plans(ctx, `where state in ('assembling', 'queued') order by created`) } diff --git a/internal/inventory/builds.go b/internal/inventory/builds.go index a6b6c955..1fa1c78d 100644 --- a/internal/inventory/builds.go +++ b/internal/inventory/builds.go @@ -9,6 +9,8 @@ import ( "time" "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/chain" ) // What has been built. @@ -62,6 +64,10 @@ type Build struct { Asked time.Time // At is when the outcome was recorded — when it finished, not when it was asked. At time.Time + // ReuseKey is the build's reuse key (novox/hq ADR 0350 §1), and Published what it published: an artifact by its + // digest, or a package release by name and version (ADR 0408 §1). Kept on the bus; a build kept in the table has neither. + ReuseKey string + Published *chain.Published } // AskedOrAt is when the build was asked, or when it was recorded when that is not known — the @@ -128,6 +134,11 @@ func (b Build) Worked() bool { return b.Failed == "" } // whoever asked and once on the exchange when nobody was. Recording both would show one build as // two, and which of the two is real is not a question anybody could answer afterwards. func (i *Inventory) RecordBuild(ctx context.Context, b Build) error { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return err + } else if on { + return i.chainRecordBuild(ctx, c, b) + } made, err := json.Marshal(b.Made) if err != nil { return err @@ -207,6 +218,14 @@ func (i *Inventory) RecordMirrored(ctx context.Context, build, why string, refer // now* after something goes wrong, and *what has happened to this* when deciding whether to // trust it. func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Build, error) { + if limit <= 0 { + limit = 20 + } + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return nil, err + } else if on { + return i.chainBuildList(ctx, c, module, limit) + } if limit <= 0 { limit = 20 } @@ -249,6 +268,11 @@ func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Bui // by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the // repository, path, ref and commit from. False when no outcome with that id was recorded. func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return Build{}, false, err + } else if on { + return i.chainBuildByID(ctx, c, id) + } var b Build var made []byte var asked *time.Time @@ -351,6 +375,11 @@ func registeredOnly(held, registered map[string]string) map[string]string { // edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a // module standing on nothing looks like: an edge the mesh has not derived is not an edge. func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return nil, err + } else if on { + return i.chainBuiltAgainst(ctx, c) + } rows, err := i.store.Pool().Query(ctx, `select distinct on (module) module, built_against from build @@ -394,6 +423,11 @@ func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, erro // none — off the trunk, from a builder that predates it, or of a source not known — leaves its module read // as before: every file of a context, and its own directory or root. func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return nil, err + } else if on { + return i.chainReadRepositories(ctx, c) + } // The newest build of each module whatever its outcome: a failed one newer than the newest that // worked leaves that one's build source stale — the merge it was asked for may have changed the closure // (novox/hq ADR 0267), so its module is read whole until a build works again. @@ -580,6 +614,11 @@ func manifestOrNil(raw []byte) any { // One row per module and commit: a module built twice at the same commit is one fact, and the // row asked last is the one whose artifacts are current (novox/hq 04-ISSUES/219). func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return nil, err + } else if on { + return i.chainAnnounceable(ctx, c) + } rows, err := i.store.Pool().Query(ctx, `select distinct on (module, commit_hash) id, repository, ref, module, commit_hash, built_on, failed, made, diff --git a/internal/inventory/builds_test.go b/internal/inventory/builds_test.go index fbdfd08a..e0f53b83 100644 --- a/internal/inventory/builds_test.go +++ b/internal/inventory/builds_test.go @@ -4,6 +4,7 @@ package inventory import ( "context" + "encoding/json" "reflect" "strings" "testing" @@ -215,11 +216,15 @@ func TestABuildsBuildSourceComesBackOverWhatItRead(t *testing.T) { if got := read["build-agent"]; len(got) != 1 || got[0].Paths != nil || got[0].Own { t.Fatalf("a build that said no build source reads as %+v", got) } - var stored string - if err := inv.store.Pool().QueryRow(ctx, `select built_contexts::text from build where id = 'said'`).Scan(&stored); err != nil { + kept, found, err := inv.chain.Builds.Get(ctx, "said") + if err != nil || !found { + t.Fatalf("the build is not on the bus: %v %v", found, err) + } + storedRead, err := json.Marshal(kept.Value.Read) + if err != nil { t.Fatal(err) } - if strings.Contains(stored, "paths") || strings.Contains(stored, "own") { + if stored := string(storedRead); strings.Contains(stored, "paths") || strings.Contains(stored, "own") { t.Fatalf("the build source was stored among what the build read: %s", stored) } } diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 14c8edfd..56a11d17 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -258,15 +258,31 @@ func (i *Inventory) SourceOf(ctx context.Context, module string) (Source, error) // machines are running last week's version" is the question somebody actually has, and it is the // one novox/hq ADR 0010 names as the thing that must not be lost. func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) { - rows, err := i.store.Pool().Query(ctx, - `select m.name, coalesce(n.name, '') + query := `select m.name, coalesce(n.name, '') from module m left join assignment a on a.module = m.name left join node n on n.id = a.node where m.source is not null and m.source_head is not null and coalesce(m.built_from, '') is distinct from m.source_head - order by m.name, n.name`) + order by m.name, n.name` + _, onTheBus, err := i.ChainReads(ctx, KindAssignments) + if err != nil { + return nil, err + } + var assigned []assignRow + if onTheBus { + query = `select m.name, '' + from module m + where m.source is not null + and m.source_head is not null + and coalesce(m.built_from, '') is distinct from m.source_head + order by m.name` + if assigned, err = i.assignments(ctx); err != nil { + return nil, err + } + } + rows, err := i.store.Pool().Query(ctx, query) if err != nil { return nil, err } @@ -285,7 +301,18 @@ func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) { out[module] = append(out[module], node) } } - return out, rows.Err() + if err := rows.Err(); err != nil { + return nil, err + } + for _, a := range assigned { + if _, behind := out[a.Module]; behind { + out[a.Module] = append(out[a.Module], a.Node) + } + } + for module := range out { + sort.Strings(out[module]) + } + return out, nil } // Catalogue is every module the mesh knows about, which is what resolution needs: the question @@ -401,22 +428,7 @@ func (i *Inventory) RetireUnshipped(ctx context.Context, shipped []string) (reti // assignedOn is the machines a module is assigned to, sorted. func (i *Inventory) assignedOn(ctx context.Context, name string) ([]string, error) { - rows, err := i.store.Pool().Query(ctx, - `select n.name from assignment a join node n on n.id = a.node where a.module = $1 - order by n.name`, name) - if err != nil { - return nil, err - } - defer rows.Close() - var on []string - for rows.Next() { - var node string - if err := rows.Scan(&node); err != nil { - return nil, err - } - on = append(on, node) - } - return on, rows.Err() + return i.assignedNodes(ctx, name) } // Provided reports whether a module came with the control plane rather than from a repository. @@ -600,25 +612,10 @@ func (i *Inventory) mayForget(ctx context.Context, name string) error { "%s comes with the control plane and cannot be forgotten; unassign it instead", name) } - var on []string - rows, err := i.store.Pool().Query(ctx, - `select n.name from assignment a join node n on n.id = a.node where a.module = $1 - order by n.name`, name) + on, err := i.assignedNodes(ctx, name) if err != nil { return err } - for rows.Next() { - var node string - if err := rows.Scan(&node); err != nil { - rows.Close() - return err - } - on = append(on, node) - } - rows.Close() - if err := rows.Err(); err != nil { - return err - } if len(on) > 0 { return fmt.Errorf("%w: %s. Unassign it first", ErrStillAssigned, strings.Join(on, ", ")) } @@ -657,6 +654,11 @@ func (i *Inventory) Assign(ctx context.Context, nodeName, module string) (bool, if err := i.runsSomewhere(ctx, module); err != nil { return false, err } + if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil { + return false, err + } else if on { + return i.chainAssign(ctx, c, node, module) + } tag, err := i.store.Pool().Exec(ctx, `insert into assignment (node, module) values ($1, $2) on conflict do nothing`, node.ID, module) @@ -675,13 +677,25 @@ func (i *Inventory) Unassign(ctx context.Context, nodeName, module string) error if err != nil { return err } - tag, err := i.store.Pool().Exec(ctx, - `delete from assignment where node = $1 and module = $2`, node.ID, module) - if err != nil { + if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil { return err + } else if on { + if err := i.chainUnassign(ctx, c, node, module); err != nil { + return err + } + } else { + tag, err := i.store.Pool().Exec(ctx, + `delete from assignment where node = $1 and module = $2`, node.ID, module) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return fmt.Errorf("%s is not assigned to %s", module, nodeName) + } } - if tag.RowsAffected() == 0 { - return fmt.Errorf("%s is not assigned to %s", module, nodeName) + // The seat it held falls back to derivation rather than pointing at nothing: what the store's foreign key did. + if _, err := i.store.Pool().Exec(ctx, `delete from seat_holding where node = $1 and module = $2`, node.ID, module); err != nil { + return err } // And its ports go back. Kept-once-chosen is a promise about a module that is still here — // held past unassignment, a fixed port stays claimed in the name of something that is gone, @@ -697,22 +711,7 @@ func (i *Inventory) Assigned(ctx context.Context, nodeName string) ([]string, er if err != nil { return nil, err } - rows, err := i.store.Pool().Query(ctx, - `select module from assignment where node = $1 order by module`, node.ID) - if err != nil { - return nil, err - } - defer rows.Close() - - var out []string - for rows.Next() { - var m string - if err := rows.Scan(&m); err != nil { - return nil, err - } - out = append(out, m) - } - return out, rows.Err() + return i.assignedModules(ctx, node.Name) } // ProfileOf is what a node last said it can do, as resolution needs it: the capabilities that are @@ -1363,8 +1362,7 @@ type Entry struct { // catalogue nobody lists, and the questions here — what is this, where did it come from, who is // running it — are asked together every time. func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) { - rows, err := i.store.Pool().Query(ctx, - `select m.name, m.manifest, + query := `select m.name, m.manifest, coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''), coalesce(m.built_from, ''), coalesce(m.source_head, ''), coalesce(m.source_seen, to_timestamp(0)), @@ -1374,7 +1372,30 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) { left join node n on n.id = a.node group by m.name, m.manifest, m.source, m.source_path, m.source_seat, m.ref, m.built_from, m.source_head, m.source_seen - order by m.name`) + order by m.name` + // On the bus the assignments are read from the bucket, and the machines of a module put beside it here. + _, onTheBus, err := i.ChainReads(ctx, KindAssignments) + if err != nil { + return nil, err + } + var assignedBy map[string][]string + if onTheBus { + query = `select m.name, m.manifest, + coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''), + coalesce(m.built_from, ''), coalesce(m.source_head, ''), + coalesce(m.source_seen, to_timestamp(0)), '{}'::text[] + from module m + order by m.name` + rows, err := i.assignments(ctx) + if err != nil { + return nil, err + } + assignedBy = map[string][]string{} + for _, a := range rows { + assignedBy[a.Module] = append(assignedBy[a.Module], a.Node) + } + } + rows, err := i.store.Pool().Query(ctx, query) if err != nil { return nil, err } @@ -1393,6 +1414,9 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) { if source.Seen.Unix() == 0 { source.Seen = time.Time{} } + if onTheBus { + on = assignedBy[name] + } var m catalogue.Manifest if err := json.Unmarshal(raw, &m); err != nil { return nil, err @@ -1612,23 +1636,7 @@ func (i *Inventory) CurrentBuilds(ctx context.Context) (map[string]CurrentBuild, // exactly the machine an upgrade most needs to reach; waiting for it to report the old version // first would mean the machines furthest behind are the last to be caught up. func (i *Inventory) Running(ctx context.Context, module string) ([]string, error) { - rows, err := i.store.Pool().Query(ctx, - `select n.name from assignment a join node n on n.id = a.node - where a.module = $1 order by n.name`, module) - if err != nil { - return nil, err - } - defer rows.Close() - - var out []string - for rows.Next() { - var name string - if err := rows.Scan(&name); err != nil { - return nil, err - } - out = append(out, name) - } - return out, rows.Err() + return i.assignedNodes(ctx, module) } // runsSomewhere refuses to put a module on a machine when it would put nothing there. diff --git a/internal/inventory/catalogue_test.go b/internal/inventory/catalogue_test.go index f956ba31..24566a2b 100644 --- a/internal/inventory/catalogue_test.go +++ b/internal/inventory/catalogue_test.go @@ -113,13 +113,15 @@ func TestRemovingANodeTakesItsAssignments(t *testing.T) { t.Fatal(err) } - var left int - if err := inv.store.Pool().QueryRow(t.Context(), - `select count(*) from assignment`).Scan(&left); err != nil { + // On the bus the key outlives the node it names, and is read as no assignment: the machine is not one the store knows. + entries, err := inv.Catalogued(t.Context()) + if err != nil { t.Fatal(err) } - if left != 0 { - t.Errorf("%d assignment(s) outlived the node they were on", left) + for _, e := range entries { + if e.Manifest.Module == "thing" && len(e.On) != 0 { + t.Errorf("thing is assigned to %v, a machine that is gone", e.On) + } } // And the module itself survives, because other nodes may be running it. shelf, err := inv.Catalogue(t.Context()) diff --git a/internal/inventory/chain.go b/internal/inventory/chain.go new file mode 100644 index 00000000..148d22da --- /dev/null +++ b/internal/inventory/chain.go @@ -0,0 +1,149 @@ +package inventory + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "github.com/novox/mesh-controller/internal/chain" +) + +// **The chain moves to the bus kind by kind** (novox/hq ADR 0408 §8, issue 981). The tables of builds, rollouts and +// assignments are copied once into the chain's buckets by the serving controller at its start (MoveChain), which then +// records each kind it moved in chain_moved (migration 0104). **Until a kind is recorded there nothing reads its bucket; +// once it is, nothing reads its tables**: ChainReads answers which, and every method of a kind asks it first. + +// The kinds of the chain, as chain_moved names them. +const ( + KindBuilds = "builds" + KindRollouts = "rollouts" +) + +// testChains are the chains in memory a test made for its store, by the store's address: every inventory opened on that +// store is given the same chain, as every process of a mesh is on the same bus. Empty outside a test. +var testChains sync.Map + +// ErrNoChain is a kind moved to the bus read by an inventory that has no chain attached: a process that did not +// open the bus. Refused loudly, never answered from the tables the move left behind. +var ErrNoChain = errors.New("this kind of record lives on the bus, and this process has not opened the bus") + +// movedKinds is which kinds this process has read as moved: once moved, always moved, so it is read from the store +// until it is true and not after. +type movedKinds struct { + mu sync.Mutex + kind map[string]bool + open ChainOpener + // view is the rollouts bucket kept in memory by one watch (issue 1092); pending are the merges added to a plan that + // is not saved yet. + view *rolloutView + pending map[string]pendingMerge + // switching is set when the assignments begin to be switched to the bus and kept once they are: the store's mirror of + // them writes nothing from then on (AssignmentsMirrorMayWrite). Cleared only if the switch fails. + switching atomic.Bool +} + +// AssignmentsMirrorMayWrite says the store's mirror of the assignments may still write the bucket: the assignments are +// neither being switched to the bus nor on it. Asked before every write of the mirror's writer, not once a pass. +func (i *Inventory) AssignmentsMirrorMayWrite() bool { + i.moved.mu.Lock() + moved := i.moved.kind[KindAssignments] + i.moved.mu.Unlock() + return !moved && !i.moved.switching.Load() +} + +// ChainOpener opens the chain on the bus when a process first needs it: a command that reads and never moved anything has +// no bus connection until a kind it reads turns out to be moved. +type ChainOpener func(ctx context.Context) (*chain.Chain, error) + +// UseChainOpener gives the inventory the means to open the chain when it needs it and none is attached. +func (i *Inventory) UseChainOpener(open ChainOpener) { + i.moved.mu.Lock() + i.moved.open = open + i.moved.mu.Unlock() +} + +// UseChain attaches the chain the moved kinds are read and written through. The serving controller attaches the bus's, +// holding the lease for its writes; a test attaches one in memory. +func (i *Inventory) UseChain(c *chain.Chain) { + i.moved.mu.Lock() + defer i.moved.mu.Unlock() + i.chain = c + if i.moved.view != nil { + i.moved.view.stop() + i.moved.view = nil + } +} + +// attached is the chain this inventory has, nil when none. +func (i *Inventory) attached() *chain.Chain { + i.moved.mu.Lock() + defer i.moved.mu.Unlock() + return i.chain +} + +// ChainReads says whether a kind of the chain is read from its bucket: chain_moved names it. It answers the chain to read +// through, and ErrNoChain when the kind is moved and none is attached. +func (i *Inventory) ChainReads(ctx context.Context, kind string) (*chain.Chain, bool, error) { + i.moved.mu.Lock() + known := i.moved.kind[kind] + i.moved.mu.Unlock() + if !known { + var moved bool + err := i.store.Pool().QueryRow(ctx, `select counts ? $1 from chain_moved`, kind).Scan(&moved) + var pgErr *pgconn.PgError + if errors.Is(err, pgx.ErrNoRows) || (errors.As(err, &pgErr) && pgErr.Code == "42P01") { + // No row, or a store older than the migration that makes the table: nothing moved. + return nil, false, nil + } + if err != nil { + return nil, false, fmt.Errorf("whether %s moved to the bus cannot be read: %w", kind, err) + } + if !moved { + return nil, false, nil + } + i.moved.mu.Lock() + if i.moved.kind == nil { + i.moved.kind = map[string]bool{} + } + i.moved.kind[kind] = true + i.moved.mu.Unlock() + } + i.moved.mu.Lock() + defer i.moved.mu.Unlock() + if i.chain == nil && i.moved.open != nil { + c, err := i.moved.open(ctx) + if err != nil { + return nil, false, fmt.Errorf("%w: %s: %v", ErrNoChain, kind, err) + } + i.chain = c + } + if i.chain == nil { + return nil, false, fmt.Errorf("%w: %s", ErrNoChain, kind) + } + return i.chain, true, nil +} + +// ChainMoved is, per kind moved, how many records the move wrote; empty before any is. +func (i *Inventory) ChainMoved(ctx context.Context) (map[string]int, error) { + var counts map[string]int + err := i.store.Pool().QueryRow(ctx, `select counts from chain_moved`).Scan(&counts) + if errors.Is(err, pgx.ErrNoRows) { + return map[string]int{}, nil + } + return counts, err +} + +// NewForTest is an inventory with no store, for a test of what needs none. +func NewForTest() *Inventory { return &Inventory{moved: &movedKinds{}} } + +// BeginSwitchForTest sets the flag a kind's switch sets, as a switch under way does. +func (i *Inventory) BeginSwitchForTest(kind string) { + if kind == KindAssignments { + i.moved.switching.Store(true) + } +} diff --git a/internal/inventory/chain_assignments.go b/internal/inventory/chain_assignments.go new file mode 100644 index 00000000..958924e0 --- /dev/null +++ b/internal/inventory/chain_assignments.go @@ -0,0 +1,396 @@ +package inventory + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/chain" +) + +// Assignments on the bus (novox/hq ADR 0408, issue 1093): once the kind `assignments` is moved (chain.go) an assignment is +// a key `.` of the assignments bucket, written by compare-and-set by the controller that holds the lease, +// and every question the joins of the store answered is answered from the bucket. **No key is removed** (ADR 0408 §4): an +// unassignment writes the value again with `ended` set, and a value with `ended` or a `state` is no assignment to any reader +// here (chain.Assignment.Stands). No `pending` state is written yet (issue 1094). + +// KindAssignments is the kind of the chain that is each module assigned to a node, with the build it runs there. +const KindAssignments = "assignments" + +func assignmentKey(node, module string) string { return node + "." + module } + +// assignRow is one assignment as the joins of the store read it. +type assignRow struct { + Node, Module string + Assigned time.Time +} + +// assignments is every assignment, by node then module: from the bucket when the kind is moved, else from the store. +// A key whose value names no time assigned is a module only pulled in by another (issue 765), and one that has ended or +// waits in a state is gone: no assignment. One on a machine the store does not know is none either, as the store took a +// removed machine's. +func (i *Inventory) assignments(ctx context.Context) ([]assignRow, error) { + c, on, err := i.ChainReads(ctx, KindAssignments) + if err != nil { + return nil, err + } + if !on { + rows, err := i.store.Pool().Query(ctx, + `select n.name, a.module, a.assigned from assignment a join node n on n.id = a.node order by n.name, a.module`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []assignRow + for rows.Next() { + var a assignRow + if err := rows.Scan(&a.Node, &a.Module, &a.Assigned); err != nil { + return nil, err + } + out = append(out, a) + } + return out, rows.Err() + } + all, err := c.Assignments.All(ctx) + if err != nil { + return nil, fmt.Errorf("the assignments cannot be read from the bus: %w", err) + } + nodes, err := i.nodeNames(ctx) + if err != nil { + return nil, err + } + var out []assignRow + for _, k := range all { + if k.Value.Assigned.IsZero() || !k.Value.Stands() || !nodes[k.Value.Node] { + continue + } + out = append(out, assignRow{k.Value.Node, k.Value.Module, k.Value.Assigned}) + } + sort.Slice(out, func(a, b int) bool { + if out[a].Node != out[b].Node { + return out[a].Node < out[b].Node + } + return out[a].Module < out[b].Module + }) + return out, nil +} + +func (i *Inventory) nodeNames(ctx context.Context) (map[string]bool, error) { + rows, err := i.store.Pool().Query(ctx, `select name from node`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]bool{} + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + out[name] = true + } + return out, rows.Err() +} + +// assignedNodes is the machines a module is assigned to, sorted. +func (i *Inventory) assignedNodes(ctx context.Context, module string) ([]string, error) { + rows, err := i.assignments(ctx) + if err != nil { + return nil, err + } + var out []string + for _, a := range rows { + if a.Module == module { + out = append(out, a.Node) + } + } + return out, nil +} + +// assignedModules is the modules assigned to a machine, sorted. +func (i *Inventory) assignedModules(ctx context.Context, node string) ([]string, error) { + rows, err := i.assignments(ctx) + if err != nil { + return nil, err + } + var out []string + for _, a := range rows { + if a.Node == node { + out = append(out, a.Module) + } + } + return out, nil +} + +// isAssigned says a module is assigned to a machine. +func (i *Inventory) isAssigned(ctx context.Context, node, module string) (bool, error) { + rows, err := i.assignments(ctx) + if err != nil { + return false, err + } + for _, a := range rows { + if a.Node == node && a.Module == module { + return true, nil + } + } + return false, nil +} + +// raiseGeneration raises the assignment generation, as the store's triggers do for a change of its table (migration +// 0094): in the chain nothing in the store changes with an assignment, so the writer says it. +func (i *Inventory) raiseGeneration(ctx context.Context) error { + _, err := i.store.Pool().Exec(ctx, `update assignment_generation set generation = generation + 1`) + return err +} + +// chainAssign puts a module on a machine, and says whether that is new. +func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, module string) (bool, error) { + var known bool + if err := i.store.Pool().QueryRow(ctx, `select exists (select 1 from module where name = $1)`, module).Scan(&known); err != nil { + return false, err + } + if !known { + return false, fmt.Errorf("%w: %s", ErrNoSuchModule, module) + } + key := assignmentKey(node.Name, module) + for try := 0; try < casTries; try++ { + cur, found, err := c.Assignments.Get(ctx, key) + if err != nil { + return false, err + } + if found && !cur.Value.Assigned.IsZero() && cur.Value.Stands() { + return false, nil // one assignment of a module per machine: a repeat is absorbed (novox/hq ADR 0115) + } + now := time.Now().UTC() + fresh := chain.Assignment{Node: node.Name, Module: module, Assigned: now} + switch { + case found && cur.Value.Stands(): + // A module only pulled in by another becomes an assignment, keeping what it holds. + cur.Value.Assigned = now + _, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision) + case found: + // One that ended is assigned again: a fresh value over it, the ended one staying in history. + _, err = c.Assignments.Update(ctx, key, fresh, cur.Revision) + default: + _, err = c.Assignments.Create(ctx, key, fresh) + } + if errors.Is(err, chain.ErrMoved) || errors.Is(err, chain.ErrExists) { + continue + } + if err != nil { + return false, fmt.Errorf("%s is not assigned to %s on the bus: %w", module, node.Name, err) + } + if err := i.raiseGeneration(ctx); err != nil { + return true, err + } + return true, nil + } + return false, fmt.Errorf("the assignment of %s to %s kept moving under the write", module, node.Name) +} + +// chainUnassign takes a module off a machine: the value is written again with `ended`, the key stays. +func (i *Inventory) chainUnassign(ctx context.Context, c *chain.Chain, node Node, module string) error { + key := assignmentKey(node.Name, module) + for try := 0; try < casTries; try++ { + cur, found, err := c.Assignments.Get(ctx, key) + if err != nil { + return err + } + if !found || cur.Value.Assigned.IsZero() || !cur.Value.Stands() { + return fmt.Errorf("%s is not assigned to %s", module, node.Name) + } + cur.Value.Ended = time.Now().UTC() + if _, err := c.Assignments.Update(ctx, key, cur.Value, cur.Revision); errors.Is(err, chain.ErrMoved) { + continue + } else if err != nil { + return fmt.Errorf("%s is not taken off %s on the bus: %w", module, node.Name, err) + } + return i.raiseGeneration(ctx) + } + return fmt.Errorf("the assignment of %s to %s kept moving under the write", module, node.Name) +} + +// chainAwaitingSince is AwaitingSince over the bucket: the oldest assignment on the machine made after it was last sent. +func (i *Inventory) chainAwaitingSince(ctx context.Context, name string) (time.Time, error) { + var sentAt *time.Time + var created time.Time + err := i.store.Pool().QueryRow(ctx, `select sent_at, created from node where name = $1`, name).Scan(&sentAt, &created) + if errors.Is(err, pgx.ErrNoRows) { + return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name) + } + if err != nil { + return time.Time{}, err + } + rows, err := i.assignments(ctx) + if err != nil { + return time.Time{}, err + } + var oldest time.Time + for _, a := range rows { + if a.Node == name && (sentAt == nil || a.Assigned.After(*sentAt)) && (oldest.IsZero() || a.Assigned.Before(oldest)) { + oldest = a.Assigned + } + } + switch { + case !oldest.IsZero(): + return oldest, nil + case sentAt != nil: + return *sentAt, nil + } + return created, nil +} + +// KeepAssignmentPlans writes the provisioner's plan of each assignment into its value on the bus, and a value of its own for +// each module only pulled in by another on a machine (issue 765), and ends the value of one that is no longer pulled in. The +// plans are derived from what each machine was last sent, not part of the record of what is assigned, so they are read +// again on a tick and written where they differ: **by compare-and-set on the one field**, never laying over the build, commit +// and rollout another write put in the same value, and never ending an assignment. A plan the provisioner's rules refuse is +// refused by name and nothing is written (AssignmentValues). +func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed int, err error) { + c, on, err := i.ChainReads(ctx, KindAssignments) + if err != nil || !on { + return 0, 0, err + } + var plans map[string]Provisioning + if i.provisioningPlans != nil { + if plans, err = i.provisioningPlans(ctx); err != nil { + return 0, 0, fmt.Errorf("composing the provisioner's plans: %w", err) + } + } + assigned, err := i.assignments(ctx) + if err != nil { + return 0, 0, err + } + rows := make([]AssignmentMirrored, 0, len(assigned)) + for _, a := range assigned { + rows = append(rows, AssignmentMirrored{Node: a.Node, Module: a.Module, Assigned: a.Assigned.UTC()}) + } + want, err := AssignmentValues(rows, plans) + if err != nil { + return 0, 0, err + } + have, err := c.Assignments.All(ctx) + if err != nil { + return 0, 0, fmt.Errorf("the assignments cannot be read from the bus: %w", err) + } + for key, body := range want { + var next chain.Assignment + if err := json.Unmarshal(body, &next); err != nil { + return written, removed, err + } + cur, has := have[key] + switch keepStep(has, cur.Value, next) { + case keepSkip: + continue + case keepCreate: + next.Assigned = time.Time{} + _, err = c.Assignments.Create(ctx, key, next) + case keepFresh: + // A value that ended, pulled in again: a fresh one over it, the ended one staying in history. + next.Assigned = time.Time{} + _, err = c.Assignments.Update(ctx, key, next, cur.Revision) + default: + cur.Value.RequiredBy, cur.Value.Provisioning = next.RequiredBy, next.Provisioning + _, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision) + } + if errors.Is(err, chain.ErrMoved) || errors.Is(err, chain.ErrExists) { + continue // written meanwhile; the next pass reads it + } + if err != nil { + return written, removed, fmt.Errorf("the plan of %s is not written: %w", key, err) + } + written++ + } + for key, cur := range have { + if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() { + continue + } + // A value of a module no longer pulled in, which no assignment names: it ends, and stays. + cur.Value.Ended = time.Now().UTC() + if _, err := c.Assignments.Update(ctx, key, cur.Value, cur.Revision); errors.Is(err, chain.ErrMoved) { + continue + } else if err != nil { + return written, removed, fmt.Errorf("the plan of %s is not ended: %w", key, err) + } + removed++ + } + return written, removed, nil +} + +// sameJSON says two encoded values are the same bytes, an absent value being a null one. +func sameJSON(a, b json.RawMessage) bool { + trim := func(x json.RawMessage) string { + if s := strings.TrimSpace(string(x)); s != "null" { + return s + } + return "" + } + return trim(a) == trim(b) +} + +// KeptValue is an assignment's value as the bucket holds it, encoded, at its revision. +type KeptValue struct { + Value []byte + Revision uint64 +} + +// KeptAssignments is every key of the assignments bucket that holds a value that stands, and false when the assignments are +// not on the bus. +// What the serving controller compares from one look to the next to say a key changed (`assignment-changed`). +func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, bool, error) { + c, on, err := i.ChainReads(ctx, KindAssignments) + if err != nil || !on { + return nil, false, err + } + all, err := c.Assignments.All(ctx) + if err != nil { + return nil, true, err + } + out := make(map[string]KeptValue, len(all)) + for key, k := range all { + if !k.Value.Stands() { + continue // an ended value is no assignment: for what is said of it, the key has gone + } + body, err := json.Marshal(k.Value) + if err != nil { + return nil, true, err + } + out[key] = KeptValue{Value: body, Revision: k.Revision} + } + return out, true, nil +} + +// What a pass that keeps the plans does with one key. +const ( + keepSkip = "skip" + keepCreate = "create" + keepFresh = "fresh" + keepField = "field" +) + +// keepStep decides what the pass does with a key from what it wants there (read from the assignments) and what the bucket +// holds (read a moment later). **It never makes or revives an assignment**: that is Assign's, and an unassignment between the +// two reads leaves a key that ended while the pass still wants it as an assignment, which would come back as a value +// pulled in by another. Only a key with no time assigned, on both sides, is made or made again here. +func keepStep(has bool, cur, next chain.Assignment) string { + switch { + case !has && !next.Assigned.IsZero(): + return keepSkip // assigned since the bucket was read: its own write is not here yet + case !has: + return keepCreate + case !cur.Stands(): + if !cur.Assigned.IsZero() || !next.Assigned.IsZero() { + return keepSkip // an assignment that ended is not pulled in again by a plan + } + return keepFresh + case sameJSON(cur.Provisioning, next.Provisioning) && cur.RequiredBy == next.RequiredBy: + return keepSkip + } + return keepField +} diff --git a/internal/inventory/chain_assignments_key_test.go b/internal/inventory/chain_assignments_key_test.go new file mode 100644 index 00000000..62e87cc7 --- /dev/null +++ b/internal/inventory/chain_assignments_key_test.go @@ -0,0 +1,81 @@ +package inventory + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/chain" +) + +// **An assignment's key is `.`, as the bucket has always had it** (ADR 0408 §3): its readers parse nothing +// but it. +func TestAnAssignmentsKeyIsTheNodeAndTheModule(t *testing.T) { + if got := assignmentKey("anchor", "notes"); got != "anchor.notes" { + t.Fatalf("the key is %q", got) + } + if got := assignmentKey("g14.example", "dunst"); got != "g14.example.dunst" { + t.Fatalf("the key of a machine whose name holds a dot is %q", got) + } +} + +// **A plan that is absent and a null one are one** (the provisioner's plan in an assignment's value): the pass that keeps +// the plans writes only where they differ. +func TestAnAbsentPlanAndANullOneAreTheSame(t *testing.T) { + for name, tc := range map[string]struct { + a, b json.RawMessage + same bool + }{ + "both absent": {nil, nil, true}, + "null and absent": {json.RawMessage(`null`), nil, true}, + "the same plan": {json.RawMessage(`{"files":[]}`), json.RawMessage(` {"files":[]} `), true}, + "another plan": {json.RawMessage(`{"files":[]}`), json.RawMessage(`{"files":[1]}`), false}, + "a plan and none": {json.RawMessage(`{"files":[]}`), nil, false}, + "none and a plan": {nil, json.RawMessage(`{"files":[]}`), false}, + } { + if got := sameJSON(tc.a, tc.b); got != tc.same { + t.Errorf("%s: %v", name, got) + } + } +} + +// **A pass that keeps the plans never brings back an assignment that ended** (issue 1093): it reads what it wants and what +// the bucket holds in two reads, and an unassignment between them leaves a key that ended while the pass still wants it +// assigned. +func TestAPassThatKeepsThePlansNeverRevivesAnEndedAssignment(t *testing.T) { + at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) + plan := json.RawMessage(`{"files":[]}`) + assigned := chain.Assignment{Node: "a", Module: "m", Assigned: at, Provisioning: plan} + ended := chain.Assignment{Node: "a", Module: "m", Assigned: at, Ended: at.Add(time.Minute)} + pulledIn := chain.Assignment{Node: "a", Module: "p", RequiredBy: "m", Provisioning: plan} + pulledInEnded := chain.Assignment{Node: "a", Module: "p", Ended: at} + for name, tc := range map[string]struct { + has bool + cur, next chain.Assignment + want string + }{ + "the interleaving: wanted assigned, ended since": {true, ended, assigned, keepSkip}, + "wanted assigned, not in the bucket yet": {false, chain.Assignment{}, assigned, keepSkip}, + "a pulled-in module, not in the bucket": {false, chain.Assignment{}, pulledIn, keepCreate}, + "a pulled-in module that ended, pulled again": {true, pulledInEnded, pulledIn, keepFresh}, + "an assignment ended, a plan for it": {true, ended, chain.Assignment{Node: "a", Module: "m", RequiredBy: "x", Provisioning: plan}, keepSkip}, + "the plan is the same": {true, assigned, assigned, keepSkip}, + "the plan changed": {true, chain.Assignment{Node: "a", Module: "m", Assigned: at}, assigned, keepField}, + } { + if got := keepStep(tc.has, tc.cur, tc.next); got != tc.want { + t.Errorf("%s: %s, want %s", name, got, tc.want) + } + } +} + +// **The store's mirror of the assignments is refused to its writer once the assignments are moved** (ADR 0408 §4, issue +// 1093): asked at the moment of the write, so a tick that decided before the switch lays nothing after it. +func TestTheStoresMirrorOfTheAssignmentsIsRefusedToItsWriterOnceTheyAreMoved(t *testing.T) { + inv := &Inventory{moved: &movedKinds{kind: map[string]bool{KindAssignments: true}}} + inv.UseChain(chain.InMemory(nil)) + if _, err := inv.AssignmentsMirrorToWrite(context.Background()); !errors.Is(err, ErrAssignmentsMoved) { + t.Fatalf("the writer was given the store's rows: %v", err) + } +} diff --git a/internal/inventory/chain_builds.go b/internal/inventory/chain_builds.go new file mode 100644 index 00000000..ececd272 --- /dev/null +++ b/internal/inventory/chain_builds.go @@ -0,0 +1,570 @@ +package inventory + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/chain" +) + +// Builds on the bus (novox/hq ADR 0408, issue 1090): once the kind `builds` is moved (chain.go) every build is a key of +// the builds bucket, under its id, and every read below is made from the bucket's records. The tables are read when the +// kind is not moved, by the methods these stand beside. + +// toChainBuild is a build as the bucket keeps it. +func toChainBuild(b Build) chain.Build { + c := chain.Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On, + Path: b.Path, Against: b.Against, Mirrored: b.Mirrored, SourceFingerprint: b.SourceFingerprint, + Failed: b.Failed, Asked: b.Asked.UTC(), At: b.At.UTC(), ReuseKey: b.ReuseKey, Published: b.Published} + if len(b.Manifest) > 0 { + c.Manifest = json.RawMessage(b.Manifest) + } + for _, r := range b.Read { + c.Read = append(c.Read, chain.ReadRepository{Repository: r.Repository, Ref: r.Ref, Paths: r.Paths, Own: r.Own}) + } + for _, s := range b.Sources { + c.Sources = append(c.Sources, chain.Source{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths}) + } + for _, a := range b.Made { + c.Made = append(c.Made, chain.Artifact{Name: a.Name, Kind: a.Kind, Reference: a.Reference}) + } + if c.Published == nil { + c.Published = publishedOf(b.Made) + } + return c +} + +// fromChainBuild is a build as the mesh reads it. +func fromChainBuild(c chain.Build) Build { + b := Build{ID: c.ID, Repository: c.Repository, Ref: c.Ref, Module: c.Module, Commit: c.Commit, On: c.On, Path: c.Path, + Manifest: []byte(c.Manifest), Against: c.Against, Mirrored: c.Mirrored, SourceFingerprint: c.SourceFingerprint, + Failed: c.Failed, Asked: c.Asked, At: c.At, ReuseKey: c.ReuseKey, Published: c.Published} + if len(b.Manifest) == 0 { + b.Manifest = nil + } + for _, r := range c.Read { + b.Read = append(b.Read, ReadRepository{Repository: r.Repository, Ref: r.Ref, Paths: r.Paths, Own: r.Own}) + } + for _, s := range c.Sources { + b.Sources = append(b.Sources, BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths}) + } + for _, a := range c.Made { + b.Made = append(b.Made, Artifact{Name: a.Name, Kind: a.Kind, Reference: a.Reference}) + } + return b +} + +// publishedOf is what a build published, read from what it made: a package by name and version, and an artifact by the +// digest of the first thing it made that has one (`@sha256:…`). Nil for a build that published nothing. +func publishedOf(made []Artifact) *chain.Published { + var out chain.Published + for _, a := range made { + switch { + case a.Kind == "package": + if name, version, ok := strings.Cut(a.Reference, "@"); ok && name != "" && version != "" && out.Package == nil { + out.Package = &chain.Package{Name: name, Version: version} + } + case out.Artifact == "": + if _, digest, ok := strings.Cut(a.Reference, "@"); ok && strings.HasPrefix(digest, "sha256:") { + out.Artifact = digest + } + } + } + if out.Artifact == "" && out.Package == nil { + return nil + } + return &out +} + +// chainBuildKey is a build's key. An id the bus takes no spelling of is refused by the write, loudly. +func chainBuildKey(id string) string { return id } + +// chainRecordBuild keeps a build under its id. **Idempotent on the id**, as the table's insert was: a result arrives twice, +// and the first record stands. A request nobody has answered is the record in state asked (issue 1091); the outcome +// takes its place under the same key, and keeps the request it answers. +func (i *Inventory) chainRecordBuild(ctx context.Context, c *chain.Chain, b Build) error { + if b.At.IsZero() { + b.At = time.Now().UTC() + } + key := chainBuildKey(b.ID) + for try := 0; try < casTries; try++ { + next := toChainBuild(b) + cur, found, err := c.Builds.Get(ctx, key) + if err != nil { + return fmt.Errorf("build %s is not kept on the bus: %w", b.ID, err) + } + switch { + case !found: + _, err = c.Builds.Create(ctx, key, next) + if errors.Is(err, chain.ErrExists) { + continue + } + case cur.Value.State == chain.BuildAsked: + next.Request = cur.Value.Request + _, err = c.Builds.Update(ctx, key, next, cur.Revision) + if errors.Is(err, chain.ErrMoved) { + continue + } + default: + // An outcome stands: the first record is the record. + return i.RecordMirrored(ctx, b.ID, "", b.Mirrored) + } + if err != nil { + return fmt.Errorf("build %s is not kept on the bus: %w", b.ID, err) + } + return i.RecordMirrored(ctx, b.ID, "", b.Mirrored) + } + return fmt.Errorf("build %s is not kept on the bus: it kept moving under the write", b.ID) +} + +// casTries is how often a write that lost a compare-and-set reads the key again and tries once more. +const casTries = 8 + +// chainBuilds is every build of the bucket, whole, in no order. +func chainBuilds(ctx context.Context, c *chain.Chain) ([]Build, error) { + all, err := c.Builds.All(ctx) + if err != nil { + return nil, fmt.Errorf("the builds cannot be read from the bus: %w", err) + } + out := make([]Build, 0, len(all)) + for _, k := range all { + // A request nobody has answered is no build: it has no outcome to read. + if k.Value.State == chain.BuildAsked { + continue + } + out = append(out, fromChainBuild(k.Value)) + } + return out, nil +} + +// newestRequestOrder sorts builds as newestRequestFirst does: the newest request first, then the newest record. +func newestRequestOrder(builds []Build) { + sort.SliceStable(builds, func(a, b int) bool { + x, y := builds[a].AskedOrAt(), builds[b].AskedOrAt() + if !x.Equal(y) { + return x.After(y) + } + if !builds[a].At.Equal(builds[b].At) { + return builds[a].At.After(builds[b].At) + } + return builds[a].ID > builds[b].ID + }) +} + +// byModule sorts builds by module, then newest request first. +func byModule(builds []Build) { + newestRequestOrder(builds) + sort.SliceStable(builds, func(a, b int) bool { return builds[a].Module < builds[b].Module }) +} + +// worked is the builds that made something for a module: succeeded, and knew what they built. +func worked(all []Build) []Build { + var out []Build + for _, b := range all { + if b.Failed == "" && b.Module != "" { + out = append(out, b) + } + } + return out +} + +func (i *Inventory) chainBuildList(ctx context.Context, c *chain.Chain, module string, limit int) ([]Build, error) { + all, err := chainBuilds(ctx, c) + if err != nil { + return nil, err + } + var out []Build + for _, b := range all { + if module == "" || b.Module == module { + out = append(out, Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, + On: b.On, Failed: b.Failed, Made: b.Made, Asked: b.Asked, At: b.At, ReuseKey: b.ReuseKey, Published: b.Published}) + } + } + sort.SliceStable(out, func(a, b int) bool { + if !out[a].At.Equal(out[b].At) { + return out[a].At.After(out[b].At) + } + return out[a].ID > out[b].ID + }) + if len(out) > limit { + out = out[:limit] + } + return out, nil +} + +func (i *Inventory) chainBuildByID(ctx context.Context, c *chain.Chain, id string) (Build, bool, error) { + k, ok, err := c.Builds.Get(ctx, chainBuildKey(id)) + if err != nil || !ok || k.Value.State == chain.BuildAsked { + // A request nobody has answered has no outcome to read. + return Build{}, false, err + } + b := fromChainBuild(k.Value) + return Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On, + Failed: b.Failed, Made: b.Made, Path: b.Path, Asked: b.Asked, At: b.At}, true, nil +} + +func (i *Inventory) chainBuiltAgainst(ctx context.Context, c *chain.Chain) (map[string][]string, error) { + all, err := chainBuilds(ctx, c) + if err != nil { + return nil, err + } + ok := worked(all) + byModule(ok) + out := map[string][]string{} + seen := map[string]bool{} + for _, b := range ok { + if seen[b.Module] { + continue + } + seen[b.Module] = true + if len(b.Against) > 0 { + out[b.Module] = b.Against + } + } + return out, nil +} + +func (i *Inventory) chainReadRepositories(ctx context.Context, c *chain.Chain) (map[string][]ReadRepository, error) { + all, err := chainBuilds(ctx, c) + if err != nil { + return nil, err + } + // The newest build of each module whatever its outcome: a failed one newer than the newest that worked leaves that + // one's build source stale. + type newestOf struct { + at time.Time + failed bool + } + newest := map[string]newestOf{} + tried := make([]Build, 0, len(all)) + for _, b := range all { + if b.Module != "" { + tried = append(tried, b) + } + } + byModule(tried) + for _, b := range tried { + if _, seen := newest[b.Module]; !seen { + newest[b.Module] = newestOf{b.AskedOrAt(), b.Failed != ""} + } + } + owns, err := i.moduleSources(ctx) + if err != nil { + return nil, err + } + ok := worked(all) + byModule(ok) + read := map[string][]ReadRepository{} + seen := map[string]bool{} + for _, b := range ok { + if seen[b.Module] { + continue + } + seen[b.Module] = true + built := b.AskedOrAt() + of := append([]ReadRepository(nil), b.Read...) + sources := b.Sources + whole := "" + if n, known := newest[b.Module]; known && n.failed && n.at.After(built) && len(sources) > 0 { + sources, whole = nil, "a newer build of it failed" + } + of = WithBuildSources(of, sources) + if own, has := owns[b.Module]; has { + of = OwnRepositoryAsOwn(of, own) + } + if whole != "" { + of = append(of, ReadRepository{Own: true, Whole: whole}) + } + if len(of) > 0 { + for k := range of { + of[k].Built, of[k].Looked = built, newest[b.Module].at + } + read[b.Module] = of + } + } + return read, nil +} + +// moduleSources is the repository each registered module is registered from. +func (i *Inventory) moduleSources(ctx context.Context) (map[string]string, error) { + rows, err := i.store.Pool().Query(ctx, `select name, source from module where source is not null`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]string{} + for rows.Next() { + var name, source string + if err := rows.Scan(&name, &source); err != nil { + return nil, err + } + out[name] = source + } + return out, rows.Err() +} + +func (i *Inventory) chainAnnounceable(ctx context.Context, c *chain.Chain) ([]Build, error) { + all, err := chainBuilds(ctx, c) + if err != nil { + return nil, err + } + var ok []Build + for _, b := range worked(all) { + if b.Commit != "" { + ok = append(ok, b) + } + } + newestRequestOrder(ok) + type key struct{ module, commit string } + seen := map[key]bool{} + var out []Build + for _, b := range ok { + k := key{b.Module, b.Commit} + if seen[k] { + continue + } + seen[k] = true + out = append(out, Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, + On: b.On, Failed: b.Failed, Made: b.Made, Path: b.Path, Manifest: b.Manifest, Against: b.Against, At: b.At}) + } + sort.SliceStable(out, func(a, b int) bool { return out[a].At.Before(out[b].At) }) + return out, nil +} + +// failedAtAGate is the builds that failed their gate. The gate's verdicts stay in their table (issue 1097). +func (i *Inventory) failedAtAGate(ctx context.Context) (map[string]bool, error) { + rows, err := i.store.Pool().Query(ctx, `select build from build_gate where verdict = 'failed'`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]bool{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + out[id] = true + } + return out, rows.Err() +} + +func (i *Inventory) chainSourceRows(ctx context.Context, c *chain.Chain, module string) (map[string][]sourceRow, []string, error) { + all, err := chainBuilds(ctx, c) + if err != nil { + return nil, nil, err + } + failed, err := i.failedAtAGate(ctx) + if err != nil { + return nil, nil, err + } + ok := worked(all) + byModule(ok) + out := map[string][]sourceRow{} + var order []string + for _, b := range ok { + if module != "" && b.Module != module { + continue + } + made, err := json.Marshal(b.Made) + if err != nil { + return nil, nil, err + } + if _, seen := out[b.Module]; !seen { + order = append(order, b.Module) + } + out[b.Module] = append(out[b.Module], sourceRow{id: b.ID, commit: b.Commit, fingerprint: b.SourceFingerprint, + made: made, manifest: b.Manifest, failedGate: failed[b.ID]}) + } + return out, order, nil +} + +// canonical is a JSON document encoded the one way (objects' keys sorted, compact), so the same content hashes alike +// however the bucket's record spells it. +func canonicalJSON(raw []byte) []byte { + if len(bytes.TrimSpace(raw)) == 0 { + return nil + } + out, err := canonical(raw) + if err != nil { + return raw + } + return out +} + +// chainFingerprint is what a build would put on a machine, hashed: its artifacts and its manifest. +func chainFingerprint(b Build) string { + made, _ := json.Marshal(b.Made) + sum := sha256.Sum256(append(append(canonicalJSON(made), 0), canonicalJSON(b.Manifest)...)) + return hex.EncodeToString(sum[:]) +} + +func (i *Inventory) chainBuildFingerprints(ctx context.Context, c *chain.Chain, module string) (map[string]string, error) { + all, err := chainBuilds(ctx, c) + if err != nil { + return nil, err + } + var ok []Build + for _, b := range all { + if b.Module == module && b.Failed == "" && b.Commit != "" { + ok = append(ok, b) + } + } + sortByRecorded(ok) + if len(ok) > 50 { + ok = ok[:50] + } + out := map[string]string{} + for _, b := range ok { + if _, seen := out[b.Commit]; !seen { + out[b.Commit] = chainFingerprint(b) + } + } + return out, nil +} + +func (i *Inventory) chainFingerprints(ctx context.Context, c *chain.Chain) (map[string]map[string]string, error) { + all, err := chainBuilds(ctx, c) + if err != nil { + return nil, err + } + var ok []Build + for _, b := range all { + if b.Module != "" && b.Failed == "" && b.Commit != "" { + ok = append(ok, b) + } + } + sortByRecorded(ok) + out := map[string]map[string]string{} + for _, b := range ok { + if out[b.Module] == nil { + out[b.Module] = map[string]string{} + } + if _, seen := out[b.Module][b.Commit]; !seen { + out[b.Module][b.Commit] = chainFingerprint(b) + } + } + return out, nil +} + +// sortByRecorded is newest record first, as `order by at desc`. +func sortByRecorded(builds []Build) { + sort.SliceStable(builds, func(a, b int) bool { + if !builds[a].At.Equal(builds[b].At) { + return builds[a].At.After(builds[b].At) + } + return builds[a].ID > builds[b].ID + }) +} + +func (i *Inventory) chainBuildOf(ctx context.Context, c *chain.Chain, module, commit string) (string, error) { + all, err := chainBuilds(ctx, c) + if err != nil { + return "", err + } + var ok []Build + for _, b := range all { + if b.Module == module && b.Commit == commit && b.Failed == "" { + ok = append(ok, b) + } + } + sortByRecorded(ok) + if len(ok) == 0 { + return "", nil + } + return ok[0].ID, nil +} + +// chainManifestBuild is the newest successful build of a module from a commit (or one it abbreviates) that kept a manifest. +func (i *Inventory) chainManifestBuild(ctx context.Context, c *chain.Chain, module, commit string) (string, []byte, bool, error) { + all, err := chainBuilds(ctx, c) + if err != nil { + return "", nil, false, err + } + var ok []Build + for _, b := range all { + if b.Module == module && b.Failed == "" && len(b.Manifest) > 0 && string(bytes.TrimSpace(b.Manifest)) != "null" && + (b.Commit == commit || strings.HasPrefix(b.Commit, commit)) { + ok = append(ok, b) + } + } + sortByRecorded(ok) + if len(ok) == 0 { + return "", nil, false, nil + } + return ok[0].ID, ok[0].Manifest, true, nil +} + +// chainManifestOf is the manifest a build was recorded with. +func (i *Inventory) chainManifestOf(ctx context.Context, c *chain.Chain, id string) ([]byte, error) { + k, ok, err := c.Builds.Get(ctx, chainBuildKey(id)) + if err != nil || !ok { + return nil, err + } + return []byte(k.Value.Manifest), nil +} + +// chainRecentMade is the artifacts and bases of the KeptBuilds newest successful builds of each registered module. +func (i *Inventory) chainRecentMade(ctx context.Context, c *chain.Chain) ([]Build, error) { + all, err := chainBuilds(ctx, c) + if err != nil { + return nil, err + } + rows, err := i.store.Pool().Query(ctx, `select name from module`) + if err != nil { + return nil, err + } + registered := map[string]bool{} + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + return nil, err + } + registered[name] = true + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + var ok []Build + for _, b := range worked(all) { + if registered[b.Module] { + ok = append(ok, b) + } + } + sortByRecorded(ok) + back := map[string]int{} + var out []Build + for _, b := range ok { + back[b.Module]++ + if back[b.Module] <= KeptBuilds { + out = append(out, b) + } + } + return out, nil +} + +// chainEveryReferenceMade is every successful build's artifacts, oldest build first. +func (i *Inventory) chainEveryReferenceMade(ctx context.Context, c *chain.Chain) ([]Build, error) { + all, err := chainBuilds(ctx, c) + if err != nil { + return nil, err + } + ok := worked(all) + sort.SliceStable(ok, func(a, b int) bool { + if !ok[a].At.Equal(ok[b].At) { + return ok[a].At.Before(ok[b].At) + } + return ok[a].ID < ok[b].ID + }) + return ok, nil +} diff --git a/internal/inventory/chain_builds_test.go b/internal/inventory/chain_builds_test.go new file mode 100644 index 00000000..348b20d1 --- /dev/null +++ b/internal/inventory/chain_builds_test.go @@ -0,0 +1,131 @@ +package inventory + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/chain" +) + +// **A build keeps its reuse key and what it published, and comes back whole from the bucket** (novox/hq ADR 0408 §1, +// issue 1090): an artifact by the digest of what it made, a package release by name and version. +func TestABuildKeepsItsReuseKeyAndWhatItPublishedWhenKeptOnTheBus(t *testing.T) { + at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) + b := Build{ID: "build-1", Repository: "novox/notes", Ref: "main", Module: "notes", Commit: "c1", On: "ace", Path: "modules/notes", + Manifest: []byte(`{"module":"notes"}`), Against: []string{"artifact-store://base@sha256:aa"}, + Read: []ReadRepository{{Repository: "novox/other", Ref: "main"}}, + Sources: []BuildSource{{Paths: []string{"modules/notes/module.json"}}}, + SourceFingerprint: "fp", Asked: at, At: at.Add(time.Minute), ReuseKey: "rk-1", + Made: []Artifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes@sha256:bb"}, + {Name: "sdk", Kind: "package", Reference: "mesh-sdk@1.4.0"}}} + kept := toChainBuild(b) + if kept.ReuseKey != "rk-1" { + t.Errorf("the reuse key is %q", kept.ReuseKey) + } + if kept.Published == nil || kept.Published.Artifact != "sha256:bb" || kept.Published.Package == nil || + kept.Published.Package.Name != "mesh-sdk" || kept.Published.Package.Version != "1.4.0" { + t.Errorf("it published %+v", kept.Published) + } + // Through the bucket's bytes and back: nothing is lost. + raw, err := json.Marshal(kept) + if err != nil { + t.Fatal(err) + } + var back chain.Build + if err := json.Unmarshal(raw, &back); err != nil { + t.Fatal(err) + } + got := fromChainBuild(back) + b.Published = kept.Published + if !reflect.DeepEqual(got, b) { + t.Fatalf("the build came back as\n %+v\nnot\n %+v", got, b) + } +} + +func TestWhatABuildPublishedIsReadFromWhatItMade(t *testing.T) { + for name, tc := range map[string]struct { + made []Artifact + want *chain.Published + }{ + "nothing": {nil, nil}, + "an archive with no digest": {[]Artifact{{Name: "config", Kind: "archive", Reference: "…/blobs/sha256:…"}}, nil}, + "an image": {[]Artifact{{Name: "a", Kind: "image", Reference: "artifact-store://a@sha256:11"}}, + &chain.Published{Artifact: "sha256:11"}}, + "the first digest stands": {[]Artifact{{Kind: "image", Reference: "x@sha256:11"}, {Kind: "archive", Reference: "y@sha256:22"}}, + &chain.Published{Artifact: "sha256:11"}}, + "a package": {[]Artifact{{Name: "sdk", Kind: "package", Reference: "mesh-sdk@2.0.1"}}, + &chain.Published{Package: &chain.Package{Name: "mesh-sdk", Version: "2.0.1"}}}, + "a package with no version": {[]Artifact{{Name: "sdk", Kind: "package", Reference: "mesh-sdk"}}, nil}, + } { + if got := publishedOf(tc.made); !reflect.DeepEqual(got, tc.want) { + t.Errorf("%s: published %+v, not %+v", name, got, tc.want) + } + } +} + +// **The same content hashes alike, however the record spells it** (issue 280's fingerprints, now over the bucket's bytes +// and not the store's jsonb text): key order and white space of a manifest change nothing. +func TestTheSameBuildHashesAlikeWhateverTheSpellingOfItsRecord(t *testing.T) { + a := Build{Manifest: []byte(`{"module": "notes", "version": "1"}`), Made: []Artifact{{Name: "n", Kind: "image", Reference: "r"}}} + b := Build{Manifest: []byte("{\"version\":\"1\",\n \"module\":\"notes\"}"), Made: []Artifact{{Name: "n", Kind: "image", Reference: "r"}}} + if chainFingerprint(a) != chainFingerprint(b) { + t.Fatal("two spellings of one manifest hash apart") + } + b.Made[0].Reference = "other" + if chainFingerprint(a) == chainFingerprint(b) { + t.Fatal("two artifacts hash alike") + } +} + +// **Newest request first, as the store's ordering was**: the newest ask, then the newest record, then the newest id. +func TestBuildsAreOrderedNewestRequestFirst(t *testing.T) { + t0 := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) + builds := []Build{ + {ID: "old", Asked: t0, At: t0.Add(time.Hour)}, + {ID: "unasked", At: t0.Add(2 * time.Hour)}, + {ID: "new", Asked: t0.Add(3 * time.Hour), At: t0.Add(3*time.Hour + time.Minute)}, + {ID: "same-a", Asked: t0.Add(4 * time.Hour), At: t0.Add(5 * time.Hour)}, + {ID: "same-b", Asked: t0.Add(4 * time.Hour), At: t0.Add(5 * time.Hour)}, + } + newestRequestOrder(builds) + var got []string + for _, b := range builds { + got = append(got, b.ID) + } + if want := []string{"same-b", "same-a", "new", "unasked", "old"}; !reflect.DeepEqual(got, want) { + t.Fatalf("ordered %v, not %v", got, want) + } +} + +// **A bus that does not answer, a lease lost and a key written first are "not now"** (issue 1092): a merge or a report whose +// write failed so is held and handed over again, as it is for a store that is away. +func TestTheBusBeingAwayIsNotNowLikeTheStoreBeingAway(t *testing.T) { + for name, err := range map[string]error{ + "a timeout": fmt.Errorf("the plan is not written: %w", nats.ErrTimeout), + "no responders": fmt.Errorf("build 1: %w", nats.ErrNoResponders), + "a closed bus": nats.ErrConnectionClosed, + "the lease lost": fmt.Errorf("%w: another controller holds it", chain.ErrNotHeld), + "a key moved": chain.ErrMoved, + "a deadline": context.DeadlineExceeded, + } { + if !Unreachable(err) { + t.Errorf("%s is read as an answer", name) + } + } + for name, err := range map[string]error{ + "a value too large": chain.ErrTooLarge, + "a key not taken": chain.ErrBadKey, + "a plan moved": ErrPlanMoved, + "a key that exists": chain.ErrExists, + } { + if Unreachable(err) { + t.Errorf("%s is read as a bus being away: asking again gets the same answer", name) + } + } +} diff --git a/internal/inventory/chain_links.go b/internal/inventory/chain_links.go new file mode 100644 index 00000000..c671c93f --- /dev/null +++ b/internal/inventory/chain_links.go @@ -0,0 +1,190 @@ +package inventory + +import ( + "context" + "errors" + "fmt" + "os" + "sort" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/chain" +) + +// The links of the chain (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): a send that puts a build on a machine +// writes, in the assignment of the module there, the build, what it published, its commit and the rollout that sent it, and +// since when; the rollout lists the assignments it changed with the build before and after (the controller's send, in +// release_plan.go). One read of an assignment then names the commit, the build and the rollout that put it there, and one read +// of the rollout lists its assignments. + +type rolloutKey struct{} + +// RolloutModule is what a rollout sends of one module: the commit and the build of it. +type RolloutModule struct{ Commit, Build string } + +type rolloutOf struct { + id string + modules map[string]RolloutModule +} + +// WithRollout is a context for a send a rollout makes: what RecordSent writes in the assignments of the modules it carries +// names this rollout and the build it sent. A module the send carries beside these, that another walk is moving, is named +// by the build the mesh holds of its commit and by no rollout. +func WithRollout(ctx context.Context, id string, modules map[string]RolloutModule) context.Context { + return context.WithValue(ctx, rolloutKey{}, rolloutOf{id: id, modules: modules}) +} + +// AssignmentRuns is what each module assigned to a machine runs there, as its assignment names it: the build and the +// commit. Empty when the assignments are not on the bus, and for a module not sent yet. +func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string]chain.Ref, error) { + c, on, err := i.ChainReads(ctx, KindAssignments) + if err != nil || !on { + return nil, err + } + all, err := c.Assignments.All(ctx) + if err != nil { + return nil, err + } + out := map[string]chain.Ref{} + for _, k := range all { + a := k.Value + if a.Node == node && !a.Assigned.IsZero() && a.Stands() && (a.Build != "" || a.Commit != "") { + out[a.Module] = chain.Ref{Build: a.Build, Commit: a.Commit} + } + } + return out, nil +} + +// AssignmentsOn are the assignments of a machine as the chain names them, by module: the standing ones, with their links. +func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Assignment, error) { + c, on, err := i.ChainReads(ctx, KindAssignments) + if err != nil || !on { + return nil, err + } + all, err := c.Assignments.All(ctx) + if err != nil { + return nil, err + } + var out []chain.Assignment + for _, k := range all { + if a := k.Value; a.Node == node && !a.Assigned.IsZero() && a.Stands() { + out = append(out, a) + } + } + sort.Slice(out, func(a, b int) bool { return out[a].Module < out[b].Module }) + return out, nil +} + +// recordRuns writes what a send put on a machine into the assignments, once the send is recorded: for each module it carried +// a build of — by the commit — the build, what it published, the commit, the rollout and since when. **A module that runs the +// commit already keeps what it names**, filled where it names nothing and never blanked, so a send that carries it again, or a +// push by hand, changes nothing it knows. A module the machine is not assigned is no assignment to write. +func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[string]string, at time.Time) error { + c, on, err := i.ChainReads(ctx, KindAssignments) + if err != nil || !on || len(builds) == 0 { + return err + } + var node string + if err := i.store.Pool().QueryRow(ctx, `select name from node where id = $1`, nodeID).Scan(&node); errors.Is(err, pgx.ErrNoRows) { + return nil + } else if err != nil { + return err + } + rollout, _ := ctx.Value(rolloutKey{}).(rolloutOf) + modules := make([]string, 0, len(builds)) + for m := range builds { + modules = append(modules, m) + } + sort.Strings(modules) + var failed []error + for _, m := range modules { + commit := builds[m] + if commit == "" { + continue + } + if err := i.recordRun(ctx, c, node, m, commit, rollout, at); err != nil { + failed = append(failed, fmt.Errorf("%s on %s: %w", m, node, err)) + } + } + return errors.Join(failed...) +} + +func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, commit string, rollout rolloutOf, at time.Time) error { + key := assignmentKey(node, module) + var build, rolloutID string + if sent, ours := rollout.modules[module]; ours && sent.Commit == commit { + build, rolloutID = sent.Build, rollout.id + } + var published *chain.Published + resolved := false + for try := 0; try < casTries; try++ { + cur, found, err := c.Assignments.Get(ctx, key) + if err != nil { + return err + } + if !found || cur.Value.Assigned.IsZero() || !cur.Value.Stands() { + return nil // nothing assigned there to say what it runs + } + same := cur.Value.Commit == commit + if same && cur.Value.Build != "" && (rolloutID == "" || cur.Value.Rollout != "") { + return nil // a send that carries what the machine runs already says nothing new + } + if !resolved { + // The build of the commit is looked up once, and only for a module that moved or names no build. + if build == "" { + if build, err = i.BuildOf(ctx, module, commit); err != nil { + return err + } + } + if build != "" { + if b, found, err := i.BuildByID(ctx, build); err != nil { + return err + } else if found { + published = publishedOf(b.Made) + } + } + resolved = true + } + next := cur.Value + if same { + // Already running it: fill what is missing, blank nothing. + if next.Build == "" { + next.Build = build + } + if next.Rollout == "" { + next.Rollout = rolloutID + } + if next.Since.IsZero() { + next.Since = at.UTC() + } + } else { + next.Commit, next.Build, next.Rollout, next.Since = commit, build, rolloutID, at.UTC() + next.Artifact, next.Package = "", nil + } + if published != nil && next.Artifact == "" && next.Package == nil && next.Build == build { + next.Artifact, next.Package = published.Artifact, published.Package + } + if next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout && + next.Artifact == cur.Value.Artifact && (next.Package == nil) == (cur.Value.Package == nil) && + next.Since.Equal(cur.Value.Since) { + return nil + } + if _, err := c.Assignments.Update(ctx, key, next, cur.Revision); errors.Is(err, chain.ErrMoved) { + continue + } else { + return err + } + } + return fmt.Errorf("the assignment kept moving under the write") +} + +// saidRunsNot says a send's links could not be written, to the log of whoever sent: the send is away and is recorded, and +// a record of what it put where that could not be written is said, never a reason to call the send failed. +func saidRunsNot(node string, err error) { + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: %s was sent its declaration, and the build, commit and rollout of each module "+ + "could not be written in its assignments: %v\n", node, err) + } +} diff --git a/internal/inventory/chain_move.go b/internal/inventory/chain_move.go new file mode 100644 index 00000000..281ce98f --- /dev/null +++ b/internal/inventory/chain_move.go @@ -0,0 +1,663 @@ +package inventory + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/chain" +) + +// **The move** (novox/hq ADR 0408 §8, issue 981 note 7 Q2): the serving controller writes the rows of a kind into its +// bucket at its start, holding the lease and after the buckets exist, and records the kind in chain_moved. Idempotent: a +// key that exists is already moved and is never overwritten, so a bucket value written since is not laid over by the row +// it came from. It counts the rows against the keys, logs the largest value, and **refuses** (a Refusal, which the +// controller raises as an urgent condition) on a count that differs or a value over the bus's largest, with nothing +// switched: a kind is recorded moved only once all of it is in the bucket. + +// Refusal is a move that stopped without switching a kind, and why. It is permanent until the cause is put right; any +// other error of a move (the bus away, a write not granted) is tried again. +type Refusal struct { + Kind, Why string +} + +func (r *Refusal) Error() string { + return fmt.Sprintf("the move of %s to the bus is refused: %s", r.Kind, r.Why) +} + +// MoveReport is what a move of one kind did. +type MoveReport struct { + Kind string + // Rows is how many records the tables held, Written how many keys this move made, and Present how many already + // held a value (moved by an earlier start). + Rows, Written, Present int + // TableRows is how many rows of the tables those records came from: a request and its outcome are two rows and one + // record. + TableRows int + // Largest is the biggest value, in bytes. + Largest int + // Already is a kind an earlier start moved: nothing was done. + Already bool +} + +// mover is one kind of the chain: its rows, read from its tables, as the keys and values they become. +type mover struct { + kind string + // rows reads the tables: each record's key and its value, whole, and how many rows of the tables they came from. + rows func(ctx context.Context) ([]moveRow, int, error) + // create writes one record under its key; chain.ErrExists when the key holds a value. + create func(ctx context.Context, c *chain.Chain, row moveRow) error + // keys are the keys the bucket holds. + keys func(ctx context.Context, c *chain.Chain) ([]string, error) + // count is how many rows the tables hold now, counted as rows reports them. + count func(ctx context.Context, tx pgx.Tx) (int, error) + // lock keeps the tables from taking a row while the switch is recorded. + lock string + // settle runs under that lock, with the rows read again: it makes the bucket hold what the rows say now. A row the old + // controller changed after the first pass is in the bucket before the kind is recorded as moved, for before the switch + // the table is the only truth. + settle func(ctx context.Context, c *chain.Chain, rows []moveRow) error +} + +type moveRow struct { + key string + value any + size int +} + +func (i *Inventory) movers() []mover { + return []mover{{ + kind: KindBuilds, + rows: func(ctx context.Context) ([]moveRow, int, error) { + builds, err := i.allBuildRows(ctx) + if err != nil { + return nil, 0, err + } + requests, err := i.allRequestRows(ctx) + if err != nil { + return nil, 0, err + } + // A build request is the build's record in state asked, and the request stays beside the outcome that + // answered it: one record under one id (issue 1091). + byID := make(map[string]*chain.Build, len(builds)) + var order []string + for _, b := range builds { + v := toChainBuild(b) + byID[b.ID] = &v + order = append(order, b.ID) + } + for _, r := range requests { + if v, answered := byID[r.ID]; answered { + v.Request = toRequest(r) + continue + } + v := askedRecord(r) + if r.NotAsked != "" { + // Never handed over: asked again once, after the move (askAgain). + v.Request.Reask = chain.ReaskPending + } + byID[r.ID] = &v + order = append(order, r.ID) + } + out := make([]moveRow, 0, len(order)) + for _, id := range order { + body, err := json.Marshal(byID[id]) + if err != nil { + return nil, 0, err + } + out = append(out, moveRow{key: chainBuildKey(id), value: *byID[id], size: len(body)}) + } + return out, len(builds) + len(requests), nil + }, + create: func(ctx context.Context, c *chain.Chain, row moveRow) error { + _, err := c.Builds.Create(ctx, row.key, row.value.(chain.Build)) + return err + }, + keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Builds.Keys(ctx) }, + count: func(ctx context.Context, tx pgx.Tx) (int, error) { + var n int + err := tx.QueryRow(ctx, `select (select count(*) from build) + (select count(*) from build_request)`).Scan(&n) + return n, err + }, + lock: `lock table build, build_request in share mode`, + settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error { + return settleBucket(ctx, c.Builds, rows) + }, + }, { + kind: KindRollouts, + rows: func(ctx context.Context) ([]moveRow, int, error) { + plans, err := i.plans(ctx, `order by created`) + if err != nil { + return nil, 0, err + } + merges, err := i.merges(ctx, ``) + if err != nil { + return nil, 0, err + } + tablePlans := len(plans) + if plans, err = holdMerges(plans, merges); err != nil { + return nil, 0, err + } + out := make([]moveRow, 0, len(plans)) + for _, p := range plans { + body, err := json.Marshal(p) + if err != nil { + return nil, 0, err + } + out = append(out, moveRow{key: p.ID, value: p, size: len(body)}) + } + return out, tablePlans + len(merges), nil + }, + create: func(ctx context.Context, c *chain.Chain, row moveRow) error { + _, err := chain.RolloutsOf[Plan](c).Create(ctx, row.key, row.value.(Plan)) + return err + }, + keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { + return chain.RolloutsOf[Plan](c).Keys(ctx) + }, + count: func(ctx context.Context, tx pgx.Tx) (int, error) { + var n int + err := tx.QueryRow(ctx, `select (select count(*) from release_plan) + (select count(*) from batched_merge)`).Scan(&n) + return n, err + }, + lock: `lock table release_plan, batched_merge in share mode`, + settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error { + return settleBucket(ctx, chain.RolloutsOf[Plan](c), rows) + }, + }, { + kind: KindAssignments, + rows: func(ctx context.Context) ([]moveRow, int, error) { + assigned, err := i.assignedWithSends(ctx) + if err != nil { + return nil, 0, err + } + out := make([]moveRow, 0, len(assigned)) + for _, a := range assigned { + body, err := json.Marshal(a) + if err != nil { + return nil, 0, err + } + out = append(out, moveRow{key: assignmentKey(a.Node, a.Module), value: a, size: len(body)}) + } + return out, len(out), nil + }, + create: createAssignment, + keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Assignments.Keys(ctx) }, + count: func(ctx context.Context, tx pgx.Tx) (int, error) { + var n int + err := tx.QueryRow(ctx, `select count(*) from assignment`).Scan(&n) + return n, err + }, + lock: `lock table assignment in share mode`, + settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error { + have := make(map[string]bool, len(rows)) + for _, r := range rows { + have[r.key] = true + if err := settleAssignment(ctx, c, r); err != nil { + return err + } + } + // An assignment the store no longer holds ended since the first pass: the bucket says so. + all, err := c.Assignments.All(ctx) + if err != nil { + return err + } + for key, k := range all { + if have[key] || k.Value.Assigned.IsZero() || !k.Value.Stands() { + continue + } + k.Value.Ended = time.Now().UTC() + if _, err := c.Assignments.Update(ctx, key, k.Value, k.Revision); err != nil && !errors.Is(err, chain.ErrMoved) { + return err + } + } + return nil + }, + }} +} + +// assignedWithSends is every assignment of the store as the chain keeps it: with the commit its machine was last sent for +// the module (node.sent_builds), the build that is of it and what the build published, and since when (when the machine was +// last sent). An assignment of a module the machine was never sent has none of those. +func (i *Inventory) assignedWithSends(ctx context.Context) ([]chain.Assignment, error) { + rows, err := i.store.Pool().Query(ctx, + `select n.name, a.module, a.assigned, n.sent_at, n.sent_builds + from assignment a join node n on n.id = a.node order by n.name, a.module`) + if err != nil { + return nil, err + } + var out []chain.Assignment + for rows.Next() { + var a chain.Assignment + var sentAt *time.Time + var raw []byte + if err := rows.Scan(&a.Node, &a.Module, &a.Assigned, &sentAt, &raw); err != nil { + rows.Close() + return nil, err + } + a.Assigned = a.Assigned.UTC() + var sent map[string]string + if len(raw) > 0 { + _ = json.Unmarshal(raw, &sent) + } + if commit := sent[a.Module]; commit != "" && sentAt != nil { + a.Commit, a.Since = commit, sentAt.UTC() + } + out = append(out, a) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + // The build of each commit, read once: the newest successful build of the module at it. + type at struct{ module, commit string } + var all []Build + if _, moved, err := i.ChainReads(ctx, KindBuilds); err != nil { + return nil, err + } else if moved { + if all, err = chainBuilds(ctx, i.attached()); err != nil { + return nil, err + } + } else if all, err = i.allBuildRows(ctx); err != nil { + return nil, err + } + newest := map[at]Build{} + for _, b := range all { + k := at{b.Module, b.Commit} + if b.Failed != "" || b.Module == "" { + continue + } + if had, has := newest[k]; !has || b.At.After(had.At) || (b.At.Equal(had.At) && b.ID > had.ID) { + newest[k] = b + } + } + for k := range out { + if out[k].Commit == "" { + continue + } + b, found := newest[at{out[k].Module, out[k].Commit}] + if !found { + continue + } + out[k].Build = b.ID + if published := publishedOf(b.Made); published != nil { + out[k].Artifact, out[k].Package = published.Artifact, published.Package + } + } + return out, nil +} + +// holdMerges puts each batched merge into the record of the plan that answers it. A merge no plan answers — the plan_id +// of a merge a failed walk carried is null, or names a plan the store never kept — is held, flagged waiting, by the +// newest plan that lists it among its merges, else the record kept for the merges no plan answers: the record that held it +// last. +func holdMerges(plans []Plan, merges []BatchedMerge) ([]Plan, error) { + waitingAt := -1 + at := make(map[string]int, len(plans)) + for k, p := range plans { + at[p.ID] = k + } + newest := func(match func(Plan) bool) int { + best := -1 + for k, p := range plans { + if match(p) && (best < 0 || p.Created.After(plans[best].Created)) { + best = k + } + } + return best + } + for _, m := range merges { + held := heldOf(m, m.Heard) + holder, answered := at[m.Plan] + if m.Plan == "" || !answered { + held.Waiting = true + holder = newest(func(p Plan) bool { + if p.Delivery == nil { + return false + } + for _, listed := range p.Delivery.Merges { + if strings.EqualFold(listed.Repository, m.Repository) && listed.Commit == m.Commit { + return true + } + } + return false + }) + if holder < 0 { + // Held by the record kept for the merges no plan answers, made with the others. + if waitingAt < 0 { + plans = append(plans, Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1, + Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"}) + waitingAt = len(plans) - 1 + } + holder = waitingAt + } + } + plans[holder].Held = append(plans[holder].Held, held) + } + return plans, nil +} + +// MoveChain moves every kind not moved yet, in order, and says what it did. It needs the chain attached and the lease +// held (the chain's writes ask). It stops at the first kind that fails. +func (i *Inventory) MoveChain(ctx context.Context, say func(format string, args ...any)) ([]MoveReport, error) { + if say == nil { + say = func(string, ...any) {} + } + if i.attached() == nil { + return nil, ErrNoChain + } + var out []MoveReport + for _, m := range i.movers() { + r, err := i.moveKind(ctx, m, say) + if err != nil { + return out, err + } + out = append(out, r) + } + return out, nil +} + +// movePasses is how often a move reads the tables again when they took a row while it wrote: the controller before this +// one may still have been writing as the lease changed hands. +const movePasses = 5 + +func (i *Inventory) moveKind(ctx context.Context, m mover, say func(string, ...any)) (MoveReport, error) { + report := MoveReport{Kind: m.kind} + moved, err := i.ChainMoved(ctx) + if err != nil { + return report, err + } + if _, done := moved[m.kind]; done { + report.Already = true + return report, nil + } + for pass := 1; pass <= movePasses; pass++ { + rows, tableRows, err := m.rows(ctx) + if err != nil { + return report, fmt.Errorf("the %s cannot be read from the tables: %w", m.kind, err) + } + report.Rows, report.TableRows, report.Largest = len(rows), tableRows, 0 + // Every value measured before any is written: a value over the bus's largest stops the move with nothing + // written (ADR 0408 §6), never cut. + for _, r := range rows { + report.Largest = max(report.Largest, r.size) + if r.size > broker.ChainValueBytes { + return report, &Refusal{m.kind, fmt.Sprintf("the value of %s is %d bytes and the bus holds at most %d; "+ + "nothing is cut, so the record must be made smaller first", r.key, r.size, broker.ChainValueBytes)} + } + } + report.Written, report.Present = 0, 0 + for _, r := range rows { + switch err := m.create(ctx, i.attached(), r); { + case err == nil: + report.Written++ + case errors.Is(err, chain.ErrExists): + report.Present++ + case errors.Is(err, chain.ErrBadKey): + return report, &Refusal{m.kind, fmt.Sprintf("%s cannot be a key of the bus: %v", r.key, err)} + default: + return report, fmt.Errorf("%s %s is not written to the bus: %w", m.kind, r.key, err) + } + } + // Rows against keys: every row's key is in the bucket. + keys, err := m.keys(ctx, i.attached()) + if err != nil { + return report, fmt.Errorf("the keys of the %s cannot be read back: %w", m.kind, err) + } + have := make(map[string]bool, len(keys)) + for _, k := range keys { + have[k] = true + } + for _, r := range rows { + if !have[r.key] { + return report, &Refusal{m.kind, fmt.Sprintf("the tables hold %d records and the bucket does not hold %s", len(rows), r.key)} + } + } + settled, err := i.switchKind(ctx, m, report) + if err != nil { + return report, err + } + if settled { + say("moved the %s to the bus: %d records, %d written now and %d already there, the largest %d bytes", + m.kind, report.Rows, report.Written, report.Present, report.Largest) + return report, nil + } + say("the %s took a row while they were moved, so they are read again (pass %d of %d)", m.kind, pass, movePasses) + } + return report, &Refusal{m.kind, fmt.Sprintf("the tables kept taking rows through %d passes; another controller is still writing them", movePasses)} +} + +// switchKind records the kind as moved, in one transaction that holds the tables against writes and finds their count +// as the move read it: false when they took a row since. +func (i *Inventory) switchKind(ctx context.Context, m mover, r MoveReport) (bool, error) { + committed := false + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback(ctx) }() + if _, err := tx.Exec(ctx, m.lock); err != nil { + return false, err + } + n, err := m.count(ctx, tx) + if err != nil { + return false, err + } + if n != r.TableRows { + return false, nil + } + // The store's mirror of the assignments writes nothing from here: it would lay values without the links over the keys + // the settle writes and end the ones it ended. + if m.kind == KindAssignments { + i.moved.switching.Store(true) + defer func() { + if !committed { + i.moved.switching.Store(false) + } + }() + } + // The rows read again under the lock: nothing writes them now, so what the bucket holds is made what they say. + rows, tableRows, err := m.rows(ctx) + if err != nil { + return false, err + } + if tableRows != r.TableRows { + return false, nil + } + if err := m.settle(ctx, i.attached(), rows); err != nil { + return false, fmt.Errorf("the %s changed since they were first written, and the bucket cannot be brought to them: %w", m.kind, err) + } + counts, _ := json.Marshal(map[string]int{m.kind: r.Rows}) + largest, _ := json.Marshal(map[string]int{m.kind: r.Largest}) + if _, err := tx.Exec(ctx, + `insert into chain_moved (counts, largest) values ($1, $2) + on conflict (only_one) do update + set counts = chain_moved.counts || excluded.counts, largest = chain_moved.largest || excluded.largest, + moved_at = now()`, counts, largest); err != nil { + return false, err + } + if err := tx.Commit(ctx); err != nil { + return false, err + } + committed = true + i.moved.mu.Lock() + if i.moved.kind == nil { + i.moved.kind = map[string]bool{} + } + i.moved.kind[m.kind] = true + i.moved.mu.Unlock() + return true, nil +} + +// allBuildRows is every build of the table, whole. +func (i *Inventory) allBuildRows(ctx context.Context) ([]Build, error) { + rows, err := i.store.Pool().Query(ctx, + `select id, repository, ref, coalesce(module, ''), commit_hash, built_on, failed, made, source_path, manifest, + built_against, built_contexts, build_sources, source_fingerprint, asked, at + from build order by at, id`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Build + for rows.Next() { + var b Build + var made, manifest, against, read, sources []byte + var asked *time.Time + if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, &b.On, &b.Failed, &made, &b.Path, + &manifest, &against, &read, &sources, &b.SourceFingerprint, &asked, &b.At); err != nil { + return nil, err + } + if asked != nil { + b.Asked = *asked + } + b.Manifest = manifest + for _, part := range []struct { + raw []byte + to any + who string + }{{made, &b.Made, "made"}, {against, &b.Against, "built_against"}, {read, &b.Read, "built_contexts"}, + {sources, &b.Sources, "build_sources"}} { + if len(part.raw) == 0 { + continue + } + if err := json.Unmarshal(part.raw, part.to); err != nil { + return nil, fmt.Errorf("build %s: %s is not readable: %w", b.ID, part.who, err) + } + } + out = append(out, b) + } + return out, rows.Err() +} + +// allRequestRows is every build request of the table. +func (i *Inventory) allRequestRows(ctx context.Context) ([]BuildRequest, error) { + rows, err := i.store.Pool().Query(ctx, + `select id, repository, seat, source_path, ref, commit_hash, asked_for, coalesce(not_asked, ''), + coalesce(outcome_unknown, ''), asked_at, at_terminal + from build_request order by asked_at, id`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []BuildRequest + for rows.Next() { + var a BuildRequest + if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked, + &a.OutcomeUnknown, &a.At, &a.AtTerminal); err != nil { + return nil, err + } + out = append(out, a) + } + return out, rows.Err() +} + +// createAssignment writes an assignment's row under its key; chain.ErrExists when the key holds a value that stays. +func createAssignment(ctx context.Context, c *chain.Chain, row moveRow) error { + want := row.value.(chain.Assignment) + // The mirror of the store kept a value under this key until now, with the provisioner's plan beside it: the + // plan stays, the record's links are laid in. A value that names its build already is a later word, and stays. + for try := 0; try < casTries; try++ { + cur, found, err := c.Assignments.Get(ctx, row.key) + if err != nil { + return err + } + if !found { + if _, err := c.Assignments.Create(ctx, row.key, want); !errors.Is(err, chain.ErrExists) { + return err + } + continue + } + if cur.Value.Build != "" || cur.Value.Commit != "" || !cur.Value.Since.IsZero() || (want.Build == "" && want.Commit == "") { + return chain.ErrExists + } + merged := want + merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning + if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) { + continue + } else { + return err + } + } + return fmt.Errorf("the assignment %s kept moving under the write", row.key) +} + +// settleBucket makes a bucket of whole values hold each row's value, by compare-and-set: a key that is absent is made, one +// whose bytes differ from the row's is written over, and one that is the same is left. +func settleBucket[T any](ctx context.Context, b chain.Bucket[T], rows []moveRow) error { + for _, r := range rows { + want, err := json.Marshal(r.value) + if err != nil { + return err + } + settled := false + for try := 0; try < casTries && !settled; try++ { + e, found, err := b.GetRaw(ctx, r.key) + if err != nil { + return err + } + switch { + case !found: + _, err = b.CreateRaw(ctx, r.key, want) + if errors.Is(err, chain.ErrExists) { + continue + } + case bytes.Equal(e.Value, want): + settled = true + continue + default: + _, err = b.UpdateRaw(ctx, r.key, want, e.Revision) + if errors.Is(err, chain.ErrMoved) { + continue + } + } + if err != nil { + return err + } + settled = true + } + if !settled { + return fmt.Errorf("%s kept moving under the write", r.key) + } + } + return nil +} + +// settleAssignment makes the value under an assignment's key what its row says, keeping the plan the mirror put beside it: +// before the switch the store is the only truth of the links, so a commit it changed since the first pass is written over. +func settleAssignment(ctx context.Context, c *chain.Chain, row moveRow) error { + want := row.value.(chain.Assignment) + for try := 0; try < casTries; try++ { + cur, found, err := c.Assignments.Get(ctx, row.key) + if err != nil { + return err + } + if !found { + if _, err := c.Assignments.Create(ctx, row.key, want); errors.Is(err, chain.ErrExists) { + continue + } else { + return err + } + } + merged := want + merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning + a, _ := json.Marshal(merged) + b, _ := json.Marshal(cur.Value) + if bytes.Equal(a, b) { + return nil + } + if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) { + continue + } else { + return err + } + } + return fmt.Errorf("the assignment %s kept moving under the write", row.key) +} diff --git a/internal/inventory/chain_move_test.go b/internal/inventory/chain_move_test.go new file mode 100644 index 00000000..e72a07b9 --- /dev/null +++ b/internal/inventory/chain_move_test.go @@ -0,0 +1,949 @@ +//go:build integration + +package inventory + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "reflect" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/chain" +) + +// A build row as the store held it before the move: written by SQL, the only hand-written state these tests have, for +// it stands for what the mesh held before this release. +func aBuildRow(t *testing.T, inv *Inventory, id, module, commit, failed string) { + t.Helper() + inv.ExecForTest(t, + `insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, source_path, manifest, + built_against, built_contexts, asked, source_fingerprint, build_sources) + values ($1::text, 'novox/' || $2::text, 'main', $2::text, $3::text, 'ace', $4::text, + case when $4::text = '' + then ('[{"name":"app","kind":"image","reference":"artifact-store://' || $2::text || '@sha256:' || $1::text || '"}]')::jsonb + else '[]'::jsonb end, + 'modules/' || $2::text, ('{"module":"' || $2::text || '"}')::jsonb, + '["artifact-store://base@sha256:aa"]'::jsonb, '[{"repository":"novox/other","ref":"main"}]'::jsonb, + now() - interval '1 hour', 'fp-' || $1::text, + ('[{"paths":["modules/' || $2::text || '/module.json"]}]')::jsonb)`, + id, module, commit, failed) +} + +// inTheTables is an inventory before the move: its builds and assignments are rows of the store, which a test of a data +// migration of the store (that reads those rows) needs. +func inTheTables(t *testing.T) *Inventory { + t.Helper() + inv, finish := ForTestBefore(t, 105) + finish() + return inv +} + +// reportOf is the report of one kind among those a move made, which are one for each kind in the order they are moved. +func reportOf(t *testing.T, reports []MoveReport, kind string) MoveReport { + t.Helper() + for _, r := range reports { + if r.Kind == kind { + return r + } + } + t.Fatalf("no report of %s among %+v", kind, reports) + return MoveReport{} +} + +// allAlready says every kind was moved by an earlier start: the second run writes nothing, and reports one for each kind. +func allAlready(t *testing.T, reports []MoveReport) { + t.Helper() + if len(reports) != 3 { + t.Fatalf("the move reported %d kinds, want 3: %+v", len(reports), reports) + } + for _, r := range reports { + if !r.Already || r.Written != 0 { + t.Fatalf("the second move of %s %+v", r.Kind, r) + } + } +} + +func rowsInTheTable(t *testing.T, inv *Inventory, table string) string { + t.Helper() + rows := inv.QueryForTest(t, `select count(*) from `+table) + return rows[0][0] +} + +// **The move copies every build row into the builds bucket, and a second run writes nothing** (ADR 0408 §8, issue +// 1090): each row under its id, whole; the kind recorded in chain_moved; and from then on a build is read from the +// bucket and written to it, the table taking no more. +func TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing(t *testing.T) { + inv, finish := ForTestBefore(t, 105) + finish() + ctx := context.Background() + aBuildRow(t, inv, "build-1", "notes", "c1", "") + aBuildRow(t, inv, "build-2", "notes", "c2", "") + aBuildRow(t, inv, "build-3", "wiki", "", "compile error") + + // Before the move the tables are read, and the bucket is empty. + if before, err := inv.Builds(ctx, "", 10); err != nil || len(before) != 3 { + t.Fatalf("before the move the tables hold %d builds: %v", len(before), err) + } + if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { + t.Fatalf("moved before the move: %v", moved) + } + + reports, err := inv.MoveChain(ctx, nil) + if err != nil { + t.Fatal(err) + } + if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.Written != 3 || b.Largest == 0 { + t.Fatalf("the move of the builds reported %+v", reports) + } + moved, err := inv.ChainMoved(ctx) + if err != nil || moved[KindBuilds] != 3 { + t.Fatalf("chain_moved says %v: %v", moved, err) + } + kept, found, err := inv.chain.Builds.Get(ctx, "build-2") + if err != nil || !found { + t.Fatalf("build-2 is not in the bucket: %v %v", found, err) + } + if v := kept.Value; v.Module != "notes" || v.Commit != "c2" || v.Repository != "novox/notes" || v.Path != "modules/notes" || + v.SourceFingerprint != "fp-build-2" || len(v.Made) != 1 || len(v.Against) != 1 || len(v.Read) != 1 || len(v.Sources) != 1 || + v.Published == nil || v.Published.Artifact != "sha256:build-2" || v.Asked.IsZero() || v.At.IsZero() || len(v.Manifest) == 0 { + t.Fatalf("build-2 was moved as %+v", v) + } + + // From now on the bucket is what is read and written. + if err := inv.RecordBuild(ctx, aBuild("build-4", "notes", "")); err != nil { + t.Fatal(err) + } + if n := rowsInTheTable(t, inv, "build"); n != "3" { + t.Fatalf("the table took a build after the move: %s rows", n) + } + after, err := inv.Builds(ctx, "notes", 10) + if err != nil || len(after) != 3 { + t.Fatalf("notes has %d builds on the bus: %v", len(after), err) + } + + // A second run finds the kind moved and writes nothing: every key still has the one value it was made with. + again, err := inv.MoveChain(ctx, nil) + if err != nil { + t.Fatal(err) + } + allAlready(t, again) + for _, id := range []string{"build-1", "build-2", "build-3", "build-4"} { + if history, err := inv.chain.Builds.History(ctx, chainBuildKey(id)); err != nil || len(history) != 1 { + t.Fatalf("%s has %d values: %v", id, len(history), err) + } + } +} + +// failingToCreate is a bucket that fails to make one key while told to: the bus going away half way through a move. +type failingToCreate struct { + chain.Backend + key string + fail *atomic.Bool +} + +func (f failingToCreate) Create(ctx context.Context, key string, value []byte) (uint64, error) { + if key == f.key && f.fail.Load() { + return 0, errors.New("the bus went away") + } + return f.Backend.Create(ctx, key, value) +} + +// **A controller restarted half way through the move finishes it: the bucket equals the table, with no key made twice and +// no refusal** (ADR 0408 §8). Before a kind is recorded as moved nothing but the move writes its keys, and every write of +// the kind goes to its table (each method of a kind asks ChainReads first and writes the store until it says moved), so the +// table is the only truth: a key the first start made is the row it was, and the second start settles it to the row it is. +func TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing(t *testing.T) { + inv, finish := ForTestBefore(t, 105) + finish() + ctx := context.Background() + aBuildRow(t, inv, "build-1", "notes", "c1", "") + aBuildRow(t, inv, "build-2", "notes", "c2", "") + aBuildRow(t, inv, "build-3", "wiki", "c3", "") + var away atomic.Bool + away.Store(true) + builds := chain.NewMemory() + inv.UseChain(chain.New(failingToCreate{builds, "build-3", &away}, chain.NewMemory(), chain.NewMemory(), + func(context.Context) error { return nil })) + // The first start made two of the three keys and the bus went away. + _, err := inv.MoveChain(ctx, nil) + var refusal *Refusal + if err == nil || errors.As(err, &refusal) { + t.Fatalf("the first start answered %v: it should fail to be tried again, not be refused", err) + } + if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { + t.Fatalf("a move that did not finish recorded %v", moved) + } + if keys, _ := builds.Keys(ctx); len(keys) != 2 { + t.Fatalf("the first start made %v", keys) + } + // The controller is started again, the bus is back, and the move finishes. + away.Store(false) + reports, err := inv.MoveChain(ctx, nil) + if err != nil { + t.Fatalf("the second start was refused or failed: %v", err) + } + if b := reportOf(t, reports, KindBuilds); b.Written != 1 || b.Present != 2 || b.Rows != 3 { + t.Fatalf("the move reported %+v", b) + } + if moved, _ := inv.ChainMoved(ctx); moved[KindBuilds] != 3 { + t.Fatalf("chain_moved says %v", moved) + } + // The bucket is the table: three keys, each made once and not again, each the row's own. + for _, id := range []string{"build-1", "build-2", "build-3"} { + history, err := builds.History(ctx, id) + if err != nil || len(history) != 1 { + t.Fatalf("%s has %d values: %v", id, len(history), err) + } + } + rows, err := inv.allBuildRows(ctx) + if err != nil || len(rows) != 3 { + t.Fatalf("the table holds %d builds: %v", len(rows), err) + } + for _, row := range rows { + got, found, err := inv.chain.Builds.Get(ctx, row.ID) + want, _ := json.Marshal(toChainBuild(row)) + have, _ := json.Marshal(got.Value) + if err != nil || !found || string(want) != string(have) { + t.Fatalf("%s in the bucket is %s, the row says %s (%v)", row.ID, have, want, err) + } + } +} + +// **A value over the bus's largest message stops the move with nothing written and nothing switched** (ADR 0408 §6): +// a Refusal, no key in the bucket, no kind recorded, and the tables still what is read. +func TestTheMoveRefusesAValueOverTheLargestMessage(t *testing.T) { + inv, finish := ForTestBefore(t, 105) + finish() + ctx := context.Background() + aBuildRow(t, inv, "build-1", "notes", "c1", "") + inv.ExecForTest(t, `insert into build (id, repository, module, commit_hash, manifest) + values ('build-big', 'novox/wiki', 'wiki', 'c9', ('{"x":"' || repeat('a', 1100000) || '"}')::jsonb)`) + _, err := inv.MoveChain(ctx, nil) + var refusal *Refusal + if !errors.As(err, &refusal) || refusal.Kind != KindBuilds { + t.Fatalf("the move answered %v", err) + } + if keys, _ := inv.chain.Builds.Keys(ctx); len(keys) != 0 { + t.Fatalf("the bucket holds %v after a refused move", keys) + } + if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { + t.Fatalf("a refused move recorded %v", moved) + } + if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 2 { + t.Fatalf("the tables are no longer read: %d builds, %v", len(builds), err) + } +} + +// losing is a bucket that says it wrote a key and kept nothing: the move's count of keys against rows must see it. +type losing struct { + chain.Backend + lose string +} + +func (l losing) Create(ctx context.Context, key string, value []byte) (uint64, error) { + if key == l.lose { + return 1, nil + } + return l.Backend.Create(ctx, key, value) +} + +// **The move refuses on a count that differs** (ADR 0408 §8): a row whose key is not in the bucket afterwards stops it, +// nothing is recorded, and the tables are still read. +func TestTheMoveRefusesOnACountThatDiffers(t *testing.T) { + inv, finish := ForTestBefore(t, 105) + finish() + ctx := context.Background() + aBuildRow(t, inv, "build-1", "notes", "c1", "") + aBuildRow(t, inv, "build-2", "notes", "c2", "") + inv.UseChain(chain.New(losing{chain.NewMemory(), "build-2"}, chain.NewMemory(), chain.NewMemory(), + func(context.Context) error { return nil })) + _, err := inv.MoveChain(ctx, nil) + var refusal *Refusal + if !errors.As(err, &refusal) { + t.Fatalf("the move answered %v", err) + } + if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { + t.Fatalf("a refused move recorded %v", moved) + } +} + +// **Without the lease the move writes nothing**, and fails the way the bus being away does — tried again, not refused: +// the tables are still read. +func TestTheMoveWithoutTheLeaseWritesNothing(t *testing.T) { + inv, finish := ForTestBefore(t, 105) + finish() + ctx := context.Background() + aBuildRow(t, inv, "build-1", "notes", "c1", "") + mem := chain.NewMemory() + inv.UseChain(chain.New(mem, chain.NewMemory(), chain.NewMemory(), + func(context.Context) error { return errors.New("another controller holds the lease") })) + _, err := inv.MoveChain(ctx, nil) + var refusal *Refusal + if err == nil || errors.As(err, &refusal) || !errors.Is(err, chain.ErrNotHeld) { + t.Fatalf("the move answered %v", err) + } + if mem.Writes() != 0 { + t.Fatalf("the bucket took %d writes without the lease", mem.Writes()) + } + if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { + t.Fatalf("recorded %v", moved) + } +} + +// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its +// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none. +func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) { + inv := fresh(t) + ctx := context.Background() + first := aBuild("build-1", "notes", "") + first.Commit = "c1" + first.Against = []string{"artifact-store://base@sha256:aa"} + first.ReuseKey = "rk-1" + second := aBuild("build-2", "notes", "") + second.Commit = "c2" + failed := aBuild("build-3", "notes", "compile error") + other := aBuild("build-4", "wiki", "") + for _, b := range []Build{first, second, failed, other, first} { + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + if n := rowsInTheTable(t, inv, "build"); n != "0" { + t.Fatalf("the table took %s builds", n) + } + kept, found, err := inv.chain.Builds.Get(ctx, "build-1") + if err != nil || !found || kept.Value.ReuseKey != "rk-1" { + t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err) + } + if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 1 { + t.Fatalf("a build recorded twice has %d values", len(history)) + } + list, err := inv.Builds(ctx, "notes", 10) + if err != nil || len(list) != 3 { + t.Fatalf("notes has %d builds: %v", len(list), err) + } + one, found, err := inv.BuildByID(ctx, "build-2") + if err != nil || !found || one.Commit != "c2" || one.Module != "notes" { + t.Fatalf("build-2 reads as %+v %v %v", one, found, err) + } + if _, found, _ := inv.BuildByID(ctx, "build-nope"); found { + t.Fatal("a build nobody recorded was found") + } + if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" { + t.Fatalf("the build of notes at c1 is %q: %v", id, err) + } + held, err := inv.Held(ctx) + if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" { + t.Fatalf("held %v: %v", held, err) + } + against, err := inv.BuiltAgainst(ctx) + if err != nil || len(against["notes"]) != 0 { + // The newest successful build of notes, build-2, stood on nothing. + t.Fatalf("built against %v: %v", against, err) + } + announced, err := inv.Announceable(ctx) + if err != nil || len(announced) != 3 { + t.Fatalf("announceable %d: %v", len(announced), err) + } + prints, err := inv.Fingerprints(ctx) + if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" { + t.Fatalf("fingerprints %v: %v", prints, err) + } +} + +// A build request row as the store held it before the move. +func aRequestRow(t *testing.T, inv *Inventory, id, notAsked string) { + t.Helper() + inv.ExecForTest(t, + `insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at) + values ($1::text, 'novox/notes', '', 'modules/notes', 'main', 'c1', 'merge', nullif($2::text, ''), now() - interval '1 hour')`, + id, notAsked) +} + +// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue +// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none +// after the outcome is heard. The table of requests takes none. +func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) { + inv := fresh(t) + ctx := context.Background() + ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true} + if err := inv.RecordBuildRequest(ctx, ask); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuildRequest(ctx, ask); err != nil { + t.Fatalf("a request kept twice: %v", err) + } + if n := rowsInTheTable(t, inv, "build_request"); n != "0" { + t.Fatalf("the table took %s requests", n) + } + if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 { + t.Fatalf("a request nobody answered is a build: %+v %v", builds, err) + } + if _, found, _ := inv.BuildByID(ctx, "build-1"); found { + t.Fatal("a request nobody answered has an outcome") + } + got, found, err := inv.BuildRequestByID(ctx, "build-1") + if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() { + t.Fatalf("the request reads back as %+v %v %v", got, found, err) + } + if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil { + t.Fatal(err) + } + asked, err := inv.RequestsNamed(ctx, "notes") + if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" { + t.Fatalf("the request is %+v: %v", asked, err) + } + if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) { + t.Fatalf("requested names %v: %v", names, err) + } + + // The outcome comes: the same key, now a build, with the request beside it. + built := aBuild("build-1", "notes", "") + if err := inv.RecordBuild(ctx, built); err != nil { + t.Fatal(err) + } + if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" { + t.Fatalf("the outcome reads as %+v %v %v", one, found, err) + } + answered, err := inv.RequestsNamed(ctx, "notes") + if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" { + t.Fatalf("the request answered is %+v: %v", answered, err) + } + if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" { + t.Fatalf("the request was lost with the outcome: %+v", again) + } + // Heard: the marks of an unanswered request change nothing. + if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil { + t.Fatal(err) + } + if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" { + t.Fatalf("an answered request was marked not asked: %+v", again) + } + if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 3 { + // asked, marked unknown, answered. + t.Fatalf("the record has %d values", len(history)) + } + // An outcome heard before its request is kept, and the request lands beside it. + if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil { + t.Fatal(err) + } + if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" { + t.Fatalf("the request kept after its outcome reads %+v", late) + } + // A request older than the time they are kept is no longer listed. + if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old", + At: time.Now().Add(-2 * KeptFor)}); err != nil { + t.Fatal(err) + } + if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") { + t.Fatalf("a request past its time is listed: %v", names) + } +} + +// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a +// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it +// is asked and so asked at most once. +func TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain(t *testing.T) { + inv, finish := ForTestBefore(t, 105) + finish() + ctx := context.Background() + aBuildRow(t, inv, "build-1", "notes", "c1", "") + aRequestRow(t, inv, "build-1", "") + aRequestRow(t, inv, "build-2", "") + aRequestRow(t, inv, "not-asked-c1-modules-notes", "no seat holder") + + if requests, err := inv.RequestsNamed(ctx, "notes"); err != nil || len(requests) != 3 { + t.Fatalf("before the move the table holds %d requests: %v", len(requests), err) + } + reports, err := inv.MoveChain(ctx, nil) + if err != nil { + t.Fatal(err) + } + if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.TableRows != 4 { + t.Fatalf("the move reported %+v", b) + } + one, _, _ := inv.chain.Builds.Get(ctx, "build-1") + if one.Value.State != "" || one.Value.Request == nil || one.Value.Request.For != "merge" || one.Value.Module != "notes" { + t.Fatalf("an answered request moved as %+v", one.Value) + } + two, _, _ := inv.chain.Builds.Get(ctx, "build-2") + if two.Value.State != chain.BuildAsked || two.Value.Request.Reask != "" { + t.Fatalf("an unanswered request moved as %+v", two.Value) + } + never, _, _ := inv.chain.Builds.Get(ctx, "not-asked-c1-modules-notes") + if never.Value.State != chain.BuildAsked || never.Value.Request.NotAsked != "no seat holder" || never.Value.Request.Reask != chain.ReaskPending { + t.Fatalf("a request never handed over moved as %+v", never.Value) + } + requests, err := inv.RequestsNamed(ctx, "notes") + if err != nil || len(requests) != 3 { + t.Fatalf("after the move the bucket holds %d requests: %v", len(requests), err) + } + toAsk, err := inv.RequestsToAskAgain(ctx) + if err != nil || len(toAsk) != 1 || toAsk[0].ID != "not-asked-c1-modules-notes" { + t.Fatalf("to ask again: %+v %v", toAsk, err) + } + first, err := inv.ClaimToAskAgain(ctx, toAsk[0].ID) + second, err2 := inv.ClaimToAskAgain(ctx, toAsk[0].ID) + if err != nil || err2 != nil || !first || second { + t.Fatalf("claimed %v then %v: %v %v", first, second, err, err2) + } + if left, _ := inv.RequestsToAskAgain(ctx); len(left) != 0 { + t.Fatalf("a claimed request is still to be asked: %+v", left) + } + if err := inv.AskedAgainAs(ctx, toAsk[0].ID, "build-9"); err != nil { + t.Fatal(err) + } + if done, _, _ := inv.BuildRequestByID(ctx, toAsk[0].ID); done.NotAsked != "no seat holder" { + t.Fatalf("the request lost its words: %+v", done) + } +} + +// **The move carries every plan with the merges it answers, and the plans read the same from the bucket as from the +// table** (ADR 0408 §8, ADR 0276, issue 1092): a batch with a merge, a walk that was built and rolled, a merge waiting +// for a walk of its own, and a merge whose plan the store never kept — each in the record of the plan that held it last. +func TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame(t *testing.T) { + inv, finish := ForTestBefore(t, 105) + finish() + ctx := context.Background() + batch := aBatch("plan-1", time.Now().Add(-3*time.Hour).UTC()) + walk := aBatch("plan-2", time.Now().Add(-2*time.Hour).UTC()) + walk.State, walk.Tier, walk.Tiers = PlanRolling, 1, [][]string{{"a"}, {"b"}} + walk.Modules = map[string]*PlanModule{"a": {State: "built"}, "b": {}} + walk.Delivery = &PlanDelivery{Awaits: "mesh-delivery", Merges: []PlanMerge{{Repository: "novox/mesh-catalog", Commit: "m-waiting"}}} + for _, p := range []*Plan{batch, walk} { + if err := inv.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + } + walk.Note = "second save" + if err := inv.SavePlan(ctx, walk); err != nil { + t.Fatal(err) + } + for _, m := range []BatchedMerge{aMerge("m-batch", "plan-1"), aMerge("m-walk", "plan-2"), aMerge("m-dangling", "plan-never-kept")} { + if added, err := inv.AddMerge(ctx, m); err != nil || !added { + t.Fatalf("added %v: %v", added, err) + } + } + waiting := aMerge("m-waiting", "plan-2") + if _, err := inv.AddMerge(ctx, waiting); err != nil { + t.Fatal(err) + } + if err := inv.AnswerMerge(ctx, waiting.Repository, waiting.Commit, "", true); err != nil { + t.Fatal(err) + } + wasPlans, err := inv.RecentPlans(ctx, 10) + if err != nil || len(wasPlans) != 2 { + t.Fatalf("the table holds %d plans: %v", len(wasPlans), err) + } + wasMerge, _, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk") + + reports, err := inv.MoveChain(ctx, nil) + if err != nil { + t.Fatal(err) + } + var moved MoveReport + for _, r := range reports { + if r.Kind == KindRollouts { + moved = r + } + } + // Two plans, and the record of the merge whose plan the store never kept. + if moved.Rows != 3 || moved.TableRows != 6 || moved.Written != 3 { + t.Fatalf("the move of the rollouts reported %+v", moved) + } + nowPlans, err := inv.RecentPlans(ctx, 10) + if err != nil || len(nowPlans) != 2 { + t.Fatalf("the bucket holds %d plans: %v", len(nowPlans), err) + } + for k := range wasPlans { + was, now := wasPlans[k], nowPlans[k] + if was.ID != now.ID || was.State != now.State || was.Tier != now.Tier || was.Revision != now.Revision || was.Note != now.Note || + !was.Created.Equal(now.Created) || !reflect.DeepEqual(was.Tiers, now.Tiers) || len(was.Modules) != len(now.Modules) { + t.Errorf("plan %s was %+v and is %+v", was.ID, was, now) + } + } + if open, _ := inv.OpenPlans(ctx); len(open) != 1 || open[0].ID != "plan-2" { + t.Fatalf("open plans %+v", open) + } + if batches, _ := inv.Batches(ctx); len(batches) != 1 || batches[0].ID != "plan-1" { + t.Fatalf("batches %+v", batches) + } + if got, _ := inv.MergesOf(ctx, "plan-1"); len(got) != 1 || got[0].Commit != "m-batch" { + t.Fatalf("the batch answers %+v", got) + } + nowMerge, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk") + if !known || nowMerge.Plan != "plan-2" || nowMerge.Branch != wasMerge.Branch || !nowMerge.Merged.Equal(wasMerge.Merged) || + !nowMerge.Heard.Equal(wasMerge.Heard) || !sameEvent(nowMerge.Event, wasMerge.Event) { + t.Fatalf("a merge was %+v and is %+v", wasMerge, nowMerge) + } + alone, _ := inv.AloneMerges(ctx) + if len(alone) != 1 || alone[0].Commit != "m-waiting" || alone[0].Plan != "" { + t.Fatalf("waiting: %+v", alone) + } + if dangling, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-dangling"); !known || dangling.Plan != "" || dangling.Alone { + t.Fatalf("a merge whose plan was never kept reads as %+v %v", dangling, known) + } + if n := rowsInTheTable(t, inv, "release_plan"); n != "2" { + t.Fatalf("the table is %s plans", n) + } + // From now on a plan is saved to the bucket and the table takes none. + later := aBatch("plan-3", time.Now().UTC()) + if err := inv.SavePlan(ctx, later); err != nil { + t.Fatal(err) + } + if n := rowsInTheTable(t, inv, "release_plan"); n != "2" { + t.Fatalf("the table took a plan after the move: %s", n) + } + again, err := inv.MoveChain(ctx, nil) + if err != nil { + t.Fatal(err) + } + allAlready(t, again) +} + +// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a +// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module +// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the +// assignment goes with it. +func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres", "web") + ctx := t.Context() + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + start := generationNow(t, inv) + if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added { + t.Fatalf("assigned %v: %v", added, err) + } + if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added { + t.Fatalf("assigned again %v: %v", added, err) + } + if n := rowsInTheTable(t, inv, "assignment"); n != "0" { + t.Fatalf("the table took %s assignments", n) + } + kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres") + if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() { + t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err) + } + if generationNow(t, inv) <= start { + t.Fatal("an assignment left the generation where it was") + } + if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) { + t.Fatalf("a module the mesh does not know: %v", err) + } + if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) { + t.Fatalf("assigned on %s: %v %v", node, on, err) + } + if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) { + t.Fatalf("running: %v %v", running, err) + } + entries, err := inv.Catalogued(ctx) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + switch e.Manifest.Module { + case "postgres": + if !reflect.DeepEqual(e.On, []string{node}) { + t.Errorf("postgres is on %v", e.On) + } + case "web": + if len(e.On) != 0 { + t.Errorf("web is on %v", e.On) + } + } + } + if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) { + t.Fatalf("a module still assigned was forgotten: %v", err) + } + // A seat is held by an assignment, or by nothing. + if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil { + t.Fatal("a seat was handed to a module that is not assigned") + } + if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil { + t.Fatalf("a seat handed to an assignment: %v", err) + } + + before := generationNow(t, inv) + if err := inv.Unassign(ctx, node, "postgres"); err != nil { + t.Fatal(err) + } + if generationNow(t, inv) <= before { + t.Fatal("an unassignment left the generation where it was") + } + // No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history. + gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres") + if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() { + t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err) + } + if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() || + history[1].Value.Stands() { + t.Fatalf("the earlier values of the key: %+v", history) + } + if on, _ := inv.Assigned(ctx, node); len(on) != 0 { + t.Fatalf("an ended assignment is still assigned: %v", on) + } + if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 { + t.Fatalf("an ended assignment is running: %v", running) + } + if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 { + t.Fatalf("the seat the assignment held is held by %+v: %v", held, err) + } + if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" { + t.Fatalf("an unassignment of nothing: %v", err) + } + if err := inv.ForgetModule(ctx, "postgres"); err != nil { + t.Fatalf("a module nothing is assigned could not be forgotten: %v", err) + } + // Made again after it ended: a fresh value over the ended one, which stays in history. + if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { + t.Fatal(added, err) + } + if added, err := inv.Assign(ctx, node, "web"); err != nil || added { + t.Fatal(added, err) + } + if err := inv.Unassign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { + t.Fatalf("assigned again after it ended: %v %v", added, err) + } + again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web") + if !again.Value.Stands() || !again.Value.Ended.IsZero() { + t.Fatalf("the assignment made again is %+v", again.Value) + } + if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() { + t.Fatalf("the values of the key: %+v", history) + } +} + +// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the +// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay, +// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here +// ends an assignment. +func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) { + inv, node := aNodeWithModules(t, "dunst", "web") + ctx := t.Context() + for _, m := range []string{"dunst", "web"} { + if _, err := inv.Assign(ctx, node, m); err != nil { + t.Fatal(err) + } + } + // A build and a commit put in the value by another write. + cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") + cur.Value.Build, cur.Value.Commit = "build-1", "c1" + if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil { + t.Fatal(err) + } + record, _ := inv.NodeByName(ctx, node) + digest := "sha256:" + strings.Repeat("d", 64) + plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest, + Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc", + Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}} + raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`) + if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil { + t.Fatal(err) + } + inv.SetProvisioningPlans(inv.SentProvisioningPlans) + written, removed, err := inv.KeepAssignmentPlans(ctx) + if err != nil || written != 2 || removed != 0 { + t.Fatalf("kept: %d written, %d ended: %v", written, removed, err) + } + dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") + if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() { + t.Fatalf("dunst's value is %+v", dunst.Value) + } + if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil { + t.Fatalf("web has a plan: %+v", web.Value) + } + notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify") + if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil { + t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found) + } + if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) { + t.Fatalf("a pulled-in module is assigned: %v", on) + } + if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 { + t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err) + } + // Sent again without the pulled-in module: its value ends, the assignments stay. + only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`) + if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil { + t.Fatal(err) + } + if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 { + t.Fatalf("ended %d: %v", removed, err) + } + if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() { + t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value) + } + if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) { + t.Fatalf("assigned: %v", on) + } +} + +// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value** +// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build +// already is left as it is. +func TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent(t *testing.T) { + inv, finish := ForTestBefore(t, 105) + finish() + ctx := context.Background() + for _, name := range []string{"anchor", "laptop"} { + if _, err := inv.AddNode(ctx, name); err != nil { + t.Fatal(err) + } + } + for _, m := range []string{"notes", "wiki"} { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil { + t.Fatal(err) + } + } + for _, a := range [][2]string{{"anchor", "notes"}, {"anchor", "wiki"}, {"laptop", "notes"}} { + if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { + t.Fatal(err) + } + } + aBuildRow(t, inv, "build-notes", "notes", "c-notes", "") + anchor, _ := inv.NodeByName(ctx, "anchor") + if err := inv.RecordSent(ctx, anchor.ID, "sent-1", map[string]string{"notes": "c-notes"}); err != nil { + t.Fatal(err) + } + // The mirror of the store put a value under one key, with a plan. + if _, err := inv.chain.Assignments.Create(ctx, "anchor.wiki", chain.Assignment{Node: "anchor", Module: "wiki", + Assigned: time.Now().UTC(), Provisioning: json.RawMessage(`{"plan":1}`)}); err != nil { + t.Fatal(err) + } + reports, err := inv.MoveChain(ctx, nil) + if err != nil { + t.Fatal(err) + } + var moved MoveReport + for _, r := range reports { + if r.Kind == KindAssignments { + moved = r + } + } + if moved.Rows != 3 || moved.Written+moved.Present != 3 { + t.Fatalf("the move of the assignments reported %+v", moved) + } + notes, _, _ := inv.chain.Assignments.Get(ctx, "anchor.notes") + if notes.Value.Commit != "c-notes" || notes.Value.Build != "build-notes" || notes.Value.Since.IsZero() || notes.Value.Assigned.IsZero() { + t.Fatalf("anchor.notes moved as %+v", notes.Value) + } + if notes.Value.Artifact != "sha256:build-notes" { + t.Fatalf("what the build published: %+v", notes.Value) + } + laptop, _, _ := inv.chain.Assignments.Get(ctx, "laptop.notes") + if laptop.Value.Commit != "" || laptop.Value.Build != "" { + t.Fatalf("a machine never sent has a build: %+v", laptop.Value) + } + wiki, _, _ := inv.chain.Assignments.Get(ctx, "anchor.wiki") + if string(wiki.Value.Provisioning) != `{"plan":1}` { + t.Fatalf("the plan the mirror put in the value was lost: %+v", wiki.Value) + } + // From now on the bucket is what is written. + if _, err := inv.Assign(ctx, "laptop", "wiki"); err != nil { + t.Fatal(err) + } + if n := rowsInTheTable(t, inv, "assignment"); n != "3" { + t.Fatalf("the table took an assignment after the move: %s", n) + } + if on, _ := inv.Assigned(ctx, "laptop"); !reflect.DeepEqual(on, []string{"notes", "wiki"}) { + t.Fatalf("assigned on laptop: %v", on) + } +} + +// hooked runs a function the first time a key is made in a bucket: the old controller writing the store between the move's +// first pass and its switch. +type hooked struct { + chain.Backend + once *sync.Once + then func() +} + +func (h hooked) Create(ctx context.Context, key string, value []byte) (uint64, error) { + rev, err := h.Backend.Create(ctx, key, value) + h.once.Do(h.then) + return rev, err +} + +// **A row the old controller changes after the move's first pass is in the bucket when the kind is recorded as moved** +// (ADR 0408 §8, issue 981 note 7 Q2): before the switch the store is the only truth, so under the lock the move reads the +// rows again and writes over every key that differs — a plan's state and a build's failure — and ends an assignment the +// store dropped. The count alone would have kept the older value and refused the next save of the plan. +func TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved(t *testing.T) { + inv, finish := ForTestBefore(t, 105) + finish() + ctx := context.Background() + aBuildRow(t, inv, "build-1", "notes", "c1", "") + for _, n := range []string{"anchor"} { + if _, err := inv.AddNode(ctx, n); err != nil { + t.Fatal(err) + } + } + for _, m := range []string{"notes", "wiki"} { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil { + t.Fatal(err) + } + } + for _, m := range []string{"notes", "wiki"} { + if _, err := inv.Assign(ctx, "anchor", m); err != nil { + t.Fatal(err) + } + } + p := aBatch("plan-1", time.Now().Add(-time.Hour).UTC()) + p.State, p.Tiers = PlanBuilding, [][]string{{"notes"}} + if err := inv.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + // Between a kind's first pass and its switch the old controller saves the plan again, fails the build and unassigns wiki: + // each is done when the first key of its kind is made, so the first pass has read the rows as they were. + after := func(sql string) hooked { + return hooked{chain.NewMemory(), &sync.Once{}, func() { inv.ExecForTest(t, sql) }} + } + inv.UseChain(chain.New( + after(`update build set failed = 'it broke later' where id = 'build-1'`), + after(`update release_plan set state = 'rolling', revision = revision + 1, note = 'the old controller was here' where id = 'plan-1'`), + after(`delete from assignment where module = 'wiki'`), + func(context.Context) error { return nil })) + if _, err := inv.MoveChain(ctx, nil); err != nil { + t.Fatal(err) + } + now, err := inv.PlanByID(ctx, "plan-1") + if err != nil { + t.Fatal(err) + } + if now.State != PlanRolling || now.Note != "the old controller was here" || now.Revision != 2 { + t.Fatalf("the bucket holds a plan from before the change: %s %q revision %d", now.State, now.Note, now.Revision) + } + if b, found, _ := inv.BuildByID(ctx, "build-1"); !found || b.Failed != "it broke later" { + t.Fatalf("the bucket holds the build as it was: %+v", b) + } + if on, _ := inv.Assigned(ctx, "anchor"); !slices.Equal(on, []string{"notes"}) { + t.Fatalf("anchor is assigned %v, though the store dropped wiki", on) + } + // The plan can be saved at the revision the store had. + now.Note = "saved after the move" + if err := inv.SavePlan(ctx, &now); err != nil { + t.Fatalf("the next save of the plan is refused: %v", err) + } +} + +// sameEvent says two encodings of the forge's announcement are one: the store keeps it as jsonb, which spells it with its own +// white space, and whoever reads it (the cut, mesh-delivery) decodes it, so the white space is no part of what it says. +func sameEvent(a, b json.RawMessage) bool { + var x, y bytes.Buffer + return json.Compact(&x, a) == nil && json.Compact(&y, b) == nil && x.String() == y.String() +} diff --git a/internal/inventory/chain_moved_test.go b/internal/inventory/chain_moved_test.go new file mode 100644 index 00000000..3ea88b25 --- /dev/null +++ b/internal/inventory/chain_moved_test.go @@ -0,0 +1,45 @@ +package inventory + +import ( + "strings" + "testing" +) + +// **The move of the chain has a switch, made by the last migration and written by no migration** (ADR 0408 §8, issue +// 1089): migration 0104 makes the one-row table chain_moved, and does not drop a table of the chain, which a later +// delivery does once the move is read back. +func TestMigration0104MakesTheSwitchOfTheChainsMoveAndNothingElse(t *testing.T) { + migrations, err := Migrations() + if err != nil { + t.Fatal(err) + } + var found bool + for _, m := range migrations { + if m.Number != 104 { + continue + } + found = true + sql := strings.ToLower(m.SQL) + if !strings.Contains(sql, "create table chain_moved") || !strings.Contains(sql, "check (only_one)") { + t.Errorf("migration 0104 does not make the one-row chain_moved:\n%s", m.SQL) + } + for _, line := range strings.Split(sql, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "--") { + continue + } + for _, verb := range []string{"drop table", "insert ", "delete ", "update "} { + if strings.Contains(line, verb) { + t.Errorf("migration 0104 does more than make the switch: %q", line) + } + } + // The one other change: a seat's holder is no longer a foreign key to the assignment (issue 1093); the tables of + // the chain are dropped by a later migration (issue 1096). + if strings.Contains(line, "alter ") && !strings.Contains(line, "alter table seat_holding drop constraint") { + t.Errorf("migration 0104 alters more than the seat holdings' foreign key: %q", line) + } + } + } + if !found { + t.Fatal("no migration 0104") + } +} diff --git a/internal/inventory/chain_requests.go b/internal/inventory/chain_requests.go new file mode 100644 index 00000000..230ba9c3 --- /dev/null +++ b/internal/inventory/chain_requests.go @@ -0,0 +1,172 @@ +package inventory + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/chain" +) + +// Build requests on the bus (novox/hq ADR 0408, issue 1091): a request is the build record in state asked, under the id its +// outcome will carry; the outcome takes its place and keeps the request. The bucket removes no key, so a request is not +// deleted after KeptFor: it is no longer listed. + +func toRequest(a BuildRequest) *chain.Request { + return &chain.Request{Repository: a.Repository, Seat: a.Seat, Path: strings.Trim(a.Path, "/"), Ref: a.Ref, Commit: a.Commit, + For: a.For, NotAsked: a.NotAsked, OutcomeUnknown: a.OutcomeUnknown, At: a.At.UTC(), AtTerminal: a.AtTerminal} +} + +func fromRequest(id string, r chain.Request) BuildRequest { + return BuildRequest{ID: id, Repository: r.Repository, Seat: r.Seat, Path: r.Path, Ref: r.Ref, Commit: r.Commit, For: r.For, + NotAsked: r.NotAsked, OutcomeUnknown: r.OutcomeUnknown, At: r.At, AtTerminal: r.AtTerminal} +} + +// askedRecord is a request as the build record in state asked. +func askedRecord(a BuildRequest) chain.Build { + r := toRequest(a) + return chain.Build{ID: a.ID, State: chain.BuildAsked, Repository: a.Repository, Ref: a.Ref, Path: r.Path, Asked: r.At, Request: r} +} + +func (i *Inventory) chainRecordBuildRequest(ctx context.Context, c *chain.Chain, a BuildRequest) error { + if a.At.IsZero() { + a.At = time.Now().UTC() + } + key := chainBuildKey(a.ID) + for try := 0; try < casTries; try++ { + _, err := c.Builds.Create(ctx, key, askedRecord(a)) + if err == nil { + return nil + } + if !errors.Is(err, chain.ErrExists) { + return fmt.Errorf("build request %s is not kept on the bus: %w", a.ID, err) + } + // The key holds a value: the request is kept already, or its outcome came first and has no request beside it. + cur, found, err := c.Builds.Get(ctx, key) + if err != nil { + return err + } + if !found || cur.Value.Request != nil { + if !found { + continue + } + return nil + } + cur.Value.Request = toRequest(a) + if _, err := c.Builds.Update(ctx, key, cur.Value, cur.Revision); err == nil { + return nil + } else if !errors.Is(err, chain.ErrMoved) { + return fmt.Errorf("build request %s is not kept on the bus: %w", a.ID, err) + } + } + return fmt.Errorf("build request %s is not kept on the bus: it kept moving under the write", a.ID) +} + +func (i *Inventory) chainBuildRequestByID(ctx context.Context, c *chain.Chain, id string) (BuildRequest, bool, error) { + k, found, err := c.Builds.Get(ctx, chainBuildKey(id)) + if err != nil || !found || k.Value.Request == nil { + return BuildRequest{}, false, err + } + return fromRequest(id, *k.Value.Request), true, nil +} + +// chainMarkRequest changes a request that has no outcome yet, by compare-and-set; one answered, or none kept, is left. +func (i *Inventory) chainMarkRequest(ctx context.Context, c *chain.Chain, id string, mark func(*chain.Request)) error { + key := chainBuildKey(id) + for try := 0; try < casTries; try++ { + cur, found, err := c.Builds.Get(ctx, key) + if err != nil || !found || cur.Value.State != chain.BuildAsked || cur.Value.Request == nil { + return err + } + mark(cur.Value.Request) + if _, err := c.Builds.Update(ctx, key, cur.Value, cur.Revision); err == nil { + return nil + } else if !errors.Is(err, chain.ErrMoved) { + return fmt.Errorf("build request %s is not marked on the bus: %w", id, err) + } + } + return fmt.Errorf("build request %s is not marked: it kept moving under the write", id) +} + +// chainRequests is every request kept, newest first, each with its outcome where its build was heard. Requests older than +// KeptFor are not listed: the bucket keeps them, and no reader needs them. +func (i *Inventory) chainRequests(ctx context.Context, c *chain.Chain) ([]RequestOutcome, error) { + all, err := c.Builds.All(ctx) + if err != nil { + return nil, fmt.Errorf("the builds cannot be read from the bus: %w", err) + } + oldest := time.Now().Add(-KeptFor) + var out []RequestOutcome + for id, k := range all { + r := k.Value.Request + if r == nil || !r.At.After(oldest) { + continue + } + a := RequestOutcome{BuildRequest: fromRequest(id, *r), HeardAt: r.At} + if k.Value.State != chain.BuildAsked { + a.Heard, a.Failed, a.Module, a.HeardAt = true, k.Value.Failed, k.Value.Module, k.Value.At + } + out = append(out, a) + } + sort.Slice(out, func(a, b int) bool { + if !out[a].At.Equal(out[b].At) { + return out[a].At.After(out[b].At) + } + return out[a].ID > out[b].ID + }) + return out, nil +} + +// RequestsToAskAgain are the requests the move found never handed over (issue 981 note 7 Q5), not yet asked again: each is +// asked once, and never twice. +func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, error) { + c, on, err := i.ChainReads(ctx, KindBuilds) + if err != nil || !on { + return nil, err + } + all, err := c.Builds.All(ctx) + if err != nil { + return nil, err + } + var out []BuildRequest + for id, k := range all { + if k.Value.State == chain.BuildAsked && k.Value.Request != nil && k.Value.Request.Reask == chain.ReaskPending { + out = append(out, fromRequest(id, *k.Value.Request)) + } + } + sort.Slice(out, func(a, b int) bool { return out[a].ID < out[b].ID }) + return out, nil +} + +// ClaimToAskAgain takes a request to ask again, by compare-and-set: true for the one process that took it. **Claimed +// before it is asked and settled after**, so a crash between loses the ask rather than making it twice. +func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error) { + c, on, err := i.ChainReads(ctx, KindBuilds) + if err != nil || !on { + return false, err + } + cur, found, err := c.Builds.Get(ctx, chainBuildKey(id)) + if err != nil || !found || cur.Value.Request == nil || cur.Value.Request.Reask != chain.ReaskPending { + return false, err + } + cur.Value.Request.Reask = chain.ReaskClaimed + if _, err := c.Builds.Update(ctx, chainBuildKey(id), cur.Value, cur.Revision); err != nil { + if errors.Is(err, chain.ErrMoved) { + return false, nil + } + return false, err + } + return true, nil +} + +// AskedAgainAs records the build id a claimed request was asked as. +func (i *Inventory) AskedAgainAs(ctx context.Context, id, asked string) error { + c, on, err := i.ChainReads(ctx, KindBuilds) + if err != nil || !on { + return err + } + return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.Reask = asked }) +} diff --git a/internal/inventory/chain_requests_test.go b/internal/inventory/chain_requests_test.go new file mode 100644 index 00000000..6f99b143 --- /dev/null +++ b/internal/inventory/chain_requests_test.go @@ -0,0 +1,25 @@ +package inventory + +import ( + "reflect" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/chain" +) + +// **A request is the build record in state asked, under the id its outcome will carry** (ADR 0408 §1, issue 1091): the +// whole ask is in it, with a path trimmed as the table kept it. +func TestABuildRequestIsTheBuildRecordInStateAsked(t *testing.T) { + at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) + ask := BuildRequest{ID: "build-1", Repository: "https://forge.invalid/notes.git", Seat: "", Path: "/modules/notes/", Ref: "main", + Commit: "c1", For: "merge", NotAsked: "", OutcomeUnknown: "", At: at, AtTerminal: true} + record := askedRecord(ask) + if record.State != chain.BuildAsked || record.ID != "build-1" || record.Path != "modules/notes" || !record.Asked.Equal(at) { + t.Fatalf("the request is %+v", record) + } + if got := fromRequest(record.ID, *record.Request); !reflect.DeepEqual(got, BuildRequest{ID: "build-1", + Repository: ask.Repository, Path: "modules/notes", Ref: "main", Commit: "c1", For: "merge", At: at, AtTerminal: true}) { + t.Fatalf("the request reads back as %+v", got) + } +} diff --git a/internal/inventory/chain_rollouts.go b/internal/inventory/chain_rollouts.go new file mode 100644 index 00000000..98a49f86 --- /dev/null +++ b/internal/inventory/chain_rollouts.go @@ -0,0 +1,939 @@ +package inventory + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/novox/mesh-controller/internal/chain" +) + +// Rollouts on the bus (novox/hq ADR 0408, ADR 0276, issue 1092): once the kind `rollouts` is moved (chain.go) every plan +// and batch is a key of the rollouts bucket under its id, written by compare-and-set, and the merges a batch or walk +// answers are held in its record. The tables are read when the kind is not moved, by the methods these stand beside. +// +// **Two counters.** The bucket's revision is the compare-and-set of every write. The plan's own Revision, kept in its value, +// counts the saves of the plan: a save refuses (ErrPlanMoved) a plan another save moved since it was read, and takes no +// notice of a merge moving in or out of the record, which a copy read before could not know of. Held is written by the +// merge operations alone; a save keeps what the record holds. + +// rolloutView is the rollouts bucket in memory, kept current by one watch (issue 1092): the survey of the plans, the +// batches and the merges is read from here, never from the bucket one key at a time. A write is still made by +// compare-and-set on the bucket, so a stale view costs a refused write and a read again, never a wrong one. +type rolloutView struct { + chain *chain.Chain + mu sync.RWMutex + entries map[string]viewEntry + ready chan struct{} + once sync.Once + err error + dead atomic.Bool + cancel context.CancelFunc +} + +// viewEntry is a record as the bucket holds it, with what the surveys ask of it read out. +type viewEntry struct { + raw []byte + rev uint64 + sum planSummary +} + +// planSummary is the part of a plan the surveys filter and sort by. +type planSummary struct { + ID string `json:"id"` + State string `json:"state"` + Created time.Time `json:"created"` + Repository string `json:"repository"` + Branch string `json:"branch"` + Merged time.Time `json:"merged"` + Release json.RawMessage `json:"release"` + Held []chain.HeldMerge `json:"held"` +} + +func (s planSummary) open() bool { return s.State == PlanBuilding || s.State == PlanRolling } + +func (s planSummary) isRelease() bool { + r := strings.TrimSpace(string(s.Release)) + return r != "" && r != "null" +} + +func newRolloutView(c *chain.Chain) *rolloutView { + return &rolloutView{chain: c, entries: map[string]viewEntry{}, ready: make(chan struct{})} +} + +func (v *rolloutView) start() { + ctx, cancel := context.WithCancel(context.Background()) + v.cancel = cancel + go func() { + w, err := chain.RolloutsOf[Plan](v.chain).Watch(ctx) + if err != nil { + v.fail(fmt.Errorf("the rollouts cannot be watched: %w", err)) + return + } + for { + select { + case <-ctx.Done(): + v.fail(ctx.Err()) + return + case e, open := <-w: + if !open { + v.fail(errors.New("the rollouts bucket stopped answering")) + return + } + if e.Caught { + v.once.Do(func() { close(v.ready) }) + continue + } + v.apply(e.Key, e.Value, e.Revision) + } + } + }() +} + +// fail ends the view: it answers an error to whoever waits for it to fill, and is made again by the next read. +func (v *rolloutView) fail(err error) { + v.mu.Lock() + if v.err == nil { + v.err = err + } + v.mu.Unlock() + v.dead.Store(true) + v.once.Do(func() { close(v.ready) }) +} + +func (v *rolloutView) stop() { + if v.cancel != nil { + v.cancel() + } +} + +// apply keeps a record the watch handed over or a write of this process made, unless the view holds a newer one. +func (v *rolloutView) apply(key string, raw []byte, rev uint64) { + var sum planSummary + if err := json.Unmarshal(raw, &sum); err != nil || sum.ID == "" { + sum = planSummary{ID: key} + } + v.mu.Lock() + defer v.mu.Unlock() + if old, has := v.entries[key]; has && old.rev >= rev { + return + } + v.entries[key] = viewEntry{raw: append([]byte(nil), raw...), rev: rev, sum: sum} +} + +func (v *rolloutView) list() []viewEntry { + v.mu.RLock() + defer v.mu.RUnlock() + out := make([]viewEntry, 0, len(v.entries)) + for _, e := range v.entries { + out = append(out, e) + } + return out +} + +// rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole. +func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) { + i.moved.mu.Lock() + v := i.moved.view + if v == nil || v.chain != c || v.dead.Load() { + if v != nil { + v.stop() + } + v = newRolloutView(c) + i.moved.view = v + v.start() + } + i.moved.mu.Unlock() + select { + case <-v.ready: + case <-ctx.Done(): + return nil, ctx.Err() + } + v.mu.RLock() + defer v.mu.RUnlock() + if v.err != nil && len(v.entries) == 0 { + return nil, fmt.Errorf("the rollouts cannot be read from the bus: %w", v.err) + } + return v, nil +} + +// stopViews ends the view's watch: the inventory is closing. +func (i *Inventory) stopViews() { + i.moved.mu.Lock() + defer i.moved.mu.Unlock() + if i.moved.view != nil { + i.moved.view.stop() + i.moved.view = nil + } +} + +func normalizePlan(p *Plan) { + if p.Modules == nil { + p.Modules = map[string]*PlanModule{} + } + if p.TierEntered.IsZero() { + p.TierEntered = p.Created + } + if !p.Merged.IsZero() { + p.Merged = p.Merged.UTC() + } +} + +func planOf(e viewEntry) (Plan, error) { + var p Plan + if err := json.Unmarshal(e.raw, &p); err != nil { + return Plan{}, fmt.Errorf("the plan %s on the bus cannot be read: %w", e.sum.ID, err) + } + normalizePlan(&p) + return p, nil +} + +func sortCreated(entries []viewEntry) { + sort.SliceStable(entries, func(a, b int) bool { + if !entries[a].sum.Created.Equal(entries[b].sum.Created) { + return entries[a].sum.Created.Before(entries[b].sum.Created) + } + return entries[a].sum.ID < entries[b].sum.ID + }) +} + +func sortCreatedNewest(entries []viewEntry) { + sort.SliceStable(entries, func(a, b int) bool { + if !entries[a].sum.Created.Equal(entries[b].sum.Created) { + return entries[a].sum.Created.After(entries[b].sum.Created) + } + return entries[a].sum.ID > entries[b].sum.ID + }) +} + +// chainPlans is the plans that pass keep, in an order, read from the view. +func (i *Inventory) chainPlans(ctx context.Context, c *chain.Chain, keep func(viewEntry) bool, order func([]viewEntry)) ([]Plan, error) { + v, err := i.rolloutView(ctx, c) + if err != nil { + return nil, err + } + var chosen []viewEntry + for _, e := range v.list() { + if e.sum.ID != WaitingRecord && keep(e) { + chosen = append(chosen, e) + } + } + order(chosen) + out := make([]Plan, 0, len(chosen)) + for _, e := range chosen { + p, err := planOf(e) + if err != nil { + return nil, err + } + out = append(out, p) + } + return out, nil +} + +func (i *Inventory) chainNewestMergeOf(ctx context.Context, c *chain.Chain, repository, branch string) (Plan, bool, error) { + v, err := i.rolloutView(ctx, c) + if err != nil { + return Plan{}, false, err + } + var chosen []viewEntry + for _, e := range v.list() { + if strings.EqualFold(e.sum.Repository, repository) && e.sum.Branch == branch && !e.sum.Merged.IsZero() && !e.sum.isRelease() { + chosen = append(chosen, e) + } + } + sort.SliceStable(chosen, func(a, b int) bool { + if !chosen[a].sum.Merged.Equal(chosen[b].sum.Merged) { + return chosen[a].sum.Merged.After(chosen[b].sum.Merged) + } + return chosen[a].sum.Created.After(chosen[b].sum.Created) + }) + if len(chosen) == 0 { + return Plan{}, false, nil + } + p, err := planOf(chosen[0]) + return p, err == nil, err +} + +// chainPlanByID is one plan, read from the bucket itself: the caller of the verb wants the plan as it is, and the view +// learns of it. +func (i *Inventory) chainPlanByID(ctx context.Context, c *chain.Chain, id string) (Plan, error) { + if !chain.ValidKey(id) { + // No plan has an id the bus takes no spelling of; it is a plain id no plan has. + return Plan{}, fmt.Errorf("no plan %s", id) + } + e, found, err := chain.RolloutsOf[Plan](c).GetRaw(ctx, id) + if err != nil { + return Plan{}, err + } + if !found { + return Plan{}, fmt.Errorf("no plan %s", id) + } + if v, err := i.rolloutView(ctx, c); err == nil { + v.apply(id, e.Value, e.Revision) + } + return planOf(viewEntry{raw: e.Value, rev: e.Revision, sum: planSummary{ID: id}}) +} + +// ---- the merges + +// mergeHolder is a record's copy of a merge. +type mergeHolder struct { + plan string + m chain.HeldMerge + rev uint64 +} + +func mergeKey(repository, commit string) string { return strings.ToLower(repository) + "|" + commit } + +// mergeHolders is every record's copy of each merge, by the merge's key. +func mergeHolders(entries []viewEntry) map[string][]mergeHolder { + out := map[string][]mergeHolder{} + for _, e := range entries { + for _, m := range e.sum.Held { + k := mergeKey(m.Repository, m.Commit) + out[k] = append(out[k], mergeHolder{plan: e.sum.ID, m: m, rev: e.rev}) + } + } + return out +} + +// winner is the copy of a merge that counts: the one that entered its record last (ADR 0408, issue 1092 — a merge in two +// records is the half-done move of one, and the record it entered last is the one it moved to), a merge a plan answers +// before one no plan does, then the record written last. +func winner(copies []mergeHolder) mergeHolder { + best := copies[0] + for _, c := range copies[1:] { + switch { + case c.m.Placed.After(best.m.Placed): + best = c + case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting && !c.m.Waiting: + best = c + case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting == c.m.Waiting && c.rev > best.rev: + best = c + } + } + return best +} + +func winners(entries []viewEntry) map[string]mergeHolder { + out := map[string]mergeHolder{} + for k, copies := range mergeHolders(entries) { + out[k] = winner(copies) + } + return out +} + +func batchedOf(h mergeHolder) BatchedMerge { + b := BatchedMerge{Repository: strings.ToLower(h.m.Repository), Branch: h.m.Branch, Commit: h.m.Commit, + Heard: h.m.Heard.UTC(), Event: h.m.Event, Plan: h.plan, Alone: h.m.Alone} + if !h.m.Merged.IsZero() { + b.Merged = h.m.Merged.UTC() + } + if h.m.Waiting { + b.Plan = "" + } + return b +} + +func heldOf(m BatchedMerge, now time.Time) chain.HeldMerge { + h := chain.HeldMerge{Repository: strings.ToLower(m.Repository), Branch: m.Branch, Commit: m.Commit, Heard: m.Heard.UTC(), + Event: m.Event, Alone: m.Alone, Placed: now.UTC()} + if !m.Merged.IsZero() { + h.Merged = m.Merged.UTC() + } + return h +} + +func (i *Inventory) chainWinners(ctx context.Context, c *chain.Chain) (map[string]mergeHolder, error) { + v, err := i.rolloutView(ctx, c) + if err != nil { + return nil, err + } + return winners(v.list()), nil +} + +func (i *Inventory) chainMergeOf(ctx context.Context, c *chain.Chain, repository, commit string) (BatchedMerge, bool, error) { + w, err := i.chainWinners(ctx, c) + if err != nil { + return BatchedMerge{}, false, err + } + h, ok := w[mergeKey(repository, commit)] + if !ok { + return BatchedMerge{}, false, nil + } + return batchedOf(h), true, nil +} + +// merged orders merges by the time they were made, those whose time is not known last, then by when they were heard. +func sortMerges(ms []BatchedMerge, newest bool) { + sort.SliceStable(ms, func(a, b int) bool { + x, y := ms[a], ms[b] + switch { + case x.Merged.IsZero() != y.Merged.IsZero(): + return !x.Merged.IsZero() + case !x.Merged.Equal(y.Merged): + if newest { + return x.Merged.After(y.Merged) + } + return x.Merged.Before(y.Merged) + } + if newest { + return x.Heard.After(y.Heard) + } + return x.Heard.Before(y.Heard) + }) +} + +func (i *Inventory) chainMergesOf(ctx context.Context, c *chain.Chain, plan string) ([]BatchedMerge, error) { + w, err := i.chainWinners(ctx, c) + if err != nil { + return nil, err + } + var out []BatchedMerge + for _, h := range w { + if h.plan == plan && !h.m.Waiting { + out = append(out, batchedOf(h)) + } + } + sortMerges(out, false) + return out, nil +} + +func (i *Inventory) chainLaterMergesOf(ctx context.Context, c *chain.Chain, repository, branch string, after time.Time) ([]BatchedMerge, error) { + w, err := i.chainWinners(ctx, c) + if err != nil { + return nil, err + } + var out []BatchedMerge + for _, h := range w { + if !h.m.Waiting && strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && h.m.Merged.After(after) { + out = append(out, batchedOf(h)) + } + } + sort.SliceStable(out, func(a, b int) bool { return out[a].Merged.After(out[b].Merged) }) + return out, nil +} + +func (i *Inventory) chainKeptOnBranch(ctx context.Context, c *chain.Chain, repository, branch string) (bool, time.Time, error) { + w, err := i.chainWinners(ctx, c) + if err != nil { + return false, time.Time{}, err + } + var first time.Time + for _, h := range w { + if strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && (first.IsZero() || h.m.Heard.Before(first)) { + first = h.m.Heard + } + } + if first.IsZero() { + return false, time.Time{}, nil + } + return true, first.UTC(), nil +} + +func (i *Inventory) chainAloneMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) { + w, err := i.chainWinners(ctx, c) + if err != nil { + return nil, err + } + var out []BatchedMerge + for _, h := range w { + if h.m.Waiting && h.m.Alone { + out = append(out, batchedOf(h)) + } + } + sortMerges(out, true) + return out, nil +} + +// pendingMerge is a merge added to a plan that is not saved yet: carried until that plan's save writes it. +type pendingMerge struct { + plan string + m chain.HeldMerge +} + +// chainAddMerge keeps a merge for a plan. **When the plan has a record the merge is written into it at once**; when it has +// none — a new batch, saved by the caller next — the merge is carried and written by that plan's save. The forge's event is +// acknowledged only after the caller's write succeeded, and handed over again if it did not (ADR 0407 §3). +func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m BatchedMerge) (bool, error) { + waiting := m.Plan == "" + if waiting { + // A merge no plan answers is held, flagged, by the record kept for those. + m.Plan = WaitingRecord + if err := i.ensureWaitingRecord(ctx, c); err != nil { + return false, err + } + } + v, err := i.rolloutView(ctx, c) + if err != nil { + return false, err + } + key := mergeKey(m.Repository, m.Commit) + if _, known := winners(v.list())[key]; known { + return false, nil + } + held := heldOf(m, time.Now()) + held.Waiting = waiting + found, err := i.editRecord(ctx, c, m.Plan, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) { + for _, x := range h { + if mergeKey(x.Repository, x.Commit) == key { + return h, false + } + } + return append(h, held), true + }) + if err != nil { + return false, err + } + if !found { + i.moved.mu.Lock() + if i.moved.pending == nil { + i.moved.pending = map[string]pendingMerge{} + } + i.moved.pending[key] = pendingMerge{plan: m.Plan, m: held} + i.moved.mu.Unlock() + } + return true, nil +} + +// WaitingRecord is the record that holds the merges no plan has ever answered: a merge kept with no plan, which a store +// kept as a row with a null plan. It is no plan: the surveys of plans leave it out. +const WaitingRecord = "waiting-merges" + +// ensureWaitingRecord makes the record of the merges no plan answers, once. +func (i *Inventory) ensureWaitingRecord(ctx context.Context, c *chain.Chain) error { + b := chain.RolloutsOf[Plan](c) + if _, found, err := b.GetRaw(ctx, WaitingRecord); err != nil || found { + return err + } + body, err := json.Marshal(Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1, + Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"}) + if err != nil { + return err + } + rev, err := b.CreateRaw(ctx, WaitingRecord, body) + if errors.Is(err, chain.ErrExists) { + return nil + } + if err != nil { + return err + } + if v, verr := i.rolloutView(ctx, c); verr == nil { + v.apply(WaitingRecord, body, rev) + } + return nil +} + +// ErrMergeUnsaved is a merge moved to a plan that has no record to hold it. +var ErrMergeUnsaved = errors.New("a merge was moved to a plan that has not been saved") + +// editRecord changes the merges a plan's record holds and nothing else of it, by compare-and-set on the bucket, read again +// when it moved. It answers false when the plan has no record. +func (i *Inventory) editRecord(ctx context.Context, c *chain.Chain, id string, edit func([]chain.HeldMerge) ([]chain.HeldMerge, bool)) (bool, error) { + b := chain.RolloutsOf[Plan](c) + for try := 0; try < casTries; try++ { + e, found, err := b.GetRaw(ctx, id) + if err != nil || !found { + return false, err + } + var record map[string]json.RawMessage + if err := json.Unmarshal(e.Value, &record); err != nil { + return true, fmt.Errorf("the plan %s on the bus cannot be read: %w", id, err) + } + var held []chain.HeldMerge + if raw, has := record["held"]; has { + if err := json.Unmarshal(raw, &held); err != nil { + return true, fmt.Errorf("the merges of plan %s on the bus cannot be read: %w", id, err) + } + } + next, changed := edit(held) + if !changed { + return true, nil + } + if len(next) == 0 { + delete(record, "held") + } else { + raw, err := json.Marshal(next) + if err != nil { + return true, err + } + record["held"] = raw + } + body, err := json.Marshal(record) + if err != nil { + return true, err + } + rev, err := b.UpdateRaw(ctx, id, body, e.Revision) + if errors.Is(err, chain.ErrMoved) { + continue + } + if err != nil { + return true, fmt.Errorf("the merges of plan %s are not written: %w", id, err) + } + if v, verr := i.rolloutView(ctx, c); verr == nil { + v.apply(id, body, rev) + } + return true, nil + } + return true, fmt.Errorf("the merges of plan %s kept moving under the write", id) +} + +// recordEdit is what the merges of one record gain, lose and have flagged in one act. +type recordEdit struct { + add []chain.HeldMerge + drop map[string]bool + flag map[string]chain.HeldMerge +} + +func (e *recordEdit) apply(held []chain.HeldMerge) []chain.HeldMerge { + out := make([]chain.HeldMerge, 0, len(held)+len(e.add)) + for _, h := range held { + k := mergeKey(h.Repository, h.Commit) + if e.drop[k] { + continue + } + if f, has := e.flag[k]; has { + h.Waiting, h.Alone = f.Waiting, f.Alone + } + out = append(out, h) + } + for _, a := range e.add { + k := mergeKey(a.Repository, a.Commit) + dup := false + for _, h := range out { + if mergeKey(h.Repository, h.Commit) == k { + dup = true + break + } + } + if !dup { + out = append(out, a) + } + } + return out +} + +func (e *recordEdit) empty() bool { return len(e.add) == 0 && len(e.drop) == 0 && len(e.flag) == 0 } + +// ---- the save + +// tierLeftOf is the measurement a plan's save makes when it leaves a tier (planTierLeft, over the record the save +// replaces): when the plan entered the tier it is now at, and the duration of the tier it left, if it left one. +func tierLeftOf(old *Plan, p Plan, now time.Time) (time.Time, *Duration) { + if old == nil { + return now, nil // a new plan enters its first tier now + } + if old.State == PlanAssembling || old.State == PlanQueued { + return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276) + } + since := old.TierEntered + if since.IsZero() { + since = old.Created + } + wasOpen := old.State == PlanBuilding || old.State == PlanRolling + if !wasOpen || (old.Tier == p.Tier && p.Open()) { + return since, nil // still in the tier, or already ended + } + var modules []string + if old.Tier >= 0 && old.Tier < len(p.Tiers) { + modules = p.Tiers[old.Tier] + } + detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, old.Tier, len(p.Tiers), modules, p.State) + if p.Open() { + detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, old.Tier, len(p.Tiers), modules, p.Tier) + } + return now, &Duration{Kind: DurationPlanTier, Subject: p.Repository, Ref: fmt.Sprintf("%s/tier-%d", p.ID, old.Tier), + Started: since, Took: now.Sub(since), Detail: detail} +} + +// chainSavePlans writes plans, as SavePlan does each, and gives merges to the plans that answer them. **Each plan by +// compare-and-set on its own Revision**, a copy read before another save refused with ErrPlanMoved before anything is +// written; **a merge moved to a plan is written into the receiving record first, then taken from the giving one**, so a +// crash between leaves it in both and never in neither, and the one it entered last counts (ADR 0408, issue 1092). +func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []*Plan, moves []MergeMove) (err error) { + epoch, err := i.actingEpoch(ctx) + if err != nil { + if len(plans) > 0 { + return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err) + } + return err + } + v, err := i.rolloutView(ctx, c) + if err != nil { + return err + } + b := chain.RolloutsOf[Plan](c) + marked := time.Now() + undo := make([]func(), 0, len(plans)) + defer func() { + if err != nil { + for _, u := range undo { + u() + } + } + }() + for _, p := range plans { + undo = append(undo, p.markWalk(marked)) + } + + // Every plan still at the revision it was read at, before anything is written. + cur := make(map[string]chain.Kept[Plan], len(plans)) + for _, p := range plans { + k, found, err := b.Get(ctx, p.ID) + if err != nil { + return err + } + if found { + if k.Value.Revision != p.Revision { + return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) + } + cur[p.ID] = k + } + } + + // What each record gains, loses and has flagged. + now := time.Now().UTC() + edits := map[string]*recordEdit{} + editOf := func(id string) *recordEdit { + if edits[id] == nil { + edits[id] = &recordEdit{drop: map[string]bool{}, flag: map[string]chain.HeldMerge{}} + } + return edits[id] + } + copies := mergeHolders(v.list()) + for _, m := range moves { + key := mergeKey(m.Repository, m.Commit) + held, has := copies[key] + if !has { + // A merge carried for a plan not saved yet goes where it is moved. + i.moved.mu.Lock() + if pm, pending := i.moved.pending[key]; pending && m.Plan != "" { + pm.plan, pm.m.Alone = m.Plan, m.Alone + i.moved.pending[key] = pm + } + i.moved.mu.Unlock() + continue + } + w := winner(held) + if m.Plan == "" { + f := w.m + f.Waiting, f.Alone = true, m.Alone + editOf(w.plan).flag[key] = f + continue + } + if w.plan == m.Plan { + f := w.m + f.Waiting, f.Alone = false, m.Alone + editOf(w.plan).flag[key] = f + continue + } + moved := w.m + moved.Waiting, moved.Alone, moved.Placed = false, m.Alone, now + editOf(m.Plan).add = append(editOf(m.Plan).add, moved) + for _, h := range held { + editOf(h.plan).drop[key] = true + } + } + + // The records not among the plans: receivers now, givers after. + for id, e := range edits { + if _, written := cur[id]; written || planNamed(plans, id) || len(e.add) == 0 { + continue + } + e := e + found, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) { + return (&recordEdit{add: e.add}).apply(h), true + }) + if err != nil { + return err + } + if !found { + // Nothing is taken from the record that holds a merge to give it to a plan that has none. + return fmt.Errorf("%w: %s has no record to hold %d merge(s) moved to it", ErrMergeUnsaved, id, len(e.add)) + } + } + + // The plans, those that gain a merge first. + order := append([]*Plan(nil), plans...) + sort.SliceStable(order, func(a, b int) bool { + x, y := edits[order[a].ID], edits[order[b].ID] + return x != nil && len(x.add) > 0 && (y == nil || len(y.add) == 0) + }) + type written struct { + revision int64 + entered time.Time + held []chain.HeldMerge + duration *Duration + } + done := map[string]written{} + for _, p := range order { + var w written + for try := 0; ; try++ { + if try == casTries { + return fmt.Errorf("the plan for %s %s (%s) kept moving under the write", p.Repository, short(p.Commit), p.ID) + } + k, found := cur[p.ID] + if try > 0 { + fresh, still, gerr := b.Get(ctx, p.ID) + if gerr != nil { + return gerr + } + if still && fresh.Value.Revision != p.Revision { + return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) + } + k, found = fresh, still + cur[p.ID] = k + } + record := *p + var old *Plan + var held []chain.HeldMerge + if found { + o := k.Value + old, held = &o, append([]chain.HeldMerge(nil), k.Value.Held...) + record.Revision = k.Value.Revision + 1 + } else { + record.Revision = 1 + } + if e := edits[p.ID]; e != nil { + held = e.apply(held) + } + i.moved.mu.Lock() + for key, pm := range i.moved.pending { + if pm.plan == p.ID { + held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held) + _ = key + } + } + i.moved.mu.Unlock() + record.Held = held + entered, duration := tierLeftOf(old, *p, time.Now()) + record.TierEntered, record.Updated = entered, time.Now().UTC() + if epoch != nil { + record.Epoch = uint64(*epoch) + } else { + record.Epoch = 0 + } + body, merr := json.Marshal(record) + if merr != nil { + return merr + } + var rev uint64 + var werr error + if found { + rev, werr = b.UpdateRaw(ctx, p.ID, body, k.Revision) + } else { + rev, werr = createRaw(ctx, c, p.ID, body) + } + if errors.Is(werr, chain.ErrMoved) || (!found && errors.Is(werr, chain.ErrExists)) { + if !found { + return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) + } + continue + } + if werr != nil { + return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, werr) + } + v.apply(p.ID, body, rev) + w = written{revision: record.Revision, entered: entered, held: held, duration: duration} + break + } + done[p.ID] = w + } + + // The records that gave a merge up, now it is held by the one it went to. + for id, e := range edits { + if planNamed(plans, id) || (len(e.drop) == 0 && len(e.flag) == 0) { + continue + } + e := e + if _, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) { + return (&recordEdit{drop: e.drop, flag: e.flag}).apply(h), true + }); err != nil { + return err + } + } + + i.moved.mu.Lock() + for key, pm := range i.moved.pending { + if _, saved := done[pm.plan]; saved { + delete(i.moved.pending, key) + } + } + i.moved.mu.Unlock() + for _, p := range plans { + w := done[p.ID] + p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held + if epoch != nil { + p.Epoch = uint64(*epoch) + } else { + p.Epoch = 0 + } + if w.duration != nil { + // After the write: the key of the duration is the plan and its tier, so a measurement made twice is one. + record := i.RecordDuration + if i.recorded != nil { + record = i.recorded + } + if derr := record(ctx, *w.duration); derr != nil { + fmt.Printf("the time plan %s spent in a tier is not recorded: %v\n", p.ID, derr) + } + } + if PlanSaved != nil { + PlanSaved(i, *p) + } + } + return nil +} + +func planNamed(plans []*Plan, id string) bool { + for _, p := range plans { + if p.ID == id { + return true + } + } + return false +} + +// createRaw writes a plan's first record. +func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uint64, error) { + return chain.RolloutsOf[json.RawMessage](c).CreateRaw(ctx, id, body) +} + +// RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired +// at the next cut (ADR 0408, issue 1092). It says how many it repaired. +func (i *Inventory) RepairMerges(ctx context.Context) (int, error) { + c, on, err := i.ChainReads(ctx, KindRollouts) + if err != nil || !on { + return 0, err + } + v, err := i.rolloutView(ctx, c) + if err != nil { + return 0, err + } + repaired := 0 + for key, copies := range mergeHolders(v.list()) { + if len(copies) < 2 { + continue + } + w := winner(copies) + for _, h := range copies { + if h.plan == w.plan { + continue + } + key := key + if _, err := i.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) { + return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true + }); err != nil { + return repaired, err + } + repaired++ + } + } + return repaired, nil +} diff --git a/internal/inventory/chain_rollouts_test.go b/internal/inventory/chain_rollouts_test.go new file mode 100644 index 00000000..dffeeb17 --- /dev/null +++ b/internal/inventory/chain_rollouts_test.go @@ -0,0 +1,352 @@ +package inventory + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/chain" +) + +// anInventoryOnTheChain is an inventory with no store and a chain in memory: the chain's own paths, which never reach the +// store but for the durations a save measures, taken here by nothing. +func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) { + t.Helper() + c := chain.InMemory(nil) + inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }} + inv.UseChain(c) + t.Cleanup(inv.stopViews) + return inv, c +} + +var plansAt = time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) + +func aBatch(id string, at time.Time) *Plan { + return &Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c-" + id, Created: at, State: PlanAssembling, + Tiers: [][]string{}, Modules: map[string]*PlanModule{}} +} + +func aMerge(commit, plan string) BatchedMerge { + return BatchedMerge{Repository: "Novox/Mesh-Catalog", Branch: "main", Commit: commit, Merged: plansAt, Heard: plansAt.Add(time.Second), + Event: []byte(`{"commit":"` + commit + `"}`), Plan: plan} +} + +// **A plan is kept under its id; a save of a copy another save has moved since is refused and writes nothing** (ADR 0408 +// §2, issue 1092): the plan's own Revision counts its saves, 1 for the first. +func TestAPlanIsKeptUnderItsIdAndASaveOfAStaleCopyIsRefused(t *testing.T) { + inv, c := anInventoryOnTheChain(t) + ctx := context.Background() + p := aBatch("plan-1", plansAt) + if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil { + t.Fatal(err) + } + if p.Revision != 1 { + t.Fatalf("a new plan is at revision %d", p.Revision) + } + stale := *p + p.Note = "moved on" + if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil || p.Revision != 2 { + t.Fatalf("the second save: revision %d, %v", p.Revision, err) + } + stale.Note = "laid over" + if err := inv.chainSavePlans(ctx, c, []*Plan{&stale}, nil); !errors.Is(err, ErrPlanMoved) { + t.Fatalf("a stale copy was saved: %v", err) + } + got, err := inv.chainPlanByID(ctx, c, "plan-1") + if err != nil || got.Note != "moved on" || got.Revision != 2 { + t.Fatalf("the plan is %+v: %v", got, err) + } + // A plan new to the bucket that is already there is refused too. + again := aBatch("plan-1", plansAt) + if err := inv.chainSavePlans(ctx, c, []*Plan{again}, nil); !errors.Is(err, ErrPlanMoved) { + t.Fatalf("a new plan over a kept one: %v", err) + } + if _, err := inv.chainPlanByID(ctx, c, "plan-nobody"); err == nil || err.Error() != "no plan plan-nobody" { + t.Fatalf("a plan nobody kept: %v", err) + } +} + +// **A save keeps the merges the record holds, not the ones its copy read** (issue 1092): a merge that moved into the record +// after the copy was read is not laid back by the copy's save. +func TestASaveKeepsTheMergesTheRecordHoldsAndNotTheCopys(t *testing.T) { + inv, c := anInventoryOnTheChain(t) + ctx := context.Background() + batch := aBatch("plan-1", plansAt) + if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil { + t.Fatal(err) + } + if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || !added { + t.Fatalf("added %v: %v", added, err) + } + // The copy was read before the merge came in; saving it neither fails nor loses the merge. + batch.Note = "window closed" + if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil { + t.Fatalf("a copy read before a merge came in: %v", err) + } + merges, err := inv.chainMergesOf(ctx, c, "plan-1") + if err != nil || len(merges) != 1 || merges[0].Commit != "m1" || merges[0].Repository != "novox/mesh-catalog" { + t.Fatalf("the merges of the plan: %+v %v", merges, err) + } + if string(merges[0].Event) != `{"commit":"m1"}` || merges[0].Heard.IsZero() || merges[0].Plan != "plan-1" { + t.Fatalf("the merge came back as %+v", merges[0]) + } + // A merge heard is not heard twice, whatever the spelling of its repository. + if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || added { + t.Fatalf("added again %v: %v", added, err) + } +} + +// **A merge added to a plan that is not saved is written by that plan's save, and by nothing else** (issue 1092): until +// then it is in no record — so the event that brought it is handed over again if the save fails. +func TestAMergeAddedToAPlanNotSavedYetIsWrittenByItsSave(t *testing.T) { + inv, c := anInventoryOnTheChain(t) + ctx := context.Background() + if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-new")); err != nil || !added { + t.Fatalf("added %v: %v", added, err) + } + if _, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); known { + t.Fatal("a merge for a plan not saved is in a record") + } + // The save is refused: nothing is written, the merge is still carried. + held := false + away := chain.New(chain.NewMemory(), chain.NewMemory(), chain.NewMemory(), func(context.Context) error { + if held { + return nil + } + return errors.New("the lease is not held") + }) + other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded} + other.UseChain(away) + t.Cleanup(other.stopViews) + other.moved.pending = inv.moved.pending + p := aBatch("plan-new", plansAt) + if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); !errors.Is(err, chain.ErrNotHeld) { + t.Fatalf("the save answered %v", err) + } + if _, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); known { + t.Fatal("a refused save left the merge in a record") + } + held = true + if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); err != nil { + t.Fatal(err) + } + if m, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); !known || m.Plan != "plan-new" { + t.Fatalf("the save did not write the merge: %+v %v", m, known) + } +} + +// failingOn is a bucket that refuses to write one key: the second half of a move, interrupted. +type failingOn struct { + chain.Backend + key string + fail *atomic.Bool +} + +func (f failingOn) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { + if key == f.key && f.fail.Load() { + return 0, errors.New("the bus went away") + } + return f.Backend.Update(ctx, key, value, revision) +} + +// **A merge moved to a plan is written to the receiving record first, then taken from the giving one; one found in both is +// read from the one it entered last and repaired** (issue 1092, ADR 0299 §7). +func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *testing.T) { + c := chain.InMemory(nil) + var fail atomic.Bool + rollouts := failingOn{Backend: chain.NewMemory(), key: "plan-a", fail: &fail} + c = chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil }) + inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }} + inv.UseChain(c) + t.Cleanup(inv.stopViews) + ctx := context.Background() + a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute)) + for _, p := range []*Plan{a, b} { + if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil { + t.Fatal(err) + } + } + if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil { + t.Fatal(err) + } + + // The move stops before the giving record is written: the merge is in both. + fail.Store(true) + err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b"}}) + if err == nil { + t.Fatal("the move was said to be made with the giving record unwritten") + } + fail.Store(false) + if m, _, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); m.Plan != "plan-b" { + t.Fatalf("a merge in two records is read from %q, not the one it moved to", m.Plan) + } + if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 { + t.Fatalf("the giving plan still answers %+v", got) + } + // The next cut repairs it. + v, _ := inv.rolloutView(ctx, c) + if n := len(mergeHolders(v.list())["novox/mesh-catalog|m1"]); n != 2 { + t.Fatalf("the merge is held by %d records, want the two of a stopped move", n) + } + inv.moved.mu.Lock() + inv.moved.kind = map[string]bool{KindRollouts: true} + inv.moved.mu.Unlock() + repaired := 0 + for k, copies := range mergeHolders(v.list()) { + if len(copies) < 2 { + continue + } + w := winner(copies) + for _, h := range copies { + if h.plan != w.plan { + key := k + if _, err := inv.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) { + return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true + }); err != nil { + t.Fatal(err) + } + repaired++ + } + } + } + if repaired != 1 { + t.Fatalf("repaired %d", repaired) + } + waitFor(t, func() bool { + v, _ := inv.rolloutView(ctx, c) + return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1 + }) +} + +// waitFor polls a condition that a watch makes true, for a moment. +func waitFor(t *testing.T, ok func() bool) { + t.Helper() + for end := time.Now().Add(3 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) { + if ok() { + return + } + } + t.Fatal("the condition did not come true") +} + +// **A merge no plan answers stays, flagged, in the record that held it last, and is walked alone** (issue 1092): not in +// the merges of the plan, listed among the merges waiting, and answered again by the plan it is moved to. +func TestAWaitingMergeStaysInTheRecordThatHeldItLast(t *testing.T) { + inv, c := anInventoryOnTheChain(t) + ctx := context.Background() + a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute)) + for _, p := range []*Plan{a, b} { + if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil { + t.Fatal(err) + } + } + if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil { + t.Fatal(err) + } + if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "", Alone: true}}); err != nil { + t.Fatal(err) + } + if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 { + t.Fatalf("a waiting merge is answered by %+v", got) + } + alone, err := inv.chainAloneMerges(ctx, c) + if err != nil || len(alone) != 1 || alone[0].Commit != "m1" || alone[0].Plan != "" || !alone[0].Alone { + t.Fatalf("waiting: %+v %v", alone, err) + } + if m, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); !known || m.Plan != "" { + t.Fatalf("a waiting merge reads as %+v %v", m, known) + } + if later, _ := inv.chainLaterMergesOf(ctx, c, "novox/mesh-catalog", "main", plansAt.Add(-time.Hour)); len(later) != 0 { + t.Fatalf("a waiting merge is among those a plan answers: %+v", later) + } + if kept, first, _ := inv.chainKeptOnBranch(ctx, c, "novox/mesh-catalog", "main"); !kept || first.IsZero() { + t.Fatalf("a waiting merge is not kept on its branch: %v %v", kept, first) + } + // Walked alone: the plan it goes to answers it, and the record that held it lets it go. + if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b", Alone: true}}); err != nil { + t.Fatal(err) + } + if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 || !got[0].Alone { + t.Fatalf("plan-b answers %+v", got) + } + if left, _ := inv.chainAloneMerges(ctx, c); len(left) != 0 { + t.Fatalf("still waiting: %+v", left) + } + waitFor(t, func() bool { + v, _ := inv.rolloutView(ctx, c) + return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1 + }) + // A move to a plan with no record is refused and takes nothing from the one that holds the merge. + err = inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-nowhere"}}) + if !errors.Is(err, ErrMergeUnsaved) { + t.Fatalf("a move to a plan with no record: %v", err) + } + if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 { + t.Fatalf("the merge was taken from plan-b: %+v", got) + } +} + +// countingBackend counts the reads a bucket answers. +type countingBackend struct { + chain.Backend + gets, keys atomic.Int32 +} + +func (c *countingBackend) Get(ctx context.Context, key string) (chain.Entry, bool, error) { + c.gets.Add(1) + return c.Backend.Get(ctx, key) +} + +func (c *countingBackend) Keys(ctx context.Context) ([]string, error) { + c.keys.Add(1) + return c.Backend.Keys(ctx) +} + +// **The surveys are read from a view kept by one watch, never from the bucket key by key** (issue 1092): the plans and the +// merges are asked for many times and the bucket is read for none of them; a write by another process reaches the view +// through its watch. +func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) { + counting := &countingBackend{Backend: chain.NewMemory()} + c := chain.New(chain.NewMemory(), counting, chain.NewMemory(), func(context.Context) error { return nil }) + inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }} + inv.UseChain(c) + t.Cleanup(inv.stopViews) + ctx := context.Background() + for k, id := range []string{"plan-1", "plan-2", "plan-3"} { + p := aBatch(id, plansAt.Add(time.Duration(k)*time.Minute)) + if k > 0 { + p.State = PlanBuilding + } + if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil { + t.Fatal(err) + } + } + before := counting.gets.Load() + for n := 0; n < 20; n++ { + open, err := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) + if err != nil || len(open) != 2 || open[0].ID != "plan-2" { + t.Fatalf("open plans %+v: %v", open, err) + } + if _, err := inv.chainAloneMerges(ctx, c); err != nil { + t.Fatal(err) + } + } + if counting.gets.Load() != before || counting.keys.Load() != 0 { + t.Fatalf("the bucket was read %d times and its keys %d times for the surveys", counting.gets.Load()-before, counting.keys.Load()) + } + // Another process writes; the view follows. + other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded} + other.UseChain(c) + t.Cleanup(other.stopViews) + late := aBatch("plan-4", plansAt.Add(time.Hour)) + late.State = PlanRolling + if err := other.chainSavePlans(ctx, c, []*Plan{late}, nil); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { + open, _ := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) + return len(open) == 3 + }) +} diff --git a/internal/inventory/collection.go b/internal/inventory/collection.go index 6e60a872..6c0ba092 100644 --- a/internal/inventory/collection.go +++ b/internal/inventory/collection.go @@ -159,44 +159,23 @@ func (i *Inventory) KeepSet(ctx context.Context) (map[string][]string, error) { // no releases left to return between, and its build rows stay only as history. Without the // join every module ever built kept five builds' artifacts for ever — and once the store's // collector runs for real, what the keep set says is what the disk holds. - recent, err := i.store.Pool().Query(ctx, - `select made, built_against from ( - select b.made, b.built_against, - row_number() over (partition by b.module order by b.at desc, b.id desc) as back - from build b - join module m on m.name = b.module - where b.failed = '' and b.module is not null and b.module <> '' - ) ranked where back <= $1`, KeptBuilds) + recent, err := i.recentBuilds(ctx) if err != nil { return nil, err } - defer recent.Close() var stoodOn []string - for recent.Next() { - var raw, against []byte - if err := recent.Scan(&raw, &against); err != nil { - return nil, err - } - var made []Artifact - if err := json.Unmarshal(raw, &made); err == nil { - for _, a := range made { - if reference := asRecorded(a.Reference); reference != "" { - because(reference, KeptByRecentBuild) - } + for _, r := range recent { + for _, a := range r.made { + if reference := asRecorded(a.Reference); reference != "" { + because(reference, KeptByRecentBuild) } } - var bases []string - if err := json.Unmarshal(against, &bases); err == nil { - for _, b := range bases { - if reference := asRecorded(b); reference != "" { - stoodOn = append(stoodOn, reference) - } + for _, b := range r.against { + if reference := asRecorded(b); reference != "" { + stoodOn = append(stoodOn, reference) } } } - if err := recent.Err(); err != nil { - return nil, err - } // **A copied base is kept by the kept builds that stood on it, and by nothing else** (novox/hq // ADR 0257). It has no builds of its own to count five of, and no machine is ever handed a base, @@ -340,27 +319,97 @@ func (i *Inventory) mirroredSet(ctx context.Context) (map[string]bool, error) { return out, rows.Err() } -// everyReferenceMade is every artifact reference any successful build recorded, oldest build first. -func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) { +// recentBuild is what one of the KeptBuilds newest successful builds of a module made and stood on. +type recentBuild struct { + made []Artifact + against []string +} + +// recentBuilds is, of each module the mesh still holds, its KeptBuilds newest successful builds' artifacts and bases. +func (i *Inventory) recentBuilds(ctx context.Context) ([]recentBuild, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return nil, err + } else if on { + builds, err := i.chainRecentMade(ctx, c) + if err != nil { + return nil, err + } + out := make([]recentBuild, 0, len(builds)) + for _, b := range builds { + out = append(out, recentBuild{made: b.Made, against: b.Against}) + } + return out, nil + } rows, err := i.store.Pool().Query(ctx, - `select made from build where failed = '' and module is not null and module <> '' - order by at asc, id asc`) + `select made, built_against from ( + select b.made, b.built_against, + row_number() over (partition by b.module order by b.at desc, b.id desc) as back + from build b + join module m on m.name = b.module + where b.failed = '' and b.module is not null and b.module <> '' + ) ranked where back <= $1`, KeptBuilds) if err != nil { return nil, err } defer rows.Close() - seen := map[string]bool{} - var out []string + var out []recentBuild for rows.Next() { - var raw []byte - if err := rows.Scan(&raw); err != nil { + var raw, against []byte + if err := rows.Scan(&raw, &against); err != nil { return nil, err } - var made []Artifact - if err := json.Unmarshal(raw, &made); err != nil { - continue + var r recentBuild + if err := json.Unmarshal(raw, &r.made); err != nil { + r.made = nil } - for _, a := range made { + if err := json.Unmarshal(against, &r.against); err != nil { + r.against = nil + } + out = append(out, r) + } + return out, rows.Err() +} + +// everyReferenceMade is every artifact reference any successful build recorded, oldest build first. +func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) { + var made [][]Artifact + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return nil, err + } else if on { + builds, err := i.chainEveryReferenceMade(ctx, c) + if err != nil { + return nil, err + } + for _, b := range builds { + made = append(made, b.Made) + } + } else { + rows, err := i.store.Pool().Query(ctx, + `select made from build where failed = '' and module is not null and module <> '' + order by at asc, id asc`) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var raw []byte + if err := rows.Scan(&raw); err != nil { + return nil, err + } + var a []Artifact + if err := json.Unmarshal(raw, &a); err != nil { + continue + } + made = append(made, a) + } + if err := rows.Err(); err != nil { + return nil, err + } + } + seen := map[string]bool{} + var out []string + for _, row := range made { + for _, a := range row { reference := asRecorded(a.Reference) if reference == "" || seen[reference] { continue @@ -369,7 +418,7 @@ func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) { out = append(out, reference) } } - return out, rows.Err() + return out, nil } // asRecorded is an artifact reference in the one vocabulary the sweep speaks (novox/hq issue 226). diff --git a/internal/inventory/fortest.go b/internal/inventory/fortest.go index cbd7f73a..5a2067d0 100644 --- a/internal/inventory/fortest.go +++ b/internal/inventory/fortest.go @@ -12,6 +12,7 @@ import ( "github.com/jackc/pgx/v5" "golang.org/x/crypto/bcrypt" + "github.com/novox/mesh-controller/internal/chain" "github.com/novox/mesh-controller/internal/store" ) @@ -64,6 +65,16 @@ func ForTest(t *testing.T) *Inventory { if _, err := inv.store.Migrate(t.Context(), migrations); err != nil { t.Fatal(err) } + // The chain is on the bus in a mesh; in a test it is in memory, with every kind moved, as a mesh raised after this + // release has them (ADR 0408, issue 1089). The tests of the move itself use ForTestBefore. + c := chain.InMemory(nil) + inv.UseChain(c) + testChains.Store(os.Getenv(store.Variable(Name)), c) + t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) }) + if _, err := inv.store.Pool().Exec(t.Context(), + `insert into chain_moved (counts, largest) values ($1, $1)`, `{"builds": 0, "rollouts": 0, "assignments": 0}`); err != nil { + t.Fatal(err) + } return inv } @@ -121,6 +132,11 @@ func ForTestBefore(t *testing.T, number int) (*Inventory, func()) { if err := inv.Ready(t.Context(), 20*time.Second); err != nil { t.Fatal(err) } + // The chain is in memory and nothing is moved: the tests of a move seed the tables and move them. + c := chain.InMemory(nil) + inv.UseChain(c) + testChains.Store(os.Getenv(store.Variable(Name)), c) + t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) }) return inv, func() { t.Helper() if _, err := inv.store.Migrate(t.Context(), all); err != nil { diff --git a/internal/inventory/gate.go b/internal/inventory/gate.go index b5b76768..9932b559 100644 --- a/internal/inventory/gate.go +++ b/internal/inventory/gate.go @@ -187,8 +187,8 @@ func (i *Inventory) PreviousBuild(ctx context.Context, module, commit string, fa } else if bad { continue } - var manifest []byte - if err := i.store.Pool().QueryRow(ctx, `select manifest from build where id = $1`, b.ID).Scan(&manifest); err != nil { + manifest, err := i.manifestOf(ctx, b.ID) + if err != nil { return Build{}, false, err } if len(manifest) == 0 || string(manifest) == "null" { @@ -241,6 +241,11 @@ func (i *Inventory) RestoreModule(ctx context.Context, b Build) error { // machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told // from one that does (the bus's planned step, ADR 0236). func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return nil, err + } else if on { + return i.chainBuildFingerprints(ctx, c, module) + } rows, err := i.store.Pool().Query(ctx, `select commit_hash, made, coalesce(manifest::text, '') from build where module = $1 and failed = '' and commit_hash <> '' order by at desc limit 50`, module) @@ -266,6 +271,11 @@ func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[s // Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint. func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return nil, err + } else if on { + return i.chainFingerprints(ctx, c) + } rows, err := i.store.Pool().Query(ctx, `select module, commit_hash, made, coalesce(manifest::text, '') from build where module is not null and failed = '' and commit_hash <> '' order by at desc`) @@ -316,6 +326,11 @@ func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bo // BuildOf is the id of the newest successful build of a module from a commit; empty when none is // recorded (a manifest handed over by hand). func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return "", err + } else if on { + return i.chainBuildOf(ctx, c, module, commit) + } var id string err := i.store.Pool().QueryRow(ctx, `select id from build where module = $1 and commit_hash = $2 and failed = '' order by at desc limit 1`, @@ -335,19 +350,13 @@ func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (cata if commit == "" { return catalogue.Manifest{}, false, nil } - var id string - var raw []byte - err := i.store.Pool().QueryRow(ctx, - `select id, manifest from build - where module = $1 and failed = '' and manifest is not null and manifest::text <> 'null' - and (commit_hash = $2 or starts_with(commit_hash, $2)) - order by at desc limit 1`, module, commit).Scan(&id, &raw) - if errors.Is(err, pgx.ErrNoRows) { - return catalogue.Manifest{}, false, nil - } + id, raw, found, err := i.newestManifestAt(ctx, module, commit) if err != nil { return catalogue.Manifest{}, false, err } + if !found { + return catalogue.Manifest{}, false, nil + } if stands, same, err := i.StandingBuild(ctx, module, id); err != nil { return catalogue.Manifest{}, false, err } else if stands != "" && stands != id && len(same) > 0 { @@ -360,3 +369,41 @@ func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (cata } return m, true, nil } + +// manifestOf is the manifest a build was recorded with; nil when it kept none. +func (i *Inventory) manifestOf(ctx context.Context, id string) ([]byte, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return nil, err + } else if on { + return i.chainManifestOf(ctx, c, id) + } + var manifest []byte + if err := i.store.Pool().QueryRow(ctx, `select manifest from build where id = $1`, id).Scan(&manifest); err != nil { + return nil, err + } + return manifest, nil +} + +// newestManifestAt is the newest successful build of a module from a commit, or one it abbreviates, that kept a manifest: +// its id and manifest; false when there is none. +func (i *Inventory) newestManifestAt(ctx context.Context, module, commit string) (string, []byte, bool, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return "", nil, false, err + } else if on { + return i.chainManifestBuild(ctx, c, module, commit) + } + var id string + var raw []byte + err := i.store.Pool().QueryRow(ctx, + `select id, manifest from build + where module = $1 and failed = '' and manifest is not null and manifest::text <> 'null' + and (commit_hash = $2 or starts_with(commit_hash, $2)) + order by at desc limit 1`, module, commit).Scan(&id, &raw) + if errors.Is(err, pgx.ErrNoRows) { + return "", nil, false, nil + } + if err != nil { + return "", nil, false, err + } + return id, raw, true, nil +} diff --git a/internal/inventory/given.go b/internal/inventory/given.go index fac8574b..20507be1 100644 --- a/internal/inventory/given.go +++ b/internal/inventory/given.go @@ -93,11 +93,28 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s if declared == "" { return nil, nil } + // Only a module assigned there: from the bucket when the assignments are on the bus, else the store's join. + _, onTheBus, err := i.ChainReads(ctx, KindAssignments) + if err != nil { + return nil, err + } + assignedHere := map[string]bool{} + join := `join assignment a on a.node = s.node and a.module = s.module` + if onTheBus { + join = `` + modules, err := i.assignedModules(ctx, node) + if err != nil { + return nil, err + } + for _, m := range modules { + assignedHere[m] = true + } + } rows, err := i.store.Pool().Query(ctx, `select s.node, s.module, s.name, s.replace_after_start from module_secret s join node n on n.id = s.node - join assignment a on a.node = s.node and a.module = s.module + `+join+` where n.name = $1 and n.sent = $2 and n.sent_at > s.replace_after_start and s.origin = 'accepted' and s.replace_after_start is not null and (not n.adopted or exists (select 1 from taken t where t.node = s.node and t.module = s.module)) @@ -117,6 +134,9 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s rows.Close() return nil, err } + if onTheBus && !assignedHere[d.module] { + continue + } dues = append(dues, d) } rows.Close() diff --git a/internal/inventory/given_test.go b/internal/inventory/given_test.go index a79a7877..ade0433a 100644 --- a/internal/inventory/given_test.go +++ b/internal/inventory/given_test.go @@ -7,8 +7,10 @@ import ( "errors" "strings" "testing" + "time" "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/chain" ) // A module with one secret of each kind the rule tells apart (novox/hq ADR 0228), assigned to the @@ -36,8 +38,9 @@ func assign(t *testing.T, inv *Inventory, ctx context.Context, node, module stri if err != nil { t.Fatal(err) } - if _, err := inv.store.Pool().Exec(ctx, - `insert into assignment (node, module) values ($1, $2) on conflict do nothing`, n.ID, module); err != nil { + // Written as the fixture it is, past the checks Assign makes: the assignment on the bus. + if _, err := inv.chain.Assignments.Create(ctx, assignmentKey(n.Name, module), + chain.Assignment{Node: n.Name, Module: module, Assigned: time.Now().UTC()}); err != nil && !errors.Is(err, chain.ErrExists) { t.Fatal(err) } } diff --git a/internal/inventory/migrations/0104-the-chain-moves-to-the-bus.sql b/internal/inventory/migrations/0104-the-chain-moves-to-the-bus.sql new file mode 100644 index 00000000..5a865465 --- /dev/null +++ b/internal/inventory/migrations/0104-the-chain-moves-to-the-bus.sql @@ -0,0 +1,21 @@ +-- Builds, rollouts and assignments move to the bus (novox/hq ADR 0408 §8, design 47 milestone 5, issue 1089). +-- +-- This migration makes the switch alone. The rows of build, build_request, release_plan, batched_merge, assignment and +-- pending_assignment are written into the chain's buckets by the serving controller at its start, holding the lease, +-- because a migration runs when the store is opened, before any connection to the bus. This row is the switch: while it +-- is absent nothing reads the buckets; once the move has counted every row against the keys it wrote, it is written, and +-- nothing reads the tables. One row at most; never written by a migration, so a delivery alone changes nothing. The +-- controller keeps no consolidated schema, so this migration is the whole of the change; the tables go by a later one. +create table chain_moved ( + only_one boolean primary key default true check (only_one), + moved_at timestamptz not null default now(), + -- The counts the move read from the tables and wrote as keys, per kind, and the largest value it wrote: what a + -- reader of the move reads back. + counts jsonb not null, + largest jsonb not null +); + +-- A seat's holder was an assignment by a foreign key; the assignments are keys of the bus once moved, so the controller says +-- it (HoldSeat refuses a holder that is not assigned) and an unassignment takes the holding with it (Unassign), as the +-- key's `on delete cascade` did. +alter table seat_holding drop constraint seat_holding_node_module_fkey; diff --git a/internal/inventory/mirrored_test.go b/internal/inventory/mirrored_test.go index e58bf53d..d258ce91 100644 --- a/internal/inventory/mirrored_test.go +++ b/internal/inventory/mirrored_test.go @@ -151,7 +151,7 @@ func TestAPersonRecordsACopyAndTheFirstRecordStands(t *testing.T) { // The migration records the copies the build records already name: only a module's own on- // repository, and only once, by the earliest build naming it. func TestTheCopiesTheRecordsAlreadyNameAreRecorded(t *testing.T) { - inv := fresh(t) + inv := inTheTables(t) ctx := context.Background() copyRef := mirrorRef("route-proxy/on-go_base", 1) notACopy := ref("go-base", "build", 2) diff --git a/internal/inventory/mirrors.go b/internal/inventory/mirrors.go index b4b09a68..fadab025 100644 --- a/internal/inventory/mirrors.go +++ b/internal/inventory/mirrors.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "time" ) @@ -69,6 +70,21 @@ type DeclaredMirrored struct { var DeclaredMirroredFields = []string{"kind", "default", "why", "type", "description", "enum", "minimum", "maximum", "pattern", "trusted", "predefined", "required", "runtime"} +// ErrAssignmentsMoved is a write of the store's mirror of the assignments, refused because the assignments are keys of the bus. +var ErrAssignmentsMoved = errors.New("the assignments are keys of the bus, and the store's rows are no longer what they say") + +// AssignmentsMirrorToWrite is AssignmentsMirror for the mirror's writer, which lays what it returns over the bucket and +// ends the keys it does not return: **refused once the assignments are moved** (ADR 0408 §4), asked at the moment of the +// write, so a tick that decided before the switch cannot lay mirror-shaped values over the links after it. +func (i *Inventory) AssignmentsMirrorToWrite(ctx context.Context) (map[string][]byte, error) { + if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil { + return nil, err + } else if on { + return nil, ErrAssignmentsMoved + } + return i.AssignmentsMirror(ctx) +} + // AssignmentsMirror is every assignment's mirrored value, keyed `.`, with the provisioner's plan of // each assignment of a module handed to it, and a value of its own for each module handed to it on a node that // nothing assigned there, only pulled in by another (issue 765). @@ -88,22 +104,15 @@ func (i *Inventory) AssignmentsMirror(ctx context.Context) (map[string][]byte, e } func (i *Inventory) assignmentsMirrored(ctx context.Context) ([]AssignmentMirrored, error) { - rows, err := i.store.Pool().Query(ctx, - `select n.name, a.module, a.assigned from assignment a join node n on n.id = a.node`) + rows, err := i.assignments(ctx) if err != nil { return nil, err } - defer rows.Close() - var out []AssignmentMirrored - for rows.Next() { - var a AssignmentMirrored - if err := rows.Scan(&a.Node, &a.Module, &a.Assigned); err != nil { - return nil, err - } - a.Assigned = a.Assigned.UTC() - out = append(out, a) + out := make([]AssignmentMirrored, 0, len(rows)) + for _, a := range rows { + out = append(out, AssignmentMirrored{Node: a.Node, Module: a.Module, Assigned: a.Assigned.UTC()}) } - return out, rows.Err() + return out, nil } // DeclaredConfigMirror is every registered module's mirrored config declarations, keyed by the module. diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 11386553..3f6c6dd2 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "os" "regexp" "slices" "sort" @@ -18,6 +19,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/chain" "github.com/novox/mesh-controller/internal/store" ) @@ -32,6 +34,13 @@ type Inventory struct { // configValues reads a module's config values from mesh-config once the move of config has run (ReadsConfigFrom, // issue 664); nil reads none. configValues ConfigValues + // chain is the controller's chain on the bus (novox/hq ADR 0408, issue 1089), once attached (UseChain); moved is which + // kinds of it have been moved from the tables, as chain_moved says (ChainReads). A kind not moved is read and written + // in the tables as before. + chain *chain.Chain + moved *movedKinds + // recorded, when set, takes the durations a save of the chain measures in place of the store: a test with no store. + recorded func(context.Context, Duration) error } // Open connects to the inventory store. @@ -40,10 +49,18 @@ func Open(ctx context.Context) (*Inventory, error) { if err != nil { return nil, err } - return &Inventory{store: s}, nil + inv := &Inventory{store: s, moved: &movedKinds{}} + // A test's store has the chain a test made for it in memory, shared by every inventory opened on that store. + if c, ok := testChains.Load(os.Getenv(store.Variable(Name))); ok { + inv.chain = c.(*chain.Chain) + } + return inv, nil } -func (i *Inventory) Close() { i.store.Close() } +func (i *Inventory) Close() { + i.stopViews() + i.store.Close() +} // Ready waits for the database to answer. func (i *Inventory) Ready(ctx context.Context, within time.Duration) error { @@ -1259,6 +1276,9 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu _, err := i.store.Pool().Exec(ctx, `update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4 where id = $1`, node, digest, carried, sentEpoch) + if err == nil { + saidRunsNot(node, i.recordRuns(ctx, node, builds, time.Now())) + } return err } @@ -1268,6 +1288,11 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu // read from: every change a push carries happened after the last push, and an assignment is the one // that says when. func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time, error) { + if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil { + return time.Time{}, err + } else if on { + return i.chainAwaitingSince(ctx, name) + } var since time.Time err := i.store.Pool().QueryRow(ctx, `select coalesce( diff --git a/internal/inventory/pending.go b/internal/inventory/pending.go index 0e761e31..7f8258d5 100644 --- a/internal/inventory/pending.go +++ b/internal/inventory/pending.go @@ -8,6 +8,8 @@ import ( "time" "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/chain" ) // What the controller asked to be built, and the assignments waiting for it (novox/hq issue 325, ADR 0261). @@ -67,6 +69,11 @@ type RequestOutcome struct { // RecordBuildRequest keeps one build request, once it is asked. Idempotent on the id. Requests older than // KeptFor are deleted in the same act. func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) error { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return err + } else if on { + return i.chainRecordBuildRequest(ctx, c, a) + } at := a.At if at.IsZero() { at = time.Now().UTC() @@ -90,6 +97,11 @@ func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) erro // BuildRequestByID is the build request kept under this id, and whether one was kept. func (i *Inventory) BuildRequestByID(ctx context.Context, id string) (BuildRequest, bool, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return BuildRequest{}, false, err + } else if on { + return i.chainBuildRequestByID(ctx, c, id) + } var a BuildRequest err := i.store.Pool().QueryRow(ctx, `select id, repository, seat, source_path, ref, commit_hash, asked_for, at_terminal, asked_at @@ -104,21 +116,40 @@ func (i *Inventory) BuildRequestByID(ctx context.Context, id string) (BuildReque // MarkNotAsked says a kept build request was never handed over: the words are kept, unless its outcome was // heard first. func (i *Inventory) MarkNotAsked(ctx context.Context, id, why string) error { - _, err := i.store.Pool().Exec(ctx, - `update build_request set not_asked = $2 - where id = $1 and not exists (select 1 from build where build.id = $1)`, id, why) + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return err + } else if on { + return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.NotAsked = why }) + } + if heard, err := i.outcomeHeard(ctx, id); err != nil || heard { + return err + } + _, err := i.store.Pool().Exec(ctx, `update build_request set not_asked = $2 where id = $1`, id, why) return err } // MarkOutcomeUnknown says the asker of a build it handed over stopped waiting for it: asked, outcome unknown. // The build may still run; its outcome, when heard, is the last word. func (i *Inventory) MarkOutcomeUnknown(ctx context.Context, id, why string) error { - _, err := i.store.Pool().Exec(ctx, - `update build_request set outcome_unknown = $2 - where id = $1 and not exists (select 1 from build where build.id = $1)`, id, why) + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return err + } else if on { + return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.OutcomeUnknown = why }) + } + if heard, err := i.outcomeHeard(ctx, id); err != nil || heard { + return err + } + _, err := i.store.Pool().Exec(ctx, `update build_request set outcome_unknown = $2 where id = $1`, id, why) return err } +// outcomeHeard says a build of this id was recorded in the table: its outcome was taken in. +func (i *Inventory) outcomeHeard(ctx context.Context, id string) (bool, error) { + var heard bool + err := i.store.Pool().QueryRow(ctx, `select exists (select 1 from build where id = $1)`, id).Scan(&heard) + return heard, err +} + // RequestsNamed is every build request kept whose directory names the module, or whose build said it built // it, newest first, each with its outcome. func (i *Inventory) RequestsNamed(ctx context.Context, name string) ([]RequestOutcome, error) { @@ -154,6 +185,11 @@ func (i *Inventory) RequestedNames(ctx context.Context) ([]string, error) { } func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return nil, err + } else if on { + return i.chainRequests(ctx, c) + } rows, err := i.store.Pool().Query(ctx, `select a.id, a.repository, a.seat, a.source_path, a.ref, a.commit_hash, a.asked_for, coalesce(a.not_asked, ''), coalesce(a.outcome_unknown, ''), a.asked_at, diff --git a/internal/inventory/pin_completed_test.go b/internal/inventory/pin_completed_test.go index 9fac8387..2cc7d37c 100644 --- a/internal/inventory/pin_completed_test.go +++ b/internal/inventory/pin_completed_test.go @@ -34,7 +34,7 @@ func completeEarlierPins(t *testing.T, inv *Inventory) { } func TestAPinMadeBeforeIsCompletedWhenTheNodeAnswersOnce(t *testing.T) { - inv := fresh(t) + inv := inTheTables(t) ctx := context.Background() for _, n := range []string{"user", "provider"} { if _, err := inv.AddNode(ctx, n); err != nil { @@ -61,7 +61,7 @@ func TestAPinMadeBeforeIsCompletedWhenTheNodeAnswersOnce(t *testing.T) { } func TestAPinMadeBeforeIsLeftOpenWhenTheNodeAnswersTwice(t *testing.T) { - inv := fresh(t) + inv := inTheTables(t) ctx := context.Background() for _, n := range []string{"user", "provider"} { if _, err := inv.AddNode(ctx, n); err != nil { diff --git a/internal/inventory/plan_id_test.go b/internal/inventory/plan_id_test.go index 88cd0021..431643b2 100644 --- a/internal/inventory/plan_id_test.go +++ b/internal/inventory/plan_id_test.go @@ -3,14 +3,17 @@ package inventory import ( + "errors" "strings" "testing" "time" + + "github.com/novox/mesh-controller/internal/chain" ) -// **A plan's id is a value, never SQL** (novox/hq issue 561): the id reaches PlanByID from a caller of the +// **A plan's id is a value, never a query** (novox/hq issue 561): the id reaches PlanByID from a caller of the // controller's plans verb, so an id holding a quote is read as a plain id — no error from the store, and no -// other plan answered for it. +// other plan answered for it. On the bus an id is a key, and one the bus takes no spelling of is refused to be kept. func TestAPlanIdHoldingAQuoteIsReadAsAPlainId(t *testing.T) { inv := ForTest(t) ctx := t.Context() @@ -32,14 +35,11 @@ func TestAPlanIdHoldingAQuoteIsReadAsAPlainId(t *testing.T) { t.Fatalf("PlanByID(%q) failed with %q, not as a plain id no plan has (%q)", id, err, want) } } - // A plan whose id holds a quote is found by it, and by nothing else. + // A plan whose id holds a quote cannot be a key of the bus: refused by name, never rewritten into another id. quoted := p quoted.ID, quoted.Revision = "plan-'561'", 0 - if err := inv.SavePlan(ctx, "ed); err != nil { - t.Fatal(err) - } - if got, err := inv.PlanByID(ctx, quoted.ID); err != nil || got.ID != quoted.ID { - t.Fatalf("the plan with a quote in its id was not read back: %v %+v", err, got.ID) + if err := inv.SavePlan(ctx, "ed); !errors.Is(err, chain.ErrBadKey) { + t.Fatalf("a plan with a quote in its id was kept, or refused for another reason: %v", err) } if recent, err := inv.RecentPlans(ctx, 1); err != nil || len(recent) != 1 || !strings.HasPrefix(recent[0].ID, "plan-") { t.Fatalf("RecentPlans with a parameter: %v %+v", err, recent) diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index da3dad4a..8ad7a0d1 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -10,6 +10,8 @@ import ( "time" "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/chain" ) // A Plan is what a merge produces (novox/hq ADR 0162): the modules it changed and everything @@ -66,6 +68,15 @@ type Plan struct { // per machine and module, what was asked and how it went, and the machines with no backup holder. Nil for a // walk that moves no module with an item protected by a backup, and for a walk kept before it was. Backups *PlanBackups `json:"backups,omitempty"` + // Links are what the rollout names of the chain (novox/hq ADR 0408 §1): the builds it carried, the machine steps it has + // taken and the assignments it changed, each with the build before and after. Kept in the rollout's record beside + // the fields above; empty in the controller's store. + chain.Links + // Held are the merges this plan or batch holds (novox/hq ADR 0276, ADR 0408): what the store kept as batched merges + // with this plan's id, and the merges no plan answers that it held last. **Written by the merge operations only**: a + // save of the plan keeps what the record holds, never what this copy read, so a copy read before a merge moved cannot + // lay it back (issue 1092). + Held []chain.HeldMerge `json:"held,omitempty"` // WalkSaid is, on the save that started or ended the walk, which of the two it was (novox/hq ADR 0308 // decision 1): set by the save, never kept, read by whoever says the save (`plan-moved`). Empty on every // other save. @@ -670,6 +681,11 @@ type MergeMove struct { // somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once // the transaction is committed, in order. func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) (err error) { + if c, on, cerr := i.ChainReads(ctx, KindRollouts); cerr != nil { + return cerr + } else if on { + return i.chainSavePlans(ctx, c, plans, moves) + } epoch, err := i.actingEpoch(ctx) if err != nil { if len(plans) > 0 { @@ -818,16 +834,35 @@ func mergedAt(t time.Time) *time.Time { // OpenPlans is every plan still being worked, oldest first. func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return nil, err + } else if on { + return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) + } return i.plans(ctx, `where state in ('building', 'rolling') order by created`) } // PlansSince is every plan made after a moment, and every plan still being worked, oldest first. func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, error) { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return nil, err + } else if on { + return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated) + } return i.plans(ctx, `where created > $1 or state in ('building', 'rolling') order by created`, since) } // RecentPlans is the last few plans, newest first, open or not — what the overview shows. func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return nil, err + } else if on { + plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest) + if err == nil && limit >= 0 && len(plans) > limit { + plans = plans[:limit] + } + return plans, err + } return i.plans(ctx, `order by created desc limit $1`, limit) } @@ -835,6 +870,11 @@ func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) // planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it // recorded when its merge was made. func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return Plan{}, false, err + } else if on { + return i.chainNewestMergeOf(ctx, c, repository, branch) + } plans, err := i.plans(ctx, `where lower(repository) = lower($1) and branch = $2 and merged_at is not null and release is null order by merged_at desc, created desc limit 1`, repository, branch) if err != nil || len(plans) == 0 { @@ -846,6 +886,11 @@ func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string // PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a // caller of the plans verb. func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) { + if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + return Plan{}, err + } else if on { + return i.chainPlanByID(ctx, c, id) + } plans, err := i.plans(ctx, `where id = $1`, id) if err != nil { return Plan{}, err diff --git a/internal/inventory/reachable.go b/internal/inventory/reachable.go index 5f53f937..41a96f45 100644 --- a/internal/inventory/reachable.go +++ b/internal/inventory/reachable.go @@ -1,11 +1,16 @@ package inventory import ( + "context" "errors" "io" "net" "github.com/jackc/pgx/v5/pgconn" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/chain" ) // Unreachable says whether an error means the store could not be asked right now, rather than @@ -29,6 +34,13 @@ func Unreachable(err error) bool { if err == nil { return false } + // The chain on the bus (novox/hq ADR 0408): a bus that does not answer, a lease this controller no longer holds, a key + // somebody else wrote first — each is "not now", as the store being away is. + if errors.Is(err, chain.ErrNotHeld) || errors.Is(err, chain.ErrMoved) || errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, nats.ErrTimeout) || errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrConnectionClosed) || + errors.Is(err, nats.ErrDisconnected) || errors.Is(err, nats.ErrNoServers) || errors.Is(err, jetstream.ErrNoStreamResponse) { + return true + } var pg *pgconn.PgError if errors.As(err, &pg) { switch pg.Code { diff --git a/internal/inventory/seats.go b/internal/inventory/seats.go index 2c7ceff0..33dc4706 100644 --- a/internal/inventory/seats.go +++ b/internal/inventory/seats.go @@ -238,6 +238,20 @@ func (i *Inventory) RenameSeat(ctx context.Context, from, to string) error { return nil } +// mustBeAssigned refuses to record a holder that is not an assignment: a seat cannot be handed to something that is not +// running anywhere. The store's foreign key said it while the assignments were rows of the store; they are keys of the bus +// now, so the controller says it. +func (i *Inventory) mustBeAssigned(ctx context.Context, seat, nodeName, module string) error { + assigned, err := i.isAssigned(ctx, nodeName, module) + if err != nil { + return err + } + if !assigned { + return fmt.Errorf("recording %s on %s as the holder of %s: %s is not assigned to %s", module, nodeName, seat, module, nodeName) + } + return nil +} + // HoldSeat records that one assignment holds a seat, replacing whoever held it — every holder, for a // replicated seat (novox/hq ADR 0223) — as one transaction, so the seat is never without a holder in // between (novox/hq ADR 0131, design 28 task 5.3). The assignment must exist; the store refuses @@ -248,6 +262,9 @@ func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module if err != nil { return err } + if err := i.mustBeAssigned(ctx, seat, nodeName, module); err != nil { + return err + } tx, err := i.store.Pool().Begin(ctx) if err != nil { return err @@ -276,6 +293,9 @@ func (i *Inventory) AddSeatHolder(ctx context.Context, seat, scope, nodeName, mo if err != nil { return err } + if err := i.mustBeAssigned(ctx, seat, nodeName, module); err != nil { + return err + } if _, err := i.store.Pool().Exec(ctx, `insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4) on conflict (seat, node, module) do nothing`, diff --git a/internal/inventory/sends.go b/internal/inventory/sends.go index 7342c3d1..f9fdcdf4 100644 --- a/internal/inventory/sends.go +++ b/internal/inventory/sends.go @@ -174,6 +174,8 @@ func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, bui if err := tx.Commit(ctx); err != nil { return err } + // And what it put on the machine, in the assignments of the modules it carried a build of (issue 1095). + saidRunsNot(node, i.recordRuns(ctx, node, builds, time.Now())) if recordErr != nil { return &SendNotRecordedError{Err: recordErr} } diff --git a/internal/inventory/sends_test.go b/internal/inventory/sends_test.go index b08ab198..aba20d88 100644 --- a/internal/inventory/sends_test.go +++ b/internal/inventory/sends_test.go @@ -22,9 +22,8 @@ func generationNow(t *testing.T, inv *Inventory) int64 { return g } -// **Every change to what is assigned where raises the generation, in its own transaction** — an -// assignment, an unassignment, and one taken by the machine's removal, which no writer in Go makes — and -// a repeated assignment, which changes nothing, does not. +// **Every change to what is assigned where raises the generation** — an assignment and an unassignment — and a repeated +// assignment, which changes nothing, does not. func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) { inv, node := aNodeWithModules(t, "postgres", "web") ctx := t.Context() @@ -52,16 +51,8 @@ func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) { if unassigned <= assigned { t.Fatalf("an unassignment left the generation at %d", unassigned) } - if _, err := inv.Assign(ctx, node, "web"); err != nil { - t.Fatal(err) - } - before := generationNow(t, inv) - if _, err := inv.RemoveNodeForTest(ctx, node); err != nil { - t.Fatal(err) - } - if removed := generationNow(t, inv); removed <= before { - t.Errorf("a machine's removal took its assignments and left the generation at %d", removed) - } + // A machine's removal, which no verb makes, is read from the machines the store knows: an assignment on a machine it + // does not know is none (assignments in the bucket have no key of the store to cascade through). } // **The generation never goes down, and is raised past what a machine applied when the mesh's own counter @@ -175,22 +166,6 @@ func TestASendRecordThatCannotBeWrittenLeavesTheMachinesRecord(t *testing.T) { } } -// **An update that leaves an assignment as it was raises nothing** (the trigger's WHEN). -func TestAnUpdateThatChangesNoAssignmentRaisesNothing(t *testing.T) { - inv, node := aNodeWithModules(t, "web") - ctx := t.Context() - if _, err := inv.Assign(ctx, node, "web"); err != nil { - t.Fatal(err) - } - before := generationNow(t, inv) - if _, err := inv.store.Pool().Exec(ctx, `update assignment set module = module`); err != nil { - t.Fatal(err) - } - if after := generationNow(t, inv); after != before { - t.Errorf("an update that changed nothing moved the generation from %d to %d", before, after) - } -} - // **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer. func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) { inv, node := aNodeWithModules(t) diff --git a/internal/inventory/source.go b/internal/inventory/source.go index 15504244..63bb9dbc 100644 --- a/internal/inventory/source.go +++ b/internal/inventory/source.go @@ -32,6 +32,11 @@ type sourceRow struct { // sourceRows is every successful build of each module (of one module, when named), newest request // first. func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][]sourceRow, []string, error) { + if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + return nil, nil, err + } else if on { + return i.chainSourceRows(ctx, c, module) + } query := `select b.module, b.id, b.commit_hash, b.source_fingerprint, b.made, b.manifest, coalesce(g.verdict = 'failed', false) from build b left join build_gate g on g.build = b.id diff --git a/internal/link/source_moved_held_test.go b/internal/link/source_moved_held_test.go new file mode 100644 index 00000000..198e6682 --- /dev/null +++ b/internal/link/source_moved_held_test.go @@ -0,0 +1,60 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "io" + "log" + "testing" + "time" +) + +// heldControl is a merge announcement that records how it was settled. +type heldControl struct { + body []byte + took, held, drops int +} + +func (c *heldControl) Kind() string { return "source-moved" } +func (c *heldControl) Body() []byte { return c.body } +func (c *heldControl) Subject() string { return "" } +func (c *heldControl) Redelivered() bool { return false } +func (c *heldControl) HeldFor() time.Duration { return 0 } +func (c *heldControl) Answer(context.Context, []byte) error { return nil } +func (c *heldControl) Took() error { c.took++; return nil } +func (c *heldControl) Hold(time.Duration) error { c.held++; return nil } +func (c *heldControl) Drop() error { c.drops++; return nil } + +type sourceMover struct { + Upgrader + answer error +} + +func (s sourceMover) SourceMoved(context.Context, SourceMoved) error { return s.answer } + +// **A merge is acknowledged only after the controller recorded it** (novox/hq ADR 0407 decision 3, issue 1092): an +// answer of "not now" — the plan it belongs to could not be saved, the bus away, the lease lost — leaves the announcement +// unacknowledged, held to be handed over again; only an answer that is not "not now" settles it. +func TestAMergeTheControllerCouldNotRecordIsHeldAndNotAcknowledged(t *testing.T) { + body, err := json.Marshal(SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1"}) + if err != nil { + t.Fatal(err) + } + s := &Server{log: log.New(io.Discard, "", 0)} + for name, tc := range map[string]struct { + answer error + took, held int + }{ + "recorded": {nil, 1, 0}, + "not now": {errors.Join(ErrTryAgain, errors.New("the plan is not saved")), 0, 1}, + "an answer, not a wait": {errors.New("the mesh could not act on it"), 1, 0}, + } { + s.upgrader = sourceMover{answer: tc.answer} + m := &heldControl{body: body} + s.sourceMoved(context.Background(), m) + if m.took != tc.took || m.held != tc.held || m.drops != 0 { + t.Errorf("%s: acknowledged %d times and held %d times, want %d and %d", name, m.took, m.held, tc.took, tc.held) + } + } +} diff --git a/internal/recordmirror/mirror.go b/internal/recordmirror/mirror.go index d23bee95..a751f73c 100644 --- a/internal/recordmirror/mirror.go +++ b/internal/recordmirror/mirror.go @@ -325,6 +325,10 @@ func validKey(key string) error { return nil } +// ChangedFields names the top-level fields whose values differ between two encoded objects, sorted; nil when either is +// not an object: what an event of a changed key says of it. +func ChangedFields(before, after []byte) []string { return changedFields(before, after) } + // changedFields names the top-level fields whose values differ between two encoded objects, sorted; nil // when either is not an object. func changedFields(before, after []byte) []string { diff --git a/internal/recordmirror/readers_test.go b/internal/recordmirror/readers_test.go index 591b57d9..864a22ea 100644 --- a/internal/recordmirror/readers_test.go +++ b/internal/recordmirror/readers_test.go @@ -20,6 +20,9 @@ import ( // (internal/catalogue/state.go), and each mirror's own writer (this package and the loop that runs it). The // audit of a mirror against its store (ADR 0289 §4, issue 419) joins this list when it is built. var mayNameAMirror = []string{ + // The chain opens its buckets, the assignments' among them, which stopped being a mirror when the chain took them + // (ADR 0408 §3, issue 1089). + "internal/chain/bus.go", "cmd/mesh-controller/record_mirrors.go", "internal/broker/controller_buckets.go", // The grant of a machine's own keys of the assignments mirror to its provisioner client (design 57 P3). diff --git a/modules/route-proxy/go.mod b/modules/route-proxy/go.mod index 54bca71c..a9826b99 100644 --- a/modules/route-proxy/go.mod +++ b/modules/route-proxy/go.mod @@ -2,4 +2,4 @@ module route-proxy go 1.25.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 diff --git a/modules/route-proxy/go.sum b/modules/route-proxy/go.sum index 63b4da41..1a769534 100644 --- a/modules/route-proxy/go.sum +++ b/modules/route-proxy/go.sum @@ -1,2 +1,4 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 h1:Sv1Qzix+U1RpaUkw5IbB0IJEyZ+FqgzHmbZrCZkcWf0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/route-proxy/vendor/git.novox.be/novox/mesh-sdk/go/provider/provider.go b/modules/route-proxy/vendor/git.novox.be/novox/mesh-sdk/go/provider/provider.go index 33bf61f7..4adee9e9 100644 --- a/modules/route-proxy/vendor/git.novox.be/novox/mesh-sdk/go/provider/provider.go +++ b/modules/route-proxy/vendor/git.novox.be/novox/mesh-sdk/go/provider/provider.go @@ -335,6 +335,9 @@ func (p *Provider) pass(ctx context.Context, assignments, entries map[string][]b var out []Outcome unknown := map[string]bool{} // consumers whose assignment was skipped: their entries are not judged for _, key := range sortedKeys(assignments) { + if !Standing(assignments[key]) { + continue // ended or pending: read as a removed key (issue 1098) + } var a planned if err := json.Unmarshal(assignments[key], &a); err != nil || a.Node+"."+a.Module != key { o := Outcome{Key: key, Status: Failed, Why: fmt.Sprintf("the assignment %s is not the controller's, so "+ @@ -744,3 +747,23 @@ func (b stateBus) Delete(_ context.Context, bucket, key string) error { } return s.Delete(key) } + +// Standing is whether an assignment's value is a standing assignment: false when `ended` is set (the module was +// taken off its node, novox/hq ADR 0408 clause 4) or `state` is not empty (a pending one). Such a value is read +// exactly as a removed key. Every other field, known or not, changes nothing (issue 1098). +func Standing(value []byte) bool { + var v struct { + Ended json.RawMessage `json:"ended"` + State json.RawMessage `json:"state"` + } + if json.Unmarshal(value, &v) != nil { + return true // not decided here: the parse that follows refuses it + } + return !fieldSet(v.Ended) && !fieldSet(v.State) +} + +// fieldSet is whether a field carries something: not absent, null or empty. +func fieldSet(f json.RawMessage) bool { + s := strings.TrimSpace(string(f)) + return s != "" && s != "null" && s != `""` +} diff --git a/modules/route-proxy/vendor/modules.txt b/modules/route-proxy/vendor/modules.txt index abae1a04..a4b722ac 100644 --- a/modules/route-proxy/vendor/modules.txt +++ b/modules/route-proxy/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 +# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 ## explicit; go 1.22 git.novox.be/novox/mesh-sdk/go git.novox.be/novox/mesh-sdk/go/provider diff --git a/testdata/store-tests b/testdata/store-tests index 589272ff..91c80206 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -139,3 +139,19 @@ ./cmd/mesh-controller TestADeliveryGroupMovingAModuleMachinesRunPassesTheGate ./cmd/mesh-controller TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt ./cmd/mesh-controller TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheBus +./internal/inventory TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing +./internal/inventory TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing +./internal/inventory TestTheMoveRefusesAValueOverTheLargestMessage +./internal/inventory TestTheMoveRefusesOnACountThatDiffers +./internal/inventory TestTheMoveWithoutTheLeaseWritesNothing +./internal/inventory TestBuildsOnTheBusAreReadBackByEveryReader +./internal/inventory TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace +./internal/inventory TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain +./internal/inventory TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame +./cmd/mesh-controller TestAMergeWhosePlanCannotBeSavedIsHeldAndRecordedWhenHandedOverAgain +./internal/inventory TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt +./internal/inventory TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves +./internal/inventory TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent +./cmd/mesh-controller TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments +./cmd/mesh-controller TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment +./internal/inventory TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved