diff --git a/cmd/mesh-controller/awaiting_push_integration_test.go b/cmd/mesh-controller/awaiting_push_integration_test.go index ee881d98..d3714080 100644 --- a/cmd/mesh-controller/awaiting_push_integration_test.go +++ b/cmd/mesh-controller/awaiting_push_integration_test.go @@ -10,6 +10,7 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/overlay" ) @@ -146,8 +147,18 @@ func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing. func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) { open := aMesh(t) ctx := t.Context() - register(t, open, aKeeper()) - pushedBy(t, open, "laptop") + // What a machine was sent is read from the commits its assignments name (issue 1109): a module is carried by a send only + // when it was built from a commit, and a hand send on a machine with nothing assigned leaves no key to mark unknown. + if err := open.inventory.RegisterModule(ctx, aKeeper(), inventory.Source{BuiltFrom: "c-keeper"}); err != nil { + t.Fatal(err) + } + record, err := open.inventory.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSent(ctx, record.ID, "carried", map[string]string{overlay.Name: "c-overlay"}); err != nil { + t.Fatal(err) + } if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { t.Fatal(err) } diff --git a/cmd/mesh-controller/awaiting_push_test.go b/cmd/mesh-controller/awaiting_push_test.go index 0ec32245..58efd557 100644 --- a/cmd/mesh-controller/awaiting_push_test.go +++ b/cmd/mesh-controller/awaiting_push_test.go @@ -46,7 +46,21 @@ func pushedBy(t *testing.T, open *stores, node string) string { t.Fatal(err) } digest := digestOf(body) - if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil { + // A push records the build of each module it carries (plan.go, Allocating); a module is carried by a send only when it was + // built from a commit, for what a machine was sent is read from the commits its assignments name (issue 1109). + builds := declared.Builds + if builds == nil { + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + t.Fatal(err) + } + names := make([]string, 0, len(plan.Modules)) + for _, m := range plan.Modules { + names = append(names, m.Module) + } + builds = carriedBuilds(names, nil, current, nil) + } + if err := open.inventory.RecordSent(ctx, record.ID, digest, builds); err != nil { t.Fatal(err) } return digest diff --git a/cmd/mesh-controller/chain.go b/cmd/mesh-controller/chain.go index 18436e8b..9af1cc87 100644 --- a/cmd/mesh-controller/chain.go +++ b/cmd/mesh-controller/chain.go @@ -13,9 +13,7 @@ import ( ) // The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the -// bus's key-value store, written by the controller that holds the lease. The tables of the controller's store are -// copied into them once, by the serving controller at its start, and a kind is read from its bucket only once the move -// has recorded it (inventory.ChainReads). +// bus's key-value store, written by the controller that holds the lease, and the controller's store keeps none of them. // chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes. func chainHolds(ctx context.Context) error { @@ -33,97 +31,7 @@ func openTheChain(ctx context.Context) (*chain.Chain, error) { return chain.OnTheBus(ctx, js.Conn(), chainHolds) } -// kindChainMoveRefused is the condition of a move that stopped without switching a kind. -const kindChainMoveRefused = "chain-move-refused" - -// chainMoveRetry is how long a move that failed for want of the bus waits before it tries again. -const chainMoveRetry = 30 * time.Second - -// movingTheChain copies the tables into the buckets and records it, at the serving controller's start: after the buckets -// exist and the lease is held, before anything reads a kind that has not moved. **The first try is made here, before the -// controller serves**, so what it serves is read from one place. A move that fails for want of the bus is tried again -// every half minute while the controller serves from the tables; one that is refused — a count that differs, a value -// over the bus's largest — is raised as an urgent condition, and the tables stay what is read: nothing is half-switched. -func movingTheChain(ctx context.Context, inv *inventory.Inventory) { - say := func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) } - // The first try is made here, before the controller serves; the tries after it, while it serves. - if movedOrStopped(ctx, inv, say) { - return - } - go func() { - for { - select { - case <-ctx.Done(): - return - case <-time.After(chainMoveRetry): - } - if movedOrStopped(ctx, inv, say) { - return - } - } - }() -} - -// movedOrStopped makes one try at the move and says whether there is nothing more to try: moved, refused, or the controller -// ending. False for a failure of the kind that is tried again (the bus away, a write not granted yet, the lease not held). -func movedOrStopped(ctx context.Context, inv *inventory.Inventory, say func(string, ...any)) bool { - _, err := inv.MoveChain(ctx, say) - var refusal *inventory.Refusal - switch { - case err == nil: - go clearChainMove(ctx) - go askAgainSoon(ctx, inv) - return true - case errors.As(err, &refusal): - say("%v", err) - go raiseChainMove(ctx, refusal) - return true - case ctx.Err() != nil: - return true - } - say("the move to the bus did not finish, and the tables are read until it does: %v", err) - return false -} - -// chainMoveObservation is the urgent condition for a refused move. -func chainMoveObservation(r *inventory.Refusal) conditions.Observation { - return conditions.Observation{Scope: conditions.ScopeBus, ID: "chain." + r.Kind, Token: kindChainMoveRefused, - Kind: kindChainMoveRefused, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, - Summary: "the controller's " + r.Kind + " were not moved to the bus: " + r.Why, Source: "chain move"} -} - -// whenKept runs f once the serving controller keeps conditions: the move runs before they are kept. -func whenKept(ctx context.Context, f func(k *conditions.Keeper)) { - for conditionsFrom == nil { - select { - case <-ctx.Done(): - return - case <-time.After(time.Second): - } - } - f(conditionsFrom) -} - -func raiseChainMove(ctx context.Context, r *inventory.Refusal) { - whenKept(ctx, func(k *conditions.Keeper) { - if _, err := k.Observe(ctx, chainMoveObservation(r)); err != nil { - fmt.Printf("chain: the condition for the refused move could not be raised: %v\n", err) - } - }) -} - -func clearChainMove(ctx context.Context) { - whenKept(ctx, func(k *conditions.Keeper) { - for _, kind := range []string{inventory.KindBuilds, inventory.KindRollouts, inventory.KindAssignments} { - key := chainMoveObservation(&inventory.Refusal{Kind: kind}).Key() - if _, err := k.Clear(ctx, key, "the "+kind+" are on the bus"); err != nil { - fmt.Printf("chain: the condition for the move of the %s could not be cleared: %v\n", kind, err) - } - } - }) -} - -// attachTheChain gives the inventory of the serving controller the chain on its connection, then moves what has not moved. +// attachTheChain gives the inventory of the serving controller the chain on its connection. func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error { bus, ok := server.Bus().(link.OverNATS) if !ok { @@ -134,7 +42,13 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link. return err } inv.UseChain(c) - movingTheChain(ctx, inv) + inv.RequireLayout() + // And the builds laid out under their modules, once, before they are read (issue 1121). + go layBuildsOutSoon(ctx, inv) + // And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving. + go askAgainSoon(ctx, inv) + // And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds). + go copySentBuildsSoon(ctx, inv) return nil } @@ -179,3 +93,79 @@ func askAgainSoon(ctx context.Context, inv *inventory.Inventory) { askAgain(ctx, inv) } } + +// kindSentBuildsNotCopied is the condition of the one-time copy of node.sent_builds onto the assignments' keys that has not +// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run. +const kindSentBuildsNotCopied = "sent-builds-not-copied" + +// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon +// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails, +// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key +// that names a commit and empties each column it has copied. +func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) { + failed := false + for { + _, err := inv.CopySentBuilds(ctx) + o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied, + Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, + Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "} + if err != nil { + fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err) + if keeper := conditionsFrom; keeper != nil && !failed { + o.Summary += err.Error() + if _, oerr := keeper.Observe(ctx, o); oerr == nil { + failed = true + } + } + } else { + if keeper := conditionsFrom; keeper != nil && failed { + _, _ = keeper.Clear(ctx, o.Key(), "copied") + } + return + } + select { + case <-ctx.Done(): + return + case <-time.After(chainMoveRetry): + } + } +} + +// chainMoveRetry is how long the copy waits before it tries again. +const chainMoveRetry = 30 * time.Second + +// kindBuildsNotLaidOut is the condition of the layout of the builds under their modules that has not succeeded: until it +// has, no build is read, and a send that needs one is refused with the reason. +const kindBuildsNotLaidOut = "builds-not-laid-out" + +// layBuildsOutSoon lays the builds out under their modules as soon as this controller holds the lease (the chain refuses a +// write before), and tries again every half minute while it fails, loudly: each failure is printed, and the first raises an +// urgent condition until it has worked. It is idempotent (LayBuildsOut). +func layBuildsOutSoon(ctx context.Context, inv *inventory.Inventory) { + failed := false + for { + err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }) + o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindBuildsNotLaidOut, Token: kindBuildsNotLaidOut, + Kind: kindBuildsNotLaidOut, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, + Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "} + if err != nil { + fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err) + if keeper := conditionsFrom; keeper != nil && !failed { + o.Summary += err.Error() + if _, oerr := keeper.Observe(ctx, o); oerr == nil { + failed = true + } + } + } else { + if keeper := conditionsFrom; keeper != nil && failed { + _, _ = keeper.Clear(ctx, o.Key(), "laid out") + } + return + } + select { + case <-ctx.Done(): + return + case <-time.After(chainMoveRetry): + } + } +} diff --git a/cmd/mesh-controller/chain_integration_test.go b/cmd/mesh-controller/chain_integration_test.go index c6a98e1c..81866f2b 100644 --- a/cmd/mesh-controller/chain_integration_test.go +++ b/cmd/mesh-controller/chain_integration_test.go @@ -176,12 +176,12 @@ func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" { t.Fatalf("the assignment names %+v", on) } - // A declaration sent by hand with no builds known says nothing of them. + // A declaration sent by hand with no builds known makes what the machine runs unknown: a time on the key, no commit. if err := inv.RecordSent(ctx, node, "d3", nil); err != nil { t.Fatal(err) } - if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c1" { - t.Fatalf("a send of unknown builds changed %+v", on) + if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "" || runOf(on, "notes").SentUnknown.IsZero() { + t.Fatalf("a send of unknown builds left %+v", on) } // A module sent a commit no build is kept of is named by its commit alone. if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil { @@ -207,3 +207,39 @@ func runOf(on []chain.Assignment, module string) chain.Assignment { a, _ := assignmentOf(on, module) return a } + +// **An assignment waiting for its build is one key of the assignments bucket** (ADR 0261, ADR 0408, issue 1094): entered +// through the `assign` verb for a module whose build is in flight, one read of `.` says state pending with the +// build it waits for, the build's outcome through the build seat's path makes the key stand with the wait carried as applied, +// and the table of pending assignments is read by nobody. No key is written by hand. +func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatalf("an assignment while its build runs was refused: %v", err) + } + waiting, found, err := inv.AssignmentValue(ctx, "laptop", "sensors") + if err != nil || !found || waiting.State != chain.StatePending || waiting.Stands() || !waiting.Assigned.IsZero() || + waiting.Pending == nil || waiting.Pending.State != chain.PendingWaiting || waiting.Pending.Build != "b-modules/sensors" || + !strings.HasPrefix(waiting.Pending.ID, "p-") { + t.Fatalf("one read of the key says %+v (found %v, %v)", waiting, found, err) + } + if got := assignedTo(t, open, "laptop"); slices.Contains(got, "sensors") { + t.Fatalf("a module only waiting is assigned: %v", got) + } + + if err := (builds{inv: inv, open: open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil { + t.Fatal(err) + } + made, found, err := inv.AssignmentValue(ctx, "laptop", "sensors") + if err != nil || !found || !made.Stands() || made.State != "" || made.Assigned.IsZero() || made.Pending == nil || + made.Pending.State != chain.PendingApplied || made.Pending.ID != waiting.Pending.ID { + t.Fatalf("one read of the key after the build says %+v (found %v, %v)", made, found, err) + } + if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("the build registered sensors and it was not assigned") + } +} diff --git a/cmd/mesh-controller/chain_no_table_integration_test.go b/cmd/mesh-controller/chain_no_table_integration_test.go new file mode 100644 index 00000000..20181348 --- /dev/null +++ b/cmd/mesh-controller/chain_no_table_integration_test.go @@ -0,0 +1,85 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/link" +) + +// **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR +// 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's +// build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the +// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays. +func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asked := asksRecorded(t) + ctx := t.Context() + inv := open.inventory + if _, err := assign(ctx, open, "anchor", "notes"); err != nil { + t.Fatal(err) + } + hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second)) + cutAt(t, open, t0.Add(91*time.Second)) + ws, _ := walks(t, open) + if len(ws) != 1 { + t.Fatalf("%d walk(s) after the cut", len(ws)) + } + rollout := ws[0].ID + asking, err := inv.PlanByID(ctx, rollout) + if err != nil { + t.Fatal(err) + } + for try := 0; try < 3 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == ""); try++ { + advancePlans(ctx, open) + if asking, err = inv.PlanByID(ctx, rollout); err != nil { + t.Fatal(err) + } + } + buildID := asking.Modules["notes"].Build + if buildID == "" || len(*asked) != 1 { + t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID) + } + digest := "sha256:" + strings.Repeat("b", 64) + result := outcome(buildID, "modules/notes", "c-notes", "notes", "") + result.Made = []link.MadeArtifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes/notes@" + digest}} + if err := (builds{inv: inv, open: open}).Built(ctx, result); err != nil { + t.Fatal(err) + } + + // What the live path made is in the chain. + if built, found, err := inv.BuildByID(ctx, buildID); err != nil || !found || built.Commit != "c-notes" { + t.Fatalf("the build is not read from the chain: %+v %v %v", built, found, err) + } + if p, err := inv.PlanByID(ctx, rollout); err != nil || p.ID != rollout { + t.Fatalf("the rollout is not read from the chain: %+v %v", p, err) + } + if on, err := inv.AssignmentsOn(ctx, "anchor"); err != nil { + t.Fatal(err) + } else if _, ok := assignmentOf(on, "notes"); !ok { + t.Fatalf("the assignment is not read from the chain: %+v", on) + } + + // And the catalog of the store holds none of them. + have := map[string]bool{} + for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) { + have[strings.Trim(row[0], `"`)] = true + } + for _, table := range []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment", "chain_moved"} { + if have[table] { + t.Errorf("the table %s is in the controller's store", table) + } + } + for _, table := range []string{"build_gate", "assignment_generation"} { + if !have[table] { + t.Errorf("the table %s, which stays, is gone", table) + } + } + if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" { + t.Errorf("the function raise_assignment_generation is in the store: %v", rows) + } +} diff --git a/cmd/mesh-controller/data_integration_test.go b/cmd/mesh-controller/data_integration_test.go index c1142df8..4d8a9853 100644 --- a/cmd/mesh-controller/data_integration_test.go +++ b/cmd/mesh-controller/data_integration_test.go @@ -56,13 +56,19 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent( if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { t.Fatal(err) } - register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) + // Built from a commit, so that a send carries them: a holder is asked only of what the machine was sent (issue 1109). + built := inventory.Source{BuiltFrom: "c1"} + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "keeper", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}, built); err != nil { + t.Fatal(err) + } house, err := catalogue.ParseManifest([]byte(houseManifest)) if err != nil { t.Fatal(err) } - register(t, open, house) + if err := inv.RegisterModule(ctx, house, built); err != nil { + t.Fatal(err) + } if _, err := assign(ctx, open, "laptop", "house"); err == nil { t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up") } diff --git a/cmd/mesh-controller/merge_gate.go b/cmd/mesh-controller/merge_gate.go index ec61888b..854f39ef 100644 --- a/cmd/mesh-controller/merge_gate.go +++ b/cmd/mesh-controller/merge_gate.go @@ -1120,6 +1120,9 @@ func throwawayStores(ctx context.Context, admin string) (*stores, func(), error) drop() return nil, nil, err } + // The chain lives on the bus, and the gate has none: this mesh is raised for a moment, with a chain of its own in memory + // that no other reads (issue 1121). + open.inventory.UseThrowawayChain() for _, m := range provided { if err := open.inventory.Provide(ctx, m); err != nil { open.Close() diff --git a/cmd/mesh-controller/merge_on_the_bus_test.go b/cmd/mesh-controller/merge_on_the_bus_test.go index df3825e2..3cfc5780 100644 --- a/cmd/mesh-controller/merge_on_the_bus_test.go +++ b/cmd/mesh-controller/merge_on_the_bus_test.go @@ -50,9 +50,17 @@ func theMergePathOn(t *testing.T, open *stores) func(link.SourceMoved) { t.Cleanup(func() { stop() <-done + lookingAtBatches.Wait() setWindowClock(nil) windowLook.Store(nil) }) + // The forge's merges are taken from a consumer made when the server starts serving: a merge published before it exists + // is not delivered to it. With the chain read from the bus the set-up above is quick enough to lose that race, so the + // publisher waits for the consumer. + becomes(t, 10*time.Second, "the merges' consumer being made", func() bool { + _, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName) + return err == nil + }) return func(m link.SourceMoved) { t.Helper() body, err := json.Marshal(m) diff --git a/cmd/mesh-controller/merge_window_bus.go b/cmd/mesh-controller/merge_window_bus.go index 5b4cb0e3..bd721841 100644 --- a/cmd/mesh-controller/merge_window_bus.go +++ b/cmd/mesh-controller/merge_window_bus.go @@ -145,9 +145,15 @@ func (w *busWindow) Forget(ctx context.Context, batch string) { } } +// lookingAtBatches counts the looks under way, so a test that ends the window's clock can wait for the last of them before the +// next test replaces what they read. +var lookingAtBatches sync.WaitGroup + // lookAtBatches is the look an expired window asks for: the plans held, as a merge heard holds them, and the batches // whose window closed cut (cutBatchesHeld). Waiting for the plans rather than giving up: an expiry is heard once. func lookAtBatches(ctx context.Context, open *stores) { + lookingAtBatches.Add(1) + defer lookingAtBatches.Done() release, err := open.inventory.HoldPlans(ctx, true) if err != nil { fmt.Printf("batches: a window closed and the plans could not be held for the look: %v\n", err) diff --git a/cmd/mesh-controller/pending.go b/cmd/mesh-controller/pending.go index 7743f165..a5ec7359 100644 --- a/cmd/mesh-controller/pending.go +++ b/cmd/mesh-controller/pending.go @@ -491,8 +491,7 @@ func settlingPending(ctx context.Context, open *stores) { // settlePending is one tick: claims left by a controller that stopped are settled from what the mesh holds; // a module registered meanwhile is assigned; a build heard and not registered, or silent past its bound, -// ends its assignment; ended ones are raised as conditions and cleared once answered; ended rows older than -// KeptFor are deleted. Returns what it did. +// ends its assignment; ended ones are raised as conditions and cleared once answered. Returns what it did. func settlePending(ctx context.Context, open *stores, now time.Time) []string { inv := open.inventory var said []string @@ -565,11 +564,6 @@ func settlePending(ctx context.Context, open *stores, now time.Time) []string { } said = append(said, raisePendingEnded(ctx, inv)...) - if n, err := inv.ForgetEndedPending(ctx, now); err != nil { - say("ended pending assignments could not be deleted: %v", err) - } else if n > 0 { - say("deleted %d pending assignment(s) ended more than %s ago", n, inventory.KeptFor) - } return said } @@ -585,7 +579,7 @@ func containsString(xs []string, x string) bool { // pendingObservation is the condition for a pending assignment that was not made: one per pending // assignment, its row's id the last part of its id, so one row's condition is never another's. func pendingObservation(p inventory.PendingAssignment) conditions.Observation { - return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%d", p.Node, p.Module, p.ID), + return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%s", p.Node, p.Module, p.ID), Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Source: "pending assignments", Summary: p.Note} @@ -671,8 +665,8 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor if err != nil { return []string{fmt.Sprintf("the open conditions could not be read to clear orphaned ones: %v", err)} } - byID := map[int64]string{} - var ids []int64 + byID := map[string]string{} + var ids []string for _, c := range open { if c.Kind != kindPendingEnded { continue @@ -682,8 +676,8 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor if len(parts) < 2 { continue } - var id int64 - if _, err := fmt.Sscan(parts[len(parts)-2], &id); err != nil { + id := parts[len(parts)-2] + if !strings.HasPrefix(id, "p-") { continue } byID[id] = c.Key @@ -701,7 +695,7 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor if known[id] { continue } - if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed"); err != nil { + if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed, or the module was assigned and the wait taken back"); err != nil { said = append(said, fmt.Sprintf("the condition %s could not be cleared: %v", key, err)) } } diff --git a/cmd/mesh-controller/pending_integration_test.go b/cmd/mesh-controller/pending_integration_test.go index bbb10749..ba6a82fe 100644 --- a/cmd/mesh-controller/pending_integration_test.go +++ b/cmd/mesh-controller/pending_integration_test.go @@ -542,23 +542,26 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { t.Fatalf("a module registered meanwhile: %+v", p) } - // Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is - // still open (second review, bug B). + // Ended waits are not deleted, for the chain removes no key (ADR 0408 section 4): after KeptFor they are no longer listed, open + // ones never are left out, nor one whose condition is still open (second review, bug B). if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil { t.Fatal(err) } - said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - rows, err := inv.Pending(ctx, time.Unix(0, 0)) + later := time.Now().Add(inventory.KeptFor + time.Hour) + settlePending(ctx, open, later) + rows, err := inv.Pending(ctx, later.Add(-inventory.KeptFor)) if err != nil { t.Fatal(err) } for _, r := range rows { if !r.Open() && (r.Raised == nil || r.Cleared != nil) { - t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said) + t.Fatalf("an ended pending assignment with no open condition is listed past %s: %+v", inventory.KeptFor, r) } } - if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") { - t.Fatalf("the applied row was not deleted: %v", said) + for _, r := range rows { + if r.Module == "heard" { + t.Fatalf("an applied wait is listed past %s: %+v", inventory.KeptFor, r) + } } } @@ -637,8 +640,9 @@ func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) { } settlePending(ctx, open, time.Now()) settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] { - t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err) + // No key of the chain is removed (ADR 0408 §4): a cleared wait stays in the value of its key, saying it was cleared. + if known, err := inv.PendingKnown(ctx, []string{row.ID}); err != nil || !known[row.ID] || rowOf(t, open, row.ID).Cleared == nil { + t.Fatalf("a cleared row is not kept, saying so: %v", err) } // On anchor, then anchor removed: the row goes, and its condition with it at the next tick. diff --git a/cmd/mesh-controller/pending_test.go b/cmd/mesh-controller/pending_test.go index d949c02c..c5171bca 100644 --- a/cmd/mesh-controller/pending_test.go +++ b/cmd/mesh-controller/pending_test.go @@ -23,7 +23,7 @@ func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool { return found } -func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment { +func rowOf(t *testing.T, open *stores, id string) inventory.PendingAssignment { t.Helper() rows, err := open.inventory.Pending(t.Context(), time.Unix(0, 0)) if err != nil { @@ -34,7 +34,7 @@ func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment { return r } } - t.Fatalf("pending assignment %d is not on record", id) + t.Fatalf("pending assignment %s is not on record", id) return inventory.PendingAssignment{} } diff --git a/cmd/mesh-controller/plan_moved_every_writer_test.go b/cmd/mesh-controller/plan_moved_every_writer_test.go index 0973ae79..be7f712b 100644 --- a/cmd/mesh-controller/plan_moved_every_writer_test.go +++ b/cmd/mesh-controller/plan_moved_every_writer_test.go @@ -12,9 +12,9 @@ import ( ) // **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan -// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there -// after the commit; so no statement that writes the plan table may stand anywhere else, and writePlan, the one that -// does, may be called from nowhere but SavePlansMoving. Checked by reading the sources, no store needed. +// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there after the +// write; so no code but chainSavePlans (which SavePlansMoving calls, and no other function) writes a plan to the rollouts +// bucket by anything but a change of the merges it holds. Checked by reading the sources, no store needed. func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) { write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`) var found []string @@ -31,11 +31,13 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) { return err } text := string(raw) - if write.MatchString(text) && filepath.ToSlash(path) != "../../internal/inventory/plans.go" { + if write.MatchString(text) { found = append(found, path+" writes the plan table") } - if strings.Contains(text, "writePlan(") && filepath.ToSlash(path) != "../../internal/inventory/plans.go" { - found = append(found, path+" calls writePlan") + // A plan record is written by the bucket's raw create and update, which only the plans' own file calls. + if filepath.ToSlash(path) != "../../internal/inventory/chain_rollouts.go" && filepath.ToSlash(path) != "../../internal/chain/chain.go" && + strings.Contains(text, "chain.RolloutsOf") { + found = append(found, path+" reaches the rollouts bucket") } return nil }) @@ -51,17 +53,26 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) { t.Fatal(err) } text := string(raw) - if n := len(write.FindAllString(text, -1)); n != 1 { - t.Errorf("plans.go has %d statements that write the plan table; want the one in writePlan", n) - } - calls := regexp.MustCompile(`writePlan\(ctx, tx`).FindAllStringIndex(text, -1) save := strings.Index(text, "func (i *Inventory) SavePlansMoving") next := strings.Index(text[save:], "\nfunc ") - if len(calls) != 1 || calls[0][0] < save || calls[0][0] > save+next { - t.Errorf("writePlan is called %d times, and not only from SavePlansMoving", len(calls)) + if next < 0 { + next = len(text) - save } - if !strings.Contains(text[save:save+next], "PlanSaved(i, *p)") { - t.Error("SavePlansMoving does not say each plan it wrote") + if !strings.Contains(text[save:save+next], "chainSavePlans(") { + t.Error("SavePlansMoving does not write through chainSavePlans") + } + chainRaw, err := os.ReadFile("../../internal/inventory/chain_rollouts.go") + if err != nil { + t.Fatal(err) + } + chainText := string(chainRaw) + at := strings.Index(chainText, "func (i *Inventory) chainSavePlans") + end := strings.Index(chainText[at:], "\nfunc ") + if !strings.Contains(chainText[at:at+end], "PlanSaved(i, *p)") { + t.Error("chainSavePlans does not say each plan it wrote") + } + if n := strings.Count(chainText, "createRaw(ctx, c, p.ID"); n != 1 { + t.Errorf("a plan is made from %d places of chain_rollouts.go, want the one in chainSavePlans", n) } } diff --git a/cmd/mesh-controller/record_mirrors.go b/cmd/mesh-controller/record_mirrors.go index 970956e8..25fd9d15 100644 --- a/cmd/mesh-controller/record_mirrors.go +++ b/cmd/mesh-controller/record_mirrors.go @@ -44,7 +44,6 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats. return e } sources := map[string]func(context.Context) (map[string][]byte, error){ - recordmirror.Assignments.Name: inv.AssignmentsMirrorToWrite, recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror, recordmirror.Predefined.Name: inv.PredefinedMirror, } @@ -55,15 +54,11 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats. events := &assignmentEvents{} for { if holds() { - // The assignments, once they are keys of the bus, are written by the verbs that change them (novox/hq ADR - // 0408): this tick keeps the provisioner's plan in their values and says what changed; the mirror of the store - // writes them only until the move. - onTheBus := keepingAssignments(ctx, inv, announce, epoch, events, failing) + // The assignments are keys of the bus, written by the verbs that change them (novox/hq ADR 0408): this tick + // keeps the provisioner's plan in their values and says what changed. They are no mirror of the store. + keepingAssignments(ctx, inv, announce, epoch, events, failing) for _, m := range recordmirror.Mirrors() { - if m.Name == recordmirror.Assignments.Name && onTheBus { - continue - } - why := mirrorOnce(ctx, inv, api, m, writers, sources[m.Name], announce, epoch, holds) + why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds) if why != failing[m.Name] { if why != "" { fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why) @@ -83,7 +78,7 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats. } // mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could. -func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer, +func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer, source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64, holds func() bool) string { reading, cancel := context.WithTimeout(ctx, 30*time.Second) @@ -95,9 +90,6 @@ func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.Jet return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err) } w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds} - if m.Name == recordmirror.Assignments.Name { - w.Holds = assignmentsHolds(inv, holds) - } writers[m.Name] = w } want, err := source(reading) @@ -128,21 +120,10 @@ type assignmentEvents struct { // the values it belongs to, and one `assignment-changed` event for each key that is new, changed or ended since the last // look — whoever wrote it, a verb in this process or a command in another. False when they are not on the bus yet. func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce recordmirror.Announcer, epoch func() uint64, - events *assignmentEvents, failing map[string]string) bool { + events *assignmentEvents, failing map[string]string) { reading, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() const name = "assignments" - if _, on, err := inv.KeptAssignments(reading); err != nil || !on { - if err != nil { - // Not known to be off the bus: the store's mirror is not written over what the bus may hold. - if failing[name] == "" { - failing[name] = err.Error() - fmt.Printf("the assignments cannot be read from the bus: %v\n", err) - } - return true - } - return false - } if _, _, err := inv.KeepAssignmentPlans(reading); err != nil { if failing[name] != err.Error() { failing[name] = err.Error() @@ -152,9 +133,9 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce fmt.Printf("the provisioner's plans are kept in the assignments again\n") failing[name] = "" } - now, _, err := inv.KeptAssignments(reading) + now, err := inv.KeptAssignments(reading) if err != nil { - return true + return } if events.known != nil { keys := make([]string, 0, len(now)+len(events.known)) @@ -194,12 +175,4 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce } } events.known = now - return true -} - -// assignmentsHolds is the assignments mirror's leave to write: the lease's, and the assignments neither being switched to -// the bus nor on it. Asked by the writer before every write and purge, so a pass that read its source before the switch -// writes nothing after it (ADR 0408 §4). -func assignmentsHolds(inv *inventory.Inventory, holds func() bool) func() bool { - return func() bool { return holds() && inv.AssignmentsMirrorMayWrite() } } diff --git a/cmd/mesh-controller/record_mirrors_switch_test.go b/cmd/mesh-controller/record_mirrors_switch_test.go deleted file mode 100644 index 88ed29a1..00000000 --- a/cmd/mesh-controller/record_mirrors_switch_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "errors" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/recordmirror" -) - -// **A writer of the store's mirror of the assignments, made before the assignments are switched to the bus, writes and -// purges nothing after** (ADR 0408 §4, issue 1093): its leave to write is asked before every write, and false once they are -// switching or moved. -func TestAMirrorWriterMadeBeforeTheSwitchWritesAndPurgesNothingAfter(t *testing.T) { - inv := inventory.NewForTest() - w := &recordmirror.Writer{Mirror: recordmirror.Assignments, Holds: assignmentsHolds(inv, func() bool { return true })} - if !w.Holds() { - t.Fatal("the writer may not write before the switch") - } - inv.BeginSwitchForTest(inventory.KindAssignments) - if w.Holds() { - t.Fatal("the writer may write while the assignments are switching") - } - // Refused before it reads the bucket (it has none here) or lays or ends a key. - r, err := w.Sync(t.Context(), map[string][]byte{"anchor.notes": []byte(`{}`)}) - if !errors.Is(err, recordmirror.ErrNotHeld) || r.Written != 0 || r.Removed != 0 { - t.Fatalf("the writer wrote after the switch: %+v %v", r, err) - } -} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index f7939794..bd7604c7 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -1589,7 +1589,9 @@ func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalog var other []string listID := "" for _, m := range plan.Modules { - if _, was := sent[m.Module]; !was { + // A module no build is recorded for is carried with no commit and names none on its key (issue 1109), so it is "sent" + // only by the comparison of the summaries below; a module with a commit that was never sent is a new assignment. + if _, was := sent[m.Module]; !was && declared.Builds[m.Module] != "" { other = append(other, m.Module+" newly assigned") } if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) { diff --git a/cmd/mesh-controller/sent_required_by_integration_test.go b/cmd/mesh-controller/sent_required_by_integration_test.go index 5cef69b8..077a4192 100644 --- a/cmd/mesh-controller/sent_required_by_integration_test.go +++ b/cmd/mesh-controller/sent_required_by_integration_test.go @@ -85,10 +85,7 @@ func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) t.Fatal(err) } inv.SetProvisioningPlans(inv.SentProvisioningPlans) - mirror, err := inv.AssignmentsMirror(ctx) - if err != nil { - t.Fatal(err) - } + mirror := inv.PlanValuesForTest(t) var value inventory.AssignmentMirrored if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil { t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err) diff --git a/internal/chain/keys.go b/internal/chain/keys.go index 0c956d72..afedd589 100644 --- a/internal/chain/keys.go +++ b/internal/chain/keys.go @@ -53,7 +53,14 @@ func ModuleFilter(module string) string { return module + ".build.>" } const ManifestPrefix = "manifest" // ReservedModules are the names that are not a module's: a key under one is not a build's record. -var ReservedModules = []string{ManifestPrefix, RequestPrefix, IDPrefix} +var ReservedModules = []string{ManifestPrefix, RequestPrefix, IDPrefix, LayoutPrefix} + +// LayoutPrefix is the name of the key that says the builds are laid out under their modules: LayoutMarkerKey, written once by +// the controller's start after it has laid the old keys out. Nothing reads the builds before it exists. +const LayoutPrefix = "layout" + +// LayoutMarkerKey is the key whose existence says the layout is made. +const LayoutMarkerKey = LayoutPrefix + ".done" // ManifestKey is where a manifest is kept: `manifest.sha256-`. func ManifestKey(raw []byte) string { diff --git a/internal/chain/pending_test.go b/internal/chain/pending_test.go new file mode 100644 index 00000000..655aae40 --- /dev/null +++ b/internal/chain/pending_test.go @@ -0,0 +1,87 @@ +package chain + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +// **A pending value is not a standing assignment** (ADR 0261, ADR 0408, issue 1094): it names its node and module, holds no +// time assigned, says `state: pending`, and every reader that skips a value with a state skips it. +func TestAPendingValueIsNotAStandingAssignment(t *testing.T) { + since := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) + a := Assignment{Node: "anchor", Module: "notes", State: StatePending, + Pending: &Pending{ID: "p-1", State: PendingWaiting, Build: "build-1", Since: since}} + if a.Stands() { + t.Fatal("a pending value stands") + } + raw, _ := json.Marshal(a) + for _, want := range []string{`"node":"anchor"`, `"module":"notes"`, `"state":"pending"`, `"pending":{"id":"p-1","state":"waiting"`} { + if !strings.Contains(string(raw), want) { + t.Errorf("the value lacks %s: %s", want, raw) + } + } + if strings.Contains(string(raw), `"assigned"`) { + t.Errorf("a pending value names a time assigned: %s", raw) + } + // The wait carried into the value that stands: no state, so it stands, and the record stays. + made := Assignment{Node: "anchor", Module: "notes", Assigned: since, Pending: &Pending{ID: "p-1", State: PendingApplied}} + if !made.Stands() || !a.Pending.Open() || made.Pending.Open() { + t.Fatalf("stands %v, open before %v, open after %v", made.Stands(), a.Pending.Open(), made.Pending.Open()) + } + var back Assignment + if err := json.Unmarshal(raw, &back); err != nil || back.Pending == nil || back.Pending.Build != "build-1" || !back.Pending.Since.Equal(since) { + t.Fatalf("read back %+v: %v", back, err) + } + now := since + raised := Pending{ID: "p-2", State: PendingExpired, Raised: &now, Acknowledged: &now, Cleared: &now} + body, _ := json.Marshal(raised) + if !strings.Contains(string(body), `"raised"`) || !strings.Contains(string(body), `"acknowledged"`) || !strings.Contains(string(body), `"cleared"`) { + t.Errorf("the marks of a wait: %s", body) + } +} + +func TestTheTransitionsOfAPendingAssignment(t *testing.T) { + ok := [][2]string{{PendingWaiting, PendingExpired}, {PendingWaiting, PendingWithdrawn}, {PendingApplying, PendingApplied}, + {PendingApplying, PendingRefused}} + for _, p := range ok { + if !CanSettle(p[0], p[1]) { + t.Errorf("%s may end as %s", p[0], p[1]) + } + } + for _, p := range [][2]string{{PendingWaiting, PendingApplied}, {PendingWaiting, PendingRefused}, {PendingApplying, PendingExpired}, + {PendingApplying, PendingWithdrawn}, {PendingApplied, PendingExpired}, {PendingExpired, PendingWithdrawn}, + {PendingWaiting, PendingWaiting}} { + if CanSettle(p[0], p[1]) { + t.Errorf("%s may not end as %s", p[0], p[1]) + } + } + if !CanMove(PendingWaiting, PendingApplying) || !CanMove(PendingApplying, PendingWaiting) || CanMove(PendingApplied, PendingWaiting) { + t.Error("a claim moves a wait to applying and back") + } +} + +// **A module only pulled in is a standing value marked pulled-in, never a state, and a hand send makes a key say its commit +// unknown with a time** (issue 1109): every reader treats a non-empty state as no assignment, so a pulled-in key must have +// none; and unknown is never written in `commit`. +func TestAPulledInValueStandsAndAHandSendIsATimeNotACommit(t *testing.T) { + at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) + pulled := Assignment{Node: "anchor", Module: "mesh-wireguard", PulledIn: true, Commit: "c1", Build: "b1", Since: at} + if !pulled.Stands() || pulled.State != "" { + t.Fatalf("a pulled-in value does not stand: %+v", pulled) + } + raw, _ := json.Marshal(pulled) + if !strings.Contains(string(raw), `"pulled-in":true`) || strings.Contains(string(raw), `"state"`) || strings.Contains(string(raw), `"assigned"`) { + t.Fatalf("the pulled-in value is %s", raw) + } + unknown := Assignment{Node: "anchor", Module: "notes", Assigned: at, SentUnknown: at} + raw, _ = json.Marshal(unknown) + if !strings.Contains(string(raw), `"sent-unknown":"2026-10-18T03:00:00Z"`) || strings.Contains(string(raw), `"commit"`) || !unknown.Stands() { + t.Fatalf("the hand-sent value is %s", raw) + } + plain, _ := json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at}) + if strings.Contains(string(plain), "pulled-in") || strings.Contains(string(plain), "sent-unknown") { + t.Fatalf("a value that is neither says so: %s", plain) + } +} diff --git a/internal/chain/records.go b/internal/chain/records.go index ceecabbb..78bdeaab 100644 --- a/internal/chain/records.go +++ b/internal/chain/records.go @@ -191,9 +191,74 @@ type Assignment struct { // Ended is when the assignment ended (an unassignment): absent while it stands. **No key of the chain is removed** // (ADR 0408 §4), so an unassigned module keeps its key, saying so; every reader treats a value with Ended, or with a - // State, as no assignment. State is, in a later delivery, the pending assignment waiting for its build (issue 1094). + // State, as no assignment. State is StatePending while the key holds an assignment waiting for its build, or one that + // was never made (issue 1094). Ended time.Time `json:"ended,omitzero"` State string `json:"state,omitempty"` + // Pending is the assignment's wait for its build (novox/hq ADR 0261), kept in the value of the key it waits for and, when + // it is made, carried into the value that stands: the record of what became of the wait. + Pending *Pending `json:"pending,omitempty"` + + // PulledIn marks a standing value of a module that is on the machine only because another requires it, or because a + // send carried it: no time assigned, but a key all the same, so what the machine was sent of it is kept and read + // (issue 1109). Never a state: every reader treats a non-empty state as no assignment, and would take it off the machine. + PulledIn bool `json:"pulled-in,omitempty"` + // SentUnknown is when a send by hand made what the machine runs unknown (a declaration sent with no builds known): the + // commit is cleared with it. Absent normally; a send that names its builds clears it. + SentUnknown time.Time `json:"sent-unknown,omitzero"` +} + +// StatePending is the State of a value that holds no assignment but a pending one. +const StatePending = "pending" + +// The states of a pending assignment (novox/hq ADR 0261). +const ( + PendingWaiting = "waiting" + PendingApplying = "applying" + PendingApplied = "applied" + PendingRefused = "refused" + PendingExpired = "expired" + PendingWithdrawn = "withdrawn" +) + +// Pending is an assignment kept until its module is registered: the build it waits for, where that build is from, and the +// times of its claim and its end. Its id is p-, unique for a key, which holds one at a time. +type Pending struct { + ID string `json:"id"` + State string `json:"state"` + Build string `json:"build,omitempty"` + Repository string `json:"repository,omitempty"` + Path string `json:"path,omitempty"` + Since time.Time `json:"since"` + Note string `json:"note,omitempty"` + Settled *time.Time `json:"settled,omitempty"` + Raised *time.Time `json:"raised,omitempty"` + Acknowledged *time.Time `json:"acknowledged,omitempty"` + Cleared *time.Time `json:"cleared,omitempty"` + // Earlier are the waits of this key that ended unmade and were replaced by this one before their condition was answered + // (taken back or cleared): each stays, with its own marks, so a replaced wait never leaves the operator without the + // condition it raised or was to raise. Dropped once answered. + Earlier []Pending `json:"earlier,omitempty"` +} + +// Open says the assignment is still to be made. +func (p Pending) Open() bool { return p.State == PendingWaiting || p.State == PendingApplying } + +// CanSettle says a pending assignment in state from may end in state to: a wait expires or is withdrawn, a claim becomes +// applied or refused, and nothing else. +func CanSettle(from, to string) bool { + switch from { + case PendingWaiting: + return to == PendingExpired || to == PendingWithdrawn + case PendingApplying: + return to == PendingApplied || to == PendingRefused + } + return false +} + +// CanMove says a pending assignment in state from may go to state to without ending: claimed, or released again. +func CanMove(from, to string) bool { + return (from == PendingWaiting && to == PendingApplying) || (from == PendingApplying && to == PendingWaiting) } // Stands says the value is an assignment that holds: it has not ended and is not waiting in a state of its own. diff --git a/internal/inventory/batches.go b/internal/inventory/batches.go index c0cd6ab2..294c5f00 100644 --- a/internal/inventory/batches.go +++ b/internal/inventory/batches.go @@ -3,11 +3,8 @@ package inventory import ( "context" "encoding/json" - "errors" "strings" "time" - - "github.com/jackc/pgx/v5" ) // The merges a batch or walk answers (novox/hq ADR 0276): every merge the controller heard into a branch a @@ -33,138 +30,78 @@ type BatchedMerge struct { // AddMerge keeps a merge for a plan; false when the merge was kept already, for whatever plan. func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { - return false, err - } else if on { - return i.chainAddMerge(ctx, c, m) - } - tag, err := i.store.Pool().Exec(ctx, - `insert into batched_merge (repository, branch, commit_hash, merged_at, heard_at, event, plan_id, alone) - values (lower($1), $2, $3, $4, $5, $6, nullif($7, ''), $8) on conflict do nothing`, - m.Repository, m.Branch, m.Commit, mergedAt(m.Merged), m.Heard, []byte(m.Event), m.Plan, m.Alone) + c, err := i.theChain(ctx) if err != nil { return false, err } - return tag.RowsAffected() == 1, nil + return i.chainAddMerge(ctx, c, m) } // MergeOf is one kept merge; false when it was never heard. func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (BatchedMerge, bool, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { - return BatchedMerge{}, false, err - } else if on { - return i.chainMergeOf(ctx, c, repository, commit) - } - ms, err := i.merges(ctx, `where repository = lower($1) and commit_hash = $2`, repository, commit) - if err != nil || len(ms) == 0 { + c, err := i.theChain(ctx) + if err != nil { return BatchedMerge{}, false, err } - return ms[0], true, nil + return i.chainMergeOf(ctx, c, repository, commit) } // MergesOf is every merge a plan answers, oldest merge first. func (i *Inventory) MergesOf(ctx context.Context, plan string) ([]BatchedMerge, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - return i.chainMergesOf(ctx, c, plan) } - return i.merges(ctx, `where plan_id = $1 order by merged_at nulls last, heard_at`, plan) + return i.chainMergesOf(ctx, c, plan) } // LaterMergesOf is every merge of a repository's branch made after a moment and answered by a plan, the // newest first. func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string, after time.Time) ([]BatchedMerge, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - return i.chainLaterMergesOf(ctx, c, repository, branch, after) } - return i.merges(ctx, `where repository = lower($1) and branch = $2 and merged_at > $3 and plan_id is not null - order by merged_at desc`, repository, branch, after) + return i.chainLaterMergesOf(ctx, c, repository, branch, after) } // KeptOnBranch says the controller keeps a merge of a repository's branch, and when it heard the branch's first: a merge of a branch it keeps merges of, made since then, is the record's to answer, whatever a // module's last look says (novox/hq ADR 0276). func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string) (bool, time.Time, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { - return false, time.Time{}, err - } else if on { - return i.chainKeptOnBranch(ctx, c, repository, branch) - } - var first *time.Time - var kept bool - err := i.store.Pool().QueryRow(ctx, - `select min(heard_at), count(*) > 0 from batched_merge where repository = lower($1) and branch = $2`, - repository, branch).Scan(&first, &kept) - if err != nil || first == nil { + c, err := i.theChain(ctx) + if err != nil { return false, time.Time{}, err } - return kept, first.UTC(), nil + return i.chainKeptOnBranch(ctx, c, repository, branch) } // AloneMerges is every merge waiting to be walked on its own commit, the newest first (ADR 0276 decision 3). func (i *Inventory) AloneMerges(ctx context.Context) ([]BatchedMerge, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - return i.chainAloneMerges(ctx, c) } - return i.merges(ctx, `where plan_id is null and alone order by merged_at desc nulls last, heard_at desc`) + return i.chainAloneMerges(ctx, c) } // AnswerMerge moves a merge to the plan that answers it now; an empty plan with alone leaves it waiting to // be walked on its own commit. func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan string, alone bool) error { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return err - } else if on { - return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}}) } - _, err := i.store.Pool().Exec(ctx, - `update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`, - repository, commit, plan, alone) - return err + return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}}) } // Batches is every batch not yet cut: assembling or queued, oldest first. One at a time is the rule; more // are read so a fault is seen. func (i *Inventory) Batches(ctx context.Context) ([]Plan, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { - return nil, err - } else if on { - return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated) - } - return i.plans(ctx, `where state in ('assembling', 'queued') order by created`) -} - -func (i *Inventory) merges(ctx context.Context, tail string, args ...any) ([]BatchedMerge, error) { - rows, err := i.store.Pool().Query(ctx, - `select repository, branch, commit_hash, merged_at, heard_at, event, coalesce(plan_id, ''), alone - from batched_merge `+tail, args...) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - var out []BatchedMerge - for rows.Next() { - var m BatchedMerge - var merged *time.Time - var event []byte - if err := rows.Scan(&m.Repository, &m.Branch, &m.Commit, &merged, &m.Heard, &event, &m.Plan, &m.Alone); err != nil { - return nil, err - } - if merged != nil { - m.Merged = merged.UTC() - } - m.Heard = m.Heard.UTC() - m.Event = event - out = append(out, m) - } - if errors.Is(rows.Err(), pgx.ErrNoRows) { - return nil, nil - } - return out, rows.Err() + return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated) } // SameRepository says two spellings of a repository are one. diff --git a/internal/inventory/builds.go b/internal/inventory/builds.go index 4d7b58ef..99a3bed4 100644 --- a/internal/inventory/builds.go +++ b/internal/inventory/builds.go @@ -2,14 +2,9 @@ package inventory import ( "context" - "encoding/json" - "errors" - "sort" "strings" "time" - "github.com/jackc/pgx/v5" - "github.com/novox/mesh-controller/internal/chain" ) @@ -136,51 +131,11 @@ func (b Build) Worked() bool { return b.Failed == "" } // whoever asked and once on the exchange when nobody was. Recording both would show one build as // two, and which of the two is real is not a question anybody could answer afterwards. func (i *Inventory) RecordBuild(ctx context.Context, b Build) error { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return err - } else if on { - return i.chainRecordBuild(ctx, c, b) - } - made, err := json.Marshal(b.Made) + c, err := i.theChain(ctx) if err != nil { return err } - against, err := json.Marshal(b.Against) - if err != nil { - return err - } - read, err := json.Marshal(b.Read) - if err != nil { - return err - } - var sources any - if len(b.Sources) > 0 { - raw, err := json.Marshal(b.Sources) - if err != nil { - return err - } - sources = raw - } - var module *string - if b.Module != "" { - module = &b.Module - } - var asked *time.Time - if !b.Asked.IsZero() { - asked = &b.Asked - } - _, err = i.store.Pool().Exec(ctx, - `insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, - source_path, manifest, built_against, built_contexts, asked, source_fingerprint, - build_sources) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) - on conflict (id) do nothing`, - b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made, - b.Path, manifestOrNil(b.Manifest), against, read, asked, b.SourceFingerprint, sources) - if err != nil { - return err - } - return i.RecordMirrored(ctx, b.ID, "", b.Mirrored) + return i.chainRecordBuild(ctx, c, b) } // RecordMirrored keeps that the mesh copied these bases into the artifact store (novox/hq ADR 0257): @@ -223,79 +178,22 @@ func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Bui if limit <= 0 { limit = 20 } - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainBuildList(ctx, c, module, limit) - } - if limit <= 0 { - limit = 20 - } - query := `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at - from build order by at desc limit $1` - args := []any{limit} - if module != "" { - query = `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at - from build where module = $2 order by at desc limit $1` - args = append(args, module) - } - - rows, err := i.store.Pool().Query(ctx, query, args...) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - - var out []Build - for rows.Next() { - var b Build - var made []byte - var asked *time.Time - if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, - &b.On, &b.Failed, &made, &asked, &b.At); err != nil { - return nil, err - } - if asked != nil { - b.Asked = *asked - } - if err := json.Unmarshal(made, &b.Made); err != nil { - return nil, err - } - out = append(out, b) - } - return out, rows.Err() + return i.chainBuildList(ctx, c, module, limit) } // BuildByID is one build's record, by the id its request carried — what a plan matches its outcome // by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the // repository, path, ref and commit from. False when no outcome with that id was recorded. func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return Build{}, false, err - } else if on { - return i.chainBuildByID(ctx, c, id) - } - var b Build - var made []byte - var asked *time.Time - err := i.store.Pool().QueryRow(ctx, - `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, - coalesce(source_path,''), asked, at - from build where id = $1`, id).Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, - &b.On, &b.Failed, &made, &b.Path, &asked, &b.At) - if errors.Is(err, pgx.ErrNoRows) { - return Build{}, false, nil - } + c, err := i.theChain(ctx) if err != nil { return Build{}, false, err } - if asked != nil { - b.Asked = *asked - } - if err := json.Unmarshal(made, &b.Made); err != nil { - return Build{}, false, err - } - return b, true, nil + return i.chainBuildByID(ctx, c, id) } // Held is every artifact this mesh has built, keyed "/". @@ -377,40 +275,11 @@ func registeredOnly(held, registered map[string]string) map[string]string { // edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a // module standing on nothing looks like: an edge the mesh has not derived is not an edge. func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainBuiltAgainst(ctx, c) - } - rows, err := i.store.Pool().Query(ctx, - `select distinct on (module) module, built_against - from build - where module is not null and module <> '' and failed = '' - order by module, `+newestRequestFirst) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - - against := map[string][]string{} - for rows.Next() { - var module string - var raw []byte - if err := rows.Scan(&module, &raw); err != nil { - return nil, err - } - if len(raw) == 0 { - continue - } - var refs []string - if err := json.Unmarshal(raw, &refs); err != nil { - continue - } - if len(refs) > 0 { - against[module] = refs - } - } - return against, rows.Err() + return i.chainBuiltAgainst(ctx, c) } // ReadRepositories is what each module's newest successful build read source from besides its own @@ -425,95 +294,11 @@ func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, erro // none — off the trunk, from a builder that predates it, or of a source not known — leaves its module read // as before: every file of a context, and its own directory or root. func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainReadRepositories(ctx, c) - } - // The newest build of each module whatever its outcome: a failed one newer than the newest that - // worked leaves that one's build source stale — the merge it was asked for may have changed the closure - // (novox/hq ADR 0267), so its module is read whole until a build works again. - newest := map[string]struct { - at time.Time - failed bool - }{} - tried, err := i.store.Pool().Query(ctx, - `select distinct on (module) module, coalesce(asked, at), failed <> '' - from build - where module is not null and module <> '' - order by module, `+newestRequestFirst) + c, err := i.theChain(ctx) if err != nil { return nil, err } - for tried.Next() { - var module string - var at time.Time - var failed bool - if err := tried.Scan(&module, &at, &failed); err != nil { - tried.Close() - return nil, err - } - newest[module] = struct { - at time.Time - failed bool - }{at, failed} - } - tried.Close() - if err := tried.Err(); err != nil { - return nil, err - } - - rows, err := i.store.Pool().Query(ctx, - `select distinct on (module) module, built_contexts, build_sources, coalesce(asked, at), - (select source from module where module.name = build.module) - from build - where module is not null and module <> '' and failed = '' - order by module, `+newestRequestFirst) - if err != nil { - return nil, err - } - defer rows.Close() - - read := map[string][]ReadRepository{} - for rows.Next() { - var module string - var raw, rawSources []byte - var built time.Time - var own *string - if err := rows.Scan(&module, &raw, &rawSources, &built, &own); err != nil { - return nil, err - } - var of []ReadRepository - if len(raw) > 0 { - if err := json.Unmarshal(raw, &of); err != nil { - of = nil - } - } - var sources []BuildSource - if len(rawSources) > 0 { - if err := json.Unmarshal(rawSources, &sources); err != nil { - sources = nil - } - } - whole := "" - if n, known := newest[module]; known && n.failed && n.at.After(built) && len(sources) > 0 { - sources, whole = nil, "a newer build of it failed" - } - of = WithBuildSources(of, sources) - if own != nil { - of = OwnRepositoryAsOwn(of, *own) - } - if whole != "" { - of = append(of, ReadRepository{Own: true, Whole: whole}) - } - if len(of) > 0 { - for k := range of { - of[k].Built, of[k].Looked = built, newest[module].at - } - read[module] = of - } - } - return read, rows.Err() + return i.chainReadRepositories(ctx, c) } // WithBuildSources lays a build's build sources over the repositories it read: a context's paths on its @@ -590,18 +375,6 @@ func namesRepository(registered, named string) bool { return reg == name || strings.HasSuffix(reg, "/"+name) } -// manifestOrNil keeps the difference between "declared nothing" and "predates this being kept". -// -// A build recorded before the mesh kept manifests has no manifest, and that is not the same as one -// whose manifest was empty. A replay can then say which it is holding instead of inventing an -// empty declaration for a module that certainly had one. -func manifestOrNil(raw []byte) any { - if len(raw) == 0 { - return nil - } - return raw -} - // Announceable is every build worth telling a catalogue about, oldest first. // // **Oldest first, because a graph is built in the order things happened.** Registering a module @@ -616,53 +389,9 @@ func manifestOrNil(raw []byte) any { // One row per module and commit: a module built twice at the same commit is one fact, and the // row asked last is the one whose artifacts are current (novox/hq 04-ISSUES/219). func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainAnnounceable(ctx, c) - } - rows, err := i.store.Pool().Query(ctx, - `select distinct on (module, commit_hash) - id, repository, ref, module, commit_hash, built_on, failed, made, - source_path, manifest, built_against, at - from build - where failed = '' and module is not null and module <> '' and commit_hash <> '' - order by module, commit_hash, `+newestRequestFirst) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - - var out []Build - for rows.Next() { - var b Build - var made []byte - var manifest, against []byte - if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, - &b.On, &b.Failed, &made, &b.Path, &manifest, &against, &b.At); err != nil { - return nil, err - } - if err := json.Unmarshal(made, &b.Made); err != nil { - return nil, err - } - // Null rather than empty is a build recorded before the mesh kept these, and saying so is - // the point of keeping them nullable: the replay carries nothing rather than an empty - // declaration for a module that certainly had one. - if len(manifest) > 0 { - b.Manifest = manifest - } - if len(against) > 0 { - if err := json.Unmarshal(against, &b.Against); err != nil { - return nil, err - } - } - out = append(out, b) - } - if err := rows.Err(); err != nil { - return nil, err - } - // Sorted here rather than in the query, because `distinct on` fixes the ordering it needs and - // the order that matters to a catalogue is a different one. - sort.Slice(out, func(a, b int) bool { return out[a].At.Before(out[b].At) }) - return out, nil + return i.chainAnnounceable(ctx, c) } diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 56a11d17..adc40bd3 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -258,31 +258,17 @@ func (i *Inventory) SourceOf(ctx context.Context, module string) (Source, error) // machines are running last week's version" is the question somebody actually has, and it is the // one novox/hq ADR 0010 names as the thing that must not be lost. func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) { - query := `select m.name, coalesce(n.name, '') - from module m - left join assignment a on a.module = m.name - left join node n on n.id = a.node - where m.source is not null - and m.source_head is not null - and coalesce(m.built_from, '') is distinct from m.source_head - order by m.name, n.name` - _, onTheBus, err := i.ChainReads(ctx, KindAssignments) + assigned, err := i.assignments(ctx) if err != nil { return nil, err } - var assigned []assignRow - if onTheBus { - query = `select m.name, '' + rows, err := i.store.Pool().Query(ctx, + `select m.name from module m where m.source is not null and m.source_head is not null and coalesce(m.built_from, '') is distinct from m.source_head - order by m.name` - if assigned, err = i.assignments(ctx); err != nil { - return nil, err - } - } - rows, err := i.store.Pool().Query(ctx, query) + order by m.name`) if err != nil { return nil, err } @@ -290,16 +276,11 @@ func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) { out := map[string][]string{} for rows.Next() { - var module, node string - if err := rows.Scan(&module, &node); err != nil { + var module string + if err := rows.Scan(&module); err != nil { return nil, err } - if _, seen := out[module]; !seen { - out[module] = nil - } - if node != "" { - out[module] = append(out[module], node) - } + out[module] = nil } if err := rows.Err(); err != nil { return nil, err @@ -654,21 +635,11 @@ func (i *Inventory) Assign(ctx context.Context, nodeName, module string) (bool, if err := i.runsSomewhere(ctx, module); err != nil { return false, err } - if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil { - return false, err - } else if on { - return i.chainAssign(ctx, c, node, module) - } - tag, err := i.store.Pool().Exec(ctx, - `insert into assignment (node, module) values ($1, $2) on conflict do nothing`, - node.ID, module) - if err != nil && strings.Contains(err.Error(), "assignment_module_fkey") { - return false, fmt.Errorf("%w: %s", ErrNoSuchModule, module) - } + c, err := i.theChain(ctx) if err != nil { return false, err } - return tag.RowsAffected() > 0, nil + return i.chainAssign(ctx, c, node, module) } // Unassign takes a module off a node. @@ -677,21 +648,12 @@ func (i *Inventory) Unassign(ctx context.Context, nodeName, module string) error if err != nil { return err } - if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil { + c, err := i.theChain(ctx) + if err != nil { + return err + } + if err := i.chainUnassign(ctx, c, node, module); err != nil { return err - } else if on { - if err := i.chainUnassign(ctx, c, node, module); err != nil { - return err - } - } else { - tag, err := i.store.Pool().Exec(ctx, - `delete from assignment where node = $1 and module = $2`, node.ID, module) - if err != nil { - return err - } - if tag.RowsAffected() == 0 { - return fmt.Errorf("%s is not assigned to %s", module, nodeName) - } } // The seat it held falls back to derivation rather than pointing at nothing: what the store's foreign key did. if _, err := i.store.Pool().Exec(ctx, `delete from seat_holding where node = $1 and module = $2`, node.ID, module); err != nil { @@ -1362,39 +1324,21 @@ type Entry struct { // catalogue nobody lists, and the questions here — what is this, where did it come from, who is // running it — are asked together every time. func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) { - query := `select m.name, m.manifest, - coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''), - coalesce(m.built_from, ''), coalesce(m.source_head, ''), - coalesce(m.source_seen, to_timestamp(0)), - coalesce(array_agg(n.name order by n.name) filter (where n.name is not null), '{}') - from module m - left join assignment a on a.module = m.name - left join node n on n.id = a.node - group by m.name, m.manifest, m.source, m.source_path, m.source_seat, m.ref, m.built_from, - m.source_head, m.source_seen - order by m.name` - // On the bus the assignments are read from the bucket, and the machines of a module put beside it here. - _, onTheBus, err := i.ChainReads(ctx, KindAssignments) + // The machines of a module are read from the assignments bucket and put beside it here. + assigned, err := i.assignments(ctx) if err != nil { return nil, err } - var assignedBy map[string][]string - if onTheBus { - query = `select m.name, m.manifest, + assignedBy := map[string][]string{} + for _, a := range assigned { + assignedBy[a.Module] = append(assignedBy[a.Module], a.Node) + } + const query = `select m.name, m.manifest, coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''), coalesce(m.built_from, ''), coalesce(m.source_head, ''), coalesce(m.source_seen, to_timestamp(0)), '{}'::text[] from module m order by m.name` - rows, err := i.assignments(ctx) - if err != nil { - return nil, err - } - assignedBy = map[string][]string{} - for _, a := range rows { - assignedBy[a.Module] = append(assignedBy[a.Module], a.Node) - } - } rows, err := i.store.Pool().Query(ctx, query) if err != nil { return nil, err @@ -1414,9 +1358,7 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) { if source.Seen.Unix() == 0 { source.Seen = time.Time{} } - if onTheBus { - on = assignedBy[name] - } + on = assignedBy[name] var m catalogue.Manifest if err := json.Unmarshal(raw, &m); err != nil { return nil, err diff --git a/internal/inventory/chain.go b/internal/inventory/chain.go index cf83baa4..1a0fbd7e 100644 --- a/internal/inventory/chain.go +++ b/internal/inventory/chain.go @@ -5,55 +5,46 @@ import ( "errors" "fmt" "sync" - "sync/atomic" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" "github.com/novox/mesh-controller/internal/chain" ) -// **The chain moves to the bus kind by kind** (novox/hq ADR 0408 §8, issue 981). The tables of builds, rollouts and -// assignments are copied once into the chain's buckets by the serving controller at its start (MoveChain), which then -// records each kind it moved in chain_moved (migration 0104). **Until a kind is recorded there nothing reads its bucket; -// once it is, nothing reads its tables**: ChainReads answers which, and every method of a kind asks it first. - -// The kinds of the chain, as chain_moved names them. -const ( - KindBuilds = "builds" - KindRollouts = "rollouts" -) +// **The chain lives on the bus** (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the +// bus's key-value store, the controller's store keeps none of them, and every method of a kind reads and writes the chain this +// inventory is given (theChain). // testChains are the chains in memory a test made for its store, by the store's address: every inventory opened on that // store is given the same chain, as every process of a mesh is on the same bus. Empty outside a test. var testChains sync.Map -// ErrNoChain is a kind moved to the bus read by an inventory that has no chain attached: a process that did not -// open the bus. Refused loudly, never answered from the tables the move left behind. +// ErrNoChain is a read or write of the chain by an inventory that has none attached: a process that did not open the bus. +// Refused loudly. var ErrNoChain = errors.New("this kind of record lives on the bus, and this process has not opened the bus") -// movedKinds is which kinds this process has read as moved: once moved, always moved, so it is read from the store -// until it is true and not after. -type movedKinds struct { +// chainState is what an inventory keeps of the chain it is given. +type chainState struct { mu sync.Mutex - kind map[string]bool open ChainOpener // view is the rollouts bucket kept in memory by one watch (issue 1092); pending are the merges added to a plan that // is not saved yet. view *rolloutView pending map[string]pendingMerge - // switching is set when the assignments begin to be switched to the bus and kept once they are: the store's mirror of - // them writes nothing from then on (AssignmentsMirrorMayWrite). Cleared only if the switch fails. - switching atomic.Bool + // requireLayout is set for a chain on the bus: its builds are read only once the controller's start has laid them out + // under their modules (issue 1121), and laid is that seen. + requireLayout, laid bool } -// AssignmentsMirrorMayWrite says the store's mirror of the assignments may still write the bucket: the assignments are -// neither being switched to the bus nor on it. Asked before every write of the mirror's writer, not once a pass. -func (i *Inventory) AssignmentsMirrorMayWrite() bool { - i.moved.mu.Lock() - moved := i.moved.kind[KindAssignments] - i.moved.mu.Unlock() - return !moved && !i.moved.switching.Load() +// RequireLayout makes every read of the chain refuse (ErrNotLaidOut) until the builds are laid out under their modules. +func (i *Inventory) RequireLayout() { + i.state.mu.Lock() + i.state.requireLayout = true + i.state.mu.Unlock() +} + +func (i *Inventory) markLaidOut() { + i.state.mu.Lock() + i.state.laid = true + i.state.mu.Unlock() } // ChainOpener opens the chain on the bus when a process first needs it: a command that reads and never moved anything has @@ -62,108 +53,63 @@ type ChainOpener func(ctx context.Context) (*chain.Chain, error) // UseChainOpener gives the inventory the means to open the chain when it needs it and none is attached. func (i *Inventory) UseChainOpener(open ChainOpener) { - i.moved.mu.Lock() - i.moved.open = open - i.moved.mu.Unlock() + i.state.mu.Lock() + i.state.open = open + i.state.requireLayout = true + i.state.mu.Unlock() } // UseChain attaches the chain the moved kinds are read and written through. The serving controller attaches the bus's, // holding the lease for its writes; a test attaches one in memory. func (i *Inventory) UseChain(c *chain.Chain) { - i.moved.mu.Lock() - defer i.moved.mu.Unlock() + i.state.mu.Lock() + defer i.state.mu.Unlock() i.chain = c - if i.moved.view != nil { - i.moved.view.stop() - i.moved.view = nil + if i.state.view != nil { + i.state.view.stop() + i.state.view = nil } } +// UseThrowawayChain gives the inventory a chain in memory that is its own, laid out already: for a mesh raised for a moment +// beside the real one, as the merge gate raises the facts snapshot's mesh in a throwaway store (issue 1121). Nothing of it +// reaches the bus. +func (i *Inventory) UseThrowawayChain() { + i.UseChain(chain.InMemory(nil)) + i.markLaidOut() +} + // attached is the chain this inventory has, nil when none. func (i *Inventory) attached() *chain.Chain { - i.moved.mu.Lock() - defer i.moved.mu.Unlock() + i.state.mu.Lock() + defer i.state.mu.Unlock() return i.chain } -// ChainReads says whether a kind of the chain is read from its bucket: chain_moved names it. It answers the chain to read -// through, and ErrNoChain when the kind is moved and none is attached. -func (i *Inventory) ChainReads(ctx context.Context, kind string) (*chain.Chain, bool, error) { - moved, err := i.kindMoved(ctx, kind) - if err != nil || !moved { - return nil, false, err - } - if kind == KindBuilds { - // **The builds are read under their modules** (issue 1121): until the layout is made, a read would find nothing. - laid, err := i.kindMoved(ctx, KindBuildsLayout) +// theChain is the chain this inventory writes and reads, opened if a command has not yet. On the bus it is refused until the +// builds are laid out under their modules. +func (i *Inventory) theChain(ctx context.Context) (*chain.Chain, error) { + i.state.mu.Lock() + if i.chain == nil && i.state.open != nil { + c, err := i.state.open(ctx) if err != nil { - return nil, false, err - } - if !laid { - return nil, false, fmt.Errorf("%w: the builds are not laid out under their modules yet; the move at the controller's start does it", ErrNoChain) - } - } - i.moved.mu.Lock() - defer i.moved.mu.Unlock() - if i.chain == nil && i.moved.open != nil { - c, err := i.moved.open(ctx) - if err != nil { - return nil, false, fmt.Errorf("%w: %s: %v", ErrNoChain, kind, err) + i.state.mu.Unlock() + return nil, fmt.Errorf("%w: %v", ErrNoChain, err) } i.chain = c } - if i.chain == nil { - return nil, false, fmt.Errorf("%w: %s", ErrNoChain, kind) - } - return i.chain, true, nil -} - -// kindMoved is whether chain_moved names a kind; once it does, always, so the store is asked until it is true. -func (i *Inventory) kindMoved(ctx context.Context, kind string) (bool, error) { - i.moved.mu.Lock() - known := i.moved.kind[kind] - i.moved.mu.Unlock() - if known { - return true, nil - } - var moved bool - err := i.store.Pool().QueryRow(ctx, `select counts ? $1 from chain_moved`, kind).Scan(&moved) - var pgErr *pgconn.PgError - if errors.Is(err, pgx.ErrNoRows) || (errors.As(err, &pgErr) && pgErr.Code == "42P01") { - // No row, or a store older than the migration that makes the table: nothing moved. - return false, nil - } - if err != nil { - return false, fmt.Errorf("whether %s moved to the bus cannot be read: %w", kind, err) - } - if !moved { - return false, nil - } - i.moved.mu.Lock() - if i.moved.kind == nil { - i.moved.kind = map[string]bool{} - } - i.moved.kind[kind] = true - i.moved.mu.Unlock() - return true, nil -} - -// ChainMoved is, per kind moved, how many records the move wrote; empty before any is. -func (i *Inventory) ChainMoved(ctx context.Context) (map[string]int, error) { - var counts map[string]int - err := i.store.Pool().QueryRow(ctx, `select counts from chain_moved`).Scan(&counts) - if errors.Is(err, pgx.ErrNoRows) { - return map[string]int{}, nil - } - return counts, err -} - -// NewForTest is an inventory with no store, for a test of what needs none. -func NewForTest() *Inventory { return &Inventory{moved: &movedKinds{}} } - -// BeginSwitchForTest sets the flag a kind's switch sets, as a switch under way does. -func (i *Inventory) BeginSwitchForTest(kind string) { - if kind == KindAssignments { - i.moved.switching.Store(true) + c, check := i.chain, i.state.requireLayout && !i.state.laid + i.state.mu.Unlock() + if c == nil { + return nil, ErrNoChain } + if check { + if _, found, err := c.Builds.GetRaw(ctx, chain.LayoutMarkerKey); err != nil { + return nil, err + } else if !found { + return nil, ErrNotLaidOut + } + i.markLaidOut() + } + return c, nil } diff --git a/internal/inventory/chain_assignments.go b/internal/inventory/chain_assignments.go index 958924e0..ff7b8a02 100644 --- a/internal/inventory/chain_assignments.go +++ b/internal/inventory/chain_assignments.go @@ -21,7 +21,6 @@ import ( // here (chain.Assignment.Stands). No `pending` state is written yet (issue 1094). // KindAssignments is the kind of the chain that is each module assigned to a node, with the build it runs there. -const KindAssignments = "assignments" func assignmentKey(node, module string) string { return node + "." + module } @@ -31,32 +30,15 @@ type assignRow struct { Assigned time.Time } -// assignments is every assignment, by node then module: from the bucket when the kind is moved, else from the store. +// assignments is every assignment, by node then module, read from the assignments bucket. // A key whose value names no time assigned is a module only pulled in by another (issue 765), and one that has ended or // waits in a state is gone: no assignment. One on a machine the store does not know is none either, as the store took a // removed machine's. func (i *Inventory) assignments(ctx context.Context) ([]assignRow, error) { - c, on, err := i.ChainReads(ctx, KindAssignments) + c, err := i.theChain(ctx) if err != nil { return nil, err } - if !on { - rows, err := i.store.Pool().Query(ctx, - `select n.name, a.module, a.assigned from assignment a join node n on n.id = a.node order by n.name, a.module`) - if err != nil { - return nil, err - } - defer rows.Close() - var out []assignRow - for rows.Next() { - var a assignRow - if err := rows.Scan(&a.Node, &a.Module, &a.Assigned); err != nil { - return nil, err - } - out = append(out, a) - } - return out, rows.Err() - } all, err := c.Assignments.All(ctx) if err != nil { return nil, fmt.Errorf("the assignments cannot be read from the bus: %w", err) @@ -172,10 +154,12 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, switch { case found && cur.Value.Stands(): // A module only pulled in by another becomes an assignment, keeping what it holds. - cur.Value.Assigned = now + cur.Value.Assigned, cur.Value.PulledIn = now, false _, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision) case found: - // One that ended is assigned again: a fresh value over it, the ended one staying in history. + // One that ended is assigned again, and one that was pending is made: a fresh value over it, the earlier one + // staying in history. The wait is carried into the value that stands (issue 1094). + fresh.Pending = carriedPending(cur.Value, now) _, err = c.Assignments.Update(ctx, key, fresh, cur.Revision) default: _, err = c.Assignments.Create(ctx, key, fresh) @@ -253,8 +237,8 @@ func (i *Inventory) chainAwaitingSince(ctx context.Context, name string) (time.T // and rollout another write put in the same value, and never ending an assignment. A plan the provisioner's rules refuse is // refused by name and nothing is written (AssignmentValues). func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed int, err error) { - c, on, err := i.ChainReads(ctx, KindAssignments) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return 0, 0, err } var plans map[string]Provisioning @@ -289,11 +273,11 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i case keepSkip: continue case keepCreate: - next.Assigned = time.Time{} + next.Assigned, next.PulledIn = time.Time{}, true _, err = c.Assignments.Create(ctx, key, next) case keepFresh: // A value that ended, pulled in again: a fresh one over it, the ended one staying in history. - next.Assigned = time.Time{} + next.Assigned, next.PulledIn = time.Time{}, true _, err = c.Assignments.Update(ctx, key, next, cur.Revision) default: cur.Value.RequiredBy, cur.Value.Provisioning = next.RequiredBy, next.Provisioning @@ -308,8 +292,8 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i written++ } for key, cur := range have { - if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() { - continue + if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() || cur.Value.Commit != "" { + continue // one a send still carries is on the machine, and ends when a send does not carry it any more } // A value of a module no longer pulled in, which no assignment names: it ends, and stays. cur.Value.Ended = time.Now().UTC() @@ -340,17 +324,16 @@ type KeptValue struct { Revision uint64 } -// KeptAssignments is every key of the assignments bucket that holds a value that stands, and false when the assignments are -// not on the bus. -// What the serving controller compares from one look to the next to say a key changed (`assignment-changed`). -func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, bool, error) { - c, on, err := i.ChainReads(ctx, KindAssignments) - if err != nil || !on { - return nil, false, err +// KeptAssignments is every key of the assignments bucket that holds a value that stands. What the serving controller +// compares from one look to the next to say a key changed (`assignment-changed`). +func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, error) { + c, err := i.theChain(ctx) + if err != nil { + return nil, err } all, err := c.Assignments.All(ctx) if err != nil { - return nil, true, err + return nil, err } out := make(map[string]KeptValue, len(all)) for key, k := range all { @@ -359,11 +342,11 @@ func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, } body, err := json.Marshal(k.Value) if err != nil { - return nil, true, err + return nil, err } out[key] = KeptValue{Value: body, Revision: k.Revision} } - return out, true, nil + return out, nil } // What a pass that keeps the plans does with one key. @@ -384,6 +367,8 @@ func keepStep(has bool, cur, next chain.Assignment) string { return keepSkip // assigned since the bucket was read: its own write is not here yet case !has: return keepCreate + case cur.State == chain.StatePending: + return keepSkip // a wait for a build is not pulled in by a plan case !cur.Stands(): if !cur.Assigned.IsZero() || !next.Assigned.IsZero() { return keepSkip // an assignment that ended is not pulled in again by a plan @@ -394,3 +379,27 @@ func keepStep(has bool, cur, next chain.Assignment) string { } return keepField } + +// carriedPending is the wait a value that does not stand carries into the assignment made over it: a wait that was applying +// stays applying, for the one making it settles it applied; one still waiting is made by hand, and says so. Nothing for a +// value that held no wait, or whose key is not one that was pending. +func carriedPending(was chain.Assignment, now time.Time) *chain.Pending { + if was.Pending == nil || was.State != chain.StatePending { + return nil + } + p := *was.Pending + if p.State == chain.PendingWaiting { + p.State, p.Note, p.Settled = chain.PendingApplied, "assigned by hand", &now + } + return &p +} + +// AssignmentValue is one key of the assignments bucket as it is held, standing or not: one read of an assignment. +func (i *Inventory) AssignmentValue(ctx context.Context, node, module string) (chain.Assignment, bool, error) { + c, err := i.theChain(ctx) + if err != nil { + return chain.Assignment{}, false, err + } + k, found, err := c.Assignments.Get(ctx, assignmentKey(node, module)) + return k.Value, found, err +} diff --git a/internal/inventory/chain_assignments_key_test.go b/internal/inventory/chain_assignments_key_test.go index 62e87cc7..aa7c3efb 100644 --- a/internal/inventory/chain_assignments_key_test.go +++ b/internal/inventory/chain_assignments_key_test.go @@ -1,9 +1,7 @@ package inventory import ( - "context" "encoding/json" - "errors" "testing" "time" @@ -51,31 +49,24 @@ func TestAPassThatKeepsThePlansNeverRevivesAnEndedAssignment(t *testing.T) { ended := chain.Assignment{Node: "a", Module: "m", Assigned: at, Ended: at.Add(time.Minute)} pulledIn := chain.Assignment{Node: "a", Module: "p", RequiredBy: "m", Provisioning: plan} pulledInEnded := chain.Assignment{Node: "a", Module: "p", Ended: at} + waiting := chain.Assignment{Node: "a", Module: "p", State: chain.StatePending, + Pending: &chain.Pending{ID: "p-1", State: chain.PendingWaiting}} for name, tc := range map[string]struct { has bool cur, next chain.Assignment want string }{ - "the interleaving: wanted assigned, ended since": {true, ended, assigned, keepSkip}, - "wanted assigned, not in the bucket yet": {false, chain.Assignment{}, assigned, keepSkip}, - "a pulled-in module, not in the bucket": {false, chain.Assignment{}, pulledIn, keepCreate}, - "a pulled-in module that ended, pulled again": {true, pulledInEnded, pulledIn, keepFresh}, - "an assignment ended, a plan for it": {true, ended, chain.Assignment{Node: "a", Module: "m", RequiredBy: "x", Provisioning: plan}, keepSkip}, - "the plan is the same": {true, assigned, assigned, keepSkip}, - "the plan changed": {true, chain.Assignment{Node: "a", Module: "m", Assigned: at}, assigned, keepField}, + "the interleaving: wanted assigned, ended since": {true, ended, assigned, keepSkip}, + "wanted assigned, not in the bucket yet": {false, chain.Assignment{}, assigned, keepSkip}, + "a pulled-in module, not in the bucket": {false, chain.Assignment{}, pulledIn, keepCreate}, + "a pulled-in module that ended, pulled again": {true, pulledInEnded, pulledIn, keepFresh}, + "an assignment ended, a plan for it": {true, ended, chain.Assignment{Node: "a", Module: "m", RequiredBy: "x", Provisioning: plan}, keepSkip}, + "a wait for a build, a plan for the module pulled in": {true, waiting, pulledIn, keepSkip}, + "the plan is the same": {true, assigned, assigned, keepSkip}, + "the plan changed": {true, chain.Assignment{Node: "a", Module: "m", Assigned: at}, assigned, keepField}, } { if got := keepStep(tc.has, tc.cur, tc.next); got != tc.want { t.Errorf("%s: %s, want %s", name, got, tc.want) } } } - -// **The store's mirror of the assignments is refused to its writer once the assignments are moved** (ADR 0408 §4, issue -// 1093): asked at the moment of the write, so a tick that decided before the switch lays nothing after it. -func TestTheStoresMirrorOfTheAssignmentsIsRefusedToItsWriterOnceTheyAreMoved(t *testing.T) { - inv := &Inventory{moved: &movedKinds{kind: map[string]bool{KindAssignments: true}}} - inv.UseChain(chain.InMemory(nil)) - if _, err := inv.AssignmentsMirrorToWrite(context.Background()); !errors.Is(err, ErrAssignmentsMoved) { - t.Fatalf("the writer was given the store's rows: %v", err) - } -} diff --git a/internal/inventory/chain_builds.go b/internal/inventory/chain_builds.go index 06f65807..c6db6b75 100644 --- a/internal/inventory/chain_builds.go +++ b/internal/inventory/chain_builds.go @@ -1146,8 +1146,35 @@ type LayoutReport struct { Records, Written, Present int } -// KindBuildsLayout is the kind chain_moved names the layout of the builds bucket under their modules by (issue 1121). -const KindBuildsLayout = "builds-layout" +// ErrNotLaidOut is a read of the builds before the controller's start has laid them out under their modules (issue 1121): +// refused loudly, never answered from keys that are not there. +var ErrNotLaidOut = errors.New("the builds are not laid out under their modules yet; the controller's start does it and says so") + +// LayBuildsOut lays the builds out under their modules once and writes the marker key (chain.LayoutMarkerKey) that lets them +// be read. It needs the chain attached and the lease held (the chain's writes ask), and is idempotent: with the marker it +// does nothing, and a run that stopped half way is finished by the next. The old keys stay (ADR 0408 §4). +func (i *Inventory) LayBuildsOut(ctx context.Context, say func(string, ...any)) error { + c := i.attached() + if c == nil { + return ErrNoChain + } + if _, found, err := c.Builds.GetRaw(ctx, chain.LayoutMarkerKey); err != nil { + return err + } else if found { + i.markLaidOut() + return nil + } + report, err := i.layoutBuilds(ctx, c, say) + if err != nil { + return err + } + body, _ := json.Marshal(map[string]any{"records": report.Records, "at": time.Now().UTC()}) + if _, err := c.Builds.CreateRaw(ctx, chain.LayoutMarkerKey, body); err != nil && !errors.Is(err, chain.ErrExists) { + return fmt.Errorf("the builds are laid out and the marker is not written: %w", err) + } + i.markLaidOut() + return nil +} // layoutBuilds lays the builds of the bucket out under their modules: each record of a key that is its id alone is written // under `.` (a request, and a build that knew no module, under `req.`), its manifest moved to the manifests @@ -1179,6 +1206,28 @@ func (i *Inventory) layoutBuilds(ctx context.Context, c *chain.Chain, say func(s if r.Records%1000 == 0 { say("laying the builds out: %d of %d records, %d keys written so far", r.Records, len(ids), r.Written) } + // A record already under its new key is moved: a run again (the start before this one laid it out) neither reads its + // manifest nor writes it, and **no pointer is set back**: they are only made where missing. + own := chain.BuildKey(rec.Module, rec.ID) + if rec.State == chain.BuildAsked { + own = chain.RequestKey(rec.ID) + } + if _, there, err := c.Builds.GetRaw(ctx, own); err != nil { + return r, err + } else if there { + r.Present++ + if rec.State != chain.BuildAsked { + if wrote, err := i.pointTo(ctx, c, chain.IDKey(rec.ID), idPointer(rec.Module), false); err != nil { + return r, err + } else if wrote { + r.Written++ + } + } + if rec.State != chain.BuildAsked && rec.Module != "" { + byModule[rec.Module] = append(byModule[rec.Module], rec) + } + continue + } // A manifest already written in this run is not written again. digest := "" if body := manifestBytes([]byte(rec.Manifest)); body != nil && c.Kept { @@ -1238,7 +1287,7 @@ func (i *Inventory) layoutBuilds(ctx context.Context, c *chain.Chain, say func(s return r, err } for _, p := range points { - wrote, err := i.pointTo(ctx, c, p.key, p.value, true) + wrote, err := i.pointTo(ctx, c, p.key, p.value, false) if err != nil { return r, err } diff --git a/internal/inventory/chain_builds_layout_test.go b/internal/inventory/chain_builds_layout_test.go index a27e8f3e..26e30f3a 100644 --- a/internal/inventory/chain_builds_layout_test.go +++ b/internal/inventory/chain_builds_layout_test.go @@ -3,6 +3,7 @@ package inventory import ( "context" "encoding/json" + "errors" "fmt" "reflect" "sort" @@ -40,7 +41,7 @@ func (r *readCounting) WatchFiltered(ctx context.Context, f string) (<-chan chai func layoutChain(builds chain.Backend) (*Inventory, *chain.Chain) { c := chain.New(builds, chain.NewMemory(), chain.NewMemory(), func(context.Context) error { return nil }). WithManifests(builds) - inv := &Inventory{moved: &movedKinds{}} + inv := &Inventory{state: &chainState{}} inv.UseChain(c) return inv, c } @@ -396,3 +397,65 @@ func TestAnOlderBuildFinishingLastLeavesThePointersOnTheNewer(t *testing.T) { t.Errorf("the older build's own reuse key finds %q", id) } } + +// Nothing reads the chain on the bus until the controller's start has laid the builds out and written the marker; the layout +// is done once. +func TestTheChainIsRefusedUntilTheBuildsAreLaidOutAndTheLayoutIsDoneOnce(t *testing.T) { + mem := chain.NewMemory() + inv, c := layoutChain(mem) + inv.RequireLayout() + ctx := context.Background() + old := legacyBuild(1, "alpha", "c1") + if _, err := c.Builds.Create(ctx, old.ID, old); err != nil { + t.Fatal(err) + } + if _, err := inv.theChain(ctx); !errors.Is(err, ErrNotLaidOut) { + t.Fatalf("a read before the layout answered %v", err) + } + if err := inv.LayBuildsOut(ctx, nil); err != nil { + t.Fatal(err) + } + if _, err := inv.theChain(ctx); err != nil { + t.Fatalf("a read after the layout answered %v", err) + } + keys := keysOf(t, mem) + if !containsKey(keys, chain.LayoutMarkerKey) { + t.Fatalf("no marker among %v", keys) + } + if err := inv.LayBuildsOut(ctx, nil); err != nil || !reflect.DeepEqual(keys, keysOf(t, mem)) { + t.Fatalf("the second layout changed the bucket: %v", err) + } +} + +// A layout run again — the live start recorded the first in chain_moved, not in the marker — never sets a pointer back to an +// older build, and does not read a manifest of a record already under its new key. +func TestALayoutRunAgainLeavesThePointersOfTheBuildsMadeSince(t *testing.T) { + counting := &readCounting{Backend: chain.NewMemory()} + inv, c := layoutChain(counting) + ctx := context.Background() + for n := 1; n <= 2; n++ { + old := legacyBuild(n, "alpha", fmt.Sprintf("c%d", n)) + if _, err := c.Builds.Create(ctx, old.ID, old); err != nil { + t.Fatal(err) + } + } + if _, err := inv.layoutBuilds(ctx, c, nil); err != nil { + t.Fatal(err) + } + if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", 3, "c3", "s3")); err != nil { + t.Fatal(err) + } + counting.gets.Store(0) + r, err := inv.layoutBuilds(ctx, c, nil) + if err != nil || r.Written != 0 { + t.Fatalf("the second layout %+v: %v", r, err) + } + for _, key := range []string{"alpha.newest", "alpha.worked", "alpha.held"} { + if e, _, _ := counting.Get(ctx, key); string(e.Value) != "build-3" { + t.Errorf("%s = %q after the second layout; want build-3", key, e.Value) + } + } + if e, _, _ := counting.Get(ctx, "alpha.commit.c3"); pointedID(e.Value) != "build-3" { + t.Errorf("commit c3 = %s", e.Value) + } +} diff --git a/internal/inventory/chain_drop_test.go b/internal/inventory/chain_drop_test.go new file mode 100644 index 00000000..12a9b3ea --- /dev/null +++ b/internal/inventory/chain_drop_test.go @@ -0,0 +1,98 @@ +//go:build integration + +package inventory + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// **Migration 0105 refuses a store whose rows were not moved, and drops the tables of a moved one** (novox/hq ADR 0408 §8, +// issue 1111). The store is built as the mesh holds it before this migration, with an assignment row in the table and no +// record of the move: the migration raises, and nothing it did is kept. Then the move is recorded, a pending assignment +// still open refuses it again, and once it has settled the tables go and build_gate stays. +func TestMigration0105RefusesAStoreWhoseRowsWereNotMovedAndDropsTheTablesOfAMovedOne(t *testing.T) { + inv, _ := ForTestBefore(t, 105) + ctx := t.Context() + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "ui", Version: "1"}, Source{}); err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `insert into assignment (node, module) select id, 'ui' from node where name = 'anchor'`) + all, err := Migrations() + if err != nil { + t.Fatal(err) + } + tables := func() map[string]bool { + t.Helper() + out := map[string]bool{} + for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) { + out[strings.Trim(row[0], `"`)] = true + } + return out + } + refused := func(why string) { + t.Helper() + _, err := inv.store.Migrate(ctx, all) + if err == nil || !strings.Contains(err.Error(), why) { + t.Fatalf("the migration was not refused for %q: %v", why, err) + } + if !tables()["assignment"] || !tables()["chain_moved"] { + t.Fatalf("a refused migration kept a drop") + } + if rows := inv.QueryForTest(t, `select count(*) from assignment`); rows[0][0] != "1" { + t.Fatalf("a refused migration lost the assignment: %v", rows) + } + } + + // No move recorded: the rows of assignments were never copied. + refused("assignments") + + // The move recorded for every kind, and a pending assignment still waiting for its build: no mover carries it. + inv.ExecForTest(t, `insert into chain_moved (counts, largest) values ($1::jsonb, $1::jsonb)`, + `{"builds": 0, "rollouts": 0, "assignments": 1}`) + inv.ExecForTest(t, `insert into pending_assignment (node, module, build_id) + select id, 'ui', 'b1' from node where name = 'anchor'`) + refused("pending assignment") + + // Expired and raised as a condition, and the condition neither answered nor cleared: still refused. + inv.ExecForTest(t, `update pending_assignment set state = 'expired', raised_at = now()`) + refused("pending assignment") + + // Its condition cleared: the guard lets the migration through. + inv.ExecForTest(t, `update pending_assignment set cleared_at = now()`) + done, err := inv.store.Migrate(ctx, all) + if err != nil || len(done) != 1 || done[0].Number != 105 { + t.Fatalf("the migration did %v (%v)", done, err) + } + left := tables() + for _, gone := range append(append([]string{}, chainTables...), "chain_moved") { + if left[gone] { + t.Errorf("the table %s is still in the store", gone) + } + } + for _, stays := range []string{"build_gate", "assignment_generation", "node", "module"} { + if !left[stays] { + t.Errorf("the table %s is gone", stays) + } + } + if rows := inv.QueryForTest(t, `select count(*) from information_schema.columns where table_name = 'node' and column_name = 'sent_builds'`); rows[0][0] != "1" { + t.Errorf("node.sent_builds, which stays as the source of the copy, is gone") + } + if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" { + t.Errorf("the function raise_assignment_generation is still in the store") + } +} + +// A store that never held a row of the chain — a mesh raised after the move — passes the guard with no record of a move. +func TestMigration0105PassesAStoreThatNeverHeldARow(t *testing.T) { + inv, migrate := ForTestBefore(t, 105) + migrate() + if rows := inv.QueryForTest(t, `select count(*) from information_schema.tables where table_name in ('build', 'release_plan', 'assignment', 'chain_moved')`); rows[0][0] != "0" { + t.Fatalf("tables of the chain remain: %v", rows) + } +} diff --git a/internal/inventory/chain_links.go b/internal/inventory/chain_links.go index c671c93f..37f8f3b7 100644 --- a/internal/inventory/chain_links.go +++ b/internal/inventory/chain_links.go @@ -2,9 +2,11 @@ package inventory import ( "context" + "encoding/json" "errors" "fmt" - "os" + "maps" + "slices" "sort" "time" @@ -39,8 +41,8 @@ func WithRollout(ctx context.Context, id string, modules map[string]RolloutModul // AssignmentRuns is what each module assigned to a machine runs there, as its assignment names it: the build and the // commit. Empty when the assignments are not on the bus, and for a module not sent yet. func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string]chain.Ref, error) { - c, on, err := i.ChainReads(ctx, KindAssignments) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return nil, err } all, err := c.Assignments.All(ctx) @@ -50,7 +52,7 @@ func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string out := map[string]chain.Ref{} for _, k := range all { a := k.Value - if a.Node == node && !a.Assigned.IsZero() && a.Stands() && (a.Build != "" || a.Commit != "") { + if a.Node == node && a.Stands() && (a.Build != "" || a.Commit != "") { out[a.Module] = chain.Ref{Build: a.Build, Commit: a.Commit} } } @@ -59,8 +61,8 @@ func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string // AssignmentsOn are the assignments of a machine as the chain names them, by module: the standing ones, with their links. func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Assignment, error) { - c, on, err := i.ChainReads(ctx, KindAssignments) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return nil, err } all, err := c.Assignments.All(ctx) @@ -77,14 +79,18 @@ func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Ass return out, nil } -// recordRuns writes what a send put on a machine into the assignments, once the send is recorded: for each module it carried -// a build of — by the commit — the build, what it published, the commit, the rollout and since when. **A module that runs the -// commit already keeps what it names**, filled where it names nothing and never blanked, so a send that carries it again, or a -// push by hand, changes nothing it knows. A module the machine is not assigned is no assignment to write. +// recordRuns writes what a send put on a machine into the assignments, once the send is recorded, and it is how the mesh +// knows what a machine was sent (SentBuilds). **A send that names its builds** writes, for each module it carried a build of +// — by the commit — the build, what it published, the commit, the rollout and since when, on the module's key: filled where +// the key names nothing, replaced when the commit moved, never blanked by a push by hand; a module nothing assigns, carried +// because another requires it, gets a standing key marked pulled-in; and every other key of the machine loses what it named, +// for the machine is not sent it any more. **A send that names none** (a declaration by hand) makes what the machine runs +// unknown: every key of it says so (sent-unknown) and names no commit. **An error here fails the record of the send**, for +// what a machine was sent is read from these keys and from nowhere else. func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[string]string, at time.Time) error { - c, on, err := i.ChainReads(ctx, KindAssignments) - if err != nil || !on || len(builds) == 0 { - return err + c, err := i.theChain(ctx) + if err != nil { + return fmt.Errorf("what the machine runs cannot be written: %w", err) } var node string if err := i.store.Pool().QueryRow(ctx, `select name from node where id = $1`, nodeID).Scan(&node); errors.Is(err, pgx.ErrNoRows) { @@ -93,24 +99,81 @@ func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[st return err } rollout, _ := ctx.Value(rolloutKey{}).(rolloutOf) + all, err := c.Assignments.All(ctx) + if err != nil { + return fmt.Errorf("the assignments of %s cannot be read: %w", node, err) + } + var failed []error + for _, k := range sortedValues(all) { + a := k.Value + if a.Node != node || !a.Stands() { + continue + } + if builds[a.Module] != "" { + continue // carried, and its commit is written below + } + if err := i.forgetRun(ctx, c, k, builds == nil, at); err != nil { + failed = append(failed, fmt.Errorf("%s on %s: %w", a.Module, node, err)) + } + } modules := make([]string, 0, len(builds)) for m := range builds { modules = append(modules, m) } sort.Strings(modules) - var failed []error for _, m := range modules { - commit := builds[m] - if commit == "" { + if builds[m] == "" { continue } - if err := i.recordRun(ctx, c, node, m, commit, rollout, at); err != nil { + if err := i.recordRun(ctx, c, node, m, builds[m], rollout, at); err != nil { failed = append(failed, fmt.Errorf("%s on %s: %w", m, node, err)) } } return errors.Join(failed...) } +func sortedValues(all map[string]chain.Kept[chain.Assignment]) []chain.Kept[chain.Assignment] { + out := make([]chain.Kept[chain.Assignment], 0, len(all)) + for key, k := range all { + k.Key = key + out = append(out, k) + } + sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key }) + return out +} + +// forgetRun writes that a machine is no longer sent what a key named: its links are cleared, and for a send by hand the key +// says what the machine runs is unknown. +func (i *Inventory) forgetRun(ctx context.Context, c *chain.Chain, k chain.Kept[chain.Assignment], hand bool, at time.Time) error { + for try := 0; try < casTries; try++ { + next := k.Value + next.Build, next.Artifact, next.Package, next.Commit, next.Rollout, next.Since = "", "", nil, "", "", time.Time{} + next.SentUnknown = time.Time{} + if hand { + next.SentUnknown = at.UTC() + } + if next.Build == k.Value.Build && next.Commit == k.Value.Commit && next.Rollout == k.Value.Rollout && + next.Artifact == k.Value.Artifact && next.Package == k.Value.Package && next.Since.Equal(k.Value.Since) && + next.SentUnknown.Equal(k.Value.SentUnknown) { + return nil + } + if _, err := c.Assignments.Update(ctx, k.Key, next, k.Revision); errors.Is(err, chain.ErrMoved) { + cur, found, gerr := c.Assignments.Get(ctx, k.Key) + if gerr != nil { + return gerr + } + if !found || !cur.Value.Stands() { + return nil + } + k = chain.Kept[chain.Assignment]{Key: k.Key, Value: cur.Value, Revision: cur.Revision} + continue + } else { + return err + } + } + return fmt.Errorf("the assignment kept moving under the write") +} + func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, commit string, rollout rolloutOf, at time.Time) error { key := assignmentKey(node, module) var build, rolloutID string @@ -124,11 +187,12 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, if err != nil { return err } - if !found || cur.Value.Assigned.IsZero() || !cur.Value.Stands() { - return nil // nothing assigned there to say what it runs + if found && cur.Value.State != "" { + return nil // a wait for a build: the module is not on the machine by it } - same := cur.Value.Commit == commit - if same && cur.Value.Build != "" && (rolloutID == "" || cur.Value.Rollout != "") { + standing := found && cur.Value.Stands() + same := standing && cur.Value.Commit == commit + if same && cur.Value.Build != "" && cur.Value.SentUnknown.IsZero() && (rolloutID == "" || cur.Value.Rollout != "") { return nil // a send that carries what the machine runs already says nothing new } if !resolved { @@ -148,7 +212,13 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, resolved = true } next := cur.Value - if same { + switch { + case !standing: + // Carried and assigned to nobody here: a standing key, no time assigned, marked pulled in. An ended one is + // written over, the ended one staying in history. + next = chain.Assignment{Node: node, Module: module, PulledIn: true, Commit: commit, Build: build, Rollout: rolloutID, + Since: at.UTC()} + case same: // Already running it: fill what is missing, blank nothing. if next.Build == "" { next.Build = build @@ -159,18 +229,26 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, if next.Since.IsZero() { next.Since = at.UTC() } - } else { + default: next.Commit, next.Build, next.Rollout, next.Since = commit, build, rolloutID, at.UTC() next.Artifact, next.Package = "", nil } + next.SentUnknown = time.Time{} if published != nil && next.Artifact == "" && next.Package == nil && next.Build == build { next.Artifact, next.Package = published.Artifact, published.Package } - if next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout && + if standing && next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout && next.Artifact == cur.Value.Artifact && (next.Package == nil) == (cur.Value.Package == nil) && - next.Since.Equal(cur.Value.Since) { + next.Since.Equal(cur.Value.Since) && next.SentUnknown.Equal(cur.Value.SentUnknown) { return nil } + if !found { + if _, err := c.Assignments.Create(ctx, key, next); errors.Is(err, chain.ErrExists) { + continue + } else { + return err + } + } if _, err := c.Assignments.Update(ctx, key, next, cur.Revision); errors.Is(err, chain.ErrMoved) { continue } else { @@ -180,11 +258,105 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, return fmt.Errorf("the assignment kept moving under the write") } -// saidRunsNot says a send's links could not be written, to the log of whoever sent: the send is away and is recorded, and -// a record of what it put where that could not be written is said, never a reason to call the send failed. -func saidRunsNot(node string, err error) { +// CopySentBuilds copies what delivery 1 recorded in node.sent_builds onto the keys of the assignments, once, at the start of +// the controller that holds the lease and has attached the chain (issue 1109, review round 1): delivery 1 wrote no commit onto +// the key of a module an assignment pulled in, so without it those modules would be missing from SentBuilds although the +// machine is known to have been sent. A key that names a commit is never overwritten, so it is idempotent; once a node's keys +// are written its column is emptied, so a later restart cannot lay a stale commit over a key a later send cleared or ended. +// It answers how many keys it wrote; an error is raised by the caller as a condition. +func (i *Inventory) CopySentBuilds(ctx context.Context) (int, error) { + c, err := i.theChain(ctx) if err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: %s was sent its declaration, and the build, commit and rollout of each module "+ - "could not be written in its assignments: %v\n", node, err) + return 0, err } + rows, err := i.store.Pool().Query(ctx, `select name, sent_builds from node where sent_builds is not null`) + if err != nil { + return 0, err + } + type kept struct { + name string + sent map[string]string + } + var all []kept + for rows.Next() { + var name string + var raw []byte + if err := rows.Scan(&name, &raw); err != nil { + rows.Close() + return 0, err + } + sent := map[string]string{} + if err := json.Unmarshal(raw, &sent); err != nil { + rows.Close() + return 0, fmt.Errorf("what %s was sent, in the store, cannot be read: %w", name, err) + } + all = append(all, kept{name, sent}) + } + rows.Close() + if err := rows.Err(); err != nil { + return 0, err + } + now := time.Now().UTC() + for _, k := range all { + if err := i.copySentBuildsOf(ctx, c, k.name, k.sent, now); err != nil { + return 0, fmt.Errorf("what %s was sent cannot be copied onto its assignments: %w", k.name, err) + } + if _, err := i.store.Pool().Exec(ctx, `update node set sent_builds = null where name = $1`, k.name); err != nil { + return 0, err + } + } + return len(all), nil +} + +// copySentBuildsOf is CopySentBuilds for one node, over a chain: for each module the node was sent, a standing key that +// names no commit gets it (and the build, when one is known; no rollout), and where no key exists a standing pulled-in key is +// made. A key with a commit, an ended key, a hand send's unknown and a wait for a build are left as they are. +func (i *Inventory) copySentBuildsOf(ctx context.Context, c *chain.Chain, node string, sent map[string]string, at time.Time) error { + for _, module := range slices.Sorted(maps.Keys(sent)) { + commit := sent[module] + if commit == "" { + continue + } + key := assignmentKey(node, module) + build, err := i.chainBuildOf(ctx, c, module, commit) + if err != nil { + return err + } + for try := 0; ; try++ { + if try == casTries { + return fmt.Errorf("the assignment %s kept moving under the write", key) + } + cur, found, err := c.Assignments.Get(ctx, key) + if err != nil { + return err + } + if found && (cur.Value.State != "" || !cur.Value.Ended.IsZero() || cur.Value.Commit != "" || !cur.Value.SentUnknown.IsZero()) { + break // a wait, an ended key (never revived), one that names a commit, or a hand send's unknown + } + next := cur.Value + if !found { + next = chain.Assignment{Node: node, Module: module, PulledIn: true, Since: at} + } + next.Commit, next.Build = commit, build + if next.Since.IsZero() { + next.Since = at + } + if !found { + _, err = c.Assignments.Create(ctx, key, next) + if errors.Is(err, chain.ErrExists) { + continue + } + } else { + _, err = c.Assignments.Update(ctx, key, next, cur.Revision) + if errors.Is(err, chain.ErrMoved) { + continue + } + } + if err != nil { + return err + } + break + } + } + return nil } diff --git a/internal/inventory/chain_move.go b/internal/inventory/chain_move.go deleted file mode 100644 index acf7bde5..00000000 --- a/internal/inventory/chain_move.go +++ /dev/null @@ -1,701 +0,0 @@ -package inventory - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "time" - - "github.com/jackc/pgx/v5" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/chain" -) - -// **The move** (novox/hq ADR 0408 §8, issue 981 note 7 Q2): the serving controller writes the rows of a kind into its -// bucket at its start, holding the lease and after the buckets exist, and records the kind in chain_moved. Idempotent: a -// key that exists is already moved and is never overwritten, so a bucket value written since is not laid over by the row -// it came from. It counts the rows against the keys, logs the largest value, and **refuses** (a Refusal, which the -// controller raises as an urgent condition) on a count that differs or a value over the bus's largest, with nothing -// switched: a kind is recorded moved only once all of it is in the bucket. - -// Refusal is a move that stopped without switching a kind, and why. It is permanent until the cause is put right; any -// other error of a move (the bus away, a write not granted) is tried again. -type Refusal struct { - Kind, Why string -} - -func (r *Refusal) Error() string { - return fmt.Sprintf("the move of %s to the bus is refused: %s", r.Kind, r.Why) -} - -// MoveReport is what a move of one kind did. -type MoveReport struct { - Kind string - // Rows is how many records the tables held, Written how many keys this move made, and Present how many already - // held a value (moved by an earlier start). - Rows, Written, Present int - // TableRows is how many rows of the tables those records came from: a request and its outcome are two rows and one - // record. - TableRows int - // Largest is the biggest value, in bytes. - Largest int - // Already is a kind an earlier start moved: nothing was done. - Already bool -} - -// mover is one kind of the chain: its rows, read from its tables, as the keys and values they become. -type mover struct { - kind string - // rows reads the tables: each record's key and its value, whole, and how many rows of the tables they came from. - rows func(ctx context.Context) ([]moveRow, int, error) - // create writes one record under its key; chain.ErrExists when the key holds a value. - create func(ctx context.Context, c *chain.Chain, row moveRow) error - // keys are the keys the bucket holds. - keys func(ctx context.Context, c *chain.Chain) ([]string, error) - // count is how many rows the tables hold now, counted as rows reports them. - count func(ctx context.Context, tx pgx.Tx) (int, error) - // lock keeps the tables from taking a row while the switch is recorded. - lock string - // settle runs under that lock, with the rows read again: it makes the bucket hold what the rows say now. A row the old - // controller changed after the first pass is in the bucket before the kind is recorded as moved, for before the switch - // the table is the only truth. - settle func(ctx context.Context, c *chain.Chain, rows []moveRow) error -} - -type moveRow struct { - key string - value any - size int -} - -func (i *Inventory) movers() []mover { - return []mover{{ - kind: KindBuilds, - rows: func(ctx context.Context) ([]moveRow, int, error) { - builds, err := i.allBuildRows(ctx) - if err != nil { - return nil, 0, err - } - requests, err := i.allRequestRows(ctx) - if err != nil { - return nil, 0, err - } - // A build request is the build's record in state asked, and the request stays beside the outcome that - // answered it: one record under one id (issue 1091). - byID := make(map[string]*chain.Build, len(builds)) - var order []string - for _, b := range builds { - v := toChainBuild(b) - byID[b.ID] = &v - order = append(order, b.ID) - } - for _, r := range requests { - if v, answered := byID[r.ID]; answered { - v.Request = toRequest(r) - continue - } - v := askedRecord(r) - if r.NotAsked != "" { - // Never handed over: asked again once, after the move (askAgain). - v.Request.Reask = chain.ReaskPending - } - byID[r.ID] = &v - order = append(order, r.ID) - } - out := make([]moveRow, 0, len(order)) - for _, id := range order { - body, err := json.Marshal(byID[id]) - if err != nil { - return nil, 0, err - } - out = append(out, moveRow{key: legacyBuildKey(id), value: *byID[id], size: len(body)}) - } - return out, len(builds) + len(requests), nil - }, - create: func(ctx context.Context, c *chain.Chain, row moveRow) error { - _, err := c.Builds.Create(ctx, row.key, row.value.(chain.Build)) - return err - }, - keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Builds.Keys(ctx) }, - count: func(ctx context.Context, tx pgx.Tx) (int, error) { - var n int - err := tx.QueryRow(ctx, `select (select count(*) from build) + (select count(*) from build_request)`).Scan(&n) - return n, err - }, - lock: `lock table build, build_request in share mode`, - settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error { - return settleBucket(ctx, c.Builds, rows) - }, - }, { - kind: KindRollouts, - rows: func(ctx context.Context) ([]moveRow, int, error) { - plans, err := i.plans(ctx, `order by created`) - if err != nil { - return nil, 0, err - } - merges, err := i.merges(ctx, ``) - if err != nil { - return nil, 0, err - } - tablePlans := len(plans) - if plans, err = holdMerges(plans, merges); err != nil { - return nil, 0, err - } - out := make([]moveRow, 0, len(plans)) - for _, p := range plans { - body, err := json.Marshal(p) - if err != nil { - return nil, 0, err - } - out = append(out, moveRow{key: p.ID, value: p, size: len(body)}) - } - return out, tablePlans + len(merges), nil - }, - create: func(ctx context.Context, c *chain.Chain, row moveRow) error { - _, err := chain.RolloutsOf[Plan](c).Create(ctx, row.key, row.value.(Plan)) - return err - }, - keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { - return chain.RolloutsOf[Plan](c).Keys(ctx) - }, - count: func(ctx context.Context, tx pgx.Tx) (int, error) { - var n int - err := tx.QueryRow(ctx, `select (select count(*) from release_plan) + (select count(*) from batched_merge)`).Scan(&n) - return n, err - }, - lock: `lock table release_plan, batched_merge in share mode`, - settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error { - return settleBucket(ctx, chain.RolloutsOf[Plan](c), rows) - }, - }, { - kind: KindAssignments, - rows: func(ctx context.Context) ([]moveRow, int, error) { - assigned, err := i.assignedWithSends(ctx) - if err != nil { - return nil, 0, err - } - out := make([]moveRow, 0, len(assigned)) - for _, a := range assigned { - body, err := json.Marshal(a) - if err != nil { - return nil, 0, err - } - out = append(out, moveRow{key: assignmentKey(a.Node, a.Module), value: a, size: len(body)}) - } - return out, len(out), nil - }, - create: createAssignment, - keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Assignments.Keys(ctx) }, - count: func(ctx context.Context, tx pgx.Tx) (int, error) { - var n int - err := tx.QueryRow(ctx, `select count(*) from assignment`).Scan(&n) - return n, err - }, - lock: `lock table assignment in share mode`, - settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error { - have := make(map[string]bool, len(rows)) - for _, r := range rows { - have[r.key] = true - if err := settleAssignment(ctx, c, r); err != nil { - return err - } - } - // An assignment the store no longer holds ended since the first pass: the bucket says so. - all, err := c.Assignments.All(ctx) - if err != nil { - return err - } - for key, k := range all { - if have[key] || k.Value.Assigned.IsZero() || !k.Value.Stands() { - continue - } - k.Value.Ended = time.Now().UTC() - if _, err := c.Assignments.Update(ctx, key, k.Value, k.Revision); err != nil && !errors.Is(err, chain.ErrMoved) { - return err - } - } - return nil - }, - }} -} - -// assignedWithSends is every assignment of the store as the chain keeps it: with the commit its machine was last sent for -// the module (node.sent_builds), the build that is of it and what the build published, and since when (when the machine was -// last sent). An assignment of a module the machine was never sent has none of those. -func (i *Inventory) assignedWithSends(ctx context.Context) ([]chain.Assignment, error) { - rows, err := i.store.Pool().Query(ctx, - `select n.name, a.module, a.assigned, n.sent_at, n.sent_builds - from assignment a join node n on n.id = a.node order by n.name, a.module`) - if err != nil { - return nil, err - } - var out []chain.Assignment - for rows.Next() { - var a chain.Assignment - var sentAt *time.Time - var raw []byte - if err := rows.Scan(&a.Node, &a.Module, &a.Assigned, &sentAt, &raw); err != nil { - rows.Close() - return nil, err - } - a.Assigned = a.Assigned.UTC() - var sent map[string]string - if len(raw) > 0 { - _ = json.Unmarshal(raw, &sent) - } - if commit := sent[a.Module]; commit != "" && sentAt != nil { - a.Commit, a.Since = commit, sentAt.UTC() - } - out = append(out, a) - } - rows.Close() - if err := rows.Err(); err != nil { - return nil, err - } - // The build of each commit, read once: the newest successful build of the module at it. - type at struct{ module, commit string } - var all []Build - if _, moved, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if moved { - if all, err = chainBuilds(ctx, i.attached()); err != nil { - return nil, err - } - } else if all, err = i.allBuildRows(ctx); err != nil { - return nil, err - } - newest := map[at]Build{} - for _, b := range all { - k := at{b.Module, b.Commit} - if b.Failed != "" || b.Module == "" { - continue - } - if had, has := newest[k]; !has || b.At.After(had.At) || (b.At.Equal(had.At) && b.ID > had.ID) { - newest[k] = b - } - } - for k := range out { - if out[k].Commit == "" { - continue - } - b, found := newest[at{out[k].Module, out[k].Commit}] - if !found { - continue - } - out[k].Build = b.ID - if published := publishedOf(b.Made); published != nil { - out[k].Artifact, out[k].Package = published.Artifact, published.Package - } - } - return out, nil -} - -// holdMerges puts each batched merge into the record of the plan that answers it. A merge no plan answers — the plan_id -// of a merge a failed walk carried is null, or names a plan the store never kept — is held, flagged waiting, by the -// newest plan that lists it among its merges, else the record kept for the merges no plan answers: the record that held it -// last. -func holdMerges(plans []Plan, merges []BatchedMerge) ([]Plan, error) { - waitingAt := -1 - at := make(map[string]int, len(plans)) - for k, p := range plans { - at[p.ID] = k - } - newest := func(match func(Plan) bool) int { - best := -1 - for k, p := range plans { - if match(p) && (best < 0 || p.Created.After(plans[best].Created)) { - best = k - } - } - return best - } - for _, m := range merges { - held := heldOf(m, m.Heard) - holder, answered := at[m.Plan] - if m.Plan == "" || !answered { - held.Waiting = true - holder = newest(func(p Plan) bool { - if p.Delivery == nil { - return false - } - for _, listed := range p.Delivery.Merges { - if strings.EqualFold(listed.Repository, m.Repository) && listed.Commit == m.Commit { - return true - } - } - return false - }) - if holder < 0 { - // Held by the record kept for the merges no plan answers, made with the others. - if waitingAt < 0 { - plans = append(plans, Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1, - Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"}) - waitingAt = len(plans) - 1 - } - holder = waitingAt - } - } - plans[holder].Held = append(plans[holder].Held, held) - } - return plans, nil -} - -// MoveChain moves every kind not moved yet, in order, and says what it did. It needs the chain attached and the lease -// held (the chain's writes ask). It stops at the first kind that fails. -func (i *Inventory) MoveChain(ctx context.Context, say func(format string, args ...any)) ([]MoveReport, error) { - if say == nil { - say = func(string, ...any) {} - } - if i.attached() == nil { - return nil, ErrNoChain - } - var out []MoveReport - for _, m := range i.movers() { - r, err := i.moveKind(ctx, m, say) - if err != nil { - return out, err - } - out = append(out, r) - if m.kind == KindBuilds { - // The builds are laid out under their modules before anything reads them (issue 1121). - if err := i.moveBuildsLayout(ctx, say); err != nil { - return out, err - } - } - } - return out, nil -} - -// movePasses is how often a move reads the tables again when they took a row while it wrote: the controller before this -// one may still have been writing as the lease changed hands. -const movePasses = 5 - -func (i *Inventory) moveKind(ctx context.Context, m mover, say func(string, ...any)) (MoveReport, error) { - report := MoveReport{Kind: m.kind} - moved, err := i.ChainMoved(ctx) - if err != nil { - return report, err - } - if _, done := moved[m.kind]; done { - report.Already = true - return report, nil - } - for pass := 1; pass <= movePasses; pass++ { - rows, tableRows, err := m.rows(ctx) - if err != nil { - return report, fmt.Errorf("the %s cannot be read from the tables: %w", m.kind, err) - } - report.Rows, report.TableRows, report.Largest = len(rows), tableRows, 0 - // Every value measured before any is written: a value over the bus's largest stops the move with nothing - // written (ADR 0408 §6), never cut. - for _, r := range rows { - report.Largest = max(report.Largest, r.size) - if r.size > broker.ChainValueBytes { - return report, &Refusal{m.kind, fmt.Sprintf("the value of %s is %d bytes and the bus holds at most %d; "+ - "nothing is cut, so the record must be made smaller first", r.key, r.size, broker.ChainValueBytes)} - } - } - report.Written, report.Present = 0, 0 - for _, r := range rows { - switch err := m.create(ctx, i.attached(), r); { - case err == nil: - report.Written++ - case errors.Is(err, chain.ErrExists): - report.Present++ - case errors.Is(err, chain.ErrBadKey): - return report, &Refusal{m.kind, fmt.Sprintf("%s cannot be a key of the bus: %v", r.key, err)} - default: - return report, fmt.Errorf("%s %s is not written to the bus: %w", m.kind, r.key, err) - } - } - // Rows against keys: every row's key is in the bucket. - keys, err := m.keys(ctx, i.attached()) - if err != nil { - return report, fmt.Errorf("the keys of the %s cannot be read back: %w", m.kind, err) - } - have := make(map[string]bool, len(keys)) - for _, k := range keys { - have[k] = true - } - for _, r := range rows { - if !have[r.key] { - return report, &Refusal{m.kind, fmt.Sprintf("the tables hold %d records and the bucket does not hold %s", len(rows), r.key)} - } - } - settled, err := i.switchKind(ctx, m, report) - if err != nil { - return report, err - } - if settled { - say("moved the %s to the bus: %d records, %d written now and %d already there, the largest %d bytes", - m.kind, report.Rows, report.Written, report.Present, report.Largest) - return report, nil - } - say("the %s took a row while they were moved, so they are read again (pass %d of %d)", m.kind, pass, movePasses) - } - return report, &Refusal{m.kind, fmt.Sprintf("the tables kept taking rows through %d passes; another controller is still writing them", movePasses)} -} - -// switchKind records the kind as moved, in one transaction that holds the tables against writes and finds their count -// as the move read it: false when they took a row since. -func (i *Inventory) switchKind(ctx context.Context, m mover, r MoveReport) (bool, error) { - committed := false - tx, err := i.store.Pool().Begin(ctx) - if err != nil { - return false, err - } - defer func() { _ = tx.Rollback(ctx) }() - if _, err := tx.Exec(ctx, m.lock); err != nil { - return false, err - } - n, err := m.count(ctx, tx) - if err != nil { - return false, err - } - if n != r.TableRows { - return false, nil - } - // The store's mirror of the assignments writes nothing from here: it would lay values without the links over the keys - // the settle writes and end the ones it ended. - if m.kind == KindAssignments { - i.moved.switching.Store(true) - defer func() { - if !committed { - i.moved.switching.Store(false) - } - }() - } - // The rows read again under the lock: nothing writes them now, so what the bucket holds is made what they say. - rows, tableRows, err := m.rows(ctx) - if err != nil { - return false, err - } - if tableRows != r.TableRows { - return false, nil - } - if err := m.settle(ctx, i.attached(), rows); err != nil { - return false, fmt.Errorf("the %s changed since they were first written, and the bucket cannot be brought to them: %w", m.kind, err) - } - counts, _ := json.Marshal(map[string]int{m.kind: r.Rows}) - largest, _ := json.Marshal(map[string]int{m.kind: r.Largest}) - if _, err := tx.Exec(ctx, - `insert into chain_moved (counts, largest) values ($1, $2) - on conflict (only_one) do update - set counts = chain_moved.counts || excluded.counts, largest = chain_moved.largest || excluded.largest, - moved_at = now()`, counts, largest); err != nil { - return false, err - } - if err := tx.Commit(ctx); err != nil { - return false, err - } - committed = true - i.moved.mu.Lock() - if i.moved.kind == nil { - i.moved.kind = map[string]bool{} - } - i.moved.kind[m.kind] = true - i.moved.mu.Unlock() - return true, nil -} - -// allBuildRows is every build of the table, whole. -func (i *Inventory) allBuildRows(ctx context.Context) ([]Build, error) { - rows, err := i.store.Pool().Query(ctx, - `select id, repository, ref, coalesce(module, ''), commit_hash, built_on, failed, made, source_path, manifest, - built_against, built_contexts, build_sources, source_fingerprint, asked, at - from build order by at, id`) - if err != nil { - return nil, err - } - defer rows.Close() - var out []Build - for rows.Next() { - var b Build - var made, manifest, against, read, sources []byte - var asked *time.Time - if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, &b.On, &b.Failed, &made, &b.Path, - &manifest, &against, &read, &sources, &b.SourceFingerprint, &asked, &b.At); err != nil { - return nil, err - } - if asked != nil { - b.Asked = *asked - } - b.Manifest = manifest - for _, part := range []struct { - raw []byte - to any - who string - }{{made, &b.Made, "made"}, {against, &b.Against, "built_against"}, {read, &b.Read, "built_contexts"}, - {sources, &b.Sources, "build_sources"}} { - if len(part.raw) == 0 { - continue - } - if err := json.Unmarshal(part.raw, part.to); err != nil { - return nil, fmt.Errorf("build %s: %s is not readable: %w", b.ID, part.who, err) - } - } - out = append(out, b) - } - return out, rows.Err() -} - -// allRequestRows is every build request of the table. -func (i *Inventory) allRequestRows(ctx context.Context) ([]BuildRequest, error) { - rows, err := i.store.Pool().Query(ctx, - `select id, repository, seat, source_path, ref, commit_hash, asked_for, coalesce(not_asked, ''), - coalesce(outcome_unknown, ''), asked_at, at_terminal - from build_request order by asked_at, id`) - if err != nil { - return nil, err - } - defer rows.Close() - var out []BuildRequest - for rows.Next() { - var a BuildRequest - if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked, - &a.OutcomeUnknown, &a.At, &a.AtTerminal); err != nil { - return nil, err - } - out = append(out, a) - } - return out, rows.Err() -} - -// createAssignment writes an assignment's row under its key; chain.ErrExists when the key holds a value that stays. -func createAssignment(ctx context.Context, c *chain.Chain, row moveRow) error { - want := row.value.(chain.Assignment) - // The mirror of the store kept a value under this key until now, with the provisioner's plan beside it: the - // plan stays, the record's links are laid in. A value that names its build already is a later word, and stays. - for try := 0; try < casTries; try++ { - cur, found, err := c.Assignments.Get(ctx, row.key) - if err != nil { - return err - } - if !found { - if _, err := c.Assignments.Create(ctx, row.key, want); !errors.Is(err, chain.ErrExists) { - return err - } - continue - } - if cur.Value.Build != "" || cur.Value.Commit != "" || !cur.Value.Since.IsZero() || (want.Build == "" && want.Commit == "") { - return chain.ErrExists - } - merged := want - merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning - if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) { - continue - } else { - return err - } - } - return fmt.Errorf("the assignment %s kept moving under the write", row.key) -} - -// settleBucket makes a bucket of whole values hold each row's value, by compare-and-set: a key that is absent is made, one -// whose bytes differ from the row's is written over, and one that is the same is left. -func settleBucket[T any](ctx context.Context, b chain.Bucket[T], rows []moveRow) error { - for _, r := range rows { - want, err := json.Marshal(r.value) - if err != nil { - return err - } - settled := false - for try := 0; try < casTries && !settled; try++ { - e, found, err := b.GetRaw(ctx, r.key) - if err != nil { - return err - } - switch { - case !found: - _, err = b.CreateRaw(ctx, r.key, want) - if errors.Is(err, chain.ErrExists) { - continue - } - case bytes.Equal(e.Value, want): - settled = true - continue - default: - _, err = b.UpdateRaw(ctx, r.key, want, e.Revision) - if errors.Is(err, chain.ErrMoved) { - continue - } - } - if err != nil { - return err - } - settled = true - } - if !settled { - return fmt.Errorf("%s kept moving under the write", r.key) - } - } - return nil -} - -// settleAssignment makes the value under an assignment's key what its row says, keeping the plan the mirror put beside it: -// before the switch the store is the only truth of the links, so a commit it changed since the first pass is written over. -func settleAssignment(ctx context.Context, c *chain.Chain, row moveRow) error { - want := row.value.(chain.Assignment) - for try := 0; try < casTries; try++ { - cur, found, err := c.Assignments.Get(ctx, row.key) - if err != nil { - return err - } - if !found { - if _, err := c.Assignments.Create(ctx, row.key, want); errors.Is(err, chain.ErrExists) { - continue - } else { - return err - } - } - merged := want - merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning - a, _ := json.Marshal(merged) - b, _ := json.Marshal(cur.Value) - if bytes.Equal(a, b) { - return nil - } - if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) { - continue - } else { - return err - } - } - return fmt.Errorf("the assignment %s kept moving under the write", row.key) -} - -// moveBuildsLayout lays the builds out under their modules once, and records the kind builds-layout in chain_moved. A key -// the old layout made stays (ADR 0408 §4). -func (i *Inventory) moveBuildsLayout(ctx context.Context, say func(string, ...any)) error { - moved, err := i.ChainMoved(ctx) - if err != nil { - return err - } - if _, done := moved[KindBuildsLayout]; done { - return nil - } - report, err := i.layoutBuilds(ctx, i.attached(), say) - if err != nil { - return err - } - counts, _ := json.Marshal(map[string]int{KindBuildsLayout: report.Records}) - largest, _ := json.Marshal(map[string]int{KindBuildsLayout: 0}) - if _, err := i.store.Pool().Exec(ctx, - `insert into chain_moved (counts, largest) values ($1, $2) - on conflict (only_one) do update - set counts = chain_moved.counts || excluded.counts, largest = chain_moved.largest || excluded.largest, - moved_at = now()`, counts, largest); err != nil { - return err - } - i.moved.mu.Lock() - if i.moved.kind == nil { - i.moved.kind = map[string]bool{} - } - i.moved.kind[KindBuildsLayout] = true - i.moved.mu.Unlock() - return nil -} diff --git a/internal/inventory/chain_move_test.go b/internal/inventory/chain_move_test.go deleted file mode 100644 index ab3297e8..00000000 --- a/internal/inventory/chain_move_test.go +++ /dev/null @@ -1,950 +0,0 @@ -//go:build integration - -package inventory - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "reflect" - "slices" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/chain" -) - -// A build row as the store held it before the move: written by SQL, the only hand-written state these tests have, for -// it stands for what the mesh held before this release. -func aBuildRow(t *testing.T, inv *Inventory, id, module, commit, failed string) { - t.Helper() - inv.ExecForTest(t, - `insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, source_path, manifest, - built_against, built_contexts, asked, source_fingerprint, build_sources) - values ($1::text, 'novox/' || $2::text, 'main', $2::text, $3::text, 'ace', $4::text, - case when $4::text = '' - then ('[{"name":"app","kind":"image","reference":"artifact-store://' || $2::text || '@sha256:' || $1::text || '"}]')::jsonb - else '[]'::jsonb end, - 'modules/' || $2::text, ('{"module":"' || $2::text || '"}')::jsonb, - '["artifact-store://base@sha256:aa"]'::jsonb, '[{"repository":"novox/other","ref":"main"}]'::jsonb, - now() - interval '1 hour', 'fp-' || $1::text, - ('[{"paths":["modules/' || $2::text || '/module.json"]}]')::jsonb)`, - id, module, commit, failed) -} - -// inTheTables is an inventory before the move: its builds and assignments are rows of the store, which a test of a data -// migration of the store (that reads those rows) needs. -func inTheTables(t *testing.T) *Inventory { - t.Helper() - inv, finish := ForTestBefore(t, 105) - finish() - return inv -} - -// reportOf is the report of one kind among those a move made, which are one for each kind in the order they are moved. -func reportOf(t *testing.T, reports []MoveReport, kind string) MoveReport { - t.Helper() - for _, r := range reports { - if r.Kind == kind { - return r - } - } - t.Fatalf("no report of %s among %+v", kind, reports) - return MoveReport{} -} - -// allAlready says every kind was moved by an earlier start: the second run writes nothing, and reports one for each kind. -func allAlready(t *testing.T, reports []MoveReport) { - t.Helper() - if len(reports) != 3 { - t.Fatalf("the move reported %d kinds, want 3: %+v", len(reports), reports) - } - for _, r := range reports { - if !r.Already || r.Written != 0 { - t.Fatalf("the second move of %s %+v", r.Kind, r) - } - } -} - -func rowsInTheTable(t *testing.T, inv *Inventory, table string) string { - t.Helper() - rows := inv.QueryForTest(t, `select count(*) from `+table) - return rows[0][0] -} - -// **The move copies every build row into the builds bucket, and a second run writes nothing** (ADR 0408 §8, issue -// 1090): each row under its id, whole; the kind recorded in chain_moved; and from then on a build is read from the -// bucket and written to it, the table taking no more. -func TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - aBuildRow(t, inv, "build-2", "notes", "c2", "") - aBuildRow(t, inv, "build-3", "wiki", "", "compile error") - - // Before the move the tables are read, and the bucket is empty. - if before, err := inv.Builds(ctx, "", 10); err != nil || len(before) != 3 { - t.Fatalf("before the move the tables hold %d builds: %v", len(before), err) - } - if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { - t.Fatalf("moved before the move: %v", moved) - } - - reports, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatal(err) - } - if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.Written != 3 || b.Largest == 0 { - t.Fatalf("the move of the builds reported %+v", reports) - } - moved, err := inv.ChainMoved(ctx) - if err != nil || moved[KindBuilds] != 3 { - t.Fatalf("chain_moved says %v: %v", moved, err) - } - kept, found, err := inv.chain.Builds.Get(ctx, "build-2") - if err != nil || !found { - t.Fatalf("build-2 is not in the bucket: %v %v", found, err) - } - if v := kept.Value; v.Module != "notes" || v.Commit != "c2" || v.Repository != "novox/notes" || v.Path != "modules/notes" || - v.SourceFingerprint != "fp-build-2" || len(v.Made) != 1 || len(v.Against) != 1 || len(v.Read) != 1 || len(v.Sources) != 1 || - v.Published == nil || v.Published.Artifact != "sha256:build-2" || v.Asked.IsZero() || v.At.IsZero() || len(v.Manifest) == 0 { - t.Fatalf("build-2 was moved as %+v", v) - } - - // From now on the bucket is what is read and written. - if err := inv.RecordBuild(ctx, aBuild("build-4", "notes", "")); err != nil { - t.Fatal(err) - } - if n := rowsInTheTable(t, inv, "build"); n != "3" { - t.Fatalf("the table took a build after the move: %s rows", n) - } - after, err := inv.Builds(ctx, "notes", 10) - if err != nil || len(after) != 3 { - t.Fatalf("notes has %d builds on the bus: %v", len(after), err) - } - - // A second run finds the kind moved and writes nothing: every key still has the one value it was made with. - again, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatal(err) - } - allAlready(t, again) - for _, key := range []string{legacyBuildKey("build-1"), legacyBuildKey("build-2"), legacyBuildKey("build-3"), - chain.BuildKey("notes", "build-4")} { - if history, err := inv.chain.Builds.History(ctx, key); err != nil || len(history) != 1 { - t.Fatalf("%s has %d values: %v", key, len(history), err) - } - } -} - -// failingToCreate is a bucket that fails to make one key while told to: the bus going away half way through a move. -type failingToCreate struct { - chain.Backend - key string - fail *atomic.Bool -} - -func (f failingToCreate) Create(ctx context.Context, key string, value []byte) (uint64, error) { - if key == f.key && f.fail.Load() { - return 0, errors.New("the bus went away") - } - return f.Backend.Create(ctx, key, value) -} - -// **A controller restarted half way through the move finishes it: the bucket equals the table, with no key made twice and -// no refusal** (ADR 0408 §8). Before a kind is recorded as moved nothing but the move writes its keys, and every write of -// the kind goes to its table (each method of a kind asks ChainReads first and writes the store until it says moved), so the -// table is the only truth: a key the first start made is the row it was, and the second start settles it to the row it is. -func TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - aBuildRow(t, inv, "build-2", "notes", "c2", "") - aBuildRow(t, inv, "build-3", "wiki", "c3", "") - var away atomic.Bool - away.Store(true) - builds := chain.NewMemory() - inv.UseChain(chain.New(failingToCreate{builds, "build-3", &away}, chain.NewMemory(), chain.NewMemory(), - func(context.Context) error { return nil })) - // The first start made two of the three keys and the bus went away. - _, err := inv.MoveChain(ctx, nil) - var refusal *Refusal - if err == nil || errors.As(err, &refusal) { - t.Fatalf("the first start answered %v: it should fail to be tried again, not be refused", err) - } - if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { - t.Fatalf("a move that did not finish recorded %v", moved) - } - if keys, _ := builds.Keys(ctx); len(keys) != 2 { - t.Fatalf("the first start made %v", keys) - } - // The controller is started again, the bus is back, and the move finishes. - away.Store(false) - reports, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatalf("the second start was refused or failed: %v", err) - } - if b := reportOf(t, reports, KindBuilds); b.Written != 1 || b.Present != 2 || b.Rows != 3 { - t.Fatalf("the move reported %+v", b) - } - if moved, _ := inv.ChainMoved(ctx); moved[KindBuilds] != 3 { - t.Fatalf("chain_moved says %v", moved) - } - // The bucket is the table: three keys, each made once and not again, each the row's own. - for _, id := range []string{"build-1", "build-2", "build-3"} { - history, err := builds.History(ctx, id) - if err != nil || len(history) != 1 { - t.Fatalf("%s has %d values: %v", id, len(history), err) - } - } - rows, err := inv.allBuildRows(ctx) - if err != nil || len(rows) != 3 { - t.Fatalf("the table holds %d builds: %v", len(rows), err) - } - for _, row := range rows { - got, found, err := inv.chain.Builds.Get(ctx, row.ID) - want, _ := json.Marshal(toChainBuild(row)) - have, _ := json.Marshal(got.Value) - if err != nil || !found || string(want) != string(have) { - t.Fatalf("%s in the bucket is %s, the row says %s (%v)", row.ID, have, want, err) - } - } -} - -// **A value over the bus's largest message stops the move with nothing written and nothing switched** (ADR 0408 §6): -// a Refusal, no key in the bucket, no kind recorded, and the tables still what is read. -func TestTheMoveRefusesAValueOverTheLargestMessage(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - inv.ExecForTest(t, `insert into build (id, repository, module, commit_hash, manifest) - values ('build-big', 'novox/wiki', 'wiki', 'c9', ('{"x":"' || repeat('a', 1100000) || '"}')::jsonb)`) - _, err := inv.MoveChain(ctx, nil) - var refusal *Refusal - if !errors.As(err, &refusal) || refusal.Kind != KindBuilds { - t.Fatalf("the move answered %v", err) - } - if keys, _ := inv.chain.Builds.Keys(ctx); len(keys) != 0 { - t.Fatalf("the bucket holds %v after a refused move", keys) - } - if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { - t.Fatalf("a refused move recorded %v", moved) - } - if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 2 { - t.Fatalf("the tables are no longer read: %d builds, %v", len(builds), err) - } -} - -// losing is a bucket that says it wrote a key and kept nothing: the move's count of keys against rows must see it. -type losing struct { - chain.Backend - lose string -} - -func (l losing) Create(ctx context.Context, key string, value []byte) (uint64, error) { - if key == l.lose { - return 1, nil - } - return l.Backend.Create(ctx, key, value) -} - -// **The move refuses on a count that differs** (ADR 0408 §8): a row whose key is not in the bucket afterwards stops it, -// nothing is recorded, and the tables are still read. -func TestTheMoveRefusesOnACountThatDiffers(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - aBuildRow(t, inv, "build-2", "notes", "c2", "") - inv.UseChain(chain.New(losing{chain.NewMemory(), "build-2"}, chain.NewMemory(), chain.NewMemory(), - func(context.Context) error { return nil })) - _, err := inv.MoveChain(ctx, nil) - var refusal *Refusal - if !errors.As(err, &refusal) { - t.Fatalf("the move answered %v", err) - } - if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { - t.Fatalf("a refused move recorded %v", moved) - } -} - -// **Without the lease the move writes nothing**, and fails the way the bus being away does — tried again, not refused: -// the tables are still read. -func TestTheMoveWithoutTheLeaseWritesNothing(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - mem := chain.NewMemory() - inv.UseChain(chain.New(mem, chain.NewMemory(), chain.NewMemory(), - func(context.Context) error { return errors.New("another controller holds the lease") })) - _, err := inv.MoveChain(ctx, nil) - var refusal *Refusal - if err == nil || errors.As(err, &refusal) || !errors.Is(err, chain.ErrNotHeld) { - t.Fatalf("the move answered %v", err) - } - if mem.Writes() != 0 { - t.Fatalf("the bucket took %d writes without the lease", mem.Writes()) - } - if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { - t.Fatalf("recorded %v", moved) - } -} - -// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its -// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none. -func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) { - inv := fresh(t) - ctx := context.Background() - first := aBuild("build-1", "notes", "") - first.Commit = "c1" - first.Against = []string{"artifact-store://base@sha256:aa"} - first.ReuseKey = "rk-1" - second := aBuild("build-2", "notes", "") - second.Commit = "c2" - failed := aBuild("build-3", "notes", "compile error") - other := aBuild("build-4", "wiki", "") - for _, b := range []Build{first, second, failed, other, first} { - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - if n := rowsInTheTable(t, inv, "build"); n != "0" { - t.Fatalf("the table took %s builds", n) - } - kept, found, err := inv.chain.Builds.Get(ctx, chain.BuildKey("notes", "build-1")) - if err != nil || !found || kept.Value.ReuseKey != "rk-1" { - t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err) - } - if history, _ := inv.chain.Builds.History(ctx, chain.BuildKey("notes", "build-1")); len(history) != 1 { - t.Fatalf("a build recorded twice has %d values", len(history)) - } - list, err := inv.Builds(ctx, "notes", 10) - if err != nil || len(list) != 3 { - t.Fatalf("notes has %d builds: %v", len(list), err) - } - one, found, err := inv.BuildByID(ctx, "build-2") - if err != nil || !found || one.Commit != "c2" || one.Module != "notes" { - t.Fatalf("build-2 reads as %+v %v %v", one, found, err) - } - if _, found, _ := inv.BuildByID(ctx, "build-nope"); found { - t.Fatal("a build nobody recorded was found") - } - if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" { - t.Fatalf("the build of notes at c1 is %q: %v", id, err) - } - held, err := inv.Held(ctx) - if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" { - t.Fatalf("held %v: %v", held, err) - } - against, err := inv.BuiltAgainst(ctx) - if err != nil || len(against["notes"]) != 0 { - // The newest successful build of notes, build-2, stood on nothing. - t.Fatalf("built against %v: %v", against, err) - } - announced, err := inv.Announceable(ctx) - if err != nil || len(announced) != 3 { - t.Fatalf("announceable %d: %v", len(announced), err) - } - prints, err := inv.Fingerprints(ctx) - if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" { - t.Fatalf("fingerprints %v: %v", prints, err) - } -} - -// A build request row as the store held it before the move. -func aRequestRow(t *testing.T, inv *Inventory, id, notAsked string) { - t.Helper() - inv.ExecForTest(t, - `insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at) - values ($1::text, 'novox/notes', '', 'modules/notes', 'main', 'c1', 'merge', nullif($2::text, ''), now() - interval '1 hour')`, - id, notAsked) -} - -// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue -// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none -// after the outcome is heard. The table of requests takes none. -func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) { - inv := fresh(t) - ctx := context.Background() - ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true} - if err := inv.RecordBuildRequest(ctx, ask); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuildRequest(ctx, ask); err != nil { - t.Fatalf("a request kept twice: %v", err) - } - if n := rowsInTheTable(t, inv, "build_request"); n != "0" { - t.Fatalf("the table took %s requests", n) - } - if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 { - t.Fatalf("a request nobody answered is a build: %+v %v", builds, err) - } - if _, found, _ := inv.BuildByID(ctx, "build-1"); found { - t.Fatal("a request nobody answered has an outcome") - } - got, found, err := inv.BuildRequestByID(ctx, "build-1") - if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() { - t.Fatalf("the request reads back as %+v %v %v", got, found, err) - } - if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil { - t.Fatal(err) - } - asked, err := inv.RequestsNamed(ctx, "notes") - if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" { - t.Fatalf("the request is %+v: %v", asked, err) - } - if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) { - t.Fatalf("requested names %v: %v", names, err) - } - - // The outcome comes: the same key, now a build, with the request beside it. - built := aBuild("build-1", "notes", "") - if err := inv.RecordBuild(ctx, built); err != nil { - t.Fatal(err) - } - if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" { - t.Fatalf("the outcome reads as %+v %v %v", one, found, err) - } - answered, err := inv.RequestsNamed(ctx, "notes") - if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" { - t.Fatalf("the request answered is %+v: %v", answered, err) - } - if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" { - t.Fatalf("the request was lost with the outcome: %+v", again) - } - // Heard: the marks of an unanswered request change nothing. - if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil { - t.Fatal(err) - } - if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" { - t.Fatalf("an answered request was marked not asked: %+v", again) - } - if history, _ := inv.chain.Builds.History(ctx, chain.RequestKey("build-1")); len(history) != 3 { - // asked, marked unknown, answered. - t.Fatalf("the record has %d values", len(history)) - } - // An outcome heard before its request is kept, and the request lands beside it. - if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil { - t.Fatal(err) - } - if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" { - t.Fatalf("the request kept after its outcome reads %+v", late) - } - // A request older than the time they are kept is no longer listed. - if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old", - At: time.Now().Add(-2 * KeptFor)}); err != nil { - t.Fatal(err) - } - if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") { - t.Fatalf("a request past its time is listed: %v", names) - } -} - -// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a -// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it -// is asked and so asked at most once. -func TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - aRequestRow(t, inv, "build-1", "") - aRequestRow(t, inv, "build-2", "") - aRequestRow(t, inv, "not-asked-c1-modules-notes", "no seat holder") - - if requests, err := inv.RequestsNamed(ctx, "notes"); err != nil || len(requests) != 3 { - t.Fatalf("before the move the table holds %d requests: %v", len(requests), err) - } - reports, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatal(err) - } - if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.TableRows != 4 { - t.Fatalf("the move reported %+v", b) - } - one, _, _ := inv.chain.Builds.Get(ctx, "build-1") - if one.Value.State != "" || one.Value.Request == nil || one.Value.Request.For != "merge" || one.Value.Module != "notes" { - t.Fatalf("an answered request moved as %+v", one.Value) - } - two, _, _ := inv.chain.Builds.Get(ctx, "build-2") - if two.Value.State != chain.BuildAsked || two.Value.Request.Reask != "" { - t.Fatalf("an unanswered request moved as %+v", two.Value) - } - never, _, _ := inv.chain.Builds.Get(ctx, "not-asked-c1-modules-notes") - if never.Value.State != chain.BuildAsked || never.Value.Request.NotAsked != "no seat holder" || never.Value.Request.Reask != chain.ReaskPending { - t.Fatalf("a request never handed over moved as %+v", never.Value) - } - requests, err := inv.RequestsNamed(ctx, "notes") - if err != nil || len(requests) != 3 { - t.Fatalf("after the move the bucket holds %d requests: %v", len(requests), err) - } - toAsk, err := inv.RequestsToAskAgain(ctx) - if err != nil || len(toAsk) != 1 || toAsk[0].ID != "not-asked-c1-modules-notes" { - t.Fatalf("to ask again: %+v %v", toAsk, err) - } - first, err := inv.ClaimToAskAgain(ctx, toAsk[0].ID) - second, err2 := inv.ClaimToAskAgain(ctx, toAsk[0].ID) - if err != nil || err2 != nil || !first || second { - t.Fatalf("claimed %v then %v: %v %v", first, second, err, err2) - } - if left, _ := inv.RequestsToAskAgain(ctx); len(left) != 0 { - t.Fatalf("a claimed request is still to be asked: %+v", left) - } - if err := inv.AskedAgainAs(ctx, toAsk[0].ID, "build-9"); err != nil { - t.Fatal(err) - } - if done, _, _ := inv.BuildRequestByID(ctx, toAsk[0].ID); done.NotAsked != "no seat holder" { - t.Fatalf("the request lost its words: %+v", done) - } -} - -// **The move carries every plan with the merges it answers, and the plans read the same from the bucket as from the -// table** (ADR 0408 §8, ADR 0276, issue 1092): a batch with a merge, a walk that was built and rolled, a merge waiting -// for a walk of its own, and a merge whose plan the store never kept — each in the record of the plan that held it last. -func TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - batch := aBatch("plan-1", time.Now().Add(-3*time.Hour).UTC()) - walk := aBatch("plan-2", time.Now().Add(-2*time.Hour).UTC()) - walk.State, walk.Tier, walk.Tiers = PlanRolling, 1, [][]string{{"a"}, {"b"}} - walk.Modules = map[string]*PlanModule{"a": {State: "built"}, "b": {}} - walk.Delivery = &PlanDelivery{Awaits: "mesh-delivery", Merges: []PlanMerge{{Repository: "novox/mesh-catalog", Commit: "m-waiting"}}} - for _, p := range []*Plan{batch, walk} { - if err := inv.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - } - walk.Note = "second save" - if err := inv.SavePlan(ctx, walk); err != nil { - t.Fatal(err) - } - for _, m := range []BatchedMerge{aMerge("m-batch", "plan-1"), aMerge("m-walk", "plan-2"), aMerge("m-dangling", "plan-never-kept")} { - if added, err := inv.AddMerge(ctx, m); err != nil || !added { - t.Fatalf("added %v: %v", added, err) - } - } - waiting := aMerge("m-waiting", "plan-2") - if _, err := inv.AddMerge(ctx, waiting); err != nil { - t.Fatal(err) - } - if err := inv.AnswerMerge(ctx, waiting.Repository, waiting.Commit, "", true); err != nil { - t.Fatal(err) - } - wasPlans, err := inv.RecentPlans(ctx, 10) - if err != nil || len(wasPlans) != 2 { - t.Fatalf("the table holds %d plans: %v", len(wasPlans), err) - } - wasMerge, _, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk") - - reports, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatal(err) - } - var moved MoveReport - for _, r := range reports { - if r.Kind == KindRollouts { - moved = r - } - } - // Two plans, and the record of the merge whose plan the store never kept. - if moved.Rows != 3 || moved.TableRows != 6 || moved.Written != 3 { - t.Fatalf("the move of the rollouts reported %+v", moved) - } - nowPlans, err := inv.RecentPlans(ctx, 10) - if err != nil || len(nowPlans) != 2 { - t.Fatalf("the bucket holds %d plans: %v", len(nowPlans), err) - } - for k := range wasPlans { - was, now := wasPlans[k], nowPlans[k] - if was.ID != now.ID || was.State != now.State || was.Tier != now.Tier || was.Revision != now.Revision || was.Note != now.Note || - !was.Created.Equal(now.Created) || !reflect.DeepEqual(was.Tiers, now.Tiers) || len(was.Modules) != len(now.Modules) { - t.Errorf("plan %s was %+v and is %+v", was.ID, was, now) - } - } - if open, _ := inv.OpenPlans(ctx); len(open) != 1 || open[0].ID != "plan-2" { - t.Fatalf("open plans %+v", open) - } - if batches, _ := inv.Batches(ctx); len(batches) != 1 || batches[0].ID != "plan-1" { - t.Fatalf("batches %+v", batches) - } - if got, _ := inv.MergesOf(ctx, "plan-1"); len(got) != 1 || got[0].Commit != "m-batch" { - t.Fatalf("the batch answers %+v", got) - } - nowMerge, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk") - if !known || nowMerge.Plan != "plan-2" || nowMerge.Branch != wasMerge.Branch || !nowMerge.Merged.Equal(wasMerge.Merged) || - !nowMerge.Heard.Equal(wasMerge.Heard) || !sameEvent(nowMerge.Event, wasMerge.Event) { - t.Fatalf("a merge was %+v and is %+v", wasMerge, nowMerge) - } - alone, _ := inv.AloneMerges(ctx) - if len(alone) != 1 || alone[0].Commit != "m-waiting" || alone[0].Plan != "" { - t.Fatalf("waiting: %+v", alone) - } - if dangling, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-dangling"); !known || dangling.Plan != "" || dangling.Alone { - t.Fatalf("a merge whose plan was never kept reads as %+v %v", dangling, known) - } - if n := rowsInTheTable(t, inv, "release_plan"); n != "2" { - t.Fatalf("the table is %s plans", n) - } - // From now on a plan is saved to the bucket and the table takes none. - later := aBatch("plan-3", time.Now().UTC()) - if err := inv.SavePlan(ctx, later); err != nil { - t.Fatal(err) - } - if n := rowsInTheTable(t, inv, "release_plan"); n != "2" { - t.Fatalf("the table took a plan after the move: %s", n) - } - again, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatal(err) - } - allAlready(t, again) -} - -// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a -// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module -// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the -// assignment goes with it. -func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { - inv, node := aNodeWithModules(t, "postgres", "web") - ctx := t.Context() - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - start := generationNow(t, inv) - if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added { - t.Fatalf("assigned %v: %v", added, err) - } - if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added { - t.Fatalf("assigned again %v: %v", added, err) - } - if n := rowsInTheTable(t, inv, "assignment"); n != "0" { - t.Fatalf("the table took %s assignments", n) - } - kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres") - if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() { - t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err) - } - if generationNow(t, inv) <= start { - t.Fatal("an assignment left the generation where it was") - } - if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) { - t.Fatalf("a module the mesh does not know: %v", err) - } - if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) { - t.Fatalf("assigned on %s: %v %v", node, on, err) - } - if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) { - t.Fatalf("running: %v %v", running, err) - } - entries, err := inv.Catalogued(ctx) - if err != nil { - t.Fatal(err) - } - for _, e := range entries { - switch e.Manifest.Module { - case "postgres": - if !reflect.DeepEqual(e.On, []string{node}) { - t.Errorf("postgres is on %v", e.On) - } - case "web": - if len(e.On) != 0 { - t.Errorf("web is on %v", e.On) - } - } - } - if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) { - t.Fatalf("a module still assigned was forgotten: %v", err) - } - // A seat is held by an assignment, or by nothing. - if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil { - t.Fatal("a seat was handed to a module that is not assigned") - } - if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil { - t.Fatalf("a seat handed to an assignment: %v", err) - } - - before := generationNow(t, inv) - if err := inv.Unassign(ctx, node, "postgres"); err != nil { - t.Fatal(err) - } - if generationNow(t, inv) <= before { - t.Fatal("an unassignment left the generation where it was") - } - // No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history. - gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres") - if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() { - t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err) - } - if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() || - history[1].Value.Stands() { - t.Fatalf("the earlier values of the key: %+v", history) - } - if on, _ := inv.Assigned(ctx, node); len(on) != 0 { - t.Fatalf("an ended assignment is still assigned: %v", on) - } - if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 { - t.Fatalf("an ended assignment is running: %v", running) - } - if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 { - t.Fatalf("the seat the assignment held is held by %+v: %v", held, err) - } - if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" { - t.Fatalf("an unassignment of nothing: %v", err) - } - if err := inv.ForgetModule(ctx, "postgres"); err != nil { - t.Fatalf("a module nothing is assigned could not be forgotten: %v", err) - } - // Made again after it ended: a fresh value over the ended one, which stays in history. - if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { - t.Fatal(added, err) - } - if added, err := inv.Assign(ctx, node, "web"); err != nil || added { - t.Fatal(added, err) - } - if err := inv.Unassign(ctx, node, "web"); err != nil { - t.Fatal(err) - } - if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { - t.Fatalf("assigned again after it ended: %v %v", added, err) - } - again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web") - if !again.Value.Stands() || !again.Value.Ended.IsZero() { - t.Fatalf("the assignment made again is %+v", again.Value) - } - if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() { - t.Fatalf("the values of the key: %+v", history) - } -} - -// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the -// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay, -// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here -// ends an assignment. -func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) { - inv, node := aNodeWithModules(t, "dunst", "web") - ctx := t.Context() - for _, m := range []string{"dunst", "web"} { - if _, err := inv.Assign(ctx, node, m); err != nil { - t.Fatal(err) - } - } - // A build and a commit put in the value by another write. - cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") - cur.Value.Build, cur.Value.Commit = "build-1", "c1" - if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil { - t.Fatal(err) - } - record, _ := inv.NodeByName(ctx, node) - digest := "sha256:" + strings.Repeat("d", 64) - plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest, - Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc", - Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}} - raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`) - if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil { - t.Fatal(err) - } - inv.SetProvisioningPlans(inv.SentProvisioningPlans) - written, removed, err := inv.KeepAssignmentPlans(ctx) - if err != nil || written != 2 || removed != 0 { - t.Fatalf("kept: %d written, %d ended: %v", written, removed, err) - } - dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") - if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() { - t.Fatalf("dunst's value is %+v", dunst.Value) - } - if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil { - t.Fatalf("web has a plan: %+v", web.Value) - } - notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify") - if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil { - t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found) - } - if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) { - t.Fatalf("a pulled-in module is assigned: %v", on) - } - if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 { - t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err) - } - // Sent again without the pulled-in module: its value ends, the assignments stay. - only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`) - if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil { - t.Fatal(err) - } - if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 { - t.Fatalf("ended %d: %v", removed, err) - } - if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() { - t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value) - } - if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) { - t.Fatalf("assigned: %v", on) - } -} - -// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value** -// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build -// already is left as it is. -func TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - for _, name := range []string{"anchor", "laptop"} { - if _, err := inv.AddNode(ctx, name); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"notes", "wiki"} { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil { - t.Fatal(err) - } - } - for _, a := range [][2]string{{"anchor", "notes"}, {"anchor", "wiki"}, {"laptop", "notes"}} { - if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { - t.Fatal(err) - } - } - aBuildRow(t, inv, "build-notes", "notes", "c-notes", "") - anchor, _ := inv.NodeByName(ctx, "anchor") - if err := inv.RecordSent(ctx, anchor.ID, "sent-1", map[string]string{"notes": "c-notes"}); err != nil { - t.Fatal(err) - } - // The mirror of the store put a value under one key, with a plan. - if _, err := inv.chain.Assignments.Create(ctx, "anchor.wiki", chain.Assignment{Node: "anchor", Module: "wiki", - Assigned: time.Now().UTC(), Provisioning: json.RawMessage(`{"plan":1}`)}); err != nil { - t.Fatal(err) - } - reports, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatal(err) - } - var moved MoveReport - for _, r := range reports { - if r.Kind == KindAssignments { - moved = r - } - } - if moved.Rows != 3 || moved.Written+moved.Present != 3 { - t.Fatalf("the move of the assignments reported %+v", moved) - } - notes, _, _ := inv.chain.Assignments.Get(ctx, "anchor.notes") - if notes.Value.Commit != "c-notes" || notes.Value.Build != "build-notes" || notes.Value.Since.IsZero() || notes.Value.Assigned.IsZero() { - t.Fatalf("anchor.notes moved as %+v", notes.Value) - } - if notes.Value.Artifact != "sha256:build-notes" { - t.Fatalf("what the build published: %+v", notes.Value) - } - laptop, _, _ := inv.chain.Assignments.Get(ctx, "laptop.notes") - if laptop.Value.Commit != "" || laptop.Value.Build != "" { - t.Fatalf("a machine never sent has a build: %+v", laptop.Value) - } - wiki, _, _ := inv.chain.Assignments.Get(ctx, "anchor.wiki") - if string(wiki.Value.Provisioning) != `{"plan":1}` { - t.Fatalf("the plan the mirror put in the value was lost: %+v", wiki.Value) - } - // From now on the bucket is what is written. - if _, err := inv.Assign(ctx, "laptop", "wiki"); err != nil { - t.Fatal(err) - } - if n := rowsInTheTable(t, inv, "assignment"); n != "3" { - t.Fatalf("the table took an assignment after the move: %s", n) - } - if on, _ := inv.Assigned(ctx, "laptop"); !reflect.DeepEqual(on, []string{"notes", "wiki"}) { - t.Fatalf("assigned on laptop: %v", on) - } -} - -// hooked runs a function the first time a key is made in a bucket: the old controller writing the store between the move's -// first pass and its switch. -type hooked struct { - chain.Backend - once *sync.Once - then func() -} - -func (h hooked) Create(ctx context.Context, key string, value []byte) (uint64, error) { - rev, err := h.Backend.Create(ctx, key, value) - h.once.Do(h.then) - return rev, err -} - -// **A row the old controller changes after the move's first pass is in the bucket when the kind is recorded as moved** -// (ADR 0408 §8, issue 981 note 7 Q2): before the switch the store is the only truth, so under the lock the move reads the -// rows again and writes over every key that differs — a plan's state and a build's failure — and ends an assignment the -// store dropped. The count alone would have kept the older value and refused the next save of the plan. -func TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - for _, n := range []string{"anchor"} { - if _, err := inv.AddNode(ctx, n); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"notes", "wiki"} { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"notes", "wiki"} { - if _, err := inv.Assign(ctx, "anchor", m); err != nil { - t.Fatal(err) - } - } - p := aBatch("plan-1", time.Now().Add(-time.Hour).UTC()) - p.State, p.Tiers = PlanBuilding, [][]string{{"notes"}} - if err := inv.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - // Between a kind's first pass and its switch the old controller saves the plan again, fails the build and unassigns wiki: - // each is done when the first key of its kind is made, so the first pass has read the rows as they were. - after := func(sql string) hooked { - return hooked{chain.NewMemory(), &sync.Once{}, func() { inv.ExecForTest(t, sql) }} - } - inv.UseChain(chain.New( - after(`update build set failed = 'it broke later' where id = 'build-1'`), - after(`update release_plan set state = 'rolling', revision = revision + 1, note = 'the old controller was here' where id = 'plan-1'`), - after(`delete from assignment where module = 'wiki'`), - func(context.Context) error { return nil })) - if _, err := inv.MoveChain(ctx, nil); err != nil { - t.Fatal(err) - } - now, err := inv.PlanByID(ctx, "plan-1") - if err != nil { - t.Fatal(err) - } - if now.State != PlanRolling || now.Note != "the old controller was here" || now.Revision != 2 { - t.Fatalf("the bucket holds a plan from before the change: %s %q revision %d", now.State, now.Note, now.Revision) - } - if b, found, _ := inv.BuildByID(ctx, "build-1"); !found || b.Failed != "it broke later" { - t.Fatalf("the bucket holds the build as it was: %+v", b) - } - if on, _ := inv.Assigned(ctx, "anchor"); !slices.Equal(on, []string{"notes"}) { - t.Fatalf("anchor is assigned %v, though the store dropped wiki", on) - } - // The plan can be saved at the revision the store had. - now.Note = "saved after the move" - if err := inv.SavePlan(ctx, &now); err != nil { - t.Fatalf("the next save of the plan is refused: %v", err) - } -} - -// sameEvent says two encodings of the forge's announcement are one: the store keeps it as jsonb, which spells it with its own -// white space, and whoever reads it (the cut, mesh-delivery) decodes it, so the white space is no part of what it says. -func sameEvent(a, b json.RawMessage) bool { - var x, y bytes.Buffer - return json.Compact(&x, a) == nil && json.Compact(&y, b) == nil && x.String() == y.String() -} diff --git a/internal/inventory/chain_moved_test.go b/internal/inventory/chain_moved_test.go deleted file mode 100644 index 3ea88b25..00000000 --- a/internal/inventory/chain_moved_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package inventory - -import ( - "strings" - "testing" -) - -// **The move of the chain has a switch, made by the last migration and written by no migration** (ADR 0408 §8, issue -// 1089): migration 0104 makes the one-row table chain_moved, and does not drop a table of the chain, which a later -// delivery does once the move is read back. -func TestMigration0104MakesTheSwitchOfTheChainsMoveAndNothingElse(t *testing.T) { - migrations, err := Migrations() - if err != nil { - t.Fatal(err) - } - var found bool - for _, m := range migrations { - if m.Number != 104 { - continue - } - found = true - sql := strings.ToLower(m.SQL) - if !strings.Contains(sql, "create table chain_moved") || !strings.Contains(sql, "check (only_one)") { - t.Errorf("migration 0104 does not make the one-row chain_moved:\n%s", m.SQL) - } - for _, line := range strings.Split(sql, "\n") { - if strings.HasPrefix(strings.TrimSpace(line), "--") { - continue - } - for _, verb := range []string{"drop table", "insert ", "delete ", "update "} { - if strings.Contains(line, verb) { - t.Errorf("migration 0104 does more than make the switch: %q", line) - } - } - // The one other change: a seat's holder is no longer a foreign key to the assignment (issue 1093); the tables of - // the chain are dropped by a later migration (issue 1096). - if strings.Contains(line, "alter ") && !strings.Contains(line, "alter table seat_holding drop constraint") { - t.Errorf("migration 0104 alters more than the seat holdings' foreign key: %q", line) - } - } - } - if !found { - t.Fatal("no migration 0104") - } -} diff --git a/internal/inventory/chain_pending_test.go b/internal/inventory/chain_pending_test.go new file mode 100644 index 00000000..3d48ba19 --- /dev/null +++ b/internal/inventory/chain_pending_test.go @@ -0,0 +1,272 @@ +//go:build integration + +package inventory + +import ( + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/chain" +) + +// **A pending assignment is the value of the key of the assignment it waits for** (ADR 0261, ADR 0408, issue 1094): state +// pending, the wait in its `pending` object under an id that is a string, no table, and one open wait per machine and module. +func TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket(t *testing.T) { + inv, node := aNodeWithModules(t, "web") + ctx := t.Context() + p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-1", Repository: "novox/mesh-catalog", + Path: "/modules/sensors/"}) + if err != nil { + t.Fatal(err) + } + if want := fmt.Sprintf("p-%d", p.Since.UnixNano()); p.ID != want || p.State != PendingWaiting || p.Path != "modules/sensors" { + t.Fatalf("recorded %+v, want id %s", p, want) + } + kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.sensors") + if err != nil || !found || kept.Value.State != chain.StatePending || kept.Value.Stands() || !kept.Value.Assigned.IsZero() || + kept.Value.Pending == nil || kept.Value.Pending.ID != p.ID || kept.Value.Pending.Build != "build-1" { + t.Fatalf("the key holds %+v (found %v, %v)", kept.Value, found, err) + } + rows, err := inv.Pending(ctx, time.Time{}) + if err != nil || len(rows) != 1 || rows[0].ID != p.ID || rows[0].Node != "anchor" || rows[0].Build != "build-1" { + t.Fatalf("waiting: %+v %v", rows, err) + } + if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"}); !errors.Is(err, ErrAlreadyPending) { + t.Fatalf("a second open wait: %v", err) + } + if _, err := inv.Assign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "build-3"}); !errors.Is(err, ErrOnTheMachine) { + t.Fatalf("a wait for a module that is on the machine: %v", err) + } + // Written over once it ended: a settled wait is replaced by a new one under a new id. + if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil || !ok { + t.Fatalf("settled %v %v", ok, err) + } + next, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"}) + if err != nil || next.ID == p.ID { + t.Fatalf("a wait after one ended: %+v %v", next, err) + } + // The earlier wait ended unmade and nobody answered its condition, so the new wait carries it (issue 1108): both are on record. + if known, _ := inv.PendingKnown(ctx, []string{p.ID, next.ID}); !known[p.ID] || !known[next.ID] { + t.Fatalf("known %v: the key holds the newer one and carries the unanswered earlier one", known) + } + // The first wait, its settling and the new wait: three values, the earlier wait still in the history. + if history, _ := inv.chain.Assignments.History(ctx, "anchor.sensors"); len(history) != 3 { + t.Fatalf("the earlier wait is in the key's history: %d values", len(history)) + } +} + +// **A pending value is skipped by what reads assignments** (issue 1094): the machine is assigned nothing by it, the module +// runs nowhere, and a module only waiting for a build is not forgotten as assigned. +func TestARecordedPendingIsSkippedByAssignedAndTheJoins(t *testing.T) { + inv, node := aNodeWithModules(t, "web") + ctx := t.Context() + if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"}); err != nil { + t.Fatal(err) + } + if on, _ := inv.Assigned(ctx, node); len(on) != 0 { + t.Fatalf("assigned %v", on) + } + if running, _ := inv.Running(ctx, "web"); len(running) != 0 { + t.Fatalf("running %v", running) + } + if runs, _ := inv.AssignmentRuns(ctx, node); len(runs) != 0 { + t.Fatalf("runs %v", runs) + } + entries, _ := inv.Catalogued(ctx) + for _, e := range entries { + if e.Manifest.Module == "web" && len(e.On) != 0 { + t.Fatalf("web is on %v", e.On) + } + } + if err := inv.ForgetModule(ctx, "web"); errors.Is(err, ErrStillAssigned) { + t.Fatalf("a module only waiting is assigned: %v", err) + } +} + +// **Two claims of one wait make one, and the other sees false** (issue 1094): by compare-and-set on the key's revision. +func TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"}) + if err != nil { + t.Fatal(err) + } + var won atomic.Int32 + var wg sync.WaitGroup + for k := 0; k < 8; k++ { + wg.Add(1) + go func() { + defer wg.Done() + if ok, err := inv.ClaimPending(ctx, p.ID); err == nil && ok { + won.Add(1) + } + }() + } + wg.Wait() + if won.Load() != 1 { + t.Fatalf("%d claims won", won.Load()) + } + if again, err := inv.ClaimPending(ctx, p.ID); err != nil || again { + t.Fatalf("a claim of a claimed wait: %v %v", again, err) + } + if rows, _ := inv.Stuck(ctx, time.Now().Add(time.Minute)); len(rows) != 1 || rows[0].State != PendingApplying || rows[0].Settled == nil { + t.Fatalf("stuck %+v", rows) + } + if err := inv.ReleaseClaim(ctx, p.ID); err != nil { + t.Fatal(err) + } + if rows, _ := inv.Pending(ctx, time.Time{}); len(rows) != 1 || rows[0].Settled != nil { + t.Fatalf("released %+v", rows) + } + // Only a wait that waits is withdrawn or expired, only a claim is applied. + if ok, _ := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingApplied, "x"); ok { + t.Fatal("a wait was applied") + } + if ok, _ := inv.SettlePending(ctx, p.ID, PendingApplying, PendingRefused, "x"); ok { + t.Fatal("a wait that is not claimed was refused as a claim") + } + if ok, err := inv.RepointPending(ctx, p.ID, "b-2", "novox/other", "x/"); err != nil || !ok { + t.Fatalf("repoint %v %v", ok, err) + } + if rows, _ := inv.Pending(ctx, time.Time{}); rows[0].Build != "b-2" || rows[0].Path != "x" { + t.Fatalf("repointed %+v", rows) + } +} + +// **Assigning over a wait carries the wait into the value that stands** (issue 1094): the claim stays applying for the one +// making it to settle applied, and one still waiting is made by hand, and says so. +func TestAssignOverAPendingKeepsItsRecordAsApplied(t *testing.T) { + inv, node := aNodeWithModules(t, "web", "cache") + ctx := t.Context() + claimed, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"}) + if ok, err := inv.ClaimPending(ctx, claimed.ID); err != nil || !ok { + t.Fatal(ok, err) + } + if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { + t.Fatalf("assigned %v %v", added, err) + } + standing, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web") + if !standing.Value.Stands() || standing.Value.Assigned.IsZero() || standing.Value.Pending == nil || + standing.Value.Pending.State != PendingApplying { + t.Fatalf("the value that stands: %+v", standing.Value) + } + if ok, err := inv.SettlePending(ctx, claimed.ID, PendingApplying, PendingApplied, "made"); err != nil || !ok { + t.Fatalf("settled %v %v", ok, err) + } + if on, _ := inv.Assigned(ctx, node); len(on) != 1 || on[0] != "web" { + t.Fatalf("assigned %v", on) + } + rows, _ := inv.Pending(ctx, time.Now().Add(-time.Hour)) + if len(rows) != 1 || rows[0].State != PendingApplied || rows[0].Note != "made" { + t.Fatalf("what became of the wait: %+v", rows) + } + + waiting, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "cache", Build: "b-2"}) + if _, err := inv.Assign(ctx, node, "cache"); err != nil { + t.Fatal(err) + } + got, _ := inv.PendingFor(ctx, node, "cache") + if len(got) != 1 || got[0].ID != waiting.ID || got[0].State != PendingApplied || !strings.Contains(got[0].Note, "by hand") { + t.Fatalf("a wait assigned by hand: %+v", got) + } +} + +// **Marks are stamped once, and a wait of a machine the store dropped is no wait** (issue 1094). +func TestAPendingIsMarkedOnceAndGoesWithItsMachine(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + p, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"}) + if _, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil { + t.Fatal(err) + } + if rows, _ := inv.ToRaise(ctx); len(rows) != 1 { + t.Fatalf("to raise %+v", rows) + } + if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil { + t.Fatal(err) + } + first, _ := inv.Pending(ctx, time.Now().Add(-time.Hour)) + time.Sleep(2 * time.Millisecond) + if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil { + t.Fatal(err) + } + second, _ := inv.Pending(ctx, time.Now().Add(-time.Hour)) + if !first[0].Raised.Equal(*second[0].Raised) { + t.Fatalf("the first stamp did not stand: %v then %v", first[0].Raised, second[0].Raised) + } + if err := inv.MarkPending(ctx, p.ID, "frobnicated"); err == nil { + t.Fatal("a mark nobody knows") + } + if rows, _ := inv.ToRaise(ctx); len(rows) != 0 { + t.Fatalf("to raise %+v", rows) + } + if rows, _ := inv.ToClear(ctx); len(rows) != 1 { + t.Fatalf("to clear %+v", rows) + } + if _, err := inv.RemoveNodeForTest(ctx, node); err != nil { + t.Fatal(err) + } + if rows, _ := inv.Pending(ctx, time.Unix(0, 0)); len(rows) != 0 { + t.Fatalf("a wait outlived its machine: %+v", rows) + } +} + +// **A wait replaced by a newer one never leaves the operator without the condition it raised, or was to raise** (issue +// 1108): the key carries the replaced wait while its condition is unanswered, still known, still to clear, and a newer +// wait takes only the unanswered ones forward. +func TestAReplacedWaitNeverLeavesTheOperatorWithoutItsCondition(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record := func(build string) PendingAssignment { + p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: build}) + if err != nil { + t.Fatal(err) + } + return p + } + end := func(p PendingAssignment) { + if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom "+p.Build); err != nil || !ok { + t.Fatal(ok, err) + } + } + first := record("b-1") + end(first) + if err := inv.MarkPending(ctx, first.ID, "raised"); err != nil { + t.Fatal(err) + } + second := record("b-2") + if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID}); !known[first.ID] || !known[second.ID] { + t.Fatalf("known %v: the replaced wait's condition is unanswered", known) + } + if clear, _ := inv.ToClear(ctx); len(clear) != 1 || clear[0].ID != first.ID { + t.Fatalf("to clear %+v: the first's condition is still to clear", clear) + } + rows, _ := inv.PendingFor(ctx, node, "sensors") + if len(rows) != 2 || rows[0].ID != second.ID || rows[1].ID != first.ID { + t.Fatalf("the key says %+v, newest first", rows) + } + // The second ends unmade before the first was answered: both are to be said, the first's condition not cleared by it. + end(second) + if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID { + t.Fatalf("to raise %+v", raise) + } + // A third replaces the second, which was not raised yet: it is carried, with the first (still unanswered). + if err := inv.MarkPending(ctx, first.ID, "acknowledged"); err != nil { + t.Fatal(err) + } + third := record("b-3") + if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID { + t.Fatalf("the second was never raised and is to be: %+v", raise) + } + if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID, third.ID}); known[first.ID] || !known[second.ID] || !known[third.ID] { + t.Fatalf("known %v: the first was answered (taken back) and is let go, the second is not", known) + } +} diff --git a/internal/inventory/chain_requests.go b/internal/inventory/chain_requests.go index 808ffd08..4772757a 100644 --- a/internal/inventory/chain_requests.go +++ b/internal/inventory/chain_requests.go @@ -163,8 +163,8 @@ func requestRecords(ctx context.Context, c *chain.Chain) ([]chain.Kept[chain.Bui // RequestsToAskAgain are the requests the move found never handed over (issue 981 note 7 Q5), not yet asked again: each is // asked once, and never twice. func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, error) { - c, on, err := i.ChainReads(ctx, KindBuilds) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return nil, err } all, err := requestRecords(ctx, c) @@ -185,8 +185,8 @@ func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, err // ClaimToAskAgain takes a request to ask again, by compare-and-set: true for the one process that took it. **Claimed // before it is asked and settled after**, so a crash between loses the ask rather than making it twice. func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error) { - c, on, err := i.ChainReads(ctx, KindBuilds) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return false, err } cur, found, err := c.Builds.Get(ctx, chain.RequestKey(id)) @@ -205,8 +205,8 @@ func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error // AskedAgainAs records the build id a claimed request was asked as. func (i *Inventory) AskedAgainAs(ctx context.Context, id, asked string) error { - c, on, err := i.ChainReads(ctx, KindBuilds) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return err } return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.Reask = asked }) diff --git a/internal/inventory/chain_rollouts.go b/internal/inventory/chain_rollouts.go index 98a49f86..b7a9e839 100644 --- a/internal/inventory/chain_rollouts.go +++ b/internal/inventory/chain_rollouts.go @@ -139,17 +139,17 @@ func (v *rolloutView) list() []viewEntry { // rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole. func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) { - i.moved.mu.Lock() - v := i.moved.view + i.state.mu.Lock() + v := i.state.view if v == nil || v.chain != c || v.dead.Load() { if v != nil { v.stop() } v = newRolloutView(c) - i.moved.view = v + i.state.view = v v.start() } - i.moved.mu.Unlock() + i.state.mu.Unlock() select { case <-v.ready: case <-ctx.Done(): @@ -165,11 +165,11 @@ func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutVi // stopViews ends the view's watch: the inventory is closing. func (i *Inventory) stopViews() { - i.moved.mu.Lock() - defer i.moved.mu.Unlock() - if i.moved.view != nil { - i.moved.view.stop() - i.moved.view = nil + i.state.mu.Lock() + defer i.state.mu.Unlock() + if i.state.view != nil { + i.state.view.stop() + i.state.view = nil } } @@ -492,12 +492,12 @@ func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m Batched return false, err } if !found { - i.moved.mu.Lock() - if i.moved.pending == nil { - i.moved.pending = map[string]pendingMerge{} + i.state.mu.Lock() + if i.state.pending == nil { + i.state.pending = map[string]pendingMerge{} } - i.moved.pending[key] = pendingMerge{plan: m.Plan, m: held} - i.moved.mu.Unlock() + i.state.pending[key] = pendingMerge{plan: m.Plan, m: held} + i.state.mu.Unlock() } return true, nil } @@ -712,12 +712,12 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans [] held, has := copies[key] if !has { // A merge carried for a plan not saved yet goes where it is moved. - i.moved.mu.Lock() - if pm, pending := i.moved.pending[key]; pending && m.Plan != "" { + i.state.mu.Lock() + if pm, pending := i.state.pending[key]; pending && m.Plan != "" { pm.plan, pm.m.Alone = m.Plan, m.Alone - i.moved.pending[key] = pm + i.state.pending[key] = pm } - i.moved.mu.Unlock() + i.state.mu.Unlock() continue } w := winner(held) @@ -803,14 +803,14 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans [] if e := edits[p.ID]; e != nil { held = e.apply(held) } - i.moved.mu.Lock() - for key, pm := range i.moved.pending { + i.state.mu.Lock() + for key, pm := range i.state.pending { if pm.plan == p.ID { held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held) _ = key } } - i.moved.mu.Unlock() + i.state.mu.Unlock() record.Held = held entered, duration := tierLeftOf(old, *p, time.Now()) record.TierEntered, record.Updated = entered, time.Now().UTC() @@ -859,13 +859,13 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans [] } } - i.moved.mu.Lock() - for key, pm := range i.moved.pending { + i.state.mu.Lock() + for key, pm := range i.state.pending { if _, saved := done[pm.plan]; saved { - delete(i.moved.pending, key) + delete(i.state.pending, key) } } - i.moved.mu.Unlock() + i.state.mu.Unlock() for _, p := range plans { w := done[p.ID] p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held @@ -908,8 +908,8 @@ func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uin // RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired // at the next cut (ADR 0408, issue 1092). It says how many it repaired. func (i *Inventory) RepairMerges(ctx context.Context) (int, error) { - c, on, err := i.ChainReads(ctx, KindRollouts) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return 0, err } v, err := i.rolloutView(ctx, c) diff --git a/internal/inventory/chain_rollouts_test.go b/internal/inventory/chain_rollouts_test.go index dffeeb17..fffc8576 100644 --- a/internal/inventory/chain_rollouts_test.go +++ b/internal/inventory/chain_rollouts_test.go @@ -15,7 +15,7 @@ import ( func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) { t.Helper() c := chain.InMemory(nil) - inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }} + inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }} inv.UseChain(c) t.Cleanup(inv.stopViews) return inv, c @@ -117,10 +117,10 @@ func TestAMergeAddedToAPlanNotSavedYetIsWrittenByItsSave(t *testing.T) { } return errors.New("the lease is not held") }) - other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded} + other := &Inventory{state: &chainState{}, recorded: inv.recorded} other.UseChain(away) t.Cleanup(other.stopViews) - other.moved.pending = inv.moved.pending + other.state.pending = inv.state.pending p := aBatch("plan-new", plansAt) if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); !errors.Is(err, chain.ErrNotHeld) { t.Fatalf("the save answered %v", err) @@ -158,7 +158,7 @@ func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *test var fail atomic.Bool rollouts := failingOn{Backend: chain.NewMemory(), key: "plan-a", fail: &fail} c = chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil }) - inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }} + inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }} inv.UseChain(c) t.Cleanup(inv.stopViews) ctx := context.Background() @@ -190,9 +190,6 @@ func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *test if n := len(mergeHolders(v.list())["novox/mesh-catalog|m1"]); n != 2 { t.Fatalf("the merge is held by %d records, want the two of a stopped move", n) } - inv.moved.mu.Lock() - inv.moved.kind = map[string]bool{KindRollouts: true} - inv.moved.mu.Unlock() repaired := 0 for k, copies := range mergeHolders(v.list()) { if len(copies) < 2 { @@ -310,7 +307,7 @@ func (c *countingBackend) Keys(ctx context.Context) ([]string, error) { func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) { counting := &countingBackend{Backend: chain.NewMemory()} c := chain.New(chain.NewMemory(), counting, chain.NewMemory(), func(context.Context) error { return nil }) - inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }} + inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }} inv.UseChain(c) t.Cleanup(inv.stopViews) ctx := context.Background() @@ -337,7 +334,7 @@ func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) { t.Fatalf("the bucket was read %d times and its keys %d times for the surveys", counting.gets.Load()-before, counting.keys.Load()) } // Another process writes; the view follows. - other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded} + other := &Inventory{state: &chainState{}, recorded: inv.recorded} other.UseChain(c) t.Cleanup(other.stopViews) late := aBatch("plan-4", plansAt.Add(time.Hour)) diff --git a/internal/inventory/chain_sent_copy_test.go b/internal/inventory/chain_sent_copy_test.go new file mode 100644 index 00000000..3920c150 --- /dev/null +++ b/internal/inventory/chain_sent_copy_test.go @@ -0,0 +1,111 @@ +package inventory + +import ( + "testing" + "time" + + "github.com/novox/mesh-controller/internal/chain" +) + +// **What delivery 1 recorded in node.sent_builds is copied onto the keys of the assignments, once** (issue 1109, review +// round 1): a module pulled in by an assignment had no commit on its key, so it would fall out of SentBuilds. The copy +// creates a standing pulled-in key where none stands, fills a standing key that names no commit (commit, and the build +// when one is known, no rollout), and overwrites nothing: a key with a commit, a hand send's unknown, a wait for a build. +func TestWhatTheColumnRecordedIsCopiedOntoTheKeysAndOverwritesNothing(t *testing.T) { + inv, c := anInventoryOnTheChain(t) + ctx := t.Context() + at := time.Date(2026, 10, 18, 4, 0, 0, 0, time.UTC) + if err := inv.chainRecordBuild(ctx, c, Build{ID: "b-wg", Module: "mesh-wireguard", Commit: "c-wg"}); err != nil { + t.Fatal(err) + } + put := func(module string, a chain.Assignment) { + a.Node, a.Module = "laptop", module + if _, err := c.Assignments.Create(ctx, assignmentKey("laptop", module), a); err != nil { + t.Fatal(err) + } + } + put("notes", chain.Assignment{Assigned: at}) // assigned, no commit yet + put("web", chain.Assignment{Assigned: at, Commit: "c-web-new", Build: "b1"}) // already has one + put("hand", chain.Assignment{Assigned: at, SentUnknown: at}) // a hand send since + put("waits", chain.Assignment{Assigned: at, State: chain.StatePending}) // a wait for a build + put("ended", chain.Assignment{Assigned: at, Ended: at}) // unassigned + sent := map[string]string{"notes": "c-notes", "web": "c-web-old", "mesh-wireguard": "c-wg", "hand": "c-hand", + "waits": "c-waits", "ended": "c-ended"} + + for pass := 0; pass < 2; pass++ { // and the second pass changes nothing + if err := inv.copySentBuildsOf(ctx, c, "laptop", sent, at); err != nil { + t.Fatal(err) + } + get := func(module string) chain.Assignment { + t.Helper() + k, found, err := c.Assignments.Get(ctx, assignmentKey("laptop", module)) + if err != nil || !found { + t.Fatalf("%s: %v %v", module, found, err) + } + return k.Value + } + if a := get("notes"); a.Commit != "c-notes" || a.Rollout != "" || !a.Stands() || a.PulledIn { + t.Errorf("notes: %+v", a) + } + if a := get("web"); a.Commit != "c-web-new" || a.Build != "b1" { + t.Errorf("a key with a commit was overwritten: %+v", a) + } + if a := get("hand"); a.Commit != "" || a.SentUnknown.IsZero() { + t.Errorf("a hand send's unknown was overwritten: %+v", a) + } + if a := get("waits"); a.Commit != "" || a.State != chain.StatePending { + t.Errorf("a wait was overwritten: %+v", a) + } + if a := get("mesh-wireguard"); a.Commit != "c-wg" || a.Build != "b-wg" || !a.PulledIn || !a.Stands() || a.Rollout != "" { + t.Errorf("the pulled-in module has no standing key of its own: %+v", a) + } + if a := get("ended"); a.Commit != "" || a.Stands() { + t.Errorf("an ended key was revived or given a commit: %+v", a) + } + } +} + +// The copy reads the column of the store, writes the keys, and empties the column it has copied: SentBuilds then answers the +// pulled-in module, and a second copy finds nothing to do. +func TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + if _, err := inv.AddNode(ctx, "laptop"); err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `update node set sent_builds = '{"mesh-wireguard": "c-wg"}'::jsonb, sent_at = now() where name = 'laptop'`) + if n, err := inv.CopySentBuilds(ctx); err != nil || n != 1 { + t.Fatalf("the copy did %d nodes (%v)", n, err) + } + got, known, err := inv.SentBuilds(ctx, "laptop") + if err != nil || !known || got["mesh-wireguard"] != "c-wg" { + t.Fatalf("SentBuilds answers %v, known %v (%v)", got, known, err) + } + if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" { + t.Fatalf("the column was not emptied: %v", rows) + } + if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 { + t.Fatalf("a second copy did %d nodes (%v)", n, err) + } +} + +// A machine delivery 2 has sent has its column emptied by the send itself, so a copy after it leaves a key the send cleared +// without a commit. +func TestASendEmptiesTheColumnSoALaterCopyLeavesAClearedKeyBare(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + record, err := inv.AddNode(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `update node set sent_builds = '{"notes": "c-old"}'::jsonb where name = 'laptop'`) + if err := inv.RecordSentUnder(ctx, record.ID, "d1", map[string]string{}, 1); err != nil { + t.Fatal(err) + } + if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" { + t.Fatalf("a send left the column: %v", rows) + } + if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 { + t.Fatalf("the copy did %d nodes (%v)", n, err) + } +} diff --git a/internal/inventory/chain_sent_test.go b/internal/inventory/chain_sent_test.go new file mode 100644 index 00000000..a62ea017 --- /dev/null +++ b/internal/inventory/chain_sent_test.go @@ -0,0 +1,116 @@ +//go:build integration + +package inventory + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/novox/mesh-controller/internal/chain" +) + +// **What a machine was sent is read from the keys of its assignments that name a commit** (ADR 0408, issue 1109): an assigned +// module's, and a module only pulled in, whose key is a standing value marked pulled-in and never a state. +func TestSentBuildsAnswersFromTheAssignmentsKeysOfTheNode(t *testing.T) { + inv, node := aNodeWithModules(t, "web", "cache") + ctx := t.Context() + record, _ := inv.NodeByName(ctx, node) + if _, err := inv.Assign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil { + t.Fatal(err) + } + if builds, known, err := inv.SentBuilds(ctx, node); err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"web": "c1", "cache": "c2"}) { + t.Fatalf("sent %v %v %v", builds, known, err) + } + pulled, found, _ := inv.AssignmentValue(ctx, node, "cache") + if !found || !pulled.Stands() || pulled.State != "" || !pulled.PulledIn || !pulled.Assigned.IsZero() || pulled.Commit != "c2" { + t.Fatalf("the module only carried has the key %+v", pulled) + } + if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"web"}) { + t.Fatalf("a module only pulled in is assigned: %v", on) + } + // Assigned later, it keeps what it was sent and is pulled in no longer. + if _, err := inv.Assign(ctx, node, "cache"); err != nil { + t.Fatal(err) + } + made, _, _ := inv.AssignmentValue(ctx, node, "cache") + if made.PulledIn || made.Assigned.IsZero() || made.Commit != "c2" { + t.Fatalf("assigned over a pulled-in key: %+v", made) + } +} + +// **A send replaces the whole map of the node, and a hand send makes it unknown until the next send that names its builds** +// (issue 1109): a module no longer carried loses its commit, the node reads unknown when any key says so. +func TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown(t *testing.T) { + inv, node := aNodeWithModules(t, "web", "cache") + ctx := t.Context() + record, _ := inv.NodeByName(ctx, node) + if _, err := inv.Assign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, record.ID, "d2", map[string]string{"web": "c3"}); err != nil { + t.Fatal(err) + } + if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c3"}) { + t.Fatalf("the second send did not replace the map: %v %v", builds, known) + } + if old, _, _ := inv.AssignmentValue(ctx, node, "cache"); old.Commit != "" || old.Build != "" { + t.Fatalf("a module no longer carried keeps %+v", old) + } + // By hand: unknown, with a time and no commit anywhere. + if err := inv.RecordSent(ctx, record.ID, "d3", nil); err != nil { + t.Fatal(err) + } + if builds, known, _ := inv.SentBuilds(ctx, node); known || builds != nil { + t.Fatalf("a hand send reads as %v %v", builds, known) + } + web, _, _ := inv.AssignmentValue(ctx, node, "web") + if web.SentUnknown.IsZero() || web.Commit != "" || !web.Stands() { + t.Fatalf("web after a hand send: %+v", web) + } + if sent, _ := inv.Outstanding(ctx, node); sent != "d3" { + t.Fatalf("the digest of the hand send: %q", sent) + } + // A send that names its builds makes it known again. + if err := inv.RecordSent(ctx, record.ID, "d4", map[string]string{"web": "c4"}); err != nil { + t.Fatal(err) + } + if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c4"}) { + t.Fatalf("after a send that names its builds: %v %v", builds, known) + } + if again, _, _ := inv.AssignmentValue(ctx, node, "web"); !again.SentUnknown.IsZero() { + t.Fatalf("web still says unknown: %+v", again) + } +} + +// failingToWrite is an assignments bucket that fails every write. +type failingToWrite struct{ chain.Backend } + +func (f failingToWrite) Create(context.Context, string, []byte) (uint64, error) { + return 0, errors.New("the bus went away") +} + +func (f failingToWrite) Update(context.Context, string, []byte, uint64) (uint64, error) { + return 0, errors.New("the bus went away") +} + +// **A links write that fails fails the record of the send, loudly** (issue 1109): what a machine was sent is read from the +// keys, so a send recorded without them would read as one that never happened. +func TestAFailedWriteOfTheLinksFailsTheRecordOfTheSend(t *testing.T) { + inv, node := aNodeWithModules(t, "web") + ctx := t.Context() + record, _ := inv.NodeByName(ctx, node) + inv.UseChain(chain.New(chain.NewMemory(), chain.NewMemory(), failingToWrite{chain.NewMemory()}, + func(context.Context) error { return nil })) + err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1"}) + if err == nil { + t.Fatal("a send whose links could not be written was recorded") + } +} diff --git a/internal/inventory/chain_source_test.go b/internal/inventory/chain_source_test.go new file mode 100644 index 00000000..d5f5cac2 --- /dev/null +++ b/internal/inventory/chain_source_test.go @@ -0,0 +1,146 @@ +package inventory + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" +) + +// The tables of the chain, and the column, that the controller's store no longer holds (novox/hq ADR 0408 §8, issue 1096). +var chainTables = []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment"} + +// sqlStatement is what makes a literal a statement of the store: it opens with a verb of SQL. +var sqlStatement = regexp.MustCompile(`(?is)^\s*(select|insert|update|delete|alter|drop|create|with|truncate|from|join|where|and|set|values)\b|\b(select\s.+\sfrom|insert\s+into|delete\s+from)\b`) + +// **No code names a table of the chain** (acceptance 12, issue 1096), found by reading the sources: a string literal of any +// non-test file of internal/ or cmd/ that names one after from, into, update, join, delete from, references or table — +// build_gate and port_assignment are other words —. This is the static half of the proof; the +// integration test reads the catalog. +func TestNoCodeNamesATableOfTheChain(t *testing.T) { + names := regexp.MustCompile(`(?i)\b(from|into|update|join|delete\s+from|references|table)\s+(` + strings.Join(chainTables, "|") + `)\b`) + var found []string + for _, root := range []string{"../../internal", "../../cmd"} { + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() && d.Name() == "vendor" { + return filepath.SkipDir + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + return err + } + ast.Inspect(file, func(n ast.Node) bool { + lit, ok := n.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return true + } + text, err := strconv.Unquote(lit.Value) + if err != nil { + return true + } + if !sqlStatement.MatchString(text) { + return true // prose that happens to say "from assignment" is no statement + } + if m := names.FindString(text); m != "" { + found = append(found, filepath.ToSlash(path)+": "+strings.Join(strings.Fields(m), " ")) + } + return true + }) + return nil + }) + if err != nil { + t.Fatal(err) + } + } + if len(found) > 0 { + t.Fatalf("code still names a table of the chain, which the store no longer holds:\n %s", strings.Join(found, "\n ")) + } +} + +// The scan sees what it says it sees: planted, a string naming a table is found, and a comment or a word beside is not. +func TestTheSourceScanFindsATablePlanted(t *testing.T) { + names := regexp.MustCompile(`(?i)\b(from|into|update|join|delete\s+from|references|table)\s+(` + strings.Join(chainTables, "|") + `)\b`) + for text, want := range map[string]bool{ + "select id from build where id = $1": true, + "insert into assignment (node, module)": true, + "select 1 from build_gate": false, + "delete from port_assignment": false, + "select a from build_request a join build b": true, + "an assignment from build to build": true, + "the assignment generation": false, + "select count(*) from pending_assignment": true, + } { + if got := names.MatchString(text); got != want { + t.Errorf("%q: %v, want %v", text, got, want) + } + } +} + +// **The last migration drops the tables of the chain, behind its guard** (issue 1096): it names each table, the column and +// the function, drops no build_gate, and raises before it drops anything. +func TestTheLastMigrationDropsTheTablesOfTheChain(t *testing.T) { + migrations, err := Migrations() + if err != nil { + t.Fatal(err) + } + last := migrations[len(migrations)-1] + if last.Number < 105 { + t.Fatalf("the last migration is %d", last.Number) + } + raw, err := os.ReadFile(filepath.Join("migrations", migrationFile(t, last.Number))) + if err != nil { + t.Fatal(err) + } + var code []string + for _, line := range strings.Split(string(raw), "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "--") { + code = append(code, line) + } + } + sql := strings.ToLower(strings.Join(code, "\n")) + for _, table := range append(append([]string{}, chainTables...), "chain_moved") { + if !strings.Contains(sql, "drop table "+table+";") { + t.Errorf("migration %d does not drop %s", last.Number, table) + } + } + for _, want := range []string{"drop function raise_assignment_generation()"} { + if !strings.Contains(sql, want) { + t.Errorf("migration %d lacks %q", last.Number, want) + } + } + if strings.Contains(sql, "sent_builds") { + t.Errorf("migration %d touches node.sent_builds, which stays as the source of the copy (issue 1112 drops it)", last.Number) + } + if strings.Contains(sql, "build_gate") || strings.Contains(sql, "assignment_generation;") { + t.Errorf("migration %d touches a table that stays", last.Number) + } + if raise, drop := strings.Index(sql, "raise exception"), strings.Index(sql, "drop table"); raise < 0 || drop < 0 || raise > drop { + t.Errorf("migration %d does not raise before it drops (%d, %d)", last.Number, raise, drop) + } + for _, kind := range []string{"'builds'", "'rollouts'", "'assignments'", "pending_assignment", "chain_moved"} { + if !strings.Contains(sql[:strings.Index(sql, "drop table")], kind) { + t.Errorf("the guard of migration %d does not look at %s", last.Number, kind) + } + } +} + +func migrationFile(t *testing.T, number int) string { + t.Helper() + files, err := filepath.Glob(filepath.Join("migrations", strconv.Itoa(number/1000%10)+strconv.Itoa(number/100%10)+strconv.Itoa(number/10%10)+strconv.Itoa(number%10)+"-*.sql")) + if err != nil || len(files) != 1 { + t.Fatalf("the file of migration %d: %v %v", number, files, err) + } + return filepath.Base(files[0]) +} diff --git a/internal/inventory/chain_store_test.go b/internal/inventory/chain_store_test.go new file mode 100644 index 00000000..11d8d68c --- /dev/null +++ b/internal/inventory/chain_store_test.go @@ -0,0 +1,348 @@ +//go:build integration + +package inventory + +import ( + "context" + "errors" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/chain" +) + +// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its +// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none. +func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) { + inv := fresh(t) + ctx := context.Background() + first := aBuild("build-1", "notes", "") + first.Commit = "c1" + first.Against = []string{"artifact-store://base@sha256:aa"} + first.ReuseKey = "rk-1" + second := aBuild("build-2", "notes", "") + second.Commit = "c2" + failed := aBuild("build-3", "notes", "compile error") + other := aBuild("build-4", "wiki", "") + for _, b := range []Build{first, second, failed, other, first} { + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + kept, found, err := inv.chain.Builds.Get(ctx, chain.BuildKey("notes", "build-1")) + if err != nil || !found || kept.Value.ReuseKey != "rk-1" { + t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err) + } + if history, _ := inv.chain.Builds.History(ctx, chain.BuildKey("notes", "build-1")); len(history) != 1 { + t.Fatalf("a build recorded twice has %d values", len(history)) + } + list, err := inv.Builds(ctx, "notes", 10) + if err != nil || len(list) != 3 { + t.Fatalf("notes has %d builds: %v", len(list), err) + } + one, found, err := inv.BuildByID(ctx, "build-2") + if err != nil || !found || one.Commit != "c2" || one.Module != "notes" { + t.Fatalf("build-2 reads as %+v %v %v", one, found, err) + } + if _, found, _ := inv.BuildByID(ctx, "build-nope"); found { + t.Fatal("a build nobody recorded was found") + } + if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" { + t.Fatalf("the build of notes at c1 is %q: %v", id, err) + } + held, err := inv.Held(ctx) + if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" { + t.Fatalf("held %v: %v", held, err) + } + against, err := inv.BuiltAgainst(ctx) + if err != nil || len(against["notes"]) != 0 { + // The newest successful build of notes, build-2, stood on nothing. + t.Fatalf("built against %v: %v", against, err) + } + announced, err := inv.Announceable(ctx) + if err != nil || len(announced) != 3 { + t.Fatalf("announceable %d: %v", len(announced), err) + } + prints, err := inv.Fingerprints(ctx) + if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" { + t.Fatalf("fingerprints %v: %v", prints, err) + } +} + +// A build request row as the store held it before the move. + +// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue +// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none +// after the outcome is heard. The table of requests takes none. +func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) { + inv := fresh(t) + ctx := context.Background() + ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true} + if err := inv.RecordBuildRequest(ctx, ask); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuildRequest(ctx, ask); err != nil { + t.Fatalf("a request kept twice: %v", err) + } + if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 { + t.Fatalf("a request nobody answered is a build: %+v %v", builds, err) + } + if _, found, _ := inv.BuildByID(ctx, "build-1"); found { + t.Fatal("a request nobody answered has an outcome") + } + got, found, err := inv.BuildRequestByID(ctx, "build-1") + if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() { + t.Fatalf("the request reads back as %+v %v %v", got, found, err) + } + if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil { + t.Fatal(err) + } + asked, err := inv.RequestsNamed(ctx, "notes") + if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" { + t.Fatalf("the request is %+v: %v", asked, err) + } + if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) { + t.Fatalf("requested names %v: %v", names, err) + } + + // The outcome comes: the same key, now a build, with the request beside it. + built := aBuild("build-1", "notes", "") + if err := inv.RecordBuild(ctx, built); err != nil { + t.Fatal(err) + } + if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" { + t.Fatalf("the outcome reads as %+v %v %v", one, found, err) + } + answered, err := inv.RequestsNamed(ctx, "notes") + if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" { + t.Fatalf("the request answered is %+v: %v", answered, err) + } + if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" { + t.Fatalf("the request was lost with the outcome: %+v", again) + } + // Heard: the marks of an unanswered request change nothing. + if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil { + t.Fatal(err) + } + if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" { + t.Fatalf("an answered request was marked not asked: %+v", again) + } + if history, _ := inv.chain.Builds.History(ctx, chain.RequestKey("build-1")); len(history) != 3 { + // asked, marked unknown, answered. + t.Fatalf("the record has %d values", len(history)) + } + // An outcome heard before its request is kept, and the request lands beside it. + if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil { + t.Fatal(err) + } + if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" { + t.Fatalf("the request kept after its outcome reads %+v", late) + } + // A request older than the time they are kept is no longer listed. + if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old", + At: time.Now().Add(-2 * KeptFor)}); err != nil { + t.Fatal(err) + } + if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") { + t.Fatalf("a request past its time is listed: %v", names) + } +} + +// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a +// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it +// is asked and so asked at most once. + +// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a +// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module +// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the +// assignment goes with it. +func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres", "web") + ctx := t.Context() + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + start := generationNow(t, inv) + if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added { + t.Fatalf("assigned %v: %v", added, err) + } + if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added { + t.Fatalf("assigned again %v: %v", added, err) + } + kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres") + if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() { + t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err) + } + if generationNow(t, inv) <= start { + t.Fatal("an assignment left the generation where it was") + } + if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) { + t.Fatalf("a module the mesh does not know: %v", err) + } + if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) { + t.Fatalf("assigned on %s: %v %v", node, on, err) + } + if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) { + t.Fatalf("running: %v %v", running, err) + } + entries, err := inv.Catalogued(ctx) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + switch e.Manifest.Module { + case "postgres": + if !reflect.DeepEqual(e.On, []string{node}) { + t.Errorf("postgres is on %v", e.On) + } + case "web": + if len(e.On) != 0 { + t.Errorf("web is on %v", e.On) + } + } + } + if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) { + t.Fatalf("a module still assigned was forgotten: %v", err) + } + // A seat is held by an assignment, or by nothing. + if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil { + t.Fatal("a seat was handed to a module that is not assigned") + } + if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil { + t.Fatalf("a seat handed to an assignment: %v", err) + } + + before := generationNow(t, inv) + if err := inv.Unassign(ctx, node, "postgres"); err != nil { + t.Fatal(err) + } + if generationNow(t, inv) <= before { + t.Fatal("an unassignment left the generation where it was") + } + // No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history. + gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres") + if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() { + t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err) + } + if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() || + history[1].Value.Stands() { + t.Fatalf("the earlier values of the key: %+v", history) + } + if on, _ := inv.Assigned(ctx, node); len(on) != 0 { + t.Fatalf("an ended assignment is still assigned: %v", on) + } + if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 { + t.Fatalf("an ended assignment is running: %v", running) + } + if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 { + t.Fatalf("the seat the assignment held is held by %+v: %v", held, err) + } + if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" { + t.Fatalf("an unassignment of nothing: %v", err) + } + if err := inv.ForgetModule(ctx, "postgres"); err != nil { + t.Fatalf("a module nothing is assigned could not be forgotten: %v", err) + } + // Made again after it ended: a fresh value over the ended one, which stays in history. + if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { + t.Fatal(added, err) + } + if added, err := inv.Assign(ctx, node, "web"); err != nil || added { + t.Fatal(added, err) + } + if err := inv.Unassign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { + t.Fatalf("assigned again after it ended: %v %v", added, err) + } + again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web") + if !again.Value.Stands() || !again.Value.Ended.IsZero() { + t.Fatalf("the assignment made again is %+v", again.Value) + } + if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() { + t.Fatalf("the values of the key: %+v", history) + } +} + +// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the +// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay, +// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here +// ends an assignment. + +// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the +// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay, +// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here +// ends an assignment. +func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) { + inv, node := aNodeWithModules(t, "dunst", "web") + ctx := t.Context() + for _, m := range []string{"dunst", "web"} { + if _, err := inv.Assign(ctx, node, m); err != nil { + t.Fatal(err) + } + } + record, _ := inv.NodeByName(ctx, node) + digest := "sha256:" + strings.Repeat("d", 64) + plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest, + Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc", + Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}} + raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`) + if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil { + t.Fatal(err) + } + // A build and a commit put in the value by another write, after the send: a send that carries no builds (a hand send, as this + // one) forgets what each assignment of its machine runs (issue 1109), so they are written once it is recorded. + cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") + cur.Value.Build, cur.Value.Commit = "build-1", "c1" + if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil { + t.Fatal(err) + } + inv.SetProvisioningPlans(inv.SentProvisioningPlans) + written, removed, err := inv.KeepAssignmentPlans(ctx) + if err != nil || written != 2 || removed != 0 { + t.Fatalf("kept: %d written, %d ended: %v", written, removed, err) + } + dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") + if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() { + t.Fatalf("dunst's value is %+v", dunst.Value) + } + if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil { + t.Fatalf("web has a plan: %+v", web.Value) + } + notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify") + if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil { + t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found) + } + if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) { + t.Fatalf("a pulled-in module is assigned: %v", on) + } + if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 { + t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err) + } + // Sent again without the pulled-in module: its value ends, the assignments stay. + only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`) + if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil { + t.Fatal(err) + } + if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 { + t.Fatalf("ended %d: %v", removed, err) + } + if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() { + t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value) + } + if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) { + t.Fatalf("assigned: %v", on) + } +} + +// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value** +// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build +// already is left as it is. diff --git a/internal/inventory/collection.go b/internal/inventory/collection.go index 6c0ba092..ebb011c4 100644 --- a/internal/inventory/collection.go +++ b/internal/inventory/collection.go @@ -2,7 +2,6 @@ package inventory import ( "context" - "encoding/json" "sort" "strings" @@ -327,84 +326,34 @@ type recentBuild struct { // recentBuilds is, of each module the mesh still holds, its KeptBuilds newest successful builds' artifacts and bases. func (i *Inventory) recentBuilds(ctx context.Context) ([]recentBuild, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - builds, err := i.chainRecentMade(ctx, c) - if err != nil { - return nil, err - } - out := make([]recentBuild, 0, len(builds)) - for _, b := range builds { - out = append(out, recentBuild{made: b.Made, against: b.Against}) - } - return out, nil - } - rows, err := i.store.Pool().Query(ctx, - `select made, built_against from ( - select b.made, b.built_against, - row_number() over (partition by b.module order by b.at desc, b.id desc) as back - from build b - join module m on m.name = b.module - where b.failed = '' and b.module is not null and b.module <> '' - ) ranked where back <= $1`, KeptBuilds) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - var out []recentBuild - for rows.Next() { - var raw, against []byte - if err := rows.Scan(&raw, &against); err != nil { - return nil, err - } - var r recentBuild - if err := json.Unmarshal(raw, &r.made); err != nil { - r.made = nil - } - if err := json.Unmarshal(against, &r.against); err != nil { - r.against = nil - } - out = append(out, r) + builds, err := i.chainRecentMade(ctx, c) + if err != nil { + return nil, err } - return out, rows.Err() + out := make([]recentBuild, 0, len(builds)) + for _, b := range builds { + out = append(out, recentBuild{made: b.Made, against: b.Against}) + } + return out, nil } // everyReferenceMade is every artifact reference any successful build recorded, oldest build first. func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) { - var made [][]Artifact - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - builds, err := i.chainEveryReferenceMade(ctx, c) - if err != nil { - return nil, err - } - for _, b := range builds { - made = append(made, b.Made) - } - } else { - rows, err := i.store.Pool().Query(ctx, - `select made from build where failed = '' and module is not null and module <> '' - order by at asc, id asc`) - if err != nil { - return nil, err - } - defer rows.Close() - for rows.Next() { - var raw []byte - if err := rows.Scan(&raw); err != nil { - return nil, err - } - var a []Artifact - if err := json.Unmarshal(raw, &a); err != nil { - continue - } - made = append(made, a) - } - if err := rows.Err(); err != nil { - return nil, err - } + } + builds, err := i.chainEveryReferenceMade(ctx, c) + if err != nil { + return nil, err + } + var made [][]Artifact + for _, b := range builds { + made = append(made, b.Made) } seen := map[string]bool{} var out []string diff --git a/internal/inventory/durations.go b/internal/inventory/durations.go index 73edbd73..5be4b96f 100644 --- a/internal/inventory/durations.go +++ b/internal/inventory/durations.go @@ -2,11 +2,7 @@ package inventory import ( "context" - "errors" - "fmt" "time" - - "github.com/jackc/pgx/v5" ) // The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066. @@ -148,41 +144,3 @@ func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) { } return tag.RowsAffected(), nil } - -// planTierLeft is the measurement a plan's save makes when it leaves a tier: the tier moved on, or -// the plan ended. Read in the save's own transaction, so two saves cannot both measure one tier. -func planTierLeft(ctx context.Context, tx pgx.Tx, p Plan, now time.Time) (entered time.Time, err error) { - var oldTier int - var oldState string - var since time.Time - err = tx.QueryRow(ctx, - `select tier, state, coalesce(tier_entered, created) from release_plan where id = $1 for update`, - p.ID).Scan(&oldTier, &oldState, &since) - if errors.Is(err, pgx.ErrNoRows) { - return now, nil // a new plan enters its first tier now - } - if err != nil { - return time.Time{}, err - } - if oldState == PlanAssembling || oldState == PlanQueued { - return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276) - } - wasOpen := oldState == PlanBuilding || oldState == PlanRolling - if !wasOpen || (oldTier == p.Tier && p.Open()) { - return since, nil // still in the tier, or already ended - } - var modules []string - if oldTier >= 0 && oldTier < len(p.Tiers) { - modules = p.Tiers[oldTier] - } - detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, oldTier, len(p.Tiers), modules, p.State) - if p.Open() { - detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, oldTier, len(p.Tiers), modules, p.Tier) - } - _, err = tx.Exec(ctx, - `insert into duration (kind, subject, node, ref, started, took_ms, detail) - values ($1, $2, '', $3, $4, $5, $6) on conflict do nothing`, - DurationPlanTier, p.Repository, fmt.Sprintf("%s/tier-%d", p.ID, oldTier), since, - now.Sub(since).Milliseconds(), detail) - return now, err -} diff --git a/internal/inventory/fortest.go b/internal/inventory/fortest.go index 3469d78d..4cecc83c 100644 --- a/internal/inventory/fortest.go +++ b/internal/inventory/fortest.go @@ -65,16 +65,15 @@ func ForTest(t *testing.T) *Inventory { if _, err := inv.store.Migrate(t.Context(), migrations); err != nil { t.Fatal(err) } - // The chain is on the bus in a mesh; in a test it is in memory, with every kind moved, as a mesh raised after this - // release has them (ADR 0408, issue 1089). The tests of the move itself use ForTestBefore. + // The chain is on the bus in a mesh; in a test it is in memory (ADR 0408). c := chain.InMemory(nil) + // A fresh test chain has nothing to lay out: it is laid out already, as the controller's start leaves a mesh (issue 1121). + if _, err := c.Builds.CreateRaw(t.Context(), chain.LayoutMarkerKey, []byte(`{}`)); err != nil { + t.Fatal(err) + } inv.UseChain(c) testChains.Store(os.Getenv(store.Variable(Name)), c) t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) }) - if _, err := inv.store.Pool().Exec(t.Context(), - `insert into chain_moved (counts, largest) values ($1, $1)`, `{"builds": 0, "builds-layout": 0, "rollouts": 0, "assignments": 0}`); err != nil { - t.Fatal(err) - } return inv } @@ -132,8 +131,12 @@ func ForTestBefore(t *testing.T, number int) (*Inventory, func()) { if err := inv.Ready(t.Context(), 20*time.Second); err != nil { t.Fatal(err) } - // The chain is in memory and nothing is moved: the tests of a move seed the tables and move them. + // The chain is in memory, for what a test of the store before a migration reads of it. c := chain.InMemory(nil) + // A fresh test chain has nothing to lay out: it is laid out already, as the controller's start leaves a mesh (issue 1121). + if _, err := c.Builds.CreateRaw(t.Context(), chain.LayoutMarkerKey, []byte(`{}`)); err != nil { + t.Fatal(err) + } inv.UseChain(c) testChains.Store(os.Getenv(store.Variable(Name)), c) t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) }) @@ -183,3 +186,21 @@ func (i *Inventory) QueryForTest(t *testing.T, sql string, args ...any) [][]stri } return out } + +// PlanValuesForTest runs the pass that keeps the provisioner's plans in the values of the assignments (KeepAssignmentPlans) +// and answers the values that stand, by key: what a reader of the bucket reads. +func (i *Inventory) PlanValuesForTest(t *testing.T) map[string][]byte { + t.Helper() + if _, _, err := i.KeepAssignmentPlans(t.Context()); err != nil { + t.Fatal(err) + } + kept, err := i.KeptAssignments(t.Context()) + if err != nil { + t.Fatal(err) + } + out := make(map[string][]byte, len(kept)) + for key, v := range kept { + out[key] = v.Value + } + return out +} diff --git a/internal/inventory/gate.go b/internal/inventory/gate.go index 9932b559..08cc9439 100644 --- a/internal/inventory/gate.go +++ b/internal/inventory/gate.go @@ -2,8 +2,6 @@ package inventory import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -241,65 +239,20 @@ func (i *Inventory) RestoreModule(ctx context.Context, b Build) error { // machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told // from one that does (the bus's planned step, ADR 0236). func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainBuildFingerprints(ctx, c, module) - } - rows, err := i.store.Pool().Query(ctx, - `select commit_hash, made, coalesce(manifest::text, '') from build - where module = $1 and failed = '' and commit_hash <> '' order by at desc limit 50`, module) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - out := map[string]string{} - for rows.Next() { - var commit, manifest string - var made []byte - if err := rows.Scan(&commit, &made, &manifest); err != nil { - return nil, err - } - if _, seen := out[commit]; seen { - continue - } - sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...)) - out[commit] = hex.EncodeToString(sum[:]) - } - return out, rows.Err() + return i.chainBuildFingerprints(ctx, c, module) } // Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint. func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainFingerprints(ctx, c) - } - rows, err := i.store.Pool().Query(ctx, - `select module, commit_hash, made, coalesce(manifest::text, '') from build - where module is not null and failed = '' and commit_hash <> '' order by at desc`) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - out := map[string]map[string]string{} - for rows.Next() { - var module, commit, manifest string - var made []byte - if err := rows.Scan(&module, &commit, &made, &manifest); err != nil { - return nil, err - } - if out[module] == nil { - out[module] = map[string]string{} - } - if _, seen := out[module][commit]; seen { - continue - } - sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...)) - out[module][commit] = hex.EncodeToString(sum[:]) - } - return out, rows.Err() + return i.chainFingerprints(ctx, c) } // PassedCommits is, per module, the commits a build of which passed its gate on some machine. @@ -326,19 +279,11 @@ func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bo // BuildOf is the id of the newest successful build of a module from a commit; empty when none is // recorded (a manifest handed over by hand). func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + c, err := i.theChain(ctx) + if err != nil { return "", err - } else if on { - return i.chainBuildOf(ctx, c, module, commit) } - var id string - err := i.store.Pool().QueryRow(ctx, - `select id from build where module = $1 and commit_hash = $2 and failed = '' order by at desc limit 1`, - module, commit).Scan(&id) - if errors.Is(err, pgx.ErrNoRows) { - return "", nil - } - return id, err + return i.chainBuildOf(ctx, c, module, commit) } // ManifestAt is the manifest a module was registered with at a commit (or a commit it abbreviates): the @@ -372,38 +317,19 @@ func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (cata // manifestOf is the manifest a build was recorded with; nil when it kept none. func (i *Inventory) manifestOf(ctx context.Context, id string) ([]byte, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainManifestOf(ctx, c, id) - } - var manifest []byte - if err := i.store.Pool().QueryRow(ctx, `select manifest from build where id = $1`, id).Scan(&manifest); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err } - return manifest, nil + return i.chainManifestOf(ctx, c, id) } // newestManifestAt is the newest successful build of a module from a commit, or one it abbreviates, that kept a manifest: // its id and manifest; false when there is none. func (i *Inventory) newestManifestAt(ctx context.Context, module, commit string) (string, []byte, bool, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return "", nil, false, err - } else if on { - return i.chainManifestBuild(ctx, c, module, commit) - } - var id string - var raw []byte - err := i.store.Pool().QueryRow(ctx, - `select id, manifest from build - where module = $1 and failed = '' and manifest is not null and manifest::text <> 'null' - and (commit_hash = $2 or starts_with(commit_hash, $2)) - order by at desc limit 1`, module, commit).Scan(&id, &raw) - if errors.Is(err, pgx.ErrNoRows) { - return "", nil, false, nil - } + c, err := i.theChain(ctx) if err != nil { return "", nil, false, err } - return id, raw, true, nil + return i.chainManifestBuild(ctx, c, module, commit) } diff --git a/internal/inventory/given.go b/internal/inventory/given.go index 20507be1..fd930fea 100644 --- a/internal/inventory/given.go +++ b/internal/inventory/given.go @@ -93,28 +93,19 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s if declared == "" { return nil, nil } - // Only a module assigned there: from the bucket when the assignments are on the bus, else the store's join. - _, onTheBus, err := i.ChainReads(ctx, KindAssignments) + // Only a module assigned there, from the assignments bucket. + assignedHere := map[string]bool{} + modules, err := i.assignedModules(ctx, node) if err != nil { return nil, err } - assignedHere := map[string]bool{} - join := `join assignment a on a.node = s.node and a.module = s.module` - if onTheBus { - join = `` - modules, err := i.assignedModules(ctx, node) - if err != nil { - return nil, err - } - for _, m := range modules { - assignedHere[m] = true - } + for _, m := range modules { + assignedHere[m] = true } rows, err := i.store.Pool().Query(ctx, `select s.node, s.module, s.name, s.replace_after_start from module_secret s join node n on n.id = s.node - `+join+` where n.name = $1 and n.sent = $2 and n.sent_at > s.replace_after_start and s.origin = 'accepted' and s.replace_after_start is not null and (not n.adopted or exists (select 1 from taken t where t.node = s.node and t.module = s.module)) @@ -134,7 +125,7 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s rows.Close() return nil, err } - if onTheBus && !assignedHere[d.module] { + if !assignedHere[d.module] { continue } dues = append(dues, d) diff --git a/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql b/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql new file mode 100644 index 00000000..411c11b3 --- /dev/null +++ b/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql @@ -0,0 +1,52 @@ +-- The tables of builds, rollouts and assignments leave the store (novox/hq ADR 0408 §8, design 47 milestone 5, issue 1096). +-- +-- Delivery 1 wrote their rows into the bus's buckets at the controller's start and recorded each kind it had counted in +-- chain_moved; from then on nothing in the controller reads or writes the tables. This drops them. **It is one-way**: a +-- controller older than this schema is refused by the put-back gate, and the rows are in the buckets only. +-- +-- **The guard comes first, in the same transaction as the drops**, because a migration runs when the store is opened, before +-- the controller of this delivery serves and before any connection to the bus: it can prove the move from the store alone. +-- A kind that holds rows and is not named in chain_moved was never moved, so dropping its tables would lose it; a pending +-- assignment still waiting, being made, or ended unmade with its condition unanswered has no mover (it is a value of the +-- assignments bucket from this delivery on), so the table must hold none. A store that never held a row of a kind — a new +-- mesh — passes. A migration that raises leaves nothing applied, `prepare` fails with the reason, and the running controller +-- keeps its tables. +do $$ +declare + kind text; + holds boolean; + missing text[] := '{}'; +begin + foreach kind in array array['builds', 'rollouts', 'assignments'] loop + holds := case kind + when 'builds' then exists (select 1 from build) or exists (select 1 from build_request) + when 'rollouts' then exists (select 1 from release_plan) or exists (select 1 from batched_merge) + when 'assignments' then exists (select 1 from assignment) + end; + if holds and not coalesce((select counts ? kind from chain_moved), false) then + missing := missing || kind; + end if; + end loop; + if array_length(missing, 1) > 0 then + raise exception 'the controller''s store still holds rows of % that chain_moved does not name as moved to the bus; ' + 'run the controller of the previous delivery and read chain_moved back first', array_to_string(missing, ', '); + end if; + if exists (select 1 from pending_assignment + where state in ('waiting', 'applying') or (raised_at is not null and acknowledged_at is null and cleared_at is null)) then + raise exception 'a pending assignment is still in the table (waiting, being made, or ended with its condition ' + 'unanswered); let it settle, then deliver again'; + end if; +end +$$; + +drop table build_request; +drop table build; +drop table batched_merge; +drop table release_plan; +drop table pending_assignment; +-- Its triggers go with it; the function they called goes here. The counter table stays: the controller raises it. +drop table assignment; +drop function raise_assignment_generation(); +-- node.sent_builds STAYS, only as the source of the one-time copy onto the assignments' keys (issue 1109); dropping it is +-- issue 1112. build_gate stays (issue 1097). +drop table chain_moved; diff --git a/internal/inventory/mirrored_test.go b/internal/inventory/mirrored_test.go index d258ce91..9f56d9b7 100644 --- a/internal/inventory/mirrored_test.go +++ b/internal/inventory/mirrored_test.go @@ -5,7 +5,6 @@ package inventory import ( "context" "fmt" - "os" "slices" "strings" "testing" @@ -148,40 +147,6 @@ func TestAPersonRecordsACopyAndTheFirstRecordStands(t *testing.T) { } } -// The migration records the copies the build records already name: only a module's own on- -// repository, and only once, by the earliest build naming it. -func TestTheCopiesTheRecordsAlreadyNameAreRecorded(t *testing.T) { - inv := inTheTables(t) - ctx := context.Background() - copyRef := mirrorRef("route-proxy/on-go_base", 1) - notACopy := ref("go-base", "build", 2) - builtOn(t, inv, "b1", "route-proxy", 1, nil, copyRef, notACopy) - builtOn(t, inv, "b2", "route-proxy", 2, nil, copyRef) - // The rows a mesh had before the table: written by builds that did not say what they copied. - if _, err := inv.store.Pool().Exec(ctx, `truncate artifact_mirrored`); err != nil { - t.Fatal(err) - } - sql, err := os.ReadFile("migrations/0081-a-mirrored-base-is-recorded.sql") - if err != nil { - t.Fatal(err) - } - _, backfill, _ := strings.Cut(string(sql), "insert into artifact_mirrored") - if _, err := inv.store.Pool().Exec(ctx, "insert into artifact_mirrored"+backfill); err != nil { - t.Fatal(err) - } - var reference, build string - var n int - if err := inv.store.Pool().QueryRow(ctx, `select count(*) from artifact_mirrored`).Scan(&n); err != nil || n != 1 { - t.Fatalf("%d copies recorded (%v); want the one", n, err) - } - if err := inv.store.Pool().QueryRow(ctx, `select reference, build from artifact_mirrored`).Scan(&reference, &build); err != nil { - t.Fatal(err) - } - if reference != copyRef || build != "b1" { - t.Fatalf("recorded %s by %s", reference, build) - } -} - // A copy let go of and copied again by a later build is held again: its record is back, and the sweep // decides about it as before. func TestACopyLetGoOfAndCopiedAgainIsHeldAgain(t *testing.T) { diff --git a/internal/inventory/mirrors.go b/internal/inventory/mirrors.go index fadab025..4359b2be 100644 --- a/internal/inventory/mirrors.go +++ b/internal/inventory/mirrors.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "time" ) @@ -70,39 +69,8 @@ type DeclaredMirrored struct { var DeclaredMirroredFields = []string{"kind", "default", "why", "type", "description", "enum", "minimum", "maximum", "pattern", "trusted", "predefined", "required", "runtime"} -// ErrAssignmentsMoved is a write of the store's mirror of the assignments, refused because the assignments are keys of the bus. -var ErrAssignmentsMoved = errors.New("the assignments are keys of the bus, and the store's rows are no longer what they say") - -// AssignmentsMirrorToWrite is AssignmentsMirror for the mirror's writer, which lays what it returns over the bucket and -// ends the keys it does not return: **refused once the assignments are moved** (ADR 0408 §4), asked at the moment of the -// write, so a tick that decided before the switch cannot lay mirror-shaped values over the links after it. -func (i *Inventory) AssignmentsMirrorToWrite(ctx context.Context) (map[string][]byte, error) { - if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil { - return nil, err - } else if on { - return nil, ErrAssignmentsMoved - } - return i.AssignmentsMirror(ctx) -} - -// AssignmentsMirror is every assignment's mirrored value, keyed `.`, with the provisioner's plan of -// each assignment of a module handed to it, and a value of its own for each module handed to it on a node that -// nothing assigned there, only pulled in by another (issue 765). -func (i *Inventory) AssignmentsMirror(ctx context.Context) (map[string][]byte, error) { - var plans map[string]Provisioning - if i.provisioningPlans != nil { - var err error - if plans, err = i.provisioningPlans(ctx); err != nil { - return nil, fmt.Errorf("composing the provisioner's plans: %w", err) - } - } - assignments, err := i.assignmentsMirrored(ctx) - if err != nil { - return nil, err - } - return AssignmentValues(assignments, plans) -} - +// assignmentsMirrored is every standing assignment as the provisioner's plans are matched to it (SentProvisioningPlans, +// KeepAssignmentPlans). func (i *Inventory) assignmentsMirrored(ctx context.Context) ([]AssignmentMirrored, error) { rows, err := i.assignments(ctx) if err != nil { diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 3f6c6dd2..26f57483 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -34,11 +34,9 @@ type Inventory struct { // configValues reads a module's config values from mesh-config once the move of config has run (ReadsConfigFrom, // issue 664); nil reads none. configValues ConfigValues - // chain is the controller's chain on the bus (novox/hq ADR 0408, issue 1089), once attached (UseChain); moved is which - // kinds of it have been moved from the tables, as chain_moved says (ChainReads). A kind not moved is read and written - // in the tables as before. + // chain is the controller's chain on the bus (novox/hq ADR 0408), once attached (UseChain). chain *chain.Chain - moved *movedKinds + state *chainState // recorded, when set, takes the durations a save of the chain measures in place of the store: a test with no store. recorded func(context.Context, Duration) error } @@ -49,7 +47,7 @@ func Open(ctx context.Context) (*Inventory, error) { if err != nil { return nil, err } - inv := &Inventory{store: s, moved: &movedKinds{}} + inv := &Inventory{store: s, state: &chainState{}} // A test's store has the chain a test made for it in memory, shared by every inventory opened on that store. if c, ok := testChains.Load(os.Getenv(store.Variable(Name))); ok { inv.chain = c.(*chain.Chain) @@ -1264,22 +1262,12 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu e := int64(epoch) sentEpoch = &e } - var carried *string - if builds != nil { - raw, err := json.Marshal(builds) - if err != nil { - return err - } - text := string(raw) - carried = &text + if _, err := i.store.Pool().Exec(ctx, + `update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3 where id = $1`, node, digest, sentEpoch); err != nil { + return err } - _, err := i.store.Pool().Exec(ctx, - `update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4 where id = $1`, - node, digest, carried, sentEpoch) - if err == nil { - saidRunsNot(node, i.recordRuns(ctx, node, builds, time.Now())) - } - return err + // What it was sent of each module is the assignments' (SentBuilds); a write of it that fails fails this record. + return i.recordRuns(ctx, node, builds, time.Now()) } // AwaitingSince is since when a machine has had something waiting for its next send (novox/hq issue @@ -1288,40 +1276,43 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu // read from: every change a push carries happened after the last push, and an assignment is the one // that says when. func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time, error) { - if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil { - return time.Time{}, err - } else if on { - return i.chainAwaitingSince(ctx, name) - } - var since time.Time - err := i.store.Pool().QueryRow(ctx, - `select coalesce( - (select min(a.assigned) from assignment a - where a.node = n.id and (n.sent_at is null or a.assigned > n.sent_at)), - n.sent_at, n.created) - from node n where n.name = $1`, name).Scan(&since) - if errors.Is(err, pgx.ErrNoRows) { - return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name) - } - return since, err + return i.chainAwaitingSince(ctx, name) } // SentBuilds is the build of each module a machine was last sent, by its name: module to the commit -// its build was made from (novox/hq issue 259). Known is false when that was not kept — a machine -// last sent before it was, sent a declaration by hand, or one the mesh does not know. +// its build was made from (novox/hq issue 259), read from the keys of the machine's assignments that name a commit — an +// assigned module and one only pulled in (ADR 0408, issue 1109). Known is false when that was not kept: a machine never sent +// anything, or last sent a declaration by hand, which makes every key of it say its commit unknown. func (i *Inventory) SentBuilds(ctx context.Context, name string) (builds map[string]string, known bool, err error) { - var raw []byte - err = i.store.Pool().QueryRow(ctx, `select sent_builds from node where name = $1`, name).Scan(&raw) + var sentAt *time.Time + err = i.store.Pool().QueryRow(ctx, `select sent_at from node where name = $1`, name).Scan(&sentAt) if errors.Is(err, pgx.ErrNoRows) { return nil, false, nil } - if err != nil || raw == nil { + if err != nil || sentAt == nil { return nil, false, err } - builds = map[string]string{} - if err := json.Unmarshal(raw, &builds); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, false, err } + all, err := c.Assignments.All(ctx) + if err != nil { + return nil, false, fmt.Errorf("what %s was sent cannot be read from the bus: %w", name, err) + } + builds = map[string]string{} + for _, k := range all { + a := k.Value + if a.Node != name || !a.Stands() { + continue + } + if !a.SentUnknown.IsZero() { + return nil, false, nil + } + if a.Commit != "" { + builds[a.Module] = a.Commit + } + } return builds, true, nil } diff --git a/internal/inventory/pending.go b/internal/inventory/pending.go index 7f8258d5..89eb8d22 100644 --- a/internal/inventory/pending.go +++ b/internal/inventory/pending.go @@ -3,12 +3,12 @@ package inventory import ( "context" "errors" + "fmt" "path" + "sort" "strings" "time" - "github.com/jackc/pgx/v5" - "github.com/novox/mesh-controller/internal/chain" ) @@ -69,85 +69,40 @@ type RequestOutcome struct { // RecordBuildRequest keeps one build request, once it is asked. Idempotent on the id. Requests older than // KeptFor are deleted in the same act. func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) error { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return err - } else if on { - return i.chainRecordBuildRequest(ctx, c, a) - } - at := a.At - if at.IsZero() { - at = time.Now().UTC() - } - var notAsked *string - if a.NotAsked != "" { - notAsked = &a.NotAsked - } - if _, err := i.store.Pool().Exec(ctx, - `insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at, - at_terminal) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - on conflict (id) do nothing`, - a.ID, a.Repository, a.Seat, strings.Trim(a.Path, "/"), a.Ref, a.Commit, a.For, notAsked, at, - a.AtTerminal); err != nil { + c, err := i.theChain(ctx) + if err != nil { return err } - _, err := i.store.Pool().Exec(ctx, `delete from build_request where asked_at < $1`, time.Now().Add(-KeptFor)) - return err + return i.chainRecordBuildRequest(ctx, c, a) } // BuildRequestByID is the build request kept under this id, and whether one was kept. func (i *Inventory) BuildRequestByID(ctx context.Context, id string) (BuildRequest, bool, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + c, err := i.theChain(ctx) + if err != nil { return BuildRequest{}, false, err - } else if on { - return i.chainBuildRequestByID(ctx, c, id) } - var a BuildRequest - err := i.store.Pool().QueryRow(ctx, - `select id, repository, seat, source_path, ref, commit_hash, asked_for, at_terminal, asked_at - from build_request where id = $1`, id).Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, - &a.For, &a.AtTerminal, &a.At) - if errors.Is(err, pgx.ErrNoRows) { - return BuildRequest{}, false, nil - } - return a, err == nil, err + return i.chainBuildRequestByID(ctx, c, id) } // MarkNotAsked says a kept build request was never handed over: the words are kept, unless its outcome was // heard first. func (i *Inventory) MarkNotAsked(ctx context.Context, id, why string) error { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return err - } else if on { - return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.NotAsked = why }) - } - if heard, err := i.outcomeHeard(ctx, id); err != nil || heard { + c, err := i.theChain(ctx) + if err != nil { return err } - _, err := i.store.Pool().Exec(ctx, `update build_request set not_asked = $2 where id = $1`, id, why) - return err + return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.NotAsked = why }) } // MarkOutcomeUnknown says the asker of a build it handed over stopped waiting for it: asked, outcome unknown. // The build may still run; its outcome, when heard, is the last word. func (i *Inventory) MarkOutcomeUnknown(ctx context.Context, id, why string) error { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return err - } else if on { - return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.OutcomeUnknown = why }) - } - if heard, err := i.outcomeHeard(ctx, id); err != nil || heard { + c, err := i.theChain(ctx) + if err != nil { return err } - _, err := i.store.Pool().Exec(ctx, `update build_request set outcome_unknown = $2 where id = $1`, id, why) - return err -} - -// outcomeHeard says a build of this id was recorded in the table: its outcome was taken in. -func (i *Inventory) outcomeHeard(ctx context.Context, id string) (bool, error) { - var heard bool - err := i.store.Pool().QueryRow(ctx, `select exists (select 1 from build where id = $1)`, id).Scan(&heard) - return heard, err + return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.OutcomeUnknown = why }) } // RequestsNamed is every build request kept whose directory names the module, or whose build said it built @@ -185,46 +140,28 @@ func (i *Inventory) RequestedNames(ctx context.Context) ([]string, error) { } func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainRequests(ctx, c) - } - rows, err := i.store.Pool().Query(ctx, - `select a.id, a.repository, a.seat, a.source_path, a.ref, a.commit_hash, a.asked_for, - coalesce(a.not_asked, ''), coalesce(a.outcome_unknown, ''), a.asked_at, - b.id is not null, coalesce(b.failed, ''), coalesce(b.module, ''), coalesce(b.at, a.asked_at) - from build_request a left join build b on b.id = a.id - order by a.asked_at desc, a.id desc`) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - var out []RequestOutcome - for rows.Next() { - var a RequestOutcome - if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked, - &a.OutcomeUnknown, &a.At, &a.Heard, &a.Failed, &a.Module, &a.HeardAt); err != nil { - return nil, err - } - out = append(out, a) - } - return out, rows.Err() + return i.chainRequests(ctx, c) } // The states of a pending assignment. const ( - PendingWaiting = "waiting" - PendingApplying = "applying" - PendingApplied = "applied" - PendingRefused = "refused" - PendingExpired = "expired" - PendingWithdrawn = "withdrawn" + PendingWaiting = chain.PendingWaiting + PendingApplying = chain.PendingApplying + PendingApplied = chain.PendingApplied + PendingRefused = chain.PendingRefused + PendingExpired = chain.PendingExpired + PendingWithdrawn = chain.PendingWithdrawn ) -// PendingAssignment is an assignment kept until its module is registered (ADR 0261). +// PendingAssignment is an assignment kept until its module is registered (ADR 0261): the value of its key in the +// assignments bucket while it waits (state pending), and the record carried into the value that stands once it is made. type PendingAssignment struct { - ID int64 + // ID is p-, made when it was recorded. + ID string Node string Module string // Build is the build it waits for; Repository and Path where that build is from. @@ -247,195 +184,348 @@ func (p PendingAssignment) Open() bool { // ErrAlreadyPending is a second pending assignment of one module to one machine. var ErrAlreadyPending = errors.New("that assignment is already waiting for its build") -const pendingColumns = `p.id, n.name, p.module, p.build_id, p.repository, p.source_path, p.since, p.state, p.note, - p.settled_at, p.raised_at, p.acknowledged_at, p.cleared_at` +// ErrOnTheMachine is a pending assignment of a module that is on the machine already. +var ErrOnTheMachine = errors.New("that module is on the machine already") -func scanPending(rows pgx.Rows) ([]PendingAssignment, error) { - defer rows.Close() - var out []PendingAssignment - for rows.Next() { - var p PendingAssignment - if err := rows.Scan(&p.ID, &p.Node, &p.Module, &p.Build, &p.Repository, &p.Path, &p.Since, &p.State, - &p.Note, &p.Settled, &p.Raised, &p.Acknowledged, &p.Cleared); err != nil { - return nil, err - } - out = append(out, p) - } - return out, rows.Err() +func pendingOf(v chain.Assignment) PendingAssignment { return pendingRecord(v, *v.Pending) } + +// pendingRecord is one wait of a key, the one it holds or one it replaced. +func pendingRecord(v chain.Assignment, p chain.Pending) PendingAssignment { + return PendingAssignment{ID: p.ID, Node: v.Node, Module: v.Module, Build: p.Build, Repository: p.Repository, Path: p.Path, + Since: p.Since, State: p.State, Note: p.Note, Settled: p.Settled, Raised: p.Raised, Acknowledged: p.Acknowledged, + Cleared: p.Cleared} } -// RecordPending keeps an assignment until its module is registered. One is open per machine and module. +// pendingValues is every value of the bucket that holds a pending assignment, on a machine the store knows (a removed +// machine's is none, as the node's foreign key made it), with its key and revision. +func (i *Inventory) pendingValues(ctx context.Context) ([]chain.Kept[chain.Assignment], error) { + c, err := i.theChain(ctx) + if err != nil { + return nil, err + } + all, err := c.Assignments.All(ctx) + if err != nil { + return nil, fmt.Errorf("the pending assignments cannot be read from the bus: %w", err) + } + nodes, err := i.nodeNames(ctx) + if err != nil { + return nil, err + } + var out []chain.Kept[chain.Assignment] + for key, k := range all { + if k.Value.Pending != nil && nodes[k.Value.Node] { + k.Key = key + out = append(out, k) + } + } + sort.Slice(out, func(a, b int) bool { + x, y := out[a].Value.Pending, out[b].Value.Pending + if !x.Since.Equal(y.Since) { + return x.Since.Before(y.Since) + } + return x.ID < y.ID + }) + return out, nil +} + +func pendingList(kept []chain.Kept[chain.Assignment], keep func(chain.Pending) bool) []PendingAssignment { + var out []PendingAssignment + for _, k := range kept { + for _, p := range pendingWaits(k.Value) { + if keep(p) { + out = append(out, pendingRecord(k.Value, p)) + } + } + } + sort.SliceStable(out, func(a, b int) bool { + if !out[a].Since.Equal(out[b].Since) { + return out[a].Since.Before(out[b].Since) + } + return out[a].ID < out[b].ID + }) + return out +} + +// pendingWaits is the wait a value holds and the earlier ones it carries, each without the others. +func pendingWaits(v chain.Assignment) []chain.Pending { + if v.Pending == nil { + return nil + } + out := make([]chain.Pending, 0, 1+len(v.Pending.Earlier)) + for _, e := range v.Pending.Earlier { + e.Earlier = nil + out = append(out, e) + } + held := *v.Pending + held.Earlier = nil + return append(out, held) +} + +// changePending changes the pending assignment of an id — the wait a key holds or one it carries from before — by +// compare-and-set on the key that holds it, read again when it moved: false when no key holds that id now, or change says no. +func (i *Inventory) changePending(ctx context.Context, id string, change func(p *chain.Pending) bool) (bool, error) { + c, err := i.theChain(ctx) + if err != nil { + return false, err + } + for try := 0; try < casTries; try++ { + held, err := i.pendingValues(ctx) + if err != nil { + return false, err + } + var at *chain.Kept[chain.Assignment] + var target *chain.Pending + for k := range held { + v := held[k].Value + p := *v.Pending + p.Earlier = append([]chain.Pending(nil), p.Earlier...) + v.Pending = &p + switch { + case p.ID == id: + at, target = &held[k], v.Pending + default: + for e := range v.Pending.Earlier { + if v.Pending.Earlier[e].ID == id { + at, target = &held[k], &v.Pending.Earlier[e] + } + } + } + if at != nil { + held[k].Value = v + break + } + } + if at == nil { + return false, nil + } + if !change(target) { + return false, nil + } + if _, err := c.Assignments.Update(ctx, at.Key, at.Value, at.Revision); errors.Is(err, chain.ErrMoved) { + continue + } else if err != nil { + return false, err + } + return true, nil + } + return false, fmt.Errorf("the pending assignment %s kept moving under the write", id) +} + +// RecordPending keeps an assignment until its module is registered, as the value of its key (state pending). One is open per +// machine and module; a key that holds the module on the machine refuses it, and one that holds an ended assignment or a +// settled wait is written over. func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (PendingAssignment, error) { node, err := i.NodeByName(ctx, p.Node) if err != nil { return PendingAssignment{}, err } - err = i.store.Pool().QueryRow(ctx, - `insert into pending_assignment (node, module, build_id, repository, source_path) - values ($1, $2, $3, $4, $5) - on conflict (node, module) where state in ('waiting', 'applying') do nothing - returning id, since`, - node.ID, p.Module, p.Build, p.Repository, strings.Trim(p.Path, "/")).Scan(&p.ID, &p.Since) - if errors.Is(err, pgx.ErrNoRows) { - return PendingAssignment{}, ErrAlreadyPending - } + c, err := i.theChain(ctx) if err != nil { return PendingAssignment{}, err } - p.State = PendingWaiting - return p, nil + key := assignmentKey(node.Name, p.Module) + for try := 0; try < casTries; try++ { + cur, found, err := c.Assignments.Get(ctx, key) + if err != nil { + return PendingAssignment{}, err + } + switch { + case found && cur.Value.Stands(): + return PendingAssignment{}, fmt.Errorf("%w: %s on %s", ErrOnTheMachine, p.Module, node.Name) + case found && cur.Value.Pending != nil && cur.Value.State == chain.StatePending && cur.Value.Pending.Open(): + return PendingAssignment{}, ErrAlreadyPending + } + since := time.Now().UTC() + p.ID, p.Since, p.State = fmt.Sprintf("p-%d", since.UnixNano()), since, PendingWaiting + p.Path = strings.Trim(p.Path, "/") + value := chain.Assignment{Node: node.Name, Module: p.Module, State: chain.StatePending, Pending: &chain.Pending{ + ID: p.ID, State: PendingWaiting, Build: p.Build, Repository: p.Repository, Path: p.Path, Since: since, + Earlier: unansweredWaits(cur.Value, found)}} + if found { + _, err = c.Assignments.Update(ctx, key, value, cur.Revision) + } else { + _, err = c.Assignments.Create(ctx, key, value) + } + if errors.Is(err, chain.ErrMoved) || errors.Is(err, chain.ErrExists) { + continue + } + if err != nil { + return PendingAssignment{}, err + } + return p, nil + } + return PendingAssignment{}, fmt.Errorf("the assignment of %s to %s kept moving under the write", p.Module, node.Name) } // RepointPending makes a waiting pending assignment wait for another build. False when it no longer waits. -func (i *Inventory) RepointPending(ctx context.Context, id int64, build, repository, sourcePath string) (bool, error) { - tag, err := i.store.Pool().Exec(ctx, - `update pending_assignment set build_id = $2, repository = $3, source_path = $4 - where id = $1 and state = 'waiting'`, id, build, repository, strings.Trim(sourcePath, "/")) - if err != nil { - return false, err - } - return tag.RowsAffected() == 1, nil +func (i *Inventory) RepointPending(ctx context.Context, id string, build, repository, sourcePath string) (bool, error) { + return i.changePending(ctx, id, func(p *chain.Pending) bool { + if p.State != PendingWaiting { + return false + } + p.Build, p.Repository, p.Path = build, repository, strings.Trim(sourcePath, "/") + return true + }) } // Pending is every pending assignment still waiting, oldest first, when endedSince is zero. Otherwise it is // every open one and every one ended since then. func (i *Inventory) Pending(ctx context.Context, endedSince time.Time) ([]PendingAssignment, error) { - if endedSince.IsZero() { - rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` - from pending_assignment p join node n on n.id = p.node - where p.state = 'waiting' order by p.since, p.id`) - if err != nil { - return nil, err - } - return scanPending(rows) - } - rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` - from pending_assignment p join node n on n.id = p.node - where p.state in ('waiting', 'applying') or p.settled_at >= $1 - order by p.since, p.id`, endedSince) + held, err := i.pendingValues(ctx) if err != nil { return nil, err } - return scanPending(rows) + if endedSince.IsZero() { + return pendingList(held, func(p chain.Pending) bool { return p.State == PendingWaiting }), nil + } + return pendingList(held, func(p chain.Pending) bool { + return p.Open() || (p.Settled != nil && !p.Settled.Before(endedSince)) + }), nil } -// PendingFor is every pending assignment of one module to one machine, newest first. +// PendingFor is every pending assignment of one module to one machine the key still has to say, newest first: the wait it +// holds now and the unanswered ones it replaced. func (i *Inventory) PendingFor(ctx context.Context, node, module string) ([]PendingAssignment, error) { - rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` - from pending_assignment p join node n on n.id = p.node - where n.name = $1 and p.module = $2 order by p.since desc, p.id desc`, node, module) + held, err := i.pendingValues(ctx) if err != nil { return nil, err } - return scanPending(rows) + var out []PendingAssignment + for _, k := range held { + if k.Value.Node == node && k.Value.Module == module { + for _, w := range pendingWaits(k.Value) { + out = append(out, pendingRecord(k.Value, w)) + } + } + } + // Newest first. + sort.SliceStable(out, func(a, b int) bool { return out[a].Since.After(out[b].Since) }) + return out, nil } // Stuck is every pending assignment left applying since before the time given: a controller that claimed // it and stopped before it said what came of it. func (i *Inventory) Stuck(ctx context.Context, before time.Time) ([]PendingAssignment, error) { - rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` - from pending_assignment p join node n on n.id = p.node - where p.state = 'applying' and p.settled_at < $1 order by p.id`, before) + held, err := i.pendingValues(ctx) if err != nil { return nil, err } - return scanPending(rows) + return pendingList(held, func(p chain.Pending) bool { + return p.State == PendingApplying && p.Settled != nil && p.Settled.Before(before) + }), nil } // ToRaise is every pending assignment that expired or was refused and has not been raised as a condition; // ToClear every one raised and not cleared. func (i *Inventory) ToRaise(ctx context.Context) ([]PendingAssignment, error) { - rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` - from pending_assignment p join node n on n.id = p.node - where p.state in ('expired', 'refused') and p.raised_at is null order by p.id`) + held, err := i.pendingValues(ctx) if err != nil { return nil, err } - return scanPending(rows) + return pendingList(held, func(p chain.Pending) bool { + return (p.State == PendingExpired || p.State == PendingRefused) && p.Raised == nil + }), nil } func (i *Inventory) ToClear(ctx context.Context) ([]PendingAssignment, error) { - rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` - from pending_assignment p join node n on n.id = p.node - where p.raised_at is not null and p.cleared_at is null order by p.id`) + held, err := i.pendingValues(ctx) if err != nil { return nil, err } - return scanPending(rows) + return pendingList(held, func(p chain.Pending) bool { return p.Raised != nil && p.Cleared == nil }), nil } // MarkPending stamps a pending assignment raised, acknowledged or cleared, now. -func (i *Inventory) MarkPending(ctx context.Context, id int64, what string) error { - column := map[string]string{"raised": "raised_at", "acknowledged": "acknowledged_at", "cleared": "cleared_at"}[what] - if column == "" { +func (i *Inventory) MarkPending(ctx context.Context, id string, what string) error { + if what != "raised" && what != "acknowledged" && what != "cleared" { return errors.New("a pending assignment is marked raised, acknowledged or cleared") } - _, err := i.store.Pool().Exec(ctx, - `update pending_assignment set `+column+` = coalesce(`+column+`, now()) where id = $1`, id) + now := time.Now().UTC() + _, err := i.changePending(ctx, id, func(p *chain.Pending) bool { + mark := map[string]**time.Time{"raised": &p.Raised, "acknowledged": &p.Acknowledged, "cleared": &p.Cleared}[what] + if *mark != nil { + return false // the first stamp stands + } + *mark = &now + return true + }) return err } // ClaimPending takes a waiting pending assignment for making, so nothing else settles or withdraws it while // it is made. False when it no longer waits: another process took it, or it was withdrawn. -func (i *Inventory) ClaimPending(ctx context.Context, id int64) (bool, error) { - tag, err := i.store.Pool().Exec(ctx, - `update pending_assignment set state = 'applying', settled_at = now() where id = $1 and state = 'waiting'`, id) - if err != nil { - return false, err - } - return tag.RowsAffected() == 1, nil +func (i *Inventory) ClaimPending(ctx context.Context, id string) (bool, error) { + now := time.Now().UTC() + return i.changePending(ctx, id, func(p *chain.Pending) bool { + if !chain.CanMove(p.State, PendingApplying) { + return false + } + p.State, p.Settled = PendingApplying, &now + return true + }) } // ReleaseClaim puts a claimed pending assignment back to waiting. -func (i *Inventory) ReleaseClaim(ctx context.Context, id int64) error { - _, err := i.store.Pool().Exec(ctx, - `update pending_assignment set state = 'waiting', settled_at = null where id = $1 and state = 'applying'`, id) +func (i *Inventory) ReleaseClaim(ctx context.Context, id string) error { + _, err := i.changePending(ctx, id, func(p *chain.Pending) bool { + if p.State != PendingApplying { + return false + } + p.State, p.Settled = PendingWaiting, nil + return true + }) return err } // SettlePending ends a pending assignment in the state it is read in (from): waiting for an expiry or a // withdrawal, applying for the outcome of making it. False when it was no longer in that state. -func (i *Inventory) SettlePending(ctx context.Context, id int64, from, state, note string) (bool, error) { +func (i *Inventory) SettlePending(ctx context.Context, id string, from, state, note string) (bool, error) { if state == PendingWaiting || state == PendingApplying { return false, errors.New("a pending assignment is settled to applied, refused, expired or withdrawn") } - tag, err := i.store.Pool().Exec(ctx, - `update pending_assignment set state = $3, note = $4, settled_at = now() - where id = $1 and state = $2`, id, from, state, note) - if err != nil { - return false, err - } - return tag.RowsAffected() == 1, nil + now := time.Now().UTC() + return i.changePending(ctx, id, func(p *chain.Pending) bool { + if p.State != from || !chain.CanSettle(from, state) { + return false + } + p.State, p.Note, p.Settled = state, note, &now + return true + }) } -// ForgetEndedPending deletes ended pending assignments settled more than KeptFor ago, and says how many. One -// raised as a condition is kept until that condition is cleared: deleting it would leave the condition -// with nothing to answer it. -func (i *Inventory) ForgetEndedPending(ctx context.Context, now time.Time) (int64, error) { - tag, err := i.store.Pool().Exec(ctx, - `delete from pending_assignment where state not in ('waiting', 'applying') and settled_at < $1 - and (raised_at is null or cleared_at is not null)`, - now.Add(-KeptFor)) - if err != nil { - return 0, err - } - return tag.RowsAffected(), nil -} - -// PendingKnown says which of these pending assignments are still on record. -func (i *Inventory) PendingKnown(ctx context.Context, ids []int64) (map[int64]bool, error) { - rows, err := i.store.Pool().Query(ctx, `select id from pending_assignment where id = any($1)`, ids) +// PendingKnown says which of these pending assignments are still on record: a key holds them now, or carries them as +// unanswered earlier waits. +func (i *Inventory) PendingKnown(ctx context.Context, ids []string) (map[string]bool, error) { + held, err := i.pendingValues(ctx) if err != nil { return nil, err } - defer rows.Close() - out := map[int64]bool{} - for rows.Next() { - var id int64 - if err := rows.Scan(&id); err != nil { - return nil, err - } - out[id] = true + want := map[string]bool{} + for _, id := range ids { + want[id] = true } - return out, rows.Err() + out := map[string]bool{} + for _, k := range held { + for _, w := range pendingWaits(k.Value) { + if want[w.ID] { + out[w.ID] = true + } + } + } + return out, nil +} + +// unansweredWaits are the waits a key holds that ended unmade and whose condition nobody has answered yet — taken back or +// cleared — which a new wait replacing them carries, so the operator is not left without it (issue 1108). +func unansweredWaits(v chain.Assignment, found bool) []chain.Pending { + if !found || v.State != chain.StatePending { + return nil + } + var out []chain.Pending + for _, w := range pendingWaits(v) { + if (w.State == PendingExpired || w.State == PendingRefused) && w.Cleared == nil && w.Acknowledged == nil { + out = append(out, w) + } + } + return out } diff --git a/internal/inventory/pin_completed_test.go b/internal/inventory/pin_completed_test.go deleted file mode 100644 index 2cc7d37c..00000000 --- a/internal/inventory/pin_completed_test.go +++ /dev/null @@ -1,88 +0,0 @@ -//go:build integration - -package inventory - -import ( - "context" - "os" - "testing" -) - -// A pin made before it named the module (migration 0051, novox/hq #258): completed where the node it -// names answers once, left for a person where it answers twice. - -func legacyPin(t *testing.T, inv *Inventory, node, provision, provider string) { - t.Helper() - _, err := inv.store.Pool().Exec(context.Background(), - `insert into provision_pin (node, name, provider) - select u.id, $2, p.id from node u, node p where u.name = $1 and p.name = $3`, - node, provision, provider) - if err != nil { - t.Fatal(err) - } -} - -func completeEarlierPins(t *testing.T, inv *Inventory) { - t.Helper() - sql, err := os.ReadFile("migrations/0051-a-pin-made-before-is-completed.sql") - if err != nil { - t.Fatal(err) - } - if _, err := inv.store.Pool().Exec(context.Background(), string(sql)); err != nil { - t.Fatal(err) - } -} - -func TestAPinMadeBeforeIsCompletedWhenTheNodeAnswersOnce(t *testing.T) { - inv := inTheTables(t) - ctx := context.Background() - for _, n := range []string{"user", "provider"} { - if _, err := inv.AddNode(ctx, n); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"postgres", "redis"} { - if err := inv.RegisterModule(ctx, manifest(m, []string{m + "-database"}, nil), Source{}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "provider", m); err != nil { - t.Fatal(err) - } - } - legacyPin(t, inv, "user", "postgres-database", "provider") - completeEarlierPins(t, inv) - pins, err := inv.PinsFor(ctx, "user") - if err != nil { - t.Fatal(err) - } - if got := pins["postgres-database"]; got.Node != "provider" || got.Module != "postgres" { - t.Fatalf("the record was not completed with the one module that answers: %+v", got) - } -} - -func TestAPinMadeBeforeIsLeftOpenWhenTheNodeAnswersTwice(t *testing.T) { - inv := inTheTables(t) - ctx := context.Background() - for _, n := range []string{"user", "provider"} { - if _, err := inv.AddNode(ctx, n); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"public-acme", "step-ca"} { - if err := inv.RegisterModule(ctx, manifest(m, []string{"acme-ca"}, nil), Source{}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "provider", m); err != nil { - t.Fatal(err) - } - } - legacyPin(t, inv, "user", "acme-ca", "provider") - completeEarlierPins(t, inv) - pins, err := inv.PinsFor(ctx, "user") - if err != nil { - t.Fatal(err) - } - if got := pins["acme-ca"]; got.Node != "provider" || got.Module != "" { - t.Fatalf("a node that answers twice was guessed for: %+v", got) - } -} diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index 8ad7a0d1..cecc0d70 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -2,15 +2,12 @@ package inventory import ( "context" - "encoding/json" "errors" "fmt" "slices" "strings" "time" - "github.com/jackc/pgx/v5" - "github.com/novox/mesh-controller/internal/chain" ) @@ -680,140 +677,12 @@ type MergeMove struct { // leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by // somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once // the transaction is committed, in order. -func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) (err error) { - if c, on, cerr := i.ChainReads(ctx, KindRollouts); cerr != nil { - return cerr - } else if on { - return i.chainSavePlans(ctx, c, plans, moves) - } - epoch, err := i.actingEpoch(ctx) - if err != nil { - if len(plans) > 0 { - return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err) - } - return err - } - // **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the - // plan moves, in the same transaction as the move, so no save can move a tier unmeasured or - // measure one twice. - tx, err := i.store.Pool().Begin(ctx) +func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error { + c, err := i.theChain(ctx) if err != nil { return err } - defer func() { _ = tx.Rollback(ctx) }() - type written struct { - revision int64 - entered time.Time - } - done := make([]written, len(plans)) - // **The save that starts or ends a walk is marked** (novox/hq ADR 0308 decision 1), in the same write, and - // unmarked should the write not be made, so the next save marks it again. - marked := time.Now() - undo := make([]func(), 0, len(plans)) - defer func() { - if err != nil { - for _, u := range undo { - u() - } - } - }() - for _, p := range plans { - undo = append(undo, p.markWalk(marked)) - } - for n, p := range plans { - revision, entered, err := writePlan(ctx, tx, p, epoch) - if err != nil { - return err - } - done[n] = written{revision, entered} - } - for _, m := range moves { - if _, err := tx.Exec(ctx, - `update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`, - m.Repository, m.Commit, m.Plan, m.Alone); err != nil { - return err - } - } - if err := tx.Commit(ctx); err != nil { - return err - } - for n, p := range plans { - p.Revision, p.TierEntered = done[n].revision, done[n].entered - if epoch != nil { - p.Epoch = uint64(*epoch) - } else { - p.Epoch = 0 - } - if PlanSaved != nil { - PlanSaved(i, *p) - } - } - return nil -} - -// writePlan writes one plan within a transaction, by compare-and-set on its revision: its new revision, and when -// it entered its tier. -func writePlan(ctx context.Context, tx pgx.Tx, p *Plan, epoch *int64) (int64, time.Time, error) { - tiers, err := json.Marshal(p.Tiers) - if err != nil { - return 0, time.Time{}, err - } - modules, err := json.Marshal(p.Modules) - if err != nil { - return 0, time.Time{}, err - } - var release, delivery, commits, times, backups []byte - if p.Backups != nil { - if backups, err = json.Marshal(p.Backups); err != nil { - return 0, time.Time{}, err - } - } - if p.Times != nil { - if times, err = json.Marshal(p.Times); err != nil { - return 0, time.Time{}, err - } - } - if len(p.Commits) > 0 { - if commits, err = json.Marshal(p.Commits); err != nil { - return 0, time.Time{}, err - } - } - if p.Release != nil { - if release, err = json.Marshal(p.Release); err != nil { - return 0, time.Time{}, err - } - } - if p.Delivery != nil { - if delivery, err = json.Marshal(p.Delivery); err != nil { - return 0, time.Time{}, err - } - } - entered, err := planTierLeft(ctx, tx, *p, time.Now()) - if err != nil { - return 0, time.Time{}, err - } - var revision int64 - err = tx.QueryRow(ctx, - `insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, - branch, tier_entered, revision, epoch, release, delivery, merged_at, commits, times, backups) - values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14, $15, $16, $17, $18, $19) - on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier, - tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch, - tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch, - release = excluded.release, delivery = excluded.delivery, repository = excluded.repository, - commit_hash = excluded.commit_hash, merged_at = excluded.merged_at, commits = excluded.commits, - created = excluded.created, times = excluded.times, backups = excluded.backups - where release_plan.revision = $12 - returning revision`, - p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered, - p.Revision, epoch, release, delivery, mergedAt(p.Merged), commits, times, backups).Scan(&revision) - if errors.Is(err, pgx.ErrNoRows) { - // The row is there and at another revision — moved since this was read, or there already - // when this one is new: either way not this writer's to overwrite. (A plan saved before plans - // had revisions is at zero, and its first save here is from a read at zero.) - return 0, time.Time{}, fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) - } - return revision, entered, err + return i.chainSavePlans(ctx, c, plans, moves) } // short is a commit as a person reads it. @@ -824,145 +693,54 @@ func short(commit string) string { return commit } -// mergedAt is a plan's merge time as the store keeps it: null when not known. -func mergedAt(t time.Time) *time.Time { - if t.IsZero() { - return nil - } - return &t -} - // OpenPlans is every plan still being worked, oldest first. func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) } - return i.plans(ctx, `where state in ('building', 'rolling') order by created`) + return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) } // PlansSince is every plan made after a moment, and every plan still being worked, oldest first. func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated) } - return i.plans(ctx, `where created > $1 or state in ('building', 'rolling') order by created`, since) + return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated) } // RecentPlans is the last few plans, newest first, open or not — what the overview shows. func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest) - if err == nil && limit >= 0 && len(plans) > limit { - plans = plans[:limit] - } - return plans, err } - return i.plans(ctx, `order by created desc limit $1`, limit) + plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest) + if err == nil && limit >= 0 && len(plans) > limit { + plans = plans[:limit] + } + return plans, err } // NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh // planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it // recorded when its merge was made. func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { - return Plan{}, false, err - } else if on { - return i.chainNewestMergeOf(ctx, c, repository, branch) - } - plans, err := i.plans(ctx, `where lower(repository) = lower($1) and branch = $2 and merged_at is not null - and release is null order by merged_at desc, created desc limit 1`, repository, branch) - if err != nil || len(plans) == 0 { + c, err := i.theChain(ctx) + if err != nil { return Plan{}, false, err } - return plans[0], true, nil + return i.chainNewestMergeOf(ctx, c, repository, branch) } // PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a // caller of the plans verb. func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { - return Plan{}, err - } else if on { - return i.chainPlanByID(ctx, c, id) - } - plans, err := i.plans(ctx, `where id = $1`, id) + c, err := i.theChain(ctx) if err != nil { return Plan{}, err } - if len(plans) == 0 { - return Plan{}, fmt.Errorf("no plan %s", id) - } - return plans[0], nil -} - -func (i *Inventory) plans(ctx context.Context, tail string, args ...any) ([]Plan, error) { - rows, err := i.store.Pool().Query(ctx, - `select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch, - coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery, merged_at, commits, times, backups - from release_plan `+tail, args...) - if err != nil { - return nil, err - } - defer rows.Close() - var out []Plan - for rows.Next() { - var p Plan - var tiers, modules, release, delivery, commits, times, backups []byte - var epoch int64 - var merged *time.Time - if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State, - &p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch, &release, - &delivery, &merged, &commits, ×, &backups); err != nil { - return nil, err - } - if len(backups) > 0 { - if err := json.Unmarshal(backups, &p.Backups); err != nil { - return nil, err - } - } - if len(times) > 0 { - if err := json.Unmarshal(times, &p.Times); err != nil { - return nil, err - } - } - if len(commits) > 0 { - if err := json.Unmarshal(commits, &p.Commits); err != nil { - return nil, err - } - } - if merged != nil { - p.Merged = merged.UTC() - } - if len(release) > 0 { - if err := json.Unmarshal(release, &p.Release); err != nil { - return nil, err - } - } - if len(delivery) > 0 { - if err := json.Unmarshal(delivery, &p.Delivery); err != nil { - return nil, err - } - } - p.Epoch = uint64(epoch) - if err := json.Unmarshal(tiers, &p.Tiers); err != nil { - return nil, err - } - if err := json.Unmarshal(modules, &p.Modules); err != nil { - return nil, err - } - if p.Modules == nil { - p.Modules = map[string]*PlanModule{} - } - out = append(out, p) - } - if errors.Is(rows.Err(), pgx.ErrNoRows) { - return nil, nil - } - return out, rows.Err() + return i.chainPlanByID(ctx, c, id) } diff --git a/internal/inventory/sends.go b/internal/inventory/sends.go index f9fdcdf4..db8b7095 100644 --- a/internal/inventory/sends.go +++ b/internal/inventory/sends.go @@ -2,7 +2,6 @@ package inventory import ( "context" - "encoding/json" "fmt" "time" @@ -150,32 +149,26 @@ func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, bui e := int64(epoch) sentEpoch = &e } - var carried *string - if builds != nil { - raw, err := json.Marshal(builds) - if err != nil { - return err - } - text := string(raw) - carried = &text - } tx, err := i.store.Pool().Begin(ctx) if err != nil { return err } defer func() { _ = tx.Rollback(ctx) }() if _, err := tx.Exec(ctx, - `update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4, sent_generation = $5, - sent_provisioning = $6::jsonb - where id = $1`, node, digest, carried, sentEpoch, nullIfZero(generation), nullIfEmpty(s.Provisioning)); err != nil { + `update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3, sent_generation = $4, + sent_provisioning = $5::jsonb + where id = $1`, node, digest, sentEpoch, nullIfZero(generation), nullIfEmpty(s.Provisioning)); err != nil { return err } recordErr := recordSend(ctx, tx, node, s) if err := tx.Commit(ctx); err != nil { return err } - // And what it put on the machine, in the assignments of the modules it carried a build of (issue 1095). - saidRunsNot(node, i.recordRuns(ctx, node, builds, time.Now())) + // And what it put on the machine, in the assignments (issues 1095, 1109): a write of it that fails fails this record, + // for what a machine was sent is read from there. + if err := i.recordRuns(ctx, node, builds, time.Now()); err != nil { + return fmt.Errorf("%s was sent its declaration, and what each module of it runs there could not be written: %w", node, err) + } if recordErr != nil { return &SendNotRecordedError{Err: recordErr} } diff --git a/internal/inventory/sent_builds_test.go b/internal/inventory/sent_builds_test.go index f28ab693..cde4b6ff 100644 --- a/internal/inventory/sent_builds_test.go +++ b/internal/inventory/sent_builds_test.go @@ -27,8 +27,10 @@ func TestASendKeepsTheBuildsItCarried(t *testing.T) { if err := inv.RecordSent(ctx, node.ID, "d1", carried); err != nil { t.Fatal(err) } + // What it was sent is read from the keys of the machine's assignments that name a commit (issue 1109): a module carried + // with no commit known names none. builds, known, err := inv.SentBuilds(ctx, "anchor") - if err != nil || !known || !reflect.DeepEqual(builds, carried) { + if err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"resolver": "abc123"}) { t.Fatalf("the builds sent were not kept: %v %v %v", builds, known, err) } diff --git a/internal/inventory/sent_plans_test.go b/internal/inventory/sent_plans_test.go index 63a82deb..fb1a081c 100644 --- a/internal/inventory/sent_plans_test.go +++ b/internal/inventory/sent_plans_test.go @@ -48,10 +48,7 @@ func TestASendKeepsThePlansItHandedAndTheMirrorCarriesThem(t *testing.T) { t.Fatalf("the plans mirrored are %+v (%v)", plans, err) } inv.SetProvisioningPlans(inv.SentProvisioningPlans) - mirror, err := inv.AssignmentsMirror(ctx) - if err != nil { - t.Fatal(err) - } + mirror := inv.PlanValuesForTest(t) var dunst, web AssignmentMirrored if err := json.Unmarshal(mirror[node+".dunst"], &dunst); err != nil { t.Fatal(err) @@ -81,8 +78,8 @@ func TestASendKeepsThePlansItHandedAndTheMirrorCarriesThem(t *testing.T) { if sent, _, err := inv.SentTo(ctx, node); err != nil || sent.Provisioned != nil { t.Fatalf("a send handing nothing says it handed %v (%v)", sent.Provisioned, err) } - if mirror, err := inv.AssignmentsMirror(ctx); err != nil || strings.Contains(string(mirror[node+".dunst"]), "provisioning") { - t.Fatalf("dunst's assignment still carries a plan: %s (%v)", mirror[node+".dunst"], err) + if mirror := inv.PlanValuesForTest(t); strings.Contains(string(mirror[node+".dunst"]), "provisioning") { + t.Fatalf("dunst's assignment still carries a plan: %s", mirror[node+".dunst"]) } else if _, has := mirror[node+".notify"]; has { t.Fatalf("a module the machine no longer runs is mirrored: %s", mirror[node+".notify"]) } diff --git a/internal/inventory/source.go b/internal/inventory/source.go index 084905b7..f58714b8 100644 --- a/internal/inventory/source.go +++ b/internal/inventory/source.go @@ -2,7 +2,6 @@ package inventory import ( "context" - "encoding/json" "github.com/novox/mesh-controller/internal/chain" ) @@ -35,39 +34,11 @@ type sourceRow struct { // sourceRows is every successful build of each module (of one module, when named), newest request // first. func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][]sourceRow, []string, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, nil, err - } else if on { - return i.chainSourceRows(ctx, c, module) - } - query := `select b.module, b.id, b.commit_hash, b.source_fingerprint, b.made, b.manifest, - coalesce(g.verdict = 'failed', false) - from build b left join build_gate g on g.build = b.id - where b.module is not null and b.module <> '' and b.failed = ''` - args := []any{} - if module != "" { - query += ` and b.module = $1` - args = append(args, module) - } - rows, err := i.store.Pool().Query(ctx, query+` order by b.module, coalesce(b.asked, b.at) desc, b.at desc`, args...) + c, err := i.theChain(ctx) if err != nil { return nil, nil, err } - defer rows.Close() - out := map[string][]sourceRow{} - var order []string - for rows.Next() { - var m string - var r sourceRow - if err := rows.Scan(&m, &r.id, &r.commit, &r.fingerprint, &r.made, &r.manifest, &r.failedGate); err != nil { - return nil, nil, err - } - if _, seen := out[m]; !seen { - order = append(order, m) - } - out[m] = append(out[m], r) - } - return out, order, rows.Err() + return i.chainSourceRows(ctx, c, module) } // standing is, of a module's builds newest first, the index of the build that stands for the build at @@ -102,8 +73,8 @@ func (i *Inventory) StandingBuild(ctx context.Context, module, build string) (st s := builds[standing(builds, k)] if len(s.manifest) == 0 && s.manifestDigest != "" { // The manifest is kept apart from the record (issue 1121): one Get. - c, on, err := i.ChainReads(ctx, KindBuilds) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return "", nil, err } raw, err := manifestOfRecord(ctx, c, chain.Build{ID: s.id, ManifestDigest: s.manifestDigest}) @@ -118,76 +89,32 @@ func (i *Inventory) StandingBuild(ctx context.Context, module, build string) (st // SourceFingerprints is, per module, per commit, the source fingerprint of the newest successful build // from it that has one: module → commit → fingerprint. A commit whose builds carry none is absent. func (i *Inventory) SourceFingerprints(ctx context.Context) (map[string]map[string]string, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - // Read from the commit pointers, which carry each commit's source fingerprint: no record is read (issue 1121). - return i.chainSourceFingerprints(ctx, c) - } - all, _, err := i.sourceRows(ctx, "") + c, err := i.theChain(ctx) if err != nil { return nil, err } - out := map[string]map[string]string{} - for m, builds := range all { - for _, b := range builds { - if b.fingerprint == "" || b.commit == "" { - continue - } - if out[m] == nil { - out[m] = map[string]string{} - } - if _, seen := out[m][b.commit]; !seen { - out[m][b.commit] = b.fingerprint - } - } - } - return out, nil + // Read from the commit pointers, which carry each commit's source fingerprint: no record is read (issue 1121). + return i.chainSourceFingerprints(ctx, c) } // BuildSourceFingerprints is SourceFingerprints for one module: commit → fingerprint. func (i *Inventory) BuildSourceFingerprints(ctx context.Context, module string) (map[string]string, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - // This module's commit pointers only (issue 1121). - return i.chainBuildSourceFingerprints(ctx, c, module) - } - all, err := i.SourceFingerprints(ctx) + c, err := i.theChain(ctx) if err != nil { return nil, err } - if all[module] == nil { - return map[string]string{}, nil - } - return all[module], nil + // This module's commit pointers only (issue 1121). + return i.chainBuildSourceFingerprints(ctx, c, module) } // heldMade is, per module, what the build standing for its newest successful build made — what Held // answers (novox/hq issue 280). func (i *Inventory) heldMade(ctx context.Context) (map[string][]Artifact, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainHeldMade(ctx, c) - } - all, _, err := i.sourceRows(ctx, "") + c, err := i.theChain(ctx) if err != nil { return nil, err } - out := map[string][]Artifact{} - for m, builds := range all { - if len(builds) == 0 { - continue - } - var made []Artifact - if err := json.Unmarshal(builds[standing(builds, 0)].made, &made); err != nil { - // Skipped rather than fatal, as Held always did: the build that needs it says it is missing. - continue - } - out[m] = made - } - return out, nil + return i.chainHeldMade(ctx, c) } // SameSourceCommits is the commits of the builds in a build's run (novox/hq issue 280): the build and diff --git a/internal/recordmirror/mirror.go b/internal/recordmirror/mirror.go index a751f73c..f330ed4e 100644 --- a/internal/recordmirror/mirror.go +++ b/internal/recordmirror/mirror.go @@ -60,7 +60,7 @@ var ( ) // Mirrors is every mirror this package writes, in the order they are written. -func Mirrors() []Mirror { return []Mirror{Assignments, DeclaredConfig, Predefined} } +func Mirrors() []Mirror { return []Mirror{DeclaredConfig, Predefined} } // The changes an event says. const ( diff --git a/internal/recordmirror/mirror_store_test.go b/internal/recordmirror/mirror_store_test.go index 34d196df..7e4cbdda 100644 --- a/internal/recordmirror/mirror_store_test.go +++ b/internal/recordmirror/mirror_store_test.go @@ -12,30 +12,19 @@ import ( "github.com/novox/mesh-controller/internal/inventory" ) -// **Design 57's F3, end to end** (novox/hq issue 652): from the controller's store to a real bus. An -// assignment changes exactly one key of the assignments mirror and raises exactly one `assignment-changed`; -// a registration changes exactly one key of the declared-config mirror and raises exactly one -// `declared-config-changed`; and the same records written again change nothing and say nothing — a module -// registered again unchanged included, although the store's registration time moved. +// **Design 57's F3, end to end** (novox/hq issue 652): from the controller's store to a real bus. A registration +// changes exactly one key of the declared-config mirror and raises exactly one `declared-config-changed`; and the same +// records written again change nothing and say nothing — a module registered again unchanged included, although the +// store's registration time moved. (The assignments are keys of the bus, no mirror of the store: ADR 0408.) func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *testing.T) { inv := inventory.ForTest(t) ctx := t.Context() js, api := aMirrorBus(t) - assignments := aWriter(t, js, api, Assignments) declared := aWriter(t, js, api, DeclaredConfig) - heardAssignments := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+Assignments.Event) heardDeclared := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+DeclaredConfig.Event) - pass := func() (Result, Result) { + pass := func() Result { t.Helper() - a, err := inv.AssignmentsMirror(ctx) - if err != nil { - t.Fatal(err) - } - ra, err := assignments.Sync(ctx, a) - if err != nil { - t.Fatal(err) - } d, err := inv.DeclaredConfigMirror(ctx) if err != nil { t.Fatal(err) @@ -44,7 +33,7 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes if err != nil { t.Fatal(err) } - return ra, rd + return rd } if _, err := inv.AddNode(ctx, "anchor"); err != nil { @@ -56,72 +45,37 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes t.Fatal(err) } pass() - assignmentWrites, declaredWrites := writes(t, api, Assignments.Bucket), writes(t, api, DeclaredConfig.Bucket) - assignmentsSaid, declaredSaid := len(settle(heardAssignments, 0)), len(settle(heardDeclared, 1)) + declaredWrites := writes(t, api, DeclaredConfig.Bucket) + declaredSaid := len(settle(heardDeclared, 1)) // **A registration**: a module new to the catalogue. if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "notes", Version: "1"}, inventory.Source{}); err != nil { t.Fatal(err) } - ra, rd := pass() - if ra != (Result{}) || rd != (Result{Written: 1}) { - t.Errorf("a registration wrote assignments %+v and declared config %+v", ra, rd) + if rd := pass(); rd != (Result{Written: 1}) { + t.Errorf("a registration wrote declared config %+v", rd) } if got := writes(t, api, DeclaredConfig.Bucket) - declaredWrites; got != 1 { t.Errorf("a registration changed %d keys of the declared config", got) } - if got := writes(t, api, Assignments.Bucket) - assignmentWrites; got != 0 { - t.Errorf("a registration changed %d keys of the assignments", got) - } said := settle(heardDeclared, declaredSaid+1)[declaredSaid:] if len(said) != 1 || said[0].Key != "notes" || said[0].Change != Added { t.Errorf("a registration raised %+v", said) } declaredWrites, declaredSaid = writes(t, api, DeclaredConfig.Bucket), declaredSaid+len(said) - // **An assignment.** - if _, err := inv.Assign(ctx, "anchor", "ui"); err != nil { - t.Fatal(err) - } - ra, rd = pass() - if ra != (Result{Written: 1}) || rd != (Result{}) { - t.Errorf("an assignment wrote assignments %+v and declared config %+v", ra, rd) - } - if got := writes(t, api, Assignments.Bucket) - assignmentWrites; got != 1 { - t.Errorf("an assignment changed %d keys of the assignments", got) - } - said = settle(heardAssignments, assignmentsSaid+1)[assignmentsSaid:] - if len(said) != 1 || said[0].Key != "anchor.ui" || said[0].Change != Added { - t.Errorf("an assignment raised %+v", said) - } - assignmentWrites, assignmentsSaid = writes(t, api, Assignments.Bucket), assignmentsSaid+len(said) - e, err := assignments.KV.Get(ctx, "anchor.ui") - if err != nil { - t.Fatal(err) - } - var value inventory.AssignmentMirrored - if err := json.Unmarshal(e.Value(), &value); err != nil || value.Node != "anchor" || value.Module != "ui" || - value.Assigned.IsZero() { - t.Errorf("the assignment is mirrored as %s (%v)", e.Value(), err) - } - - // **The same records written again change nothing**: an assignment made again, and a module registered - // again with the manifest it had. - if _, err := inv.Assign(ctx, "anchor", "ui"); err != nil { - t.Fatal(err) - } + // **The same records written again change nothing**: a module registered again with the manifest it had. if err := inv.RegisterModule(ctx, ui, inventory.Source{}); err != nil { t.Fatal(err) } - if ra, rd := pass(); ra != (Result{}) || rd != (Result{}) { - t.Errorf("unchanged records wrote assignments %+v and declared config %+v", ra, rd) + if rd := pass(); rd != (Result{}) { + t.Errorf("unchanged records wrote declared config %+v", rd) } - if writes(t, api, Assignments.Bucket) != assignmentWrites || writes(t, api, DeclaredConfig.Bucket) != declaredWrites { + if writes(t, api, DeclaredConfig.Bucket) != declaredWrites { t.Error("unchanged records changed a key") } - if a, d := settle(heardAssignments, assignmentsSaid+1), settle(heardDeclared, declaredSaid+1); len(a) != assignmentsSaid || - len(d) != declaredSaid { - t.Errorf("unchanged records raised %+v and %+v", a[assignmentsSaid:], d[declaredSaid:]) + if d := settle(heardDeclared, declaredSaid+1); len(d) != declaredSaid { + t.Errorf("unchanged records raised %+v", d[declaredSaid:]) } // **A declaration changed by a registration**: one key, one event, naming the field that moved. @@ -131,8 +85,8 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes if err := inv.RegisterModule(ctx, ui, inventory.Source{}); err != nil { t.Fatal(err) } - if ra, rd := pass(); ra != (Result{}) || rd != (Result{Written: 1}) { - t.Errorf("a changed declaration wrote assignments %+v and declared config %+v", ra, rd) + if rd := pass(); rd != (Result{Written: 1}) { + t.Errorf("a changed declaration wrote declared config %+v", rd) } said = settle(heardDeclared, declaredSaid+1)[declaredSaid:] if len(said) != 1 || said[0].Key != "ui" || said[0].Change != Changed || strings.Join(said[0].Fields, ",") != "settings" { @@ -146,25 +100,13 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes t.Errorf("the declaration is mirrored as %s, want %s", got.Value(), want) } - // **An unassignment** removes its key, said once. - if err := inv.Unassign(ctx, "anchor", "ui"); err != nil { - t.Fatal(err) - } - if ra, _ := pass(); ra != (Result{Removed: 1}) { - t.Errorf("an unassignment did %+v", ra) - } - said = settle(heardAssignments, assignmentsSaid+1)[assignmentsSaid:] - if len(said) != 1 || said[0].Key != "anchor.ui" || said[0].Change != Removed { - t.Errorf("an unassignment raised %+v", said) - } - // **A module no longer registered** has its key removed, said once as `removed` (ADR 0356 §4). if err := inv.ForgetModule(ctx, "notes"); err != nil { t.Fatal(err) } declaredWrites = writes(t, api, DeclaredConfig.Bucket) declaredSaid = len(heardDeclared.all()) - if _, rd := pass(); rd != (Result{Removed: 1}) { + if rd := pass(); rd != (Result{Removed: 1}) { t.Errorf("a module forgotten did %+v", rd) } if got := writes(t, api, DeclaredConfig.Bucket) - declaredWrites; got != 1 { diff --git a/testdata/store-tests b/testdata/store-tests index 91c80206..3d2db5be 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -139,19 +139,25 @@ ./cmd/mesh-controller TestADeliveryGroupMovingAModuleMachinesRunPassesTheGate ./cmd/mesh-controller TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt ./cmd/mesh-controller TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheBus -./internal/inventory TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing -./internal/inventory TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing -./internal/inventory TestTheMoveRefusesAValueOverTheLargestMessage -./internal/inventory TestTheMoveRefusesOnACountThatDiffers -./internal/inventory TestTheMoveWithoutTheLeaseWritesNothing ./internal/inventory TestBuildsOnTheBusAreReadBackByEveryReader ./internal/inventory TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace -./internal/inventory TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain -./internal/inventory TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame ./cmd/mesh-controller TestAMergeWhosePlanCannotBeSavedIsHeldAndRecordedWhenHandedOverAgain ./internal/inventory TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt ./internal/inventory TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves -./internal/inventory TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent ./cmd/mesh-controller TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments ./cmd/mesh-controller TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment -./internal/inventory TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved +./internal/inventory TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket +./internal/inventory TestARecordedPendingIsSkippedByAssignedAndTheJoins +./internal/inventory TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse +./internal/inventory TestAssignOverAPendingKeepsItsRecordAsApplied +./internal/inventory TestAPendingIsMarkedOnceAndGoesWithItsMachine +./internal/inventory TestAReplacedWaitNeverLeavesTheOperatorWithoutItsCondition +./cmd/mesh-controller TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket +./internal/inventory TestSentBuildsAnswersFromTheAssignmentsKeysOfTheNode +./internal/inventory TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown +./internal/inventory TestAFailedWriteOfTheLinksFailsTheRecordOfTheSend +./internal/inventory TestMigration0105RefusesAStoreWhoseRowsWereNotMovedAndDropsTheTablesOfAMovedOne +./internal/inventory TestMigration0105PassesAStoreThatNeverHeldARow +./cmd/mesh-controller TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments +./internal/inventory TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied +./internal/inventory TestASendEmptiesTheColumnSoALaterCopyLeavesAClearedKeyBare