novox/mesh-controller
13
0
Fork 0
Code Issues Pull Requests 3 Releases Wiki Activity

Merge pull request 'The chain lives on the bus alone: the store's tables of builds, rollouts and assignments are dropped (design 47 milestone 5, delivery 2: issues 1094, 1108, 1109, 1110, 1096, 1111)' (#381) from feat/chain-on-the-bus-d2 into main

Browse Source
This commit was merged in pull request #381.
This commit is contained in:
mesh-tools
2026-10-18 12:30:42 +00:00
parent da6d038e07 68db18bdfc
commit d3556f452e
60 changed files with 2633 additions and 3665 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/awaiting_push_integration_test.go
+13 -2
View File
@@ -10,6 +10,7 @@ import (
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/overlay"
)
@@ -146,8 +147,18 @@ func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
pushedBy(t, open, "laptop")
// What a machine was sent is read from the commits its assignments name (issue 1109): a module is carried by a send only
// when it was built from a commit, and a hand send on a machine with nothing assigned leaves no key to mark unknown.
if err := open.inventory.RegisterModule(ctx, aKeeper(), inventory.Source{BuiltFrom: "c-keeper"}); err != nil {
t.Fatal(err)
}
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSent(ctx, record.ID, "carried", map[string]string{overlay.Name: "c-overlay"}); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
cmd/mesh-controller/awaiting_push_test.go
+15 -1
View File
@@ -46,7 +46,21 @@ func pushedBy(t *testing.T, open *stores, node string) string {
t.Fatal(err)
}
digest := digestOf(body)
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
// A push records the build of each module it carries (plan.go, Allocating); a module is carried by a send only when it was
// built from a commit, for what a machine was sent is read from the commits its assignments name (issue 1109).
builds := declared.Builds
if builds == nil {
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
t.Fatal(err)
}
names := make([]string, 0, len(plan.Modules))
for _, m := range plan.Modules {
names = append(names, m.Module)
}
builds = carriedBuilds(names, nil, current, nil)
}
if err := open.inventory.RecordSent(ctx, record.ID, digest, builds); err != nil {
t.Fatal(err)
}
return digest
cmd/mesh-controller/chain.go
+85 -95
View File
@@ -13,9 +13,7 @@ import (
)
// The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
// bus's key-value store, written by the controller that holds the lease. The tables of the controller's store are
// copied into them once, by the serving controller at its start, and a kind is read from its bucket only once the move
// has recorded it (inventory.ChainReads).
// bus's key-value store, written by the controller that holds the lease, and the controller's store keeps none of them.
// chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes.
func chainHolds(ctx context.Context) error {
@@ -33,97 +31,7 @@ func openTheChain(ctx context.Context) (*chain.Chain, error) {
return chain.OnTheBus(ctx, js.Conn(), chainHolds)
}
// kindChainMoveRefused is the condition of a move that stopped without switching a kind.
const kindChainMoveRefused = "chain-move-refused"
// chainMoveRetry is how long a move that failed for want of the bus waits before it tries again.
const chainMoveRetry = 30 * time.Second
// movingTheChain copies the tables into the buckets and records it, at the serving controller's start: after the buckets
// exist and the lease is held, before anything reads a kind that has not moved. **The first try is made here, before the
// controller serves**, so what it serves is read from one place. A move that fails for want of the bus is tried again
// every half minute while the controller serves from the tables; one that is refused — a count that differs, a value
// over the bus's largest — is raised as an urgent condition, and the tables stay what is read: nothing is half-switched.
func movingTheChain(ctx context.Context, inv *inventory.Inventory) {
say := func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }
// The first try is made here, before the controller serves; the tries after it, while it serves.
if movedOrStopped(ctx, inv, say) {
return
}
go func() {
for {
select {
case <-ctx.Done():
return
case <-time.After(chainMoveRetry):
}
if movedOrStopped(ctx, inv, say) {
return
}
}
}()
}
// movedOrStopped makes one try at the move and says whether there is nothing more to try: moved, refused, or the controller
// ending. False for a failure of the kind that is tried again (the bus away, a write not granted yet, the lease not held).
func movedOrStopped(ctx context.Context, inv *inventory.Inventory, say func(string, ...any)) bool {
_, err := inv.MoveChain(ctx, say)
var refusal *inventory.Refusal
switch {
case err == nil:
go clearChainMove(ctx)
go askAgainSoon(ctx, inv)
return true
case errors.As(err, &refusal):
say("%v", err)
go raiseChainMove(ctx, refusal)
return true
case ctx.Err() != nil:
return true
}
say("the move to the bus did not finish, and the tables are read until it does: %v", err)
return false
}
// chainMoveObservation is the urgent condition for a refused move.
func chainMoveObservation(r *inventory.Refusal) conditions.Observation {
return conditions.Observation{Scope: conditions.ScopeBus, ID: "chain." + r.Kind, Token: kindChainMoveRefused,
Kind: kindChainMoveRefused, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
Summary: "the controller's " + r.Kind + " were not moved to the bus: " + r.Why, Source: "chain move"}
}
// whenKept runs f once the serving controller keeps conditions: the move runs before they are kept.
func whenKept(ctx context.Context, f func(k *conditions.Keeper)) {
for conditionsFrom == nil {
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
f(conditionsFrom)
}
func raiseChainMove(ctx context.Context, r *inventory.Refusal) {
whenKept(ctx, func(k *conditions.Keeper) {
if _, err := k.Observe(ctx, chainMoveObservation(r)); err != nil {
fmt.Printf("chain: the condition for the refused move could not be raised: %v\n", err)
}
})
}
func clearChainMove(ctx context.Context) {
whenKept(ctx, func(k *conditions.Keeper) {
for _, kind := range []string{inventory.KindBuilds, inventory.KindRollouts, inventory.KindAssignments} {
key := chainMoveObservation(&inventory.Refusal{Kind: kind}).Key()
if _, err := k.Clear(ctx, key, "the "+kind+" are on the bus"); err != nil {
fmt.Printf("chain: the condition for the move of the %s could not be cleared: %v\n", kind, err)
}
}
})
}
// attachTheChain gives the inventory of the serving controller the chain on its connection, then moves what has not moved.
// attachTheChain gives the inventory of the serving controller the chain on its connection.
func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error {
bus, ok := server.Bus().(link.OverNATS)
if !ok {
@@ -134,7 +42,13 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
return err
}
inv.UseChain(c)
movingTheChain(ctx, inv)
inv.RequireLayout()
// And the builds laid out under their modules, once, before they are read (issue 1121).
go layBuildsOutSoon(ctx, inv)
// And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving.
go askAgainSoon(ctx, inv)
// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
go copySentBuildsSoon(ctx, inv)
return nil
}
@@ -179,3 +93,79 @@ func askAgainSoon(ctx context.Context, inv *inventory.Inventory) {
askAgain(ctx, inv)
}
}
// kindSentBuildsNotCopied is the condition of the one-time copy of node.sent_builds onto the assignments' keys that has not
// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
const kindSentBuildsNotCopied = "sent-builds-not-copied"
// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon
// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails,
// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key
// that names a commit and empties each column it has copied.
func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) {
failed := false
for {
_, err := inv.CopySentBuilds(ctx)
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied,
Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "}
if err != nil {
fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err)
if keeper := conditionsFrom; keeper != nil && !failed {
o.Summary += err.Error()
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
failed = true
}
}
} else {
if keeper := conditionsFrom; keeper != nil && failed {
_, _ = keeper.Clear(ctx, o.Key(), "copied")
}
return
}
select {
case <-ctx.Done():
return
case <-time.After(chainMoveRetry):
}
}
}
// chainMoveRetry is how long the copy waits before it tries again.
const chainMoveRetry = 30 * time.Second
// kindBuildsNotLaidOut is the condition of the layout of the builds under their modules that has not succeeded: until it
// has, no build is read, and a send that needs one is refused with the reason.
const kindBuildsNotLaidOut = "builds-not-laid-out"
// layBuildsOutSoon lays the builds out under their modules as soon as this controller holds the lease (the chain refuses a
// write before), and tries again every half minute while it fails, loudly: each failure is printed, and the first raises an
// urgent condition until it has worked. It is idempotent (LayBuildsOut).
func layBuildsOutSoon(ctx context.Context, inv *inventory.Inventory) {
failed := false
for {
err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) })
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindBuildsNotLaidOut, Token: kindBuildsNotLaidOut,
Kind: kindBuildsNotLaidOut, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "}
if err != nil {
fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err)
if keeper := conditionsFrom; keeper != nil && !failed {
o.Summary += err.Error()
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
failed = true
}
}
} else {
if keeper := conditionsFrom; keeper != nil && failed {
_, _ = keeper.Clear(ctx, o.Key(), "laid out")
}
return
}
select {
case <-ctx.Done():
return
case <-time.After(chainMoveRetry):
}
}
}
cmd/mesh-controller/chain_integration_test.go
+39 -3
View File
@@ -176,12 +176,12 @@ func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T
if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" {
t.Fatalf("the assignment names %+v", on)
}
// A declaration sent by hand with no builds known says nothing of them.
// A declaration sent by hand with no builds known makes what the machine runs unknown: a time on the key, no commit.
if err := inv.RecordSent(ctx, node, "d3", nil); err != nil {
t.Fatal(err)
}
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c1" {
t.Fatalf("a send of unknown builds changed %+v", on)
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "" || runOf(on, "notes").SentUnknown.IsZero() {
t.Fatalf("a send of unknown builds left %+v", on)
}
// A module sent a commit no build is kept of is named by its commit alone.
if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil {
@@ -207,3 +207,39 @@ func runOf(on []chain.Assignment, module string) chain.Assignment {
a, _ := assignmentOf(on, module)
return a
}
// **An assignment waiting for its build is one key of the assignments bucket** (ADR 0261, ADR 0408, issue 1094): entered
// through the `assign` verb for a module whose build is in flight, one read of `<node>.<module>` says state pending with the
// build it waits for, the build's outcome through the build seat's path makes the key stand with the wait carried as applied,
// and the table of pending assignments is read by nobody. No key is written by hand.
func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
inv := open.inventory
asksWithPaths(t)
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
t.Fatalf("an assignment while its build runs was refused: %v", err)
}
waiting, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
if err != nil || !found || waiting.State != chain.StatePending || waiting.Stands() || !waiting.Assigned.IsZero() ||
waiting.Pending == nil || waiting.Pending.State != chain.PendingWaiting || waiting.Pending.Build != "b-modules/sensors" ||
!strings.HasPrefix(waiting.Pending.ID, "p-") {
t.Fatalf("one read of the key says %+v (found %v, %v)", waiting, found, err)
}
if got := assignedTo(t, open, "laptop"); slices.Contains(got, "sensors") {
t.Fatalf("a module only waiting is assigned: %v", got)
}
if err := (builds{inv: inv, open: open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil {
t.Fatal(err)
}
made, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
if err != nil || !found || !made.Stands() || made.State != "" || made.Assigned.IsZero() || made.Pending == nil ||
made.Pending.State != chain.PendingApplied || made.Pending.ID != waiting.Pending.ID {
t.Fatalf("one read of the key after the build says %+v (found %v, %v)", made, found, err)
}
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
t.Fatal("the build registered sensors and it was not assigned")
}
}
cmd/mesh-controller/chain_no_table_integration_test.go
+85
View File
@@ -0,0 +1,85 @@
//go:build integration
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/link"
)
// **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR
// 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's
// build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the
// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays.
func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asked := asksRecorded(t)
ctx := t.Context()
inv := open.inventory
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
t.Fatal(err)
}
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
cutAt(t, open, t0.Add(91*time.Second))
ws, _ := walks(t, open)
if len(ws) != 1 {
t.Fatalf("%d walk(s) after the cut", len(ws))
}
rollout := ws[0].ID
asking, err := inv.PlanByID(ctx, rollout)
if err != nil {
t.Fatal(err)
}
for try := 0; try < 3 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == ""); try++ {
advancePlans(ctx, open)
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
t.Fatal(err)
}
}
buildID := asking.Modules["notes"].Build
if buildID == "" || len(*asked) != 1 {
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
}
digest := "sha256:" + strings.Repeat("b", 64)
result := outcome(buildID, "modules/notes", "c-notes", "notes", "")
result.Made = []link.MadeArtifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes/notes@" + digest}}
if err := (builds{inv: inv, open: open}).Built(ctx, result); err != nil {
t.Fatal(err)
}
// What the live path made is in the chain.
if built, found, err := inv.BuildByID(ctx, buildID); err != nil || !found || built.Commit != "c-notes" {
t.Fatalf("the build is not read from the chain: %+v %v %v", built, found, err)
}
if p, err := inv.PlanByID(ctx, rollout); err != nil || p.ID != rollout {
t.Fatalf("the rollout is not read from the chain: %+v %v", p, err)
}
if on, err := inv.AssignmentsOn(ctx, "anchor"); err != nil {
t.Fatal(err)
} else if _, ok := assignmentOf(on, "notes"); !ok {
t.Fatalf("the assignment is not read from the chain: %+v", on)
}
// And the catalog of the store holds none of them.
have := map[string]bool{}
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
have[strings.Trim(row[0], `"`)] = true
}
for _, table := range []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment", "chain_moved"} {
if have[table] {
t.Errorf("the table %s is in the controller's store", table)
}
}
for _, table := range []string{"build_gate", "assignment_generation"} {
if !have[table] {
t.Errorf("the table %s, which stays, is gone", table)
}
}
if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
t.Errorf("the function raise_assignment_generation is in the store: %v", rows)
}
}
cmd/mesh-controller/data_integration_test.go
+9 -3
View File
@@ -56,13 +56,19 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
// Built from a commit, so that a send carries them: a holder is asked only of what the machine was sent (issue 1109).
built := inventory.Source{BuiltFrom: "c1"}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "keeper", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}, built); err != nil {
t.Fatal(err)
}
house, err := catalogue.ParseManifest([]byte(houseManifest))
if err != nil {
t.Fatal(err)
}
register(t, open, house)
if err := inv.RegisterModule(ctx, house, built); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
}
cmd/mesh-controller/merge_gate.go
+3
View File
@@ -1120,6 +1120,9 @@ func throwawayStores(ctx context.Context, admin string) (*stores, func(), error)
drop()
return nil, nil, err
}
// The chain lives on the bus, and the gate has none: this mesh is raised for a moment, with a chain of its own in memory
// that no other reads (issue 1121).
open.inventory.UseThrowawayChain()
for _, m := range provided {
if err := open.inventory.Provide(ctx, m); err != nil {
open.Close()
cmd/mesh-controller/merge_on_the_bus_test.go
+8
View File
@@ -50,9 +50,17 @@ func theMergePathOn(t *testing.T, open *stores) func(link.SourceMoved) {
t.Cleanup(func() {
stop()
<-done
lookingAtBatches.Wait()
setWindowClock(nil)
windowLook.Store(nil)
})
// The forge's merges are taken from a consumer made when the server starts serving: a merge published before it exists
// is not delivered to it. With the chain read from the bus the set-up above is quick enough to lose that race, so the
// publisher waits for the consumer.
becomes(t, 10*time.Second, "the merges' consumer being made", func() bool {
_, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
return err == nil
})
return func(m link.SourceMoved) {
t.Helper()
body, err := json.Marshal(m)
cmd/mesh-controller/merge_window_bus.go
+6
View File
@@ -145,9 +145,15 @@ func (w *busWindow) Forget(ctx context.Context, batch string) {
}
}
// lookingAtBatches counts the looks under way, so a test that ends the window's clock can wait for the last of them before the
// next test replaces what they read.
var lookingAtBatches sync.WaitGroup
// lookAtBatches is the look an expired window asks for: the plans held, as a merge heard holds them, and the batches
// whose window closed cut (cutBatchesHeld). Waiting for the plans rather than giving up: an expiry is heard once.
func lookAtBatches(ctx context.Context, open *stores) {
lookingAtBatches.Add(1)
defer lookingAtBatches.Done()
release, err := open.inventory.HoldPlans(ctx, true)
if err != nil {
fmt.Printf("batches: a window closed and the plans could not be held for the look: %v\n", err)
cmd/mesh-controller/pending.go
+7 -13
View File
@@ -491,8 +491,7 @@ func settlingPending(ctx context.Context, open *stores) {
// settlePending is one tick: claims left by a controller that stopped are settled from what the mesh holds;
// a module registered meanwhile is assigned; a build heard and not registered, or silent past its bound,
// ends its assignment; ended ones are raised as conditions and cleared once answered; ended rows older than
// KeptFor are deleted. Returns what it did.
// ends its assignment; ended ones are raised as conditions and cleared once answered. Returns what it did.
func settlePending(ctx context.Context, open *stores, now time.Time) []string {
inv := open.inventory
var said []string
@@ -565,11 +564,6 @@ func settlePending(ctx context.Context, open *stores, now time.Time) []string {
}
said = append(said, raisePendingEnded(ctx, inv)...)
if n, err := inv.ForgetEndedPending(ctx, now); err != nil {
say("ended pending assignments could not be deleted: %v", err)
} else if n > 0 {
say("deleted %d pending assignment(s) ended more than %s ago", n, inventory.KeptFor)
}
return said
}
@@ -585,7 +579,7 @@ func containsString(xs []string, x string) bool {
// pendingObservation is the condition for a pending assignment that was not made: one per pending
// assignment, its row's id the last part of its id, so one row's condition is never another's.
func pendingObservation(p inventory.PendingAssignment) conditions.Observation {
return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%d", p.Node, p.Module, p.ID),
return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%s", p.Node, p.Module, p.ID),
Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning,
Resolver: conditions.ResolverOperator, Source: "pending assignments",
Summary: p.Note}
@@ -671,8 +665,8 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor
if err != nil {
return []string{fmt.Sprintf("the open conditions could not be read to clear orphaned ones: %v", err)}
}
byID := map[int64]string{}
var ids []int64
byID := map[string]string{}
var ids []string
for _, c := range open {
if c.Kind != kindPendingEnded {
continue
@@ -682,8 +676,8 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor
if len(parts) < 2 {
continue
}
var id int64
if _, err := fmt.Sscan(parts[len(parts)-2], &id); err != nil {
id := parts[len(parts)-2]
if !strings.HasPrefix(id, "p-") {
continue
}
byID[id] = c.Key
@@ -701,7 +695,7 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor
if known[id] {
continue
}
if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed"); err != nil {
if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed, or the module was assigned and the wait taken back"); err != nil {
said = append(said, fmt.Sprintf("the condition %s could not be cleared: %v", key, err))
}
}
cmd/mesh-controller/pending_integration_test.go
+13 -9
View File
@@ -542,23 +542,26 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
t.Fatalf("a module registered meanwhile: %+v", p)
}
// Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is
// still open (second review, bug B).
// Ended waits are not deleted, for the chain removes no key (ADR 0408 section 4): after KeptFor they are no longer listed, open
// ones never are left out, nor one whose condition is still open (second review, bug B).
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil {
t.Fatal(err)
}
said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
rows, err := inv.Pending(ctx, time.Unix(0, 0))
later := time.Now().Add(inventory.KeptFor + time.Hour)
settlePending(ctx, open, later)
rows, err := inv.Pending(ctx, later.Add(-inventory.KeptFor))
if err != nil {
t.Fatal(err)
}
for _, r := range rows {
if !r.Open() && (r.Raised == nil || r.Cleared != nil) {
t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said)
t.Fatalf("an ended pending assignment with no open condition is listed past %s: %+v", inventory.KeptFor, r)
}
}
if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") {
t.Fatalf("the applied row was not deleted: %v", said)
for _, r := range rows {
if r.Module == "heard" {
t.Fatalf("an applied wait is listed past %s: %+v", inventory.KeptFor, r)
}
}
}
@@ -637,8 +640,9 @@ func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) {
}
settlePending(ctx, open, time.Now())
settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] {
t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err)
// No key of the chain is removed (ADR 0408 §4): a cleared wait stays in the value of its key, saying it was cleared.
if known, err := inv.PendingKnown(ctx, []string{row.ID}); err != nil || !known[row.ID] || rowOf(t, open, row.ID).Cleared == nil {
t.Fatalf("a cleared row is not kept, saying so: %v", err)
}
// On anchor, then anchor removed: the row goes, and its condition with it at the next tick.
cmd/mesh-controller/pending_test.go
+2 -2
View File
@@ -23,7 +23,7 @@ func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool {
return found
}
func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment {
func rowOf(t *testing.T, open *stores, id string) inventory.PendingAssignment {
t.Helper()
rows, err := open.inventory.Pending(t.Context(), time.Unix(0, 0))
if err != nil {
@@ -34,7 +34,7 @@ func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment {
return r
}
}
t.Fatalf("pending assignment %d is not on record", id)
t.Fatalf("pending assignment %s is not on record", id)
return inventory.PendingAssignment{}
}
cmd/mesh-controller/plan_moved_every_writer_test.go
+25 -14
View File
@@ -12,9 +12,9 @@ import (
)
// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there
// after the commit; so no statement that writes the plan table may stand anywhere else, and writePlan, the one that
// does, may be called from nowhere but SavePlansMoving. Checked by reading the sources, no store needed.
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there after the
// write; so no code but chainSavePlans (which SavePlansMoving calls, and no other function) writes a plan to the rollouts
// bucket by anything but a change of the merges it holds. Checked by reading the sources, no store needed.
func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`)
var found []string
@@ -31,11 +31,13 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
return err
}
text := string(raw)
if write.MatchString(text) && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
if write.MatchString(text) {
found = append(found, path+" writes the plan table")
}
if strings.Contains(text, "writePlan(") && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
found = append(found, path+" calls writePlan")
// A plan record is written by the bucket's raw create and update, which only the plans' own file calls.
if filepath.ToSlash(path) != "../../internal/inventory/chain_rollouts.go" && filepath.ToSlash(path) != "../../internal/chain/chain.go" &&
strings.Contains(text, "chain.RolloutsOf") {
found = append(found, path+" reaches the rollouts bucket")
}
return nil
})
@@ -51,17 +53,26 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
t.Fatal(err)
}
text := string(raw)
if n := len(write.FindAllString(text, -1)); n != 1 {
t.Errorf("plans.go has %d statements that write the plan table; want the one in writePlan", n)
}
calls := regexp.MustCompile(`writePlan\(ctx, tx`).FindAllStringIndex(text, -1)
save := strings.Index(text, "func (i *Inventory) SavePlansMoving")
next := strings.Index(text[save:], "\nfunc ")
if len(calls) != 1 || calls[0][0] < save || calls[0][0] > save+next {
t.Errorf("writePlan is called %d times, and not only from SavePlansMoving", len(calls))
if next < 0 {
next = len(text) - save
}
if !strings.Contains(text[save:save+next], "PlanSaved(i, *p)") {
t.Error("SavePlansMoving does not say each plan it wrote")
if !strings.Contains(text[save:save+next], "chainSavePlans(") {
t.Error("SavePlansMoving does not write through chainSavePlans")
}
chainRaw, err := os.ReadFile("../../internal/inventory/chain_rollouts.go")
if err != nil {
t.Fatal(err)
}
chainText := string(chainRaw)
at := strings.Index(chainText, "func (i *Inventory) chainSavePlans")
end := strings.Index(chainText[at:], "\nfunc ")
if !strings.Contains(chainText[at:at+end], "PlanSaved(i, *p)") {
t.Error("chainSavePlans does not say each plan it wrote")
}
if n := strings.Count(chainText, "createRaw(ctx, c, p.ID"); n != 1 {
t.Errorf("a plan is made from %d places of chain_rollouts.go, want the one in chainSavePlans", n)
}
}
cmd/mesh-controller/record_mirrors.go
+8 -35
View File
@@ -44,7 +44,6 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
return e
}
sources := map[string]func(context.Context) (map[string][]byte, error){
recordmirror.Assignments.Name: inv.AssignmentsMirrorToWrite,
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
recordmirror.Predefined.Name: inv.PredefinedMirror,
}
@@ -55,15 +54,11 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
events := &assignmentEvents{}
for {
if holds() {
// The assignments, once they are keys of the bus, are written by the verbs that change them (novox/hq ADR
// 0408): this tick keeps the provisioner's plan in their values and says what changed; the mirror of the store
// writes them only until the move.
onTheBus := keepingAssignments(ctx, inv, announce, epoch, events, failing)
// The assignments are keys of the bus, written by the verbs that change them (novox/hq ADR 0408): this tick
// keeps the provisioner's plan in their values and says what changed. They are no mirror of the store.
keepingAssignments(ctx, inv, announce, epoch, events, failing)
for _, m := range recordmirror.Mirrors() {
if m.Name == recordmirror.Assignments.Name && onTheBus {
continue
}
why := mirrorOnce(ctx, inv, api, m, writers, sources[m.Name], announce, epoch, holds)
why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
if why != failing[m.Name] {
if why != "" {
fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
@@ -83,7 +78,7 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
}
// mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could.
func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64,
holds func() bool) string {
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
@@ -95,9 +90,6 @@ func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.Jet
return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err)
}
w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
if m.Name == recordmirror.Assignments.Name {
w.Holds = assignmentsHolds(inv, holds)
}
writers[m.Name] = w
}
want, err := source(reading)
@@ -128,21 +120,10 @@ type assignmentEvents struct {
// the values it belongs to, and one `assignment-changed` event for each key that is new, changed or ended since the last
// look — whoever wrote it, a verb in this process or a command in another. False when they are not on the bus yet.
func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce recordmirror.Announcer, epoch func() uint64,
events *assignmentEvents, failing map[string]string) bool {
events *assignmentEvents, failing map[string]string) {
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
const name = "assignments"
if _, on, err := inv.KeptAssignments(reading); err != nil || !on {
if err != nil {
// Not known to be off the bus: the store's mirror is not written over what the bus may hold.
if failing[name] == "" {
failing[name] = err.Error()
fmt.Printf("the assignments cannot be read from the bus: %v\n", err)
}
return true
}
return false
}
if _, _, err := inv.KeepAssignmentPlans(reading); err != nil {
if failing[name] != err.Error() {
failing[name] = err.Error()
@@ -152,9 +133,9 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
fmt.Printf("the provisioner's plans are kept in the assignments again\n")
failing[name] = ""
}
now, _, err := inv.KeptAssignments(reading)
now, err := inv.KeptAssignments(reading)
if err != nil {
return true
return
}
if events.known != nil {
keys := make([]string, 0, len(now)+len(events.known))
@@ -194,12 +175,4 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
}
}
events.known = now
return true
}
// assignmentsHolds is the assignments mirror's leave to write: the lease's, and the assignments neither being switched to
// the bus nor on it. Asked by the writer before every write and purge, so a pass that read its source before the switch
// writes nothing after it (ADR 0408 §4).
func assignmentsHolds(inv *inventory.Inventory, holds func() bool) func() bool {
return func() bool { return holds() && inv.AssignmentsMirrorMayWrite() }
}
cmd/mesh-controller/record_mirrors_switch_test.go
-29
View File
@@ -1,29 +0,0 @@
package main
import (
"errors"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/recordmirror"
)
// **A writer of the store's mirror of the assignments, made before the assignments are switched to the bus, writes and
// purges nothing after** (ADR 0408 §4, issue 1093): its leave to write is asked before every write, and false once they are
// switching or moved.
func TestAMirrorWriterMadeBeforeTheSwitchWritesAndPurgesNothingAfter(t *testing.T) {
inv := inventory.NewForTest()
w := &recordmirror.Writer{Mirror: recordmirror.Assignments, Holds: assignmentsHolds(inv, func() bool { return true })}
if !w.Holds() {
t.Fatal("the writer may not write before the switch")
}
inv.BeginSwitchForTest(inventory.KindAssignments)
if w.Holds() {
t.Fatal("the writer may write while the assignments are switching")
}
// Refused before it reads the bucket (it has none here) or lays or ends a key.
r, err := w.Sync(t.Context(), map[string][]byte{"anchor.notes": []byte(`{}`)})
if !errors.Is(err, recordmirror.ErrNotHeld) || r.Written != 0 || r.Removed != 0 {
t.Fatalf("the writer wrote after the switch: %+v %v", r, err)
}
}
cmd/mesh-controller/release.go
+3 -1
View File
@@ -1589,7 +1589,9 @@ func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalog
var other []string
listID := ""
for _, m := range plan.Modules {
if _, was := sent[m.Module]; !was {
// A module no build is recorded for is carried with no commit and names none on its key (issue 1109), so it is "sent"
// only by the comparison of the summaries below; a module with a commit that was never sent is a new assignment.
if _, was := sent[m.Module]; !was && declared.Builds[m.Module] != "" {
other = append(other, m.Module+" newly assigned")
}
if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) {
cmd/mesh-controller/sent_required_by_integration_test.go
+1 -4
View File
@@ -85,10 +85,7 @@ func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T)
t.Fatal(err)
}
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
mirror, err := inv.AssignmentsMirror(ctx)
if err != nil {
t.Fatal(err)
}
mirror := inv.PlanValuesForTest(t)
var value inventory.AssignmentMirrored
if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil {
t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err)
internal/chain/keys.go
+8 -1
View File
@@ -53,7 +53,14 @@ func ModuleFilter(module string) string { return module + ".build.>" }
const ManifestPrefix = "manifest"
// ReservedModules are the names that are not a module's: a key under one is not a build's record.
var ReservedModules = []string{ManifestPrefix, RequestPrefix, IDPrefix}
var ReservedModules = []string{ManifestPrefix, RequestPrefix, IDPrefix, LayoutPrefix}
// LayoutPrefix is the name of the key that says the builds are laid out under their modules: LayoutMarkerKey, written once by
// the controller's start after it has laid the old keys out. Nothing reads the builds before it exists.
const LayoutPrefix = "layout"
// LayoutMarkerKey is the key whose existence says the layout is made.
const LayoutMarkerKey = LayoutPrefix + ".done"
// ManifestKey is where a manifest is kept: `manifest.sha256-<hex of its bytes>`.
func ManifestKey(raw []byte) string {
internal/chain/pending_test.go
+87
View File
@@ -0,0 +1,87 @@
package chain
import (
"encoding/json"
"strings"
"testing"
"time"
)
// **A pending value is not a standing assignment** (ADR 0261, ADR 0408, issue 1094): it names its node and module, holds no
// time assigned, says `state: pending`, and every reader that skips a value with a state skips it.
func TestAPendingValueIsNotAStandingAssignment(t *testing.T) {
since := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
a := Assignment{Node: "anchor", Module: "notes", State: StatePending,
Pending: &Pending{ID: "p-1", State: PendingWaiting, Build: "build-1", Since: since}}
if a.Stands() {
t.Fatal("a pending value stands")
}
raw, _ := json.Marshal(a)
for _, want := range []string{`"node":"anchor"`, `"module":"notes"`, `"state":"pending"`, `"pending":{"id":"p-1","state":"waiting"`} {
if !strings.Contains(string(raw), want) {
t.Errorf("the value lacks %s: %s", want, raw)
}
}
if strings.Contains(string(raw), `"assigned"`) {
t.Errorf("a pending value names a time assigned: %s", raw)
}
// The wait carried into the value that stands: no state, so it stands, and the record stays.
made := Assignment{Node: "anchor", Module: "notes", Assigned: since, Pending: &Pending{ID: "p-1", State: PendingApplied}}
if !made.Stands() || !a.Pending.Open() || made.Pending.Open() {
t.Fatalf("stands %v, open before %v, open after %v", made.Stands(), a.Pending.Open(), made.Pending.Open())
}
var back Assignment
if err := json.Unmarshal(raw, &back); err != nil || back.Pending == nil || back.Pending.Build != "build-1" || !back.Pending.Since.Equal(since) {
t.Fatalf("read back %+v: %v", back, err)
}
now := since
raised := Pending{ID: "p-2", State: PendingExpired, Raised: &now, Acknowledged: &now, Cleared: &now}
body, _ := json.Marshal(raised)
if !strings.Contains(string(body), `"raised"`) || !strings.Contains(string(body), `"acknowledged"`) || !strings.Contains(string(body), `"cleared"`) {
t.Errorf("the marks of a wait: %s", body)
}
}
func TestTheTransitionsOfAPendingAssignment(t *testing.T) {
ok := [][2]string{{PendingWaiting, PendingExpired}, {PendingWaiting, PendingWithdrawn}, {PendingApplying, PendingApplied},
{PendingApplying, PendingRefused}}
for _, p := range ok {
if !CanSettle(p[0], p[1]) {
t.Errorf("%s may end as %s", p[0], p[1])
}
}
for _, p := range [][2]string{{PendingWaiting, PendingApplied}, {PendingWaiting, PendingRefused}, {PendingApplying, PendingExpired},
{PendingApplying, PendingWithdrawn}, {PendingApplied, PendingExpired}, {PendingExpired, PendingWithdrawn},
{PendingWaiting, PendingWaiting}} {
if CanSettle(p[0], p[1]) {
t.Errorf("%s may not end as %s", p[0], p[1])
}
}
if !CanMove(PendingWaiting, PendingApplying) || !CanMove(PendingApplying, PendingWaiting) || CanMove(PendingApplied, PendingWaiting) {
t.Error("a claim moves a wait to applying and back")
}
}
// **A module only pulled in is a standing value marked pulled-in, never a state, and a hand send makes a key say its commit
// unknown with a time** (issue 1109): every reader treats a non-empty state as no assignment, so a pulled-in key must have
// none; and unknown is never written in `commit`.
func TestAPulledInValueStandsAndAHandSendIsATimeNotACommit(t *testing.T) {
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
pulled := Assignment{Node: "anchor", Module: "mesh-wireguard", PulledIn: true, Commit: "c1", Build: "b1", Since: at}
if !pulled.Stands() || pulled.State != "" {
t.Fatalf("a pulled-in value does not stand: %+v", pulled)
}
raw, _ := json.Marshal(pulled)
if !strings.Contains(string(raw), `"pulled-in":true`) || strings.Contains(string(raw), `"state"`) || strings.Contains(string(raw), `"assigned"`) {
t.Fatalf("the pulled-in value is %s", raw)
}
unknown := Assignment{Node: "anchor", Module: "notes", Assigned: at, SentUnknown: at}
raw, _ = json.Marshal(unknown)
if !strings.Contains(string(raw), `"sent-unknown":"2026-10-18T03:00:00Z"`) || strings.Contains(string(raw), `"commit"`) || !unknown.Stands() {
t.Fatalf("the hand-sent value is %s", raw)
}
plain, _ := json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at})
if strings.Contains(string(plain), "pulled-in") || strings.Contains(string(plain), "sent-unknown") {
t.Fatalf("a value that is neither says so: %s", plain)
}
}
internal/chain/records.go
+66 -1
View File
@@ -191,9 +191,74 @@ type Assignment struct {
// Ended is when the assignment ended (an unassignment): absent while it stands. **No key of the chain is removed**
// (ADR 0408 §4), so an unassigned module keeps its key, saying so; every reader treats a value with Ended, or with a
// State, as no assignment. State is, in a later delivery, the pending assignment waiting for its build (issue 1094).
// State, as no assignment. State is StatePending while the key holds an assignment waiting for its build, or one that
// was never made (issue 1094).
Ended time.Time `json:"ended,omitzero"`
State string `json:"state,omitempty"`
// Pending is the assignment's wait for its build (novox/hq ADR 0261), kept in the value of the key it waits for and, when
// it is made, carried into the value that stands: the record of what became of the wait.
Pending *Pending `json:"pending,omitempty"`
// PulledIn marks a standing value of a module that is on the machine only because another requires it, or because a
// send carried it: no time assigned, but a key all the same, so what the machine was sent of it is kept and read
// (issue 1109). Never a state: every reader treats a non-empty state as no assignment, and would take it off the machine.
PulledIn bool `json:"pulled-in,omitempty"`
// SentUnknown is when a send by hand made what the machine runs unknown (a declaration sent with no builds known): the
// commit is cleared with it. Absent normally; a send that names its builds clears it.
SentUnknown time.Time `json:"sent-unknown,omitzero"`
}
// StatePending is the State of a value that holds no assignment but a pending one.
const StatePending = "pending"
// The states of a pending assignment (novox/hq ADR 0261).
const (
PendingWaiting = "waiting"
PendingApplying = "applying"
PendingApplied = "applied"
PendingRefused = "refused"
PendingExpired = "expired"
PendingWithdrawn = "withdrawn"
)
// Pending is an assignment kept until its module is registered: the build it waits for, where that build is from, and the
// times of its claim and its end. Its id is p-<unix nanoseconds of Since>, unique for a key, which holds one at a time.
type Pending struct {
ID string `json:"id"`
State string `json:"state"`
Build string `json:"build,omitempty"`
Repository string `json:"repository,omitempty"`
Path string `json:"path,omitempty"`
Since time.Time `json:"since"`
Note string `json:"note,omitempty"`
Settled *time.Time `json:"settled,omitempty"`
Raised *time.Time `json:"raised,omitempty"`
Acknowledged *time.Time `json:"acknowledged,omitempty"`
Cleared *time.Time `json:"cleared,omitempty"`
// Earlier are the waits of this key that ended unmade and were replaced by this one before their condition was answered
// (taken back or cleared): each stays, with its own marks, so a replaced wait never leaves the operator without the
// condition it raised or was to raise. Dropped once answered.
Earlier []Pending `json:"earlier,omitempty"`
}
// Open says the assignment is still to be made.
func (p Pending) Open() bool { return p.State == PendingWaiting || p.State == PendingApplying }
// CanSettle says a pending assignment in state from may end in state to: a wait expires or is withdrawn, a claim becomes
// applied or refused, and nothing else.
func CanSettle(from, to string) bool {
switch from {
case PendingWaiting:
return to == PendingExpired || to == PendingWithdrawn
case PendingApplying:
return to == PendingApplied || to == PendingRefused
}
return false
}
// CanMove says a pending assignment in state from may go to state to without ending: claimed, or released again.
func CanMove(from, to string) bool {
return (from == PendingWaiting && to == PendingApplying) || (from == PendingApplying && to == PendingWaiting)
}
// Stands says the value is an assignment that holds: it has not ended and is not waiting in a state of its own.
internal/inventory/batches.go
+22 -85
View File
@@ -3,11 +3,8 @@ package inventory
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
"github.com/jackc/pgx/v5"
)
// The merges a batch or walk answers (novox/hq ADR 0276): every merge the controller heard into a branch a
@@ -33,138 +30,78 @@ type BatchedMerge struct {
// AddMerge keeps a merge for a plan; false when the merge was kept already, for whatever plan.
func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
return false, err
} else if on {
return i.chainAddMerge(ctx, c, m)
}
tag, err := i.store.Pool().Exec(ctx,
`insert into batched_merge (repository, branch, commit_hash, merged_at, heard_at, event, plan_id, alone)
values (lower($1), $2, $3, $4, $5, $6, nullif($7, ''), $8) on conflict do nothing`,
m.Repository, m.Branch, m.Commit, mergedAt(m.Merged), m.Heard, []byte(m.Event), m.Plan, m.Alone)
c, err := i.theChain(ctx)
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
return i.chainAddMerge(ctx, c, m)
}
// MergeOf is one kept merge; false when it was never heard.
func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (BatchedMerge, bool, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
return BatchedMerge{}, false, err
} else if on {
return i.chainMergeOf(ctx, c, repository, commit)
}
ms, err := i.merges(ctx, `where repository = lower($1) and commit_hash = $2`, repository, commit)
if err != nil || len(ms) == 0 {
c, err := i.theChain(ctx)
if err != nil {
return BatchedMerge{}, false, err
}
return ms[0], true, nil
return i.chainMergeOf(ctx, c, repository, commit)
}
// MergesOf is every merge a plan answers, oldest merge first.
func (i *Inventory) MergesOf(ctx context.Context, plan string) ([]BatchedMerge, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
return i.chainMergesOf(ctx, c, plan)
}
return i.merges(ctx, `where plan_id = $1 order by merged_at nulls last, heard_at`, plan)
return i.chainMergesOf(ctx, c, plan)
}
// LaterMergesOf is every merge of a repository's branch made after a moment and answered by a plan, the
// newest first.
func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string, after time.Time) ([]BatchedMerge, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
return i.chainLaterMergesOf(ctx, c, repository, branch, after)
}
return i.merges(ctx, `where repository = lower($1) and branch = $2 and merged_at > $3 and plan_id is not null
order by merged_at desc`, repository, branch, after)
return i.chainLaterMergesOf(ctx, c, repository, branch, after)
}
// KeptOnBranch says the controller keeps a merge of a repository's branch, and when it heard the branch's first: a merge of a branch it keeps merges of, made since then, is the record's to answer, whatever a
// module's last look says (novox/hq ADR 0276).
func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string) (bool, time.Time, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
return false, time.Time{}, err
} else if on {
return i.chainKeptOnBranch(ctx, c, repository, branch)
}
var first *time.Time
var kept bool
err := i.store.Pool().QueryRow(ctx,
`select min(heard_at), count(*) > 0 from batched_merge where repository = lower($1) and branch = $2`,
repository, branch).Scan(&first, &kept)
if err != nil || first == nil {
c, err := i.theChain(ctx)
if err != nil {
return false, time.Time{}, err
}
return kept, first.UTC(), nil
return i.chainKeptOnBranch(ctx, c, repository, branch)
}
// AloneMerges is every merge waiting to be walked on its own commit, the newest first (ADR 0276 decision 3).
func (i *Inventory) AloneMerges(ctx context.Context) ([]BatchedMerge, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
return i.chainAloneMerges(ctx, c)
}
return i.merges(ctx, `where plan_id is null and alone order by merged_at desc nulls last, heard_at desc`)
return i.chainAloneMerges(ctx, c)
}
// AnswerMerge moves a merge to the plan that answers it now; an empty plan with alone leaves it waiting to
// be walked on its own commit.
func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan string, alone bool) error {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return err
} else if on {
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}})
}
_, err := i.store.Pool().Exec(ctx,
`update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`,
repository, commit, plan, alone)
return err
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}})
}
// Batches is every batch not yet cut: assembling or queued, oldest first. One at a time is the rule; more
// are read so a fault is seen.
func (i *Inventory) Batches(ctx context.Context) ([]Plan, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
return nil, err
} else if on {
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated)
}
return i.plans(ctx, `where state in ('assembling', 'queued') order by created`)
}
func (i *Inventory) merges(ctx context.Context, tail string, args ...any) ([]BatchedMerge, error) {
rows, err := i.store.Pool().Query(ctx,
`select repository, branch, commit_hash, merged_at, heard_at, event, coalesce(plan_id, ''), alone
from batched_merge `+tail, args...)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
var out []BatchedMerge
for rows.Next() {
var m BatchedMerge
var merged *time.Time
var event []byte
if err := rows.Scan(&m.Repository, &m.Branch, &m.Commit, &merged, &m.Heard, &event, &m.Plan, &m.Alone); err != nil {
return nil, err
}
if merged != nil {
m.Merged = merged.UTC()
}
m.Heard = m.Heard.UTC()
m.Event = event
out = append(out, m)
}
if errors.Is(rows.Err(), pgx.ErrNoRows) {
return nil, nil
}
return out, rows.Err()
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated)
}
// SameRepository says two spellings of a repository are one.
internal/inventory/builds.go
+12 -283
View File
@@ -2,14 +2,9 @@ package inventory
import (
"context"
"encoding/json"
"errors"
"sort"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/novox/mesh-controller/internal/chain"
)
@@ -136,51 +131,11 @@ func (b Build) Worked() bool { return b.Failed == "" }
// whoever asked and once on the exchange when nobody was. Recording both would show one build as
// two, and which of the two is real is not a question anybody could answer afterwards.
func (i *Inventory) RecordBuild(ctx context.Context, b Build) error {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return err
} else if on {
return i.chainRecordBuild(ctx, c, b)
}
made, err := json.Marshal(b.Made)
c, err := i.theChain(ctx)
if err != nil {
return err
}
against, err := json.Marshal(b.Against)
if err != nil {
return err
}
read, err := json.Marshal(b.Read)
if err != nil {
return err
}
var sources any
if len(b.Sources) > 0 {
raw, err := json.Marshal(b.Sources)
if err != nil {
return err
}
sources = raw
}
var module *string
if b.Module != "" {
module = &b.Module
}
var asked *time.Time
if !b.Asked.IsZero() {
asked = &b.Asked
}
_, err = i.store.Pool().Exec(ctx,
`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made,
source_path, manifest, built_against, built_contexts, asked, source_fingerprint,
build_sources)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
on conflict (id) do nothing`,
b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made,
b.Path, manifestOrNil(b.Manifest), against, read, asked, b.SourceFingerprint, sources)
if err != nil {
return err
}
return i.RecordMirrored(ctx, b.ID, "", b.Mirrored)
return i.chainRecordBuild(ctx, c, b)
}
// RecordMirrored keeps that the mesh copied these bases into the artifact store (novox/hq ADR 0257):
@@ -223,79 +178,22 @@ func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Bui
if limit <= 0 {
limit = 20
}
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainBuildList(ctx, c, module, limit)
}
if limit <= 0 {
limit = 20
}
query := `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at
from build order by at desc limit $1`
args := []any{limit}
if module != "" {
query = `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at
from build where module = $2 order by at desc limit $1`
args = append(args, module)
}
rows, err := i.store.Pool().Query(ctx, query, args...)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Build
for rows.Next() {
var b Build
var made []byte
var asked *time.Time
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
&b.On, &b.Failed, &made, &asked, &b.At); err != nil {
return nil, err
}
if asked != nil {
b.Asked = *asked
}
if err := json.Unmarshal(made, &b.Made); err != nil {
return nil, err
}
out = append(out, b)
}
return out, rows.Err()
return i.chainBuildList(ctx, c, module, limit)
}
// BuildByID is one build's record, by the id its request carried — what a plan matches its outcome
// by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the
// repository, path, ref and commit from. False when no outcome with that id was recorded.
func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return Build{}, false, err
} else if on {
return i.chainBuildByID(ctx, c, id)
}
var b Build
var made []byte
var asked *time.Time
err := i.store.Pool().QueryRow(ctx,
`select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made,
coalesce(source_path,''), asked, at
from build where id = $1`, id).Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
&b.On, &b.Failed, &made, &b.Path, &asked, &b.At)
if errors.Is(err, pgx.ErrNoRows) {
return Build{}, false, nil
}
c, err := i.theChain(ctx)
if err != nil {
return Build{}, false, err
}
if asked != nil {
b.Asked = *asked
}
if err := json.Unmarshal(made, &b.Made); err != nil {
return Build{}, false, err
}
return b, true, nil
return i.chainBuildByID(ctx, c, id)
}
// Held is every artifact this mesh has built, keyed "<module>/<artifact>".
@@ -377,40 +275,11 @@ func registeredOnly(held, registered map[string]string) map[string]string {
// edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a
// module standing on nothing looks like: an edge the mesh has not derived is not an edge.
func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainBuiltAgainst(ctx, c)
}
rows, err := i.store.Pool().Query(ctx,
`select distinct on (module) module, built_against
from build
where module is not null and module <> '' and failed = ''
order by module, `+newestRequestFirst)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
against := map[string][]string{}
for rows.Next() {
var module string
var raw []byte
if err := rows.Scan(&module, &raw); err != nil {
return nil, err
}
if len(raw) == 0 {
continue
}
var refs []string
if err := json.Unmarshal(raw, &refs); err != nil {
continue
}
if len(refs) > 0 {
against[module] = refs
}
}
return against, rows.Err()
return i.chainBuiltAgainst(ctx, c)
}
// ReadRepositories is what each module's newest successful build read source from besides its own
@@ -425,95 +294,11 @@ func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, erro
// none — off the trunk, from a builder that predates it, or of a source not known — leaves its module read
// as before: every file of a context, and its own directory or root.
func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainReadRepositories(ctx, c)
}
// The newest build of each module whatever its outcome: a failed one newer than the newest that
// worked leaves that one's build source stale — the merge it was asked for may have changed the closure
// (novox/hq ADR 0267), so its module is read whole until a build works again.
newest := map[string]struct {
at time.Time
failed bool
}{}
tried, err := i.store.Pool().Query(ctx,
`select distinct on (module) module, coalesce(asked, at), failed <> ''
from build
where module is not null and module <> ''
order by module, `+newestRequestFirst)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
for tried.Next() {
var module string
var at time.Time
var failed bool
if err := tried.Scan(&module, &at, &failed); err != nil {
tried.Close()
return nil, err
}
newest[module] = struct {
at time.Time
failed bool
}{at, failed}
}
tried.Close()
if err := tried.Err(); err != nil {
return nil, err
}
rows, err := i.store.Pool().Query(ctx,
`select distinct on (module) module, built_contexts, build_sources, coalesce(asked, at),
(select source from module where module.name = build.module)
from build
where module is not null and module <> '' and failed = ''
order by module, `+newestRequestFirst)
if err != nil {
return nil, err
}
defer rows.Close()
read := map[string][]ReadRepository{}
for rows.Next() {
var module string
var raw, rawSources []byte
var built time.Time
var own *string
if err := rows.Scan(&module, &raw, &rawSources, &built, &own); err != nil {
return nil, err
}
var of []ReadRepository
if len(raw) > 0 {
if err := json.Unmarshal(raw, &of); err != nil {
of = nil
}
}
var sources []BuildSource
if len(rawSources) > 0 {
if err := json.Unmarshal(rawSources, &sources); err != nil {
sources = nil
}
}
whole := ""
if n, known := newest[module]; known && n.failed && n.at.After(built) && len(sources) > 0 {
sources, whole = nil, "a newer build of it failed"
}
of = WithBuildSources(of, sources)
if own != nil {
of = OwnRepositoryAsOwn(of, *own)
}
if whole != "" {
of = append(of, ReadRepository{Own: true, Whole: whole})
}
if len(of) > 0 {
for k := range of {
of[k].Built, of[k].Looked = built, newest[module].at
}
read[module] = of
}
}
return read, rows.Err()
return i.chainReadRepositories(ctx, c)
}
// WithBuildSources lays a build's build sources over the repositories it read: a context's paths on its
@@ -590,18 +375,6 @@ func namesRepository(registered, named string) bool {
return reg == name || strings.HasSuffix(reg, "/"+name)
}
// manifestOrNil keeps the difference between "declared nothing" and "predates this being kept".
//
// A build recorded before the mesh kept manifests has no manifest, and that is not the same as one
// whose manifest was empty. A replay can then say which it is holding instead of inventing an
// empty declaration for a module that certainly had one.
func manifestOrNil(raw []byte) any {
if len(raw) == 0 {
return nil
}
return raw
}
// Announceable is every build worth telling a catalogue about, oldest first.
//
// **Oldest first, because a graph is built in the order things happened.** Registering a module
@@ -616,53 +389,9 @@ func manifestOrNil(raw []byte) any {
// One row per module and commit: a module built twice at the same commit is one fact, and the
// row asked last is the one whose artifacts are current (novox/hq 04-ISSUES/219).
func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainAnnounceable(ctx, c)
}
rows, err := i.store.Pool().Query(ctx,
`select distinct on (module, commit_hash)
id, repository, ref, module, commit_hash, built_on, failed, made,
source_path, manifest, built_against, at
from build
where failed = '' and module is not null and module <> '' and commit_hash <> ''
order by module, commit_hash, `+newestRequestFirst)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Build
for rows.Next() {
var b Build
var made []byte
var manifest, against []byte
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
&b.On, &b.Failed, &made, &b.Path, &manifest, &against, &b.At); err != nil {
return nil, err
}
if err := json.Unmarshal(made, &b.Made); err != nil {
return nil, err
}
// Null rather than empty is a build recorded before the mesh kept these, and saying so is
// the point of keeping them nullable: the replay carries nothing rather than an empty
// declaration for a module that certainly had one.
if len(manifest) > 0 {
b.Manifest = manifest
}
if len(against) > 0 {
if err := json.Unmarshal(against, &b.Against); err != nil {
return nil, err
}
}
out = append(out, b)
}
if err := rows.Err(); err != nil {
return nil, err
}
// Sorted here rather than in the query, because `distinct on` fixes the ordering it needs and
// the order that matters to a catalogue is a different one.
sort.Slice(out, func(a, b int) bool { return out[a].At.Before(out[b].At) })
return out, nil
return i.chainAnnounceable(ctx, c)
}
internal/inventory/catalogue.go
+22 -80
View File
@@ -258,31 +258,17 @@ func (i *Inventory) SourceOf(ctx context.Context, module string) (Source, error)
// machines are running last week's version" is the question somebody actually has, and it is the
// one novox/hq ADR 0010 names as the thing that must not be lost.
func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) {
query := `select m.name, coalesce(n.name, '')
from module m
left join assignment a on a.module = m.name
left join node n on n.id = a.node
where m.source is not null
and m.source_head is not null
and coalesce(m.built_from, '') is distinct from m.source_head
order by m.name, n.name`
_, onTheBus, err := i.ChainReads(ctx, KindAssignments)
assigned, err := i.assignments(ctx)
if err != nil {
return nil, err
}
var assigned []assignRow
if onTheBus {
query = `select m.name, ''
rows, err := i.store.Pool().Query(ctx,
`select m.name
from module m
where m.source is not null
and m.source_head is not null
and coalesce(m.built_from, '') is distinct from m.source_head
order by m.name`
if assigned, err = i.assignments(ctx); err != nil {
return nil, err
}
}
rows, err := i.store.Pool().Query(ctx, query)
order by m.name`)
if err != nil {
return nil, err
}
@@ -290,16 +276,11 @@ func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) {
out := map[string][]string{}
for rows.Next() {
var module, node string
if err := rows.Scan(&module, &node); err != nil {
var module string
if err := rows.Scan(&module); err != nil {
return nil, err
}
if _, seen := out[module]; !seen {
out[module] = nil
}
if node != "" {
out[module] = append(out[module], node)
}
out[module] = nil
}
if err := rows.Err(); err != nil {
return nil, err
@@ -654,21 +635,11 @@ func (i *Inventory) Assign(ctx context.Context, nodeName, module string) (bool,
if err := i.runsSomewhere(ctx, module); err != nil {
return false, err
}
if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
return false, err
} else if on {
return i.chainAssign(ctx, c, node, module)
}
tag, err := i.store.Pool().Exec(ctx,
`insert into assignment (node, module) values ($1, $2) on conflict do nothing`,
node.ID, module)
if err != nil && strings.Contains(err.Error(), "assignment_module_fkey") {
return false, fmt.Errorf("%w: %s", ErrNoSuchModule, module)
}
c, err := i.theChain(ctx)
if err != nil {
return false, err
}
return tag.RowsAffected() > 0, nil
return i.chainAssign(ctx, c, node, module)
}
// Unassign takes a module off a node.
@@ -677,21 +648,12 @@ func (i *Inventory) Unassign(ctx context.Context, nodeName, module string) error
if err != nil {
return err
}
if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return err
}
if err := i.chainUnassign(ctx, c, node, module); err != nil {
return err
} else if on {
if err := i.chainUnassign(ctx, c, node, module); err != nil {
return err
}
} else {
tag, err := i.store.Pool().Exec(ctx,
`delete from assignment where node = $1 and module = $2`, node.ID, module)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return fmt.Errorf("%s is not assigned to %s", module, nodeName)
}
}
// The seat it held falls back to derivation rather than pointing at nothing: what the store's foreign key did.
if _, err := i.store.Pool().Exec(ctx, `delete from seat_holding where node = $1 and module = $2`, node.ID, module); err != nil {
@@ -1362,39 +1324,21 @@ type Entry struct {
// catalogue nobody lists, and the questions here — what is this, where did it come from, who is
// running it — are asked together every time.
func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
query := `select m.name, m.manifest,
coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''),
coalesce(m.built_from, ''), coalesce(m.source_head, ''),
coalesce(m.source_seen, to_timestamp(0)),
coalesce(array_agg(n.name order by n.name) filter (where n.name is not null), '{}')
from module m
left join assignment a on a.module = m.name
left join node n on n.id = a.node
group by m.name, m.manifest, m.source, m.source_path, m.source_seat, m.ref, m.built_from,
m.source_head, m.source_seen
order by m.name`
// On the bus the assignments are read from the bucket, and the machines of a module put beside it here.
_, onTheBus, err := i.ChainReads(ctx, KindAssignments)
// The machines of a module are read from the assignments bucket and put beside it here.
assigned, err := i.assignments(ctx)
if err != nil {
return nil, err
}
var assignedBy map[string][]string
if onTheBus {
query = `select m.name, m.manifest,
assignedBy := map[string][]string{}
for _, a := range assigned {
assignedBy[a.Module] = append(assignedBy[a.Module], a.Node)
}
const query = `select m.name, m.manifest,
coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''),
coalesce(m.built_from, ''), coalesce(m.source_head, ''),
coalesce(m.source_seen, to_timestamp(0)), '{}'::text[]
from module m
order by m.name`
rows, err := i.assignments(ctx)
if err != nil {
return nil, err
}
assignedBy = map[string][]string{}
for _, a := range rows {
assignedBy[a.Module] = append(assignedBy[a.Module], a.Node)
}
}
rows, err := i.store.Pool().Query(ctx, query)
if err != nil {
return nil, err
@@ -1414,9 +1358,7 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
if source.Seen.Unix() == 0 {
source.Seen = time.Time{}
}
if onTheBus {
on = assignedBy[name]
}
on = assignedBy[name]
var m catalogue.Manifest
if err := json.Unmarshal(raw, &m); err != nil {
return nil, err
internal/inventory/chain.go
+61 -115
View File
@@ -5,55 +5,46 @@ import (
"errors"
"fmt"
"sync"
"sync/atomic"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/novox/mesh-controller/internal/chain"
)
// **The chain moves to the bus kind by kind** (novox/hq ADR 0408 §8, issue 981). The tables of builds, rollouts and
// assignments are copied once into the chain's buckets by the serving controller at its start (MoveChain), which then
// records each kind it moved in chain_moved (migration 0104). **Until a kind is recorded there nothing reads its bucket;
// once it is, nothing reads its tables**: ChainReads answers which, and every method of a kind asks it first.
// The kinds of the chain, as chain_moved names them.
const (
KindBuilds = "builds"
KindRollouts = "rollouts"
)
// **The chain lives on the bus** (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
// bus's key-value store, the controller's store keeps none of them, and every method of a kind reads and writes the chain this
// inventory is given (theChain).
// testChains are the chains in memory a test made for its store, by the store's address: every inventory opened on that
// store is given the same chain, as every process of a mesh is on the same bus. Empty outside a test.
var testChains sync.Map
// ErrNoChain is a kind moved to the bus read by an inventory that has no chain attached: a process that did not
// open the bus. Refused loudly, never answered from the tables the move left behind.
// ErrNoChain is a read or write of the chain by an inventory that has none attached: a process that did not open the bus.
// Refused loudly.
var ErrNoChain = errors.New("this kind of record lives on the bus, and this process has not opened the bus")
// movedKinds is which kinds this process has read as moved: once moved, always moved, so it is read from the store
// until it is true and not after.
type movedKinds struct {
// chainState is what an inventory keeps of the chain it is given.
type chainState struct {
mu sync.Mutex
kind map[string]bool
open ChainOpener
// view is the rollouts bucket kept in memory by one watch (issue 1092); pending are the merges added to a plan that
// is not saved yet.
view *rolloutView
pending map[string]pendingMerge
// switching is set when the assignments begin to be switched to the bus and kept once they are: the store's mirror of
// them writes nothing from then on (AssignmentsMirrorMayWrite). Cleared only if the switch fails.
switching atomic.Bool
// requireLayout is set for a chain on the bus: its builds are read only once the controller's start has laid them out
// under their modules (issue 1121), and laid is that seen.
requireLayout, laid bool
}
// AssignmentsMirrorMayWrite says the store's mirror of the assignments may still write the bucket: the assignments are
// neither being switched to the bus nor on it. Asked before every write of the mirror's writer, not once a pass.
func (i *Inventory) AssignmentsMirrorMayWrite() bool {
i.moved.mu.Lock()
moved := i.moved.kind[KindAssignments]
i.moved.mu.Unlock()
return !moved && !i.moved.switching.Load()
// RequireLayout makes every read of the chain refuse (ErrNotLaidOut) until the builds are laid out under their modules.
func (i *Inventory) RequireLayout() {
i.state.mu.Lock()
i.state.requireLayout = true
i.state.mu.Unlock()
}
func (i *Inventory) markLaidOut() {
i.state.mu.Lock()
i.state.laid = true
i.state.mu.Unlock()
}
// ChainOpener opens the chain on the bus when a process first needs it: a command that reads and never moved anything has
@@ -62,108 +53,63 @@ type ChainOpener func(ctx context.Context) (*chain.Chain, error)
// UseChainOpener gives the inventory the means to open the chain when it needs it and none is attached.
func (i *Inventory) UseChainOpener(open ChainOpener) {
i.moved.mu.Lock()
i.moved.open = open
i.moved.mu.Unlock()
i.state.mu.Lock()
i.state.open = open
i.state.requireLayout = true
i.state.mu.Unlock()
}
// UseChain attaches the chain the moved kinds are read and written through. The serving controller attaches the bus's,
// holding the lease for its writes; a test attaches one in memory.
func (i *Inventory) UseChain(c *chain.Chain) {
i.moved.mu.Lock()
defer i.moved.mu.Unlock()
i.state.mu.Lock()
defer i.state.mu.Unlock()
i.chain = c
if i.moved.view != nil {
i.moved.view.stop()
i.moved.view = nil
if i.state.view != nil {
i.state.view.stop()
i.state.view = nil
}
}
// UseThrowawayChain gives the inventory a chain in memory that is its own, laid out already: for a mesh raised for a moment
// beside the real one, as the merge gate raises the facts snapshot's mesh in a throwaway store (issue 1121). Nothing of it
// reaches the bus.
func (i *Inventory) UseThrowawayChain() {
i.UseChain(chain.InMemory(nil))
i.markLaidOut()
}
// attached is the chain this inventory has, nil when none.
func (i *Inventory) attached() *chain.Chain {
i.moved.mu.Lock()
defer i.moved.mu.Unlock()
i.state.mu.Lock()
defer i.state.mu.Unlock()
return i.chain
}
// ChainReads says whether a kind of the chain is read from its bucket: chain_moved names it. It answers the chain to read
// through, and ErrNoChain when the kind is moved and none is attached.
func (i *Inventory) ChainReads(ctx context.Context, kind string) (*chain.Chain, bool, error) {
moved, err := i.kindMoved(ctx, kind)
if err != nil || !moved {
return nil, false, err
}
if kind == KindBuilds {
// **The builds are read under their modules** (issue 1121): until the layout is made, a read would find nothing.
laid, err := i.kindMoved(ctx, KindBuildsLayout)
// theChain is the chain this inventory writes and reads, opened if a command has not yet. On the bus it is refused until the
// builds are laid out under their modules.
func (i *Inventory) theChain(ctx context.Context) (*chain.Chain, error) {
i.state.mu.Lock()
if i.chain == nil && i.state.open != nil {
c, err := i.state.open(ctx)
if err != nil {
return nil, false, err
}
if !laid {
return nil, false, fmt.Errorf("%w: the builds are not laid out under their modules yet; the move at the controller's start does it", ErrNoChain)
}
}
i.moved.mu.Lock()
defer i.moved.mu.Unlock()
if i.chain == nil && i.moved.open != nil {
c, err := i.moved.open(ctx)
if err != nil {
return nil, false, fmt.Errorf("%w: %s: %v", ErrNoChain, kind, err)
i.state.mu.Unlock()
return nil, fmt.Errorf("%w: %v", ErrNoChain, err)
}
i.chain = c
}
if i.chain == nil {
return nil, false, fmt.Errorf("%w: %s", ErrNoChain, kind)
}
return i.chain, true, nil
}
// kindMoved is whether chain_moved names a kind; once it does, always, so the store is asked until it is true.
func (i *Inventory) kindMoved(ctx context.Context, kind string) (bool, error) {
i.moved.mu.Lock()
known := i.moved.kind[kind]
i.moved.mu.Unlock()
if known {
return true, nil
}
var moved bool
err := i.store.Pool().QueryRow(ctx, `select counts ? $1 from chain_moved`, kind).Scan(&moved)
var pgErr *pgconn.PgError
if errors.Is(err, pgx.ErrNoRows) || (errors.As(err, &pgErr) && pgErr.Code == "42P01") {
// No row, or a store older than the migration that makes the table: nothing moved.
return false, nil
}
if err != nil {
return false, fmt.Errorf("whether %s moved to the bus cannot be read: %w", kind, err)
}
if !moved {
return false, nil
}
i.moved.mu.Lock()
if i.moved.kind == nil {
i.moved.kind = map[string]bool{}
}
i.moved.kind[kind] = true
i.moved.mu.Unlock()
return true, nil
}
// ChainMoved is, per kind moved, how many records the move wrote; empty before any is.
func (i *Inventory) ChainMoved(ctx context.Context) (map[string]int, error) {
var counts map[string]int
err := i.store.Pool().QueryRow(ctx, `select counts from chain_moved`).Scan(&counts)
if errors.Is(err, pgx.ErrNoRows) {
return map[string]int{}, nil
}
return counts, err
}
// NewForTest is an inventory with no store, for a test of what needs none.
func NewForTest() *Inventory { return &Inventory{moved: &movedKinds{}} }
// BeginSwitchForTest sets the flag a kind's switch sets, as a switch under way does.
func (i *Inventory) BeginSwitchForTest(kind string) {
if kind == KindAssignments {
i.moved.switching.Store(true)
c, check := i.chain, i.state.requireLayout && !i.state.laid
i.state.mu.Unlock()
if c == nil {
return nil, ErrNoChain
}
if check {
if _, found, err := c.Builds.GetRaw(ctx, chain.LayoutMarkerKey); err != nil {
return nil, err
} else if !found {
return nil, ErrNotLaidOut
}
i.markLaidOut()
}
return c, nil
}
internal/inventory/chain_assignments.go
+47 -38
View File
@@ -21,7 +21,6 @@ import (
// here (chain.Assignment.Stands). No `pending` state is written yet (issue 1094).
// KindAssignments is the kind of the chain that is each module assigned to a node, with the build it runs there.
const KindAssignments = "assignments"
func assignmentKey(node, module string) string { return node + "." + module }
@@ -31,32 +30,15 @@ type assignRow struct {
Assigned time.Time
}
// assignments is every assignment, by node then module: from the bucket when the kind is moved, else from the store.
// assignments is every assignment, by node then module, read from the assignments bucket.
// A key whose value names no time assigned is a module only pulled in by another (issue 765), and one that has ended or
// waits in a state is gone: no assignment. One on a machine the store does not know is none either, as the store took a
// removed machine's.
func (i *Inventory) assignments(ctx context.Context) ([]assignRow, error) {
c, on, err := i.ChainReads(ctx, KindAssignments)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
if !on {
rows, err := i.store.Pool().Query(ctx,
`select n.name, a.module, a.assigned from assignment a join node n on n.id = a.node order by n.name, a.module`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []assignRow
for rows.Next() {
var a assignRow
if err := rows.Scan(&a.Node, &a.Module, &a.Assigned); err != nil {
return nil, err
}
out = append(out, a)
}
return out, rows.Err()
}
all, err := c.Assignments.All(ctx)
if err != nil {
return nil, fmt.Errorf("the assignments cannot be read from the bus: %w", err)
@@ -172,10 +154,12 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node,
switch {
case found && cur.Value.Stands():
// A module only pulled in by another becomes an assignment, keeping what it holds.
cur.Value.Assigned = now
cur.Value.Assigned, cur.Value.PulledIn = now, false
_, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision)
case found:
// One that ended is assigned again: a fresh value over it, the ended one staying in history.
// One that ended is assigned again, and one that was pending is made: a fresh value over it, the earlier one
// staying in history. The wait is carried into the value that stands (issue 1094).
fresh.Pending = carriedPending(cur.Value, now)
_, err = c.Assignments.Update(ctx, key, fresh, cur.Revision)
default:
_, err = c.Assignments.Create(ctx, key, fresh)
@@ -253,8 +237,8 @@ func (i *Inventory) chainAwaitingSince(ctx context.Context, name string) (time.T
// and rollout another write put in the same value, and never ending an assignment. A plan the provisioner's rules refuse is
// refused by name and nothing is written (AssignmentValues).
func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed int, err error) {
c, on, err := i.ChainReads(ctx, KindAssignments)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return 0, 0, err
}
var plans map[string]Provisioning
@@ -289,11 +273,11 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i
case keepSkip:
continue
case keepCreate:
next.Assigned = time.Time{}
next.Assigned, next.PulledIn = time.Time{}, true
_, err = c.Assignments.Create(ctx, key, next)
case keepFresh:
// A value that ended, pulled in again: a fresh one over it, the ended one staying in history.
next.Assigned = time.Time{}
next.Assigned, next.PulledIn = time.Time{}, true
_, err = c.Assignments.Update(ctx, key, next, cur.Revision)
default:
cur.Value.RequiredBy, cur.Value.Provisioning = next.RequiredBy, next.Provisioning
@@ -308,8 +292,8 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i
written++
}
for key, cur := range have {
if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() {
continue
if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() || cur.Value.Commit != "" {
continue // one a send still carries is on the machine, and ends when a send does not carry it any more
}
// A value of a module no longer pulled in, which no assignment names: it ends, and stays.
cur.Value.Ended = time.Now().UTC()
@@ -340,17 +324,16 @@ type KeptValue struct {
Revision uint64
}
// KeptAssignments is every key of the assignments bucket that holds a value that stands, and false when the assignments are
// not on the bus.
// What the serving controller compares from one look to the next to say a key changed (`assignment-changed`).
func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, bool, error) {
c, on, err := i.ChainReads(ctx, KindAssignments)
if err != nil || !on {
return nil, false, err
// KeptAssignments is every key of the assignments bucket that holds a value that stands. What the serving controller
// compares from one look to the next to say a key changed (`assignment-changed`).
func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
all, err := c.Assignments.All(ctx)
if err != nil {
return nil, true, err
return nil, err
}
out := make(map[string]KeptValue, len(all))
for key, k := range all {
@@ -359,11 +342,11 @@ func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue,
}
body, err := json.Marshal(k.Value)
if err != nil {
return nil, true, err
return nil, err
}
out[key] = KeptValue{Value: body, Revision: k.Revision}
}
return out, true, nil
return out, nil
}
// What a pass that keeps the plans does with one key.
@@ -384,6 +367,8 @@ func keepStep(has bool, cur, next chain.Assignment) string {
return keepSkip // assigned since the bucket was read: its own write is not here yet
case !has:
return keepCreate
case cur.State == chain.StatePending:
return keepSkip // a wait for a build is not pulled in by a plan
case !cur.Stands():
if !cur.Assigned.IsZero() || !next.Assigned.IsZero() {
return keepSkip // an assignment that ended is not pulled in again by a plan
@@ -394,3 +379,27 @@ func keepStep(has bool, cur, next chain.Assignment) string {
}
return keepField
}
// carriedPending is the wait a value that does not stand carries into the assignment made over it: a wait that was applying
// stays applying, for the one making it settles it applied; one still waiting is made by hand, and says so. Nothing for a
// value that held no wait, or whose key is not one that was pending.
func carriedPending(was chain.Assignment, now time.Time) *chain.Pending {
if was.Pending == nil || was.State != chain.StatePending {
return nil
}
p := *was.Pending
if p.State == chain.PendingWaiting {
p.State, p.Note, p.Settled = chain.PendingApplied, "assigned by hand", &now
}
return &p
}
// AssignmentValue is one key of the assignments bucket as it is held, standing or not: one read of an assignment.
func (i *Inventory) AssignmentValue(ctx context.Context, node, module string) (chain.Assignment, bool, error) {
c, err := i.theChain(ctx)
if err != nil {
return chain.Assignment{}, false, err
}
k, found, err := c.Assignments.Get(ctx, assignmentKey(node, module))
return k.Value, found, err
}
internal/inventory/chain_assignments_key_test.go
+10 -19
View File
@@ -1,9 +1,7 @@
package inventory
import (
"context"
"encoding/json"
"errors"
"testing"
"time"
@@ -51,31 +49,24 @@ func TestAPassThatKeepsThePlansNeverRevivesAnEndedAssignment(t *testing.T) {
ended := chain.Assignment{Node: "a", Module: "m", Assigned: at, Ended: at.Add(time.Minute)}
pulledIn := chain.Assignment{Node: "a", Module: "p", RequiredBy: "m", Provisioning: plan}
pulledInEnded := chain.Assignment{Node: "a", Module: "p", Ended: at}
waiting := chain.Assignment{Node: "a", Module: "p", State: chain.StatePending,
Pending: &chain.Pending{ID: "p-1", State: chain.PendingWaiting}}
for name, tc := range map[string]struct {
has bool
cur, next chain.Assignment
want string
}{
"the interleaving: wanted assigned, ended since": {true, ended, assigned, keepSkip},
"wanted assigned, not in the bucket yet": {false, chain.Assignment{}, assigned, keepSkip},
"a pulled-in module, not in the bucket": {false, chain.Assignment{}, pulledIn, keepCreate},
"a pulled-in module that ended, pulled again": {true, pulledInEnded, pulledIn, keepFresh},
"an assignment ended, a plan for it": {true, ended, chain.Assignment{Node: "a", Module: "m", RequiredBy: "x", Provisioning: plan}, keepSkip},
"the plan is the same": {true, assigned, assigned, keepSkip},
"the plan changed": {true, chain.Assignment{Node: "a", Module: "m", Assigned: at}, assigned, keepField},
"the interleaving: wanted assigned, ended since": {true, ended, assigned, keepSkip},
"wanted assigned, not in the bucket yet": {false, chain.Assignment{}, assigned, keepSkip},
"a pulled-in module, not in the bucket": {false, chain.Assignment{}, pulledIn, keepCreate},
"a pulled-in module that ended, pulled again": {true, pulledInEnded, pulledIn, keepFresh},
"an assignment ended, a plan for it": {true, ended, chain.Assignment{Node: "a", Module: "m", RequiredBy: "x", Provisioning: plan}, keepSkip},
"a wait for a build, a plan for the module pulled in": {true, waiting, pulledIn, keepSkip},
"the plan is the same": {true, assigned, assigned, keepSkip},
"the plan changed": {true, chain.Assignment{Node: "a", Module: "m", Assigned: at}, assigned, keepField},
} {
if got := keepStep(tc.has, tc.cur, tc.next); got != tc.want {
t.Errorf("%s: %s, want %s", name, got, tc.want)
}
}
}
// **The store's mirror of the assignments is refused to its writer once the assignments are moved** (ADR 0408 §4, issue
// 1093): asked at the moment of the write, so a tick that decided before the switch lays nothing after it.
func TestTheStoresMirrorOfTheAssignmentsIsRefusedToItsWriterOnceTheyAreMoved(t *testing.T) {
inv := &Inventory{moved: &movedKinds{kind: map[string]bool{KindAssignments: true}}}
inv.UseChain(chain.InMemory(nil))
if _, err := inv.AssignmentsMirrorToWrite(context.Background()); !errors.Is(err, ErrAssignmentsMoved) {
t.Fatalf("the writer was given the store's rows: %v", err)
}
}
internal/inventory/chain_builds.go
+52 -3
View File
@@ -1146,8 +1146,35 @@ type LayoutReport struct {
Records, Written, Present int
}
// KindBuildsLayout is the kind chain_moved names the layout of the builds bucket under their modules by (issue 1121).
const KindBuildsLayout = "builds-layout"
// ErrNotLaidOut is a read of the builds before the controller's start has laid them out under their modules (issue 1121):
// refused loudly, never answered from keys that are not there.
var ErrNotLaidOut = errors.New("the builds are not laid out under their modules yet; the controller's start does it and says so")
// LayBuildsOut lays the builds out under their modules once and writes the marker key (chain.LayoutMarkerKey) that lets them
// be read. It needs the chain attached and the lease held (the chain's writes ask), and is idempotent: with the marker it
// does nothing, and a run that stopped half way is finished by the next. The old keys stay (ADR 0408 §4).
func (i *Inventory) LayBuildsOut(ctx context.Context, say func(string, ...any)) error {
c := i.attached()
if c == nil {
return ErrNoChain
}
if _, found, err := c.Builds.GetRaw(ctx, chain.LayoutMarkerKey); err != nil {
return err
} else if found {
i.markLaidOut()
return nil
}
report, err := i.layoutBuilds(ctx, c, say)
if err != nil {
return err
}
body, _ := json.Marshal(map[string]any{"records": report.Records, "at": time.Now().UTC()})
if _, err := c.Builds.CreateRaw(ctx, chain.LayoutMarkerKey, body); err != nil && !errors.Is(err, chain.ErrExists) {
return fmt.Errorf("the builds are laid out and the marker is not written: %w", err)
}
i.markLaidOut()
return nil
}
// layoutBuilds lays the builds of the bucket out under their modules: each record of a key that is its id alone is written
// under `<module>.<id>` (a request, and a build that knew no module, under `req.<id>`), its manifest moved to the manifests
@@ -1179,6 +1206,28 @@ func (i *Inventory) layoutBuilds(ctx context.Context, c *chain.Chain, say func(s
if r.Records%1000 == 0 {
say("laying the builds out: %d of %d records, %d keys written so far", r.Records, len(ids), r.Written)
}
// A record already under its new key is moved: a run again (the start before this one laid it out) neither reads its
// manifest nor writes it, and **no pointer is set back**: they are only made where missing.
own := chain.BuildKey(rec.Module, rec.ID)
if rec.State == chain.BuildAsked {
own = chain.RequestKey(rec.ID)
}
if _, there, err := c.Builds.GetRaw(ctx, own); err != nil {
return r, err
} else if there {
r.Present++
if rec.State != chain.BuildAsked {
if wrote, err := i.pointTo(ctx, c, chain.IDKey(rec.ID), idPointer(rec.Module), false); err != nil {
return r, err
} else if wrote {
r.Written++
}
}
if rec.State != chain.BuildAsked && rec.Module != "" {
byModule[rec.Module] = append(byModule[rec.Module], rec)
}
continue
}
// A manifest already written in this run is not written again.
digest := ""
if body := manifestBytes([]byte(rec.Manifest)); body != nil && c.Kept {
@@ -1238,7 +1287,7 @@ func (i *Inventory) layoutBuilds(ctx context.Context, c *chain.Chain, say func(s
return r, err
}
for _, p := range points {
wrote, err := i.pointTo(ctx, c, p.key, p.value, true)
wrote, err := i.pointTo(ctx, c, p.key, p.value, false)
if err != nil {
return r, err
}
internal/inventory/chain_builds_layout_test.go
+64 -1
View File
@@ -3,6 +3,7 @@ package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"sort"
@@ -40,7 +41,7 @@ func (r *readCounting) WatchFiltered(ctx context.Context, f string) (<-chan chai
func layoutChain(builds chain.Backend) (*Inventory, *chain.Chain) {
c := chain.New(builds, chain.NewMemory(), chain.NewMemory(), func(context.Context) error { return nil }).
WithManifests(builds)
inv := &Inventory{moved: &movedKinds{}}
inv := &Inventory{state: &chainState{}}
inv.UseChain(c)
return inv, c
}
@@ -396,3 +397,65 @@ func TestAnOlderBuildFinishingLastLeavesThePointersOnTheNewer(t *testing.T) {
t.Errorf("the older build's own reuse key finds %q", id)
}
}
// Nothing reads the chain on the bus until the controller's start has laid the builds out and written the marker; the layout
// is done once.
func TestTheChainIsRefusedUntilTheBuildsAreLaidOutAndTheLayoutIsDoneOnce(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
inv.RequireLayout()
ctx := context.Background()
old := legacyBuild(1, "alpha", "c1")
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
t.Fatal(err)
}
if _, err := inv.theChain(ctx); !errors.Is(err, ErrNotLaidOut) {
t.Fatalf("a read before the layout answered %v", err)
}
if err := inv.LayBuildsOut(ctx, nil); err != nil {
t.Fatal(err)
}
if _, err := inv.theChain(ctx); err != nil {
t.Fatalf("a read after the layout answered %v", err)
}
keys := keysOf(t, mem)
if !containsKey(keys, chain.LayoutMarkerKey) {
t.Fatalf("no marker among %v", keys)
}
if err := inv.LayBuildsOut(ctx, nil); err != nil || !reflect.DeepEqual(keys, keysOf(t, mem)) {
t.Fatalf("the second layout changed the bucket: %v", err)
}
}
// A layout run again — the live start recorded the first in chain_moved, not in the marker — never sets a pointer back to an
// older build, and does not read a manifest of a record already under its new key.
func TestALayoutRunAgainLeavesThePointersOfTheBuildsMadeSince(t *testing.T) {
counting := &readCounting{Backend: chain.NewMemory()}
inv, c := layoutChain(counting)
ctx := context.Background()
for n := 1; n <= 2; n++ {
old := legacyBuild(n, "alpha", fmt.Sprintf("c%d", n))
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
t.Fatal(err)
}
}
if _, err := inv.layoutBuilds(ctx, c, nil); err != nil {
t.Fatal(err)
}
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", 3, "c3", "s3")); err != nil {
t.Fatal(err)
}
counting.gets.Store(0)
r, err := inv.layoutBuilds(ctx, c, nil)
if err != nil || r.Written != 0 {
t.Fatalf("the second layout %+v: %v", r, err)
}
for _, key := range []string{"alpha.newest", "alpha.worked", "alpha.held"} {
if e, _, _ := counting.Get(ctx, key); string(e.Value) != "build-3" {
t.Errorf("%s = %q after the second layout; want build-3", key, e.Value)
}
}
if e, _, _ := counting.Get(ctx, "alpha.commit.c3"); pointedID(e.Value) != "build-3" {
t.Errorf("commit c3 = %s", e.Value)
}
}
internal/inventory/chain_drop_test.go
+98
View File
@@ -0,0 +1,98 @@
//go:build integration
package inventory
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// **Migration 0105 refuses a store whose rows were not moved, and drops the tables of a moved one** (novox/hq ADR 0408 §8,
// issue 1111). The store is built as the mesh holds it before this migration, with an assignment row in the table and no
// record of the move: the migration raises, and nothing it did is kept. Then the move is recorded, a pending assignment
// still open refuses it again, and once it has settled the tables go and build_gate stays.
func TestMigration0105RefusesAStoreWhoseRowsWereNotMovedAndDropsTheTablesOfAMovedOne(t *testing.T) {
inv, _ := ForTestBefore(t, 105)
ctx := t.Context()
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "ui", Version: "1"}, Source{}); err != nil {
t.Fatal(err)
}
inv.ExecForTest(t, `insert into assignment (node, module) select id, 'ui' from node where name = 'anchor'`)
all, err := Migrations()
if err != nil {
t.Fatal(err)
}
tables := func() map[string]bool {
t.Helper()
out := map[string]bool{}
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
out[strings.Trim(row[0], `"`)] = true
}
return out
}
refused := func(why string) {
t.Helper()
_, err := inv.store.Migrate(ctx, all)
if err == nil || !strings.Contains(err.Error(), why) {
t.Fatalf("the migration was not refused for %q: %v", why, err)
}
if !tables()["assignment"] || !tables()["chain_moved"] {
t.Fatalf("a refused migration kept a drop")
}
if rows := inv.QueryForTest(t, `select count(*) from assignment`); rows[0][0] != "1" {
t.Fatalf("a refused migration lost the assignment: %v", rows)
}
}
// No move recorded: the rows of assignments were never copied.
refused("assignments")
// The move recorded for every kind, and a pending assignment still waiting for its build: no mover carries it.
inv.ExecForTest(t, `insert into chain_moved (counts, largest) values ($1::jsonb, $1::jsonb)`,
`{"builds": 0, "rollouts": 0, "assignments": 1}`)
inv.ExecForTest(t, `insert into pending_assignment (node, module, build_id)
select id, 'ui', 'b1' from node where name = 'anchor'`)
refused("pending assignment")
// Expired and raised as a condition, and the condition neither answered nor cleared: still refused.
inv.ExecForTest(t, `update pending_assignment set state = 'expired', raised_at = now()`)
refused("pending assignment")
// Its condition cleared: the guard lets the migration through.
inv.ExecForTest(t, `update pending_assignment set cleared_at = now()`)
done, err := inv.store.Migrate(ctx, all)
if err != nil || len(done) != 1 || done[0].Number != 105 {
t.Fatalf("the migration did %v (%v)", done, err)
}
left := tables()
for _, gone := range append(append([]string{}, chainTables...), "chain_moved") {
if left[gone] {
t.Errorf("the table %s is still in the store", gone)
}
}
for _, stays := range []string{"build_gate", "assignment_generation", "node", "module"} {
if !left[stays] {
t.Errorf("the table %s is gone", stays)
}
}
if rows := inv.QueryForTest(t, `select count(*) from information_schema.columns where table_name = 'node' and column_name = 'sent_builds'`); rows[0][0] != "1" {
t.Errorf("node.sent_builds, which stays as the source of the copy, is gone")
}
if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
t.Errorf("the function raise_assignment_generation is still in the store")
}
}
// A store that never held a row of the chain — a mesh raised after the move — passes the guard with no record of a move.
func TestMigration0105PassesAStoreThatNeverHeldARow(t *testing.T) {
inv, migrate := ForTestBefore(t, 105)
migrate()
if rows := inv.QueryForTest(t, `select count(*) from information_schema.tables where table_name in ('build', 'release_plan', 'assignment', 'chain_moved')`); rows[0][0] != "0" {
t.Fatalf("tables of the chain remain: %v", rows)
}
}
internal/inventory/chain_links.go
+202 -30
View File
@@ -2,9 +2,11 @@ package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"maps"
"slices"
"sort"
"time"
@@ -39,8 +41,8 @@ func WithRollout(ctx context.Context, id string, modules map[string]RolloutModul
// AssignmentRuns is what each module assigned to a machine runs there, as its assignment names it: the build and the
// commit. Empty when the assignments are not on the bus, and for a module not sent yet.
func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string]chain.Ref, error) {
c, on, err := i.ChainReads(ctx, KindAssignments)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
all, err := c.Assignments.All(ctx)
@@ -50,7 +52,7 @@ func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string
out := map[string]chain.Ref{}
for _, k := range all {
a := k.Value
if a.Node == node && !a.Assigned.IsZero() && a.Stands() && (a.Build != "" || a.Commit != "") {
if a.Node == node && a.Stands() && (a.Build != "" || a.Commit != "") {
out[a.Module] = chain.Ref{Build: a.Build, Commit: a.Commit}
}
}
@@ -59,8 +61,8 @@ func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string
// AssignmentsOn are the assignments of a machine as the chain names them, by module: the standing ones, with their links.
func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Assignment, error) {
c, on, err := i.ChainReads(ctx, KindAssignments)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
all, err := c.Assignments.All(ctx)
@@ -77,14 +79,18 @@ func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Ass
return out, nil
}
// recordRuns writes what a send put on a machine into the assignments, once the send is recorded: for each module it carried
// a build of — by the commit — the build, what it published, the commit, the rollout and since when. **A module that runs the
// commit already keeps what it names**, filled where it names nothing and never blanked, so a send that carries it again, or a
// push by hand, changes nothing it knows. A module the machine is not assigned is no assignment to write.
// recordRuns writes what a send put on a machine into the assignments, once the send is recorded, and it is how the mesh
// knows what a machine was sent (SentBuilds). **A send that names its builds** writes, for each module it carried a build of
// — by the commit — the build, what it published, the commit, the rollout and since when, on the module's key: filled where
// the key names nothing, replaced when the commit moved, never blanked by a push by hand; a module nothing assigns, carried
// because another requires it, gets a standing key marked pulled-in; and every other key of the machine loses what it named,
// for the machine is not sent it any more. **A send that names none** (a declaration by hand) makes what the machine runs
// unknown: every key of it says so (sent-unknown) and names no commit. **An error here fails the record of the send**, for
// what a machine was sent is read from these keys and from nowhere else.
func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[string]string, at time.Time) error {
c, on, err := i.ChainReads(ctx, KindAssignments)
if err != nil || !on || len(builds) == 0 {
return err
c, err := i.theChain(ctx)
if err != nil {
return fmt.Errorf("what the machine runs cannot be written: %w", err)
}
var node string
if err := i.store.Pool().QueryRow(ctx, `select name from node where id = $1`, nodeID).Scan(&node); errors.Is(err, pgx.ErrNoRows) {
@@ -93,24 +99,81 @@ func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[st
return err
}
rollout, _ := ctx.Value(rolloutKey{}).(rolloutOf)
all, err := c.Assignments.All(ctx)
if err != nil {
return fmt.Errorf("the assignments of %s cannot be read: %w", node, err)
}
var failed []error
for _, k := range sortedValues(all) {
a := k.Value
if a.Node != node || !a.Stands() {
continue
}
if builds[a.Module] != "" {
continue // carried, and its commit is written below
}
if err := i.forgetRun(ctx, c, k, builds == nil, at); err != nil {
failed = append(failed, fmt.Errorf("%s on %s: %w", a.Module, node, err))
}
}
modules := make([]string, 0, len(builds))
for m := range builds {
modules = append(modules, m)
}
sort.Strings(modules)
var failed []error
for _, m := range modules {
commit := builds[m]
if commit == "" {
if builds[m] == "" {
continue
}
if err := i.recordRun(ctx, c, node, m, commit, rollout, at); err != nil {
if err := i.recordRun(ctx, c, node, m, builds[m], rollout, at); err != nil {
failed = append(failed, fmt.Errorf("%s on %s: %w", m, node, err))
}
}
return errors.Join(failed...)
}
func sortedValues(all map[string]chain.Kept[chain.Assignment]) []chain.Kept[chain.Assignment] {
out := make([]chain.Kept[chain.Assignment], 0, len(all))
for key, k := range all {
k.Key = key
out = append(out, k)
}
sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key })
return out
}
// forgetRun writes that a machine is no longer sent what a key named: its links are cleared, and for a send by hand the key
// says what the machine runs is unknown.
func (i *Inventory) forgetRun(ctx context.Context, c *chain.Chain, k chain.Kept[chain.Assignment], hand bool, at time.Time) error {
for try := 0; try < casTries; try++ {
next := k.Value
next.Build, next.Artifact, next.Package, next.Commit, next.Rollout, next.Since = "", "", nil, "", "", time.Time{}
next.SentUnknown = time.Time{}
if hand {
next.SentUnknown = at.UTC()
}
if next.Build == k.Value.Build && next.Commit == k.Value.Commit && next.Rollout == k.Value.Rollout &&
next.Artifact == k.Value.Artifact && next.Package == k.Value.Package && next.Since.Equal(k.Value.Since) &&
next.SentUnknown.Equal(k.Value.SentUnknown) {
return nil
}
if _, err := c.Assignments.Update(ctx, k.Key, next, k.Revision); errors.Is(err, chain.ErrMoved) {
cur, found, gerr := c.Assignments.Get(ctx, k.Key)
if gerr != nil {
return gerr
}
if !found || !cur.Value.Stands() {
return nil
}
k = chain.Kept[chain.Assignment]{Key: k.Key, Value: cur.Value, Revision: cur.Revision}
continue
} else {
return err
}
}
return fmt.Errorf("the assignment kept moving under the write")
}
func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, commit string, rollout rolloutOf, at time.Time) error {
key := assignmentKey(node, module)
var build, rolloutID string
@@ -124,11 +187,12 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module,
if err != nil {
return err
}
if !found || cur.Value.Assigned.IsZero() || !cur.Value.Stands() {
return nil // nothing assigned there to say what it runs
if found && cur.Value.State != "" {
return nil // a wait for a build: the module is not on the machine by it
}
same := cur.Value.Commit == commit
if same && cur.Value.Build != "" && (rolloutID == "" || cur.Value.Rollout != "") {
standing := found && cur.Value.Stands()
same := standing && cur.Value.Commit == commit
if same && cur.Value.Build != "" && cur.Value.SentUnknown.IsZero() && (rolloutID == "" || cur.Value.Rollout != "") {
return nil // a send that carries what the machine runs already says nothing new
}
if !resolved {
@@ -148,7 +212,13 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module,
resolved = true
}
next := cur.Value
if same {
switch {
case !standing:
// Carried and assigned to nobody here: a standing key, no time assigned, marked pulled in. An ended one is
// written over, the ended one staying in history.
next = chain.Assignment{Node: node, Module: module, PulledIn: true, Commit: commit, Build: build, Rollout: rolloutID,
Since: at.UTC()}
case same:
// Already running it: fill what is missing, blank nothing.
if next.Build == "" {
next.Build = build
@@ -159,18 +229,26 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module,
if next.Since.IsZero() {
next.Since = at.UTC()
}
} else {
default:
next.Commit, next.Build, next.Rollout, next.Since = commit, build, rolloutID, at.UTC()
next.Artifact, next.Package = "", nil
}
next.SentUnknown = time.Time{}
if published != nil && next.Artifact == "" && next.Package == nil && next.Build == build {
next.Artifact, next.Package = published.Artifact, published.Package
}
if next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout &&
if standing && next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout &&
next.Artifact == cur.Value.Artifact && (next.Package == nil) == (cur.Value.Package == nil) &&
next.Since.Equal(cur.Value.Since) {
next.Since.Equal(cur.Value.Since) && next.SentUnknown.Equal(cur.Value.SentUnknown) {
return nil
}
if !found {
if _, err := c.Assignments.Create(ctx, key, next); errors.Is(err, chain.ErrExists) {
continue
} else {
return err
}
}
if _, err := c.Assignments.Update(ctx, key, next, cur.Revision); errors.Is(err, chain.ErrMoved) {
continue
} else {
@@ -180,11 +258,105 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module,
return fmt.Errorf("the assignment kept moving under the write")
}
// saidRunsNot says a send's links could not be written, to the log of whoever sent: the send is away and is recorded, and
// a record of what it put where that could not be written is said, never a reason to call the send failed.
func saidRunsNot(node string, err error) {
// CopySentBuilds copies what delivery 1 recorded in node.sent_builds onto the keys of the assignments, once, at the start of
// the controller that holds the lease and has attached the chain (issue 1109, review round 1): delivery 1 wrote no commit onto
// the key of a module an assignment pulled in, so without it those modules would be missing from SentBuilds although the
// machine is known to have been sent. A key that names a commit is never overwritten, so it is idempotent; once a node's keys
// are written its column is emptied, so a later restart cannot lay a stale commit over a key a later send cleared or ended.
// It answers how many keys it wrote; an error is raised by the caller as a condition.
func (i *Inventory) CopySentBuilds(ctx context.Context) (int, error) {
c, err := i.theChain(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, "mesh-controller: %s was sent its declaration, and the build, commit and rollout of each module "+
"could not be written in its assignments: %v\n", node, err)
return 0, err
}
rows, err := i.store.Pool().Query(ctx, `select name, sent_builds from node where sent_builds is not null`)
if err != nil {
return 0, err
}
type kept struct {
name string
sent map[string]string
}
var all []kept
for rows.Next() {
var name string
var raw []byte
if err := rows.Scan(&name, &raw); err != nil {
rows.Close()
return 0, err
}
sent := map[string]string{}
if err := json.Unmarshal(raw, &sent); err != nil {
rows.Close()
return 0, fmt.Errorf("what %s was sent, in the store, cannot be read: %w", name, err)
}
all = append(all, kept{name, sent})
}
rows.Close()
if err := rows.Err(); err != nil {
return 0, err
}
now := time.Now().UTC()
for _, k := range all {
if err := i.copySentBuildsOf(ctx, c, k.name, k.sent, now); err != nil {
return 0, fmt.Errorf("what %s was sent cannot be copied onto its assignments: %w", k.name, err)
}
if _, err := i.store.Pool().Exec(ctx, `update node set sent_builds = null where name = $1`, k.name); err != nil {
return 0, err
}
}
return len(all), nil
}
// copySentBuildsOf is CopySentBuilds for one node, over a chain: for each module the node was sent, a standing key that
// names no commit gets it (and the build, when one is known; no rollout), and where no key exists a standing pulled-in key is
// made. A key with a commit, an ended key, a hand send's unknown and a wait for a build are left as they are.
func (i *Inventory) copySentBuildsOf(ctx context.Context, c *chain.Chain, node string, sent map[string]string, at time.Time) error {
for _, module := range slices.Sorted(maps.Keys(sent)) {
commit := sent[module]
if commit == "" {
continue
}
key := assignmentKey(node, module)
build, err := i.chainBuildOf(ctx, c, module, commit)
if err != nil {
return err
}
for try := 0; ; try++ {
if try == casTries {
return fmt.Errorf("the assignment %s kept moving under the write", key)
}
cur, found, err := c.Assignments.Get(ctx, key)
if err != nil {
return err
}
if found && (cur.Value.State != "" || !cur.Value.Ended.IsZero() || cur.Value.Commit != "" || !cur.Value.SentUnknown.IsZero()) {
break // a wait, an ended key (never revived), one that names a commit, or a hand send's unknown
}
next := cur.Value
if !found {
next = chain.Assignment{Node: node, Module: module, PulledIn: true, Since: at}
}
next.Commit, next.Build = commit, build
if next.Since.IsZero() {
next.Since = at
}
if !found {
_, err = c.Assignments.Create(ctx, key, next)
if errors.Is(err, chain.ErrExists) {
continue
}
} else {
_, err = c.Assignments.Update(ctx, key, next, cur.Revision)
if errors.Is(err, chain.ErrMoved) {
continue
}
}
if err != nil {
return err
}
break
}
}
return nil
}
internal/inventory/chain_move.go
-701
View File
@@ -1,701 +0,0 @@
package inventory
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/chain"
)
// **The move** (novox/hq ADR 0408 §8, issue 981 note 7 Q2): the serving controller writes the rows of a kind into its
// bucket at its start, holding the lease and after the buckets exist, and records the kind in chain_moved. Idempotent: a
// key that exists is already moved and is never overwritten, so a bucket value written since is not laid over by the row
// it came from. It counts the rows against the keys, logs the largest value, and **refuses** (a Refusal, which the
// controller raises as an urgent condition) on a count that differs or a value over the bus's largest, with nothing
// switched: a kind is recorded moved only once all of it is in the bucket.
// Refusal is a move that stopped without switching a kind, and why. It is permanent until the cause is put right; any
// other error of a move (the bus away, a write not granted) is tried again.
type Refusal struct {
Kind, Why string
}
func (r *Refusal) Error() string {
return fmt.Sprintf("the move of %s to the bus is refused: %s", r.Kind, r.Why)
}
// MoveReport is what a move of one kind did.
type MoveReport struct {
Kind string
// Rows is how many records the tables held, Written how many keys this move made, and Present how many already
// held a value (moved by an earlier start).
Rows, Written, Present int
// TableRows is how many rows of the tables those records came from: a request and its outcome are two rows and one
// record.
TableRows int
// Largest is the biggest value, in bytes.
Largest int
// Already is a kind an earlier start moved: nothing was done.
Already bool
}
// mover is one kind of the chain: its rows, read from its tables, as the keys and values they become.
type mover struct {
kind string
// rows reads the tables: each record's key and its value, whole, and how many rows of the tables they came from.
rows func(ctx context.Context) ([]moveRow, int, error)
// create writes one record under its key; chain.ErrExists when the key holds a value.
create func(ctx context.Context, c *chain.Chain, row moveRow) error
// keys are the keys the bucket holds.
keys func(ctx context.Context, c *chain.Chain) ([]string, error)
// count is how many rows the tables hold now, counted as rows reports them.
count func(ctx context.Context, tx pgx.Tx) (int, error)
// lock keeps the tables from taking a row while the switch is recorded.
lock string
// settle runs under that lock, with the rows read again: it makes the bucket hold what the rows say now. A row the old
// controller changed after the first pass is in the bucket before the kind is recorded as moved, for before the switch
// the table is the only truth.
settle func(ctx context.Context, c *chain.Chain, rows []moveRow) error
}
type moveRow struct {
key string
value any
size int
}
func (i *Inventory) movers() []mover {
return []mover{{
kind: KindBuilds,
rows: func(ctx context.Context) ([]moveRow, int, error) {
builds, err := i.allBuildRows(ctx)
if err != nil {
return nil, 0, err
}
requests, err := i.allRequestRows(ctx)
if err != nil {
return nil, 0, err
}
// A build request is the build's record in state asked, and the request stays beside the outcome that
// answered it: one record under one id (issue 1091).
byID := make(map[string]*chain.Build, len(builds))
var order []string
for _, b := range builds {
v := toChainBuild(b)
byID[b.ID] = &v
order = append(order, b.ID)
}
for _, r := range requests {
if v, answered := byID[r.ID]; answered {
v.Request = toRequest(r)
continue
}
v := askedRecord(r)
if r.NotAsked != "" {
// Never handed over: asked again once, after the move (askAgain).
v.Request.Reask = chain.ReaskPending
}
byID[r.ID] = &v
order = append(order, r.ID)
}
out := make([]moveRow, 0, len(order))
for _, id := range order {
body, err := json.Marshal(byID[id])
if err != nil {
return nil, 0, err
}
out = append(out, moveRow{key: legacyBuildKey(id), value: *byID[id], size: len(body)})
}
return out, len(builds) + len(requests), nil
},
create: func(ctx context.Context, c *chain.Chain, row moveRow) error {
_, err := c.Builds.Create(ctx, row.key, row.value.(chain.Build))
return err
},
keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Builds.Keys(ctx) },
count: func(ctx context.Context, tx pgx.Tx) (int, error) {
var n int
err := tx.QueryRow(ctx, `select (select count(*) from build) + (select count(*) from build_request)`).Scan(&n)
return n, err
},
lock: `lock table build, build_request in share mode`,
settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error {
return settleBucket(ctx, c.Builds, rows)
},
}, {
kind: KindRollouts,
rows: func(ctx context.Context) ([]moveRow, int, error) {
plans, err := i.plans(ctx, `order by created`)
if err != nil {
return nil, 0, err
}
merges, err := i.merges(ctx, ``)
if err != nil {
return nil, 0, err
}
tablePlans := len(plans)
if plans, err = holdMerges(plans, merges); err != nil {
return nil, 0, err
}
out := make([]moveRow, 0, len(plans))
for _, p := range plans {
body, err := json.Marshal(p)
if err != nil {
return nil, 0, err
}
out = append(out, moveRow{key: p.ID, value: p, size: len(body)})
}
return out, tablePlans + len(merges), nil
},
create: func(ctx context.Context, c *chain.Chain, row moveRow) error {
_, err := chain.RolloutsOf[Plan](c).Create(ctx, row.key, row.value.(Plan))
return err
},
keys: func(ctx context.Context, c *chain.Chain) ([]string, error) {
return chain.RolloutsOf[Plan](c).Keys(ctx)
},
count: func(ctx context.Context, tx pgx.Tx) (int, error) {
var n int
err := tx.QueryRow(ctx, `select (select count(*) from release_plan) + (select count(*) from batched_merge)`).Scan(&n)
return n, err
},
lock: `lock table release_plan, batched_merge in share mode`,
settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error {
return settleBucket(ctx, chain.RolloutsOf[Plan](c), rows)
},
}, {
kind: KindAssignments,
rows: func(ctx context.Context) ([]moveRow, int, error) {
assigned, err := i.assignedWithSends(ctx)
if err != nil {
return nil, 0, err
}
out := make([]moveRow, 0, len(assigned))
for _, a := range assigned {
body, err := json.Marshal(a)
if err != nil {
return nil, 0, err
}
out = append(out, moveRow{key: assignmentKey(a.Node, a.Module), value: a, size: len(body)})
}
return out, len(out), nil
},
create: createAssignment,
keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Assignments.Keys(ctx) },
count: func(ctx context.Context, tx pgx.Tx) (int, error) {
var n int
err := tx.QueryRow(ctx, `select count(*) from assignment`).Scan(&n)
return n, err
},
lock: `lock table assignment in share mode`,
settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error {
have := make(map[string]bool, len(rows))
for _, r := range rows {
have[r.key] = true
if err := settleAssignment(ctx, c, r); err != nil {
return err
}
}
// An assignment the store no longer holds ended since the first pass: the bucket says so.
all, err := c.Assignments.All(ctx)
if err != nil {
return err
}
for key, k := range all {
if have[key] || k.Value.Assigned.IsZero() || !k.Value.Stands() {
continue
}
k.Value.Ended = time.Now().UTC()
if _, err := c.Assignments.Update(ctx, key, k.Value, k.Revision); err != nil && !errors.Is(err, chain.ErrMoved) {
return err
}
}
return nil
},
}}
}
// assignedWithSends is every assignment of the store as the chain keeps it: with the commit its machine was last sent for
// the module (node.sent_builds), the build that is of it and what the build published, and since when (when the machine was
// last sent). An assignment of a module the machine was never sent has none of those.
func (i *Inventory) assignedWithSends(ctx context.Context) ([]chain.Assignment, error) {
rows, err := i.store.Pool().Query(ctx,
`select n.name, a.module, a.assigned, n.sent_at, n.sent_builds
from assignment a join node n on n.id = a.node order by n.name, a.module`)
if err != nil {
return nil, err
}
var out []chain.Assignment
for rows.Next() {
var a chain.Assignment
var sentAt *time.Time
var raw []byte
if err := rows.Scan(&a.Node, &a.Module, &a.Assigned, &sentAt, &raw); err != nil {
rows.Close()
return nil, err
}
a.Assigned = a.Assigned.UTC()
var sent map[string]string
if len(raw) > 0 {
_ = json.Unmarshal(raw, &sent)
}
if commit := sent[a.Module]; commit != "" && sentAt != nil {
a.Commit, a.Since = commit, sentAt.UTC()
}
out = append(out, a)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
// The build of each commit, read once: the newest successful build of the module at it.
type at struct{ module, commit string }
var all []Build
if _, moved, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if moved {
if all, err = chainBuilds(ctx, i.attached()); err != nil {
return nil, err
}
} else if all, err = i.allBuildRows(ctx); err != nil {
return nil, err
}
newest := map[at]Build{}
for _, b := range all {
k := at{b.Module, b.Commit}
if b.Failed != "" || b.Module == "" {
continue
}
if had, has := newest[k]; !has || b.At.After(had.At) || (b.At.Equal(had.At) && b.ID > had.ID) {
newest[k] = b
}
}
for k := range out {
if out[k].Commit == "" {
continue
}
b, found := newest[at{out[k].Module, out[k].Commit}]
if !found {
continue
}
out[k].Build = b.ID
if published := publishedOf(b.Made); published != nil {
out[k].Artifact, out[k].Package = published.Artifact, published.Package
}
}
return out, nil
}
// holdMerges puts each batched merge into the record of the plan that answers it. A merge no plan answers — the plan_id
// of a merge a failed walk carried is null, or names a plan the store never kept — is held, flagged waiting, by the
// newest plan that lists it among its merges, else the record kept for the merges no plan answers: the record that held it
// last.
func holdMerges(plans []Plan, merges []BatchedMerge) ([]Plan, error) {
waitingAt := -1
at := make(map[string]int, len(plans))
for k, p := range plans {
at[p.ID] = k
}
newest := func(match func(Plan) bool) int {
best := -1
for k, p := range plans {
if match(p) && (best < 0 || p.Created.After(plans[best].Created)) {
best = k
}
}
return best
}
for _, m := range merges {
held := heldOf(m, m.Heard)
holder, answered := at[m.Plan]
if m.Plan == "" || !answered {
held.Waiting = true
holder = newest(func(p Plan) bool {
if p.Delivery == nil {
return false
}
for _, listed := range p.Delivery.Merges {
if strings.EqualFold(listed.Repository, m.Repository) && listed.Commit == m.Commit {
return true
}
}
return false
})
if holder < 0 {
// Held by the record kept for the merges no plan answers, made with the others.
if waitingAt < 0 {
plans = append(plans, Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1,
Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"})
waitingAt = len(plans) - 1
}
holder = waitingAt
}
}
plans[holder].Held = append(plans[holder].Held, held)
}
return plans, nil
}
// MoveChain moves every kind not moved yet, in order, and says what it did. It needs the chain attached and the lease
// held (the chain's writes ask). It stops at the first kind that fails.
func (i *Inventory) MoveChain(ctx context.Context, say func(format string, args ...any)) ([]MoveReport, error) {
if say == nil {
say = func(string, ...any) {}
}
if i.attached() == nil {
return nil, ErrNoChain
}
var out []MoveReport
for _, m := range i.movers() {
r, err := i.moveKind(ctx, m, say)
if err != nil {
return out, err
}
out = append(out, r)
if m.kind == KindBuilds {
// The builds are laid out under their modules before anything reads them (issue 1121).
if err := i.moveBuildsLayout(ctx, say); err != nil {
return out, err
}
}
}
return out, nil
}
// movePasses is how often a move reads the tables again when they took a row while it wrote: the controller before this
// one may still have been writing as the lease changed hands.
const movePasses = 5
func (i *Inventory) moveKind(ctx context.Context, m mover, say func(string, ...any)) (MoveReport, error) {
report := MoveReport{Kind: m.kind}
moved, err := i.ChainMoved(ctx)
if err != nil {
return report, err
}
if _, done := moved[m.kind]; done {
report.Already = true
return report, nil
}
for pass := 1; pass <= movePasses; pass++ {
rows, tableRows, err := m.rows(ctx)
if err != nil {
return report, fmt.Errorf("the %s cannot be read from the tables: %w", m.kind, err)
}
report.Rows, report.TableRows, report.Largest = len(rows), tableRows, 0
// Every value measured before any is written: a value over the bus's largest stops the move with nothing
// written (ADR 0408 §6), never cut.
for _, r := range rows {
report.Largest = max(report.Largest, r.size)
if r.size > broker.ChainValueBytes {
return report, &Refusal{m.kind, fmt.Sprintf("the value of %s is %d bytes and the bus holds at most %d; "+
"nothing is cut, so the record must be made smaller first", r.key, r.size, broker.ChainValueBytes)}
}
}
report.Written, report.Present = 0, 0
for _, r := range rows {
switch err := m.create(ctx, i.attached(), r); {
case err == nil:
report.Written++
case errors.Is(err, chain.ErrExists):
report.Present++
case errors.Is(err, chain.ErrBadKey):
return report, &Refusal{m.kind, fmt.Sprintf("%s cannot be a key of the bus: %v", r.key, err)}
default:
return report, fmt.Errorf("%s %s is not written to the bus: %w", m.kind, r.key, err)
}
}
// Rows against keys: every row's key is in the bucket.
keys, err := m.keys(ctx, i.attached())
if err != nil {
return report, fmt.Errorf("the keys of the %s cannot be read back: %w", m.kind, err)
}
have := make(map[string]bool, len(keys))
for _, k := range keys {
have[k] = true
}
for _, r := range rows {
if !have[r.key] {
return report, &Refusal{m.kind, fmt.Sprintf("the tables hold %d records and the bucket does not hold %s", len(rows), r.key)}
}
}
settled, err := i.switchKind(ctx, m, report)
if err != nil {
return report, err
}
if settled {
say("moved the %s to the bus: %d records, %d written now and %d already there, the largest %d bytes",
m.kind, report.Rows, report.Written, report.Present, report.Largest)
return report, nil
}
say("the %s took a row while they were moved, so they are read again (pass %d of %d)", m.kind, pass, movePasses)
}
return report, &Refusal{m.kind, fmt.Sprintf("the tables kept taking rows through %d passes; another controller is still writing them", movePasses)}
}
// switchKind records the kind as moved, in one transaction that holds the tables against writes and finds their count
// as the move read it: false when they took a row since.
func (i *Inventory) switchKind(ctx context.Context, m mover, r MoveReport) (bool, error) {
committed := false
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return false, err
}
defer func() { _ = tx.Rollback(ctx) }()
if _, err := tx.Exec(ctx, m.lock); err != nil {
return false, err
}
n, err := m.count(ctx, tx)
if err != nil {
return false, err
}
if n != r.TableRows {
return false, nil
}
// The store's mirror of the assignments writes nothing from here: it would lay values without the links over the keys
// the settle writes and end the ones it ended.
if m.kind == KindAssignments {
i.moved.switching.Store(true)
defer func() {
if !committed {
i.moved.switching.Store(false)
}
}()
}
// The rows read again under the lock: nothing writes them now, so what the bucket holds is made what they say.
rows, tableRows, err := m.rows(ctx)
if err != nil {
return false, err
}
if tableRows != r.TableRows {
return false, nil
}
if err := m.settle(ctx, i.attached(), rows); err != nil {
return false, fmt.Errorf("the %s changed since they were first written, and the bucket cannot be brought to them: %w", m.kind, err)
}
counts, _ := json.Marshal(map[string]int{m.kind: r.Rows})
largest, _ := json.Marshal(map[string]int{m.kind: r.Largest})
if _, err := tx.Exec(ctx,
`insert into chain_moved (counts, largest) values ($1, $2)
on conflict (only_one) do update
set counts = chain_moved.counts || excluded.counts, largest = chain_moved.largest || excluded.largest,
moved_at = now()`, counts, largest); err != nil {
return false, err
}
if err := tx.Commit(ctx); err != nil {
return false, err
}
committed = true
i.moved.mu.Lock()
if i.moved.kind == nil {
i.moved.kind = map[string]bool{}
}
i.moved.kind[m.kind] = true
i.moved.mu.Unlock()
return true, nil
}
// allBuildRows is every build of the table, whole.
func (i *Inventory) allBuildRows(ctx context.Context) ([]Build, error) {
rows, err := i.store.Pool().Query(ctx,
`select id, repository, ref, coalesce(module, ''), commit_hash, built_on, failed, made, source_path, manifest,
built_against, built_contexts, build_sources, source_fingerprint, asked, at
from build order by at, id`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Build
for rows.Next() {
var b Build
var made, manifest, against, read, sources []byte
var asked *time.Time
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, &b.On, &b.Failed, &made, &b.Path,
&manifest, &against, &read, &sources, &b.SourceFingerprint, &asked, &b.At); err != nil {
return nil, err
}
if asked != nil {
b.Asked = *asked
}
b.Manifest = manifest
for _, part := range []struct {
raw []byte
to any
who string
}{{made, &b.Made, "made"}, {against, &b.Against, "built_against"}, {read, &b.Read, "built_contexts"},
{sources, &b.Sources, "build_sources"}} {
if len(part.raw) == 0 {
continue
}
if err := json.Unmarshal(part.raw, part.to); err != nil {
return nil, fmt.Errorf("build %s: %s is not readable: %w", b.ID, part.who, err)
}
}
out = append(out, b)
}
return out, rows.Err()
}
// allRequestRows is every build request of the table.
func (i *Inventory) allRequestRows(ctx context.Context) ([]BuildRequest, error) {
rows, err := i.store.Pool().Query(ctx,
`select id, repository, seat, source_path, ref, commit_hash, asked_for, coalesce(not_asked, ''),
coalesce(outcome_unknown, ''), asked_at, at_terminal
from build_request order by asked_at, id`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []BuildRequest
for rows.Next() {
var a BuildRequest
if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked,
&a.OutcomeUnknown, &a.At, &a.AtTerminal); err != nil {
return nil, err
}
out = append(out, a)
}
return out, rows.Err()
}
// createAssignment writes an assignment's row under its key; chain.ErrExists when the key holds a value that stays.
func createAssignment(ctx context.Context, c *chain.Chain, row moveRow) error {
want := row.value.(chain.Assignment)
// The mirror of the store kept a value under this key until now, with the provisioner's plan beside it: the
// plan stays, the record's links are laid in. A value that names its build already is a later word, and stays.
for try := 0; try < casTries; try++ {
cur, found, err := c.Assignments.Get(ctx, row.key)
if err != nil {
return err
}
if !found {
if _, err := c.Assignments.Create(ctx, row.key, want); !errors.Is(err, chain.ErrExists) {
return err
}
continue
}
if cur.Value.Build != "" || cur.Value.Commit != "" || !cur.Value.Since.IsZero() || (want.Build == "" && want.Commit == "") {
return chain.ErrExists
}
merged := want
merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning
if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) {
continue
} else {
return err
}
}
return fmt.Errorf("the assignment %s kept moving under the write", row.key)
}
// settleBucket makes a bucket of whole values hold each row's value, by compare-and-set: a key that is absent is made, one
// whose bytes differ from the row's is written over, and one that is the same is left.
func settleBucket[T any](ctx context.Context, b chain.Bucket[T], rows []moveRow) error {
for _, r := range rows {
want, err := json.Marshal(r.value)
if err != nil {
return err
}
settled := false
for try := 0; try < casTries && !settled; try++ {
e, found, err := b.GetRaw(ctx, r.key)
if err != nil {
return err
}
switch {
case !found:
_, err = b.CreateRaw(ctx, r.key, want)
if errors.Is(err, chain.ErrExists) {
continue
}
case bytes.Equal(e.Value, want):
settled = true
continue
default:
_, err = b.UpdateRaw(ctx, r.key, want, e.Revision)
if errors.Is(err, chain.ErrMoved) {
continue
}
}
if err != nil {
return err
}
settled = true
}
if !settled {
return fmt.Errorf("%s kept moving under the write", r.key)
}
}
return nil
}
// settleAssignment makes the value under an assignment's key what its row says, keeping the plan the mirror put beside it:
// before the switch the store is the only truth of the links, so a commit it changed since the first pass is written over.
func settleAssignment(ctx context.Context, c *chain.Chain, row moveRow) error {
want := row.value.(chain.Assignment)
for try := 0; try < casTries; try++ {
cur, found, err := c.Assignments.Get(ctx, row.key)
if err != nil {
return err
}
if !found {
if _, err := c.Assignments.Create(ctx, row.key, want); errors.Is(err, chain.ErrExists) {
continue
} else {
return err
}
}
merged := want
merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning
a, _ := json.Marshal(merged)
b, _ := json.Marshal(cur.Value)
if bytes.Equal(a, b) {
return nil
}
if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) {
continue
} else {
return err
}
}
return fmt.Errorf("the assignment %s kept moving under the write", row.key)
}
// moveBuildsLayout lays the builds out under their modules once, and records the kind builds-layout in chain_moved. A key
// the old layout made stays (ADR 0408 §4).
func (i *Inventory) moveBuildsLayout(ctx context.Context, say func(string, ...any)) error {
moved, err := i.ChainMoved(ctx)
if err != nil {
return err
}
if _, done := moved[KindBuildsLayout]; done {
return nil
}
report, err := i.layoutBuilds(ctx, i.attached(), say)
if err != nil {
return err
}
counts, _ := json.Marshal(map[string]int{KindBuildsLayout: report.Records})
largest, _ := json.Marshal(map[string]int{KindBuildsLayout: 0})
if _, err := i.store.Pool().Exec(ctx,
`insert into chain_moved (counts, largest) values ($1, $2)
on conflict (only_one) do update
set counts = chain_moved.counts || excluded.counts, largest = chain_moved.largest || excluded.largest,
moved_at = now()`, counts, largest); err != nil {
return err
}
i.moved.mu.Lock()
if i.moved.kind == nil {
i.moved.kind = map[string]bool{}
}
i.moved.kind[KindBuildsLayout] = true
i.moved.mu.Unlock()
return nil
}
internal/inventory/chain_move_test.go
-950
View File
@@ -1,950 +0,0 @@
//go:build integration
package inventory
import (
"bytes"
"context"
"encoding/json"
"errors"
"reflect"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/chain"
)
// A build row as the store held it before the move: written by SQL, the only hand-written state these tests have, for
// it stands for what the mesh held before this release.
func aBuildRow(t *testing.T, inv *Inventory, id, module, commit, failed string) {
t.Helper()
inv.ExecForTest(t,
`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, source_path, manifest,
built_against, built_contexts, asked, source_fingerprint, build_sources)
values ($1::text, 'novox/' || $2::text, 'main', $2::text, $3::text, 'ace', $4::text,
case when $4::text = ''
then ('[{"name":"app","kind":"image","reference":"artifact-store://' || $2::text || '@sha256:' || $1::text || '"}]')::jsonb
else '[]'::jsonb end,
'modules/' || $2::text, ('{"module":"' || $2::text || '"}')::jsonb,
'["artifact-store://base@sha256:aa"]'::jsonb, '[{"repository":"novox/other","ref":"main"}]'::jsonb,
now() - interval '1 hour', 'fp-' || $1::text,
('[{"paths":["modules/' || $2::text || '/module.json"]}]')::jsonb)`,
id, module, commit, failed)
}
// inTheTables is an inventory before the move: its builds and assignments are rows of the store, which a test of a data
// migration of the store (that reads those rows) needs.
func inTheTables(t *testing.T) *Inventory {
t.Helper()
inv, finish := ForTestBefore(t, 105)
finish()
return inv
}
// reportOf is the report of one kind among those a move made, which are one for each kind in the order they are moved.
func reportOf(t *testing.T, reports []MoveReport, kind string) MoveReport {
t.Helper()
for _, r := range reports {
if r.Kind == kind {
return r
}
}
t.Fatalf("no report of %s among %+v", kind, reports)
return MoveReport{}
}
// allAlready says every kind was moved by an earlier start: the second run writes nothing, and reports one for each kind.
func allAlready(t *testing.T, reports []MoveReport) {
t.Helper()
if len(reports) != 3 {
t.Fatalf("the move reported %d kinds, want 3: %+v", len(reports), reports)
}
for _, r := range reports {
if !r.Already || r.Written != 0 {
t.Fatalf("the second move of %s %+v", r.Kind, r)
}
}
}
func rowsInTheTable(t *testing.T, inv *Inventory, table string) string {
t.Helper()
rows := inv.QueryForTest(t, `select count(*) from `+table)
return rows[0][0]
}
// **The move copies every build row into the builds bucket, and a second run writes nothing** (ADR 0408 §8, issue
// 1090): each row under its id, whole; the kind recorded in chain_moved; and from then on a build is read from the
// bucket and written to it, the table taking no more.
func TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
aBuildRow(t, inv, "build-2", "notes", "c2", "")
aBuildRow(t, inv, "build-3", "wiki", "", "compile error")
// Before the move the tables are read, and the bucket is empty.
if before, err := inv.Builds(ctx, "", 10); err != nil || len(before) != 3 {
t.Fatalf("before the move the tables hold %d builds: %v", len(before), err)
}
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
t.Fatalf("moved before the move: %v", moved)
}
reports, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatal(err)
}
if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.Written != 3 || b.Largest == 0 {
t.Fatalf("the move of the builds reported %+v", reports)
}
moved, err := inv.ChainMoved(ctx)
if err != nil || moved[KindBuilds] != 3 {
t.Fatalf("chain_moved says %v: %v", moved, err)
}
kept, found, err := inv.chain.Builds.Get(ctx, "build-2")
if err != nil || !found {
t.Fatalf("build-2 is not in the bucket: %v %v", found, err)
}
if v := kept.Value; v.Module != "notes" || v.Commit != "c2" || v.Repository != "novox/notes" || v.Path != "modules/notes" ||
v.SourceFingerprint != "fp-build-2" || len(v.Made) != 1 || len(v.Against) != 1 || len(v.Read) != 1 || len(v.Sources) != 1 ||
v.Published == nil || v.Published.Artifact != "sha256:build-2" || v.Asked.IsZero() || v.At.IsZero() || len(v.Manifest) == 0 {
t.Fatalf("build-2 was moved as %+v", v)
}
// From now on the bucket is what is read and written.
if err := inv.RecordBuild(ctx, aBuild("build-4", "notes", "")); err != nil {
t.Fatal(err)
}
if n := rowsInTheTable(t, inv, "build"); n != "3" {
t.Fatalf("the table took a build after the move: %s rows", n)
}
after, err := inv.Builds(ctx, "notes", 10)
if err != nil || len(after) != 3 {
t.Fatalf("notes has %d builds on the bus: %v", len(after), err)
}
// A second run finds the kind moved and writes nothing: every key still has the one value it was made with.
again, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatal(err)
}
allAlready(t, again)
for _, key := range []string{legacyBuildKey("build-1"), legacyBuildKey("build-2"), legacyBuildKey("build-3"),
chain.BuildKey("notes", "build-4")} {
if history, err := inv.chain.Builds.History(ctx, key); err != nil || len(history) != 1 {
t.Fatalf("%s has %d values: %v", key, len(history), err)
}
}
}
// failingToCreate is a bucket that fails to make one key while told to: the bus going away half way through a move.
type failingToCreate struct {
chain.Backend
key string
fail *atomic.Bool
}
func (f failingToCreate) Create(ctx context.Context, key string, value []byte) (uint64, error) {
if key == f.key && f.fail.Load() {
return 0, errors.New("the bus went away")
}
return f.Backend.Create(ctx, key, value)
}
// **A controller restarted half way through the move finishes it: the bucket equals the table, with no key made twice and
// no refusal** (ADR 0408 §8). Before a kind is recorded as moved nothing but the move writes its keys, and every write of
// the kind goes to its table (each method of a kind asks ChainReads first and writes the store until it says moved), so the
// table is the only truth: a key the first start made is the row it was, and the second start settles it to the row it is.
func TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
aBuildRow(t, inv, "build-2", "notes", "c2", "")
aBuildRow(t, inv, "build-3", "wiki", "c3", "")
var away atomic.Bool
away.Store(true)
builds := chain.NewMemory()
inv.UseChain(chain.New(failingToCreate{builds, "build-3", &away}, chain.NewMemory(), chain.NewMemory(),
func(context.Context) error { return nil }))
// The first start made two of the three keys and the bus went away.
_, err := inv.MoveChain(ctx, nil)
var refusal *Refusal
if err == nil || errors.As(err, &refusal) {
t.Fatalf("the first start answered %v: it should fail to be tried again, not be refused", err)
}
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
t.Fatalf("a move that did not finish recorded %v", moved)
}
if keys, _ := builds.Keys(ctx); len(keys) != 2 {
t.Fatalf("the first start made %v", keys)
}
// The controller is started again, the bus is back, and the move finishes.
away.Store(false)
reports, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatalf("the second start was refused or failed: %v", err)
}
if b := reportOf(t, reports, KindBuilds); b.Written != 1 || b.Present != 2 || b.Rows != 3 {
t.Fatalf("the move reported %+v", b)
}
if moved, _ := inv.ChainMoved(ctx); moved[KindBuilds] != 3 {
t.Fatalf("chain_moved says %v", moved)
}
// The bucket is the table: three keys, each made once and not again, each the row's own.
for _, id := range []string{"build-1", "build-2", "build-3"} {
history, err := builds.History(ctx, id)
if err != nil || len(history) != 1 {
t.Fatalf("%s has %d values: %v", id, len(history), err)
}
}
rows, err := inv.allBuildRows(ctx)
if err != nil || len(rows) != 3 {
t.Fatalf("the table holds %d builds: %v", len(rows), err)
}
for _, row := range rows {
got, found, err := inv.chain.Builds.Get(ctx, row.ID)
want, _ := json.Marshal(toChainBuild(row))
have, _ := json.Marshal(got.Value)
if err != nil || !found || string(want) != string(have) {
t.Fatalf("%s in the bucket is %s, the row says %s (%v)", row.ID, have, want, err)
}
}
}
// **A value over the bus's largest message stops the move with nothing written and nothing switched** (ADR 0408 §6):
// a Refusal, no key in the bucket, no kind recorded, and the tables still what is read.
func TestTheMoveRefusesAValueOverTheLargestMessage(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
inv.ExecForTest(t, `insert into build (id, repository, module, commit_hash, manifest)
values ('build-big', 'novox/wiki', 'wiki', 'c9', ('{"x":"' || repeat('a', 1100000) || '"}')::jsonb)`)
_, err := inv.MoveChain(ctx, nil)
var refusal *Refusal
if !errors.As(err, &refusal) || refusal.Kind != KindBuilds {
t.Fatalf("the move answered %v", err)
}
if keys, _ := inv.chain.Builds.Keys(ctx); len(keys) != 0 {
t.Fatalf("the bucket holds %v after a refused move", keys)
}
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
t.Fatalf("a refused move recorded %v", moved)
}
if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 2 {
t.Fatalf("the tables are no longer read: %d builds, %v", len(builds), err)
}
}
// losing is a bucket that says it wrote a key and kept nothing: the move's count of keys against rows must see it.
type losing struct {
chain.Backend
lose string
}
func (l losing) Create(ctx context.Context, key string, value []byte) (uint64, error) {
if key == l.lose {
return 1, nil
}
return l.Backend.Create(ctx, key, value)
}
// **The move refuses on a count that differs** (ADR 0408 §8): a row whose key is not in the bucket afterwards stops it,
// nothing is recorded, and the tables are still read.
func TestTheMoveRefusesOnACountThatDiffers(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
aBuildRow(t, inv, "build-2", "notes", "c2", "")
inv.UseChain(chain.New(losing{chain.NewMemory(), "build-2"}, chain.NewMemory(), chain.NewMemory(),
func(context.Context) error { return nil }))
_, err := inv.MoveChain(ctx, nil)
var refusal *Refusal
if !errors.As(err, &refusal) {
t.Fatalf("the move answered %v", err)
}
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
t.Fatalf("a refused move recorded %v", moved)
}
}
// **Without the lease the move writes nothing**, and fails the way the bus being away does — tried again, not refused:
// the tables are still read.
func TestTheMoveWithoutTheLeaseWritesNothing(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
mem := chain.NewMemory()
inv.UseChain(chain.New(mem, chain.NewMemory(), chain.NewMemory(),
func(context.Context) error { return errors.New("another controller holds the lease") }))
_, err := inv.MoveChain(ctx, nil)
var refusal *Refusal
if err == nil || errors.As(err, &refusal) || !errors.Is(err, chain.ErrNotHeld) {
t.Fatalf("the move answered %v", err)
}
if mem.Writes() != 0 {
t.Fatalf("the bucket took %d writes without the lease", mem.Writes())
}
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
t.Fatalf("recorded %v", moved)
}
}
// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its
// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none.
func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
first := aBuild("build-1", "notes", "")
first.Commit = "c1"
first.Against = []string{"artifact-store://base@sha256:aa"}
first.ReuseKey = "rk-1"
second := aBuild("build-2", "notes", "")
second.Commit = "c2"
failed := aBuild("build-3", "notes", "compile error")
other := aBuild("build-4", "wiki", "")
for _, b := range []Build{first, second, failed, other, first} {
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
if n := rowsInTheTable(t, inv, "build"); n != "0" {
t.Fatalf("the table took %s builds", n)
}
kept, found, err := inv.chain.Builds.Get(ctx, chain.BuildKey("notes", "build-1"))
if err != nil || !found || kept.Value.ReuseKey != "rk-1" {
t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err)
}
if history, _ := inv.chain.Builds.History(ctx, chain.BuildKey("notes", "build-1")); len(history) != 1 {
t.Fatalf("a build recorded twice has %d values", len(history))
}
list, err := inv.Builds(ctx, "notes", 10)
if err != nil || len(list) != 3 {
t.Fatalf("notes has %d builds: %v", len(list), err)
}
one, found, err := inv.BuildByID(ctx, "build-2")
if err != nil || !found || one.Commit != "c2" || one.Module != "notes" {
t.Fatalf("build-2 reads as %+v %v %v", one, found, err)
}
if _, found, _ := inv.BuildByID(ctx, "build-nope"); found {
t.Fatal("a build nobody recorded was found")
}
if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" {
t.Fatalf("the build of notes at c1 is %q: %v", id, err)
}
held, err := inv.Held(ctx)
if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" {
t.Fatalf("held %v: %v", held, err)
}
against, err := inv.BuiltAgainst(ctx)
if err != nil || len(against["notes"]) != 0 {
// The newest successful build of notes, build-2, stood on nothing.
t.Fatalf("built against %v: %v", against, err)
}
announced, err := inv.Announceable(ctx)
if err != nil || len(announced) != 3 {
t.Fatalf("announceable %d: %v", len(announced), err)
}
prints, err := inv.Fingerprints(ctx)
if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" {
t.Fatalf("fingerprints %v: %v", prints, err)
}
}
// A build request row as the store held it before the move.
func aRequestRow(t *testing.T, inv *Inventory, id, notAsked string) {
t.Helper()
inv.ExecForTest(t,
`insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at)
values ($1::text, 'novox/notes', '', 'modules/notes', 'main', 'c1', 'merge', nullif($2::text, ''), now() - interval '1 hour')`,
id, notAsked)
}
// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue
// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none
// after the outcome is heard. The table of requests takes none.
func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true}
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
t.Fatalf("a request kept twice: %v", err)
}
if n := rowsInTheTable(t, inv, "build_request"); n != "0" {
t.Fatalf("the table took %s requests", n)
}
if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 {
t.Fatalf("a request nobody answered is a build: %+v %v", builds, err)
}
if _, found, _ := inv.BuildByID(ctx, "build-1"); found {
t.Fatal("a request nobody answered has an outcome")
}
got, found, err := inv.BuildRequestByID(ctx, "build-1")
if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() {
t.Fatalf("the request reads back as %+v %v %v", got, found, err)
}
if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil {
t.Fatal(err)
}
asked, err := inv.RequestsNamed(ctx, "notes")
if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" {
t.Fatalf("the request is %+v: %v", asked, err)
}
if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) {
t.Fatalf("requested names %v: %v", names, err)
}
// The outcome comes: the same key, now a build, with the request beside it.
built := aBuild("build-1", "notes", "")
if err := inv.RecordBuild(ctx, built); err != nil {
t.Fatal(err)
}
if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" {
t.Fatalf("the outcome reads as %+v %v %v", one, found, err)
}
answered, err := inv.RequestsNamed(ctx, "notes")
if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" {
t.Fatalf("the request answered is %+v: %v", answered, err)
}
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" {
t.Fatalf("the request was lost with the outcome: %+v", again)
}
// Heard: the marks of an unanswered request change nothing.
if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil {
t.Fatal(err)
}
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" {
t.Fatalf("an answered request was marked not asked: %+v", again)
}
if history, _ := inv.chain.Builds.History(ctx, chain.RequestKey("build-1")); len(history) != 3 {
// asked, marked unknown, answered.
t.Fatalf("the record has %d values", len(history))
}
// An outcome heard before its request is kept, and the request lands beside it.
if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil {
t.Fatal(err)
}
if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" {
t.Fatalf("the request kept after its outcome reads %+v", late)
}
// A request older than the time they are kept is no longer listed.
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old",
At: time.Now().Add(-2 * KeptFor)}); err != nil {
t.Fatal(err)
}
if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") {
t.Fatalf("a request past its time is listed: %v", names)
}
}
// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a
// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it
// is asked and so asked at most once.
func TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
aRequestRow(t, inv, "build-1", "")
aRequestRow(t, inv, "build-2", "")
aRequestRow(t, inv, "not-asked-c1-modules-notes", "no seat holder")
if requests, err := inv.RequestsNamed(ctx, "notes"); err != nil || len(requests) != 3 {
t.Fatalf("before the move the table holds %d requests: %v", len(requests), err)
}
reports, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatal(err)
}
if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.TableRows != 4 {
t.Fatalf("the move reported %+v", b)
}
one, _, _ := inv.chain.Builds.Get(ctx, "build-1")
if one.Value.State != "" || one.Value.Request == nil || one.Value.Request.For != "merge" || one.Value.Module != "notes" {
t.Fatalf("an answered request moved as %+v", one.Value)
}
two, _, _ := inv.chain.Builds.Get(ctx, "build-2")
if two.Value.State != chain.BuildAsked || two.Value.Request.Reask != "" {
t.Fatalf("an unanswered request moved as %+v", two.Value)
}
never, _, _ := inv.chain.Builds.Get(ctx, "not-asked-c1-modules-notes")
if never.Value.State != chain.BuildAsked || never.Value.Request.NotAsked != "no seat holder" || never.Value.Request.Reask != chain.ReaskPending {
t.Fatalf("a request never handed over moved as %+v", never.Value)
}
requests, err := inv.RequestsNamed(ctx, "notes")
if err != nil || len(requests) != 3 {
t.Fatalf("after the move the bucket holds %d requests: %v", len(requests), err)
}
toAsk, err := inv.RequestsToAskAgain(ctx)
if err != nil || len(toAsk) != 1 || toAsk[0].ID != "not-asked-c1-modules-notes" {
t.Fatalf("to ask again: %+v %v", toAsk, err)
}
first, err := inv.ClaimToAskAgain(ctx, toAsk[0].ID)
second, err2 := inv.ClaimToAskAgain(ctx, toAsk[0].ID)
if err != nil || err2 != nil || !first || second {
t.Fatalf("claimed %v then %v: %v %v", first, second, err, err2)
}
if left, _ := inv.RequestsToAskAgain(ctx); len(left) != 0 {
t.Fatalf("a claimed request is still to be asked: %+v", left)
}
if err := inv.AskedAgainAs(ctx, toAsk[0].ID, "build-9"); err != nil {
t.Fatal(err)
}
if done, _, _ := inv.BuildRequestByID(ctx, toAsk[0].ID); done.NotAsked != "no seat holder" {
t.Fatalf("the request lost its words: %+v", done)
}
}
// **The move carries every plan with the merges it answers, and the plans read the same from the bucket as from the
// table** (ADR 0408 §8, ADR 0276, issue 1092): a batch with a merge, a walk that was built and rolled, a merge waiting
// for a walk of its own, and a merge whose plan the store never kept — each in the record of the plan that held it last.
func TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
batch := aBatch("plan-1", time.Now().Add(-3*time.Hour).UTC())
walk := aBatch("plan-2", time.Now().Add(-2*time.Hour).UTC())
walk.State, walk.Tier, walk.Tiers = PlanRolling, 1, [][]string{{"a"}, {"b"}}
walk.Modules = map[string]*PlanModule{"a": {State: "built"}, "b": {}}
walk.Delivery = &PlanDelivery{Awaits: "mesh-delivery", Merges: []PlanMerge{{Repository: "novox/mesh-catalog", Commit: "m-waiting"}}}
for _, p := range []*Plan{batch, walk} {
if err := inv.SavePlan(ctx, p); err != nil {
t.Fatal(err)
}
}
walk.Note = "second save"
if err := inv.SavePlan(ctx, walk); err != nil {
t.Fatal(err)
}
for _, m := range []BatchedMerge{aMerge("m-batch", "plan-1"), aMerge("m-walk", "plan-2"), aMerge("m-dangling", "plan-never-kept")} {
if added, err := inv.AddMerge(ctx, m); err != nil || !added {
t.Fatalf("added %v: %v", added, err)
}
}
waiting := aMerge("m-waiting", "plan-2")
if _, err := inv.AddMerge(ctx, waiting); err != nil {
t.Fatal(err)
}
if err := inv.AnswerMerge(ctx, waiting.Repository, waiting.Commit, "", true); err != nil {
t.Fatal(err)
}
wasPlans, err := inv.RecentPlans(ctx, 10)
if err != nil || len(wasPlans) != 2 {
t.Fatalf("the table holds %d plans: %v", len(wasPlans), err)
}
wasMerge, _, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk")
reports, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatal(err)
}
var moved MoveReport
for _, r := range reports {
if r.Kind == KindRollouts {
moved = r
}
}
// Two plans, and the record of the merge whose plan the store never kept.
if moved.Rows != 3 || moved.TableRows != 6 || moved.Written != 3 {
t.Fatalf("the move of the rollouts reported %+v", moved)
}
nowPlans, err := inv.RecentPlans(ctx, 10)
if err != nil || len(nowPlans) != 2 {
t.Fatalf("the bucket holds %d plans: %v", len(nowPlans), err)
}
for k := range wasPlans {
was, now := wasPlans[k], nowPlans[k]
if was.ID != now.ID || was.State != now.State || was.Tier != now.Tier || was.Revision != now.Revision || was.Note != now.Note ||
!was.Created.Equal(now.Created) || !reflect.DeepEqual(was.Tiers, now.Tiers) || len(was.Modules) != len(now.Modules) {
t.Errorf("plan %s was %+v and is %+v", was.ID, was, now)
}
}
if open, _ := inv.OpenPlans(ctx); len(open) != 1 || open[0].ID != "plan-2" {
t.Fatalf("open plans %+v", open)
}
if batches, _ := inv.Batches(ctx); len(batches) != 1 || batches[0].ID != "plan-1" {
t.Fatalf("batches %+v", batches)
}
if got, _ := inv.MergesOf(ctx, "plan-1"); len(got) != 1 || got[0].Commit != "m-batch" {
t.Fatalf("the batch answers %+v", got)
}
nowMerge, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk")
if !known || nowMerge.Plan != "plan-2" || nowMerge.Branch != wasMerge.Branch || !nowMerge.Merged.Equal(wasMerge.Merged) ||
!nowMerge.Heard.Equal(wasMerge.Heard) || !sameEvent(nowMerge.Event, wasMerge.Event) {
t.Fatalf("a merge was %+v and is %+v", wasMerge, nowMerge)
}
alone, _ := inv.AloneMerges(ctx)
if len(alone) != 1 || alone[0].Commit != "m-waiting" || alone[0].Plan != "" {
t.Fatalf("waiting: %+v", alone)
}
if dangling, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-dangling"); !known || dangling.Plan != "" || dangling.Alone {
t.Fatalf("a merge whose plan was never kept reads as %+v %v", dangling, known)
}
if n := rowsInTheTable(t, inv, "release_plan"); n != "2" {
t.Fatalf("the table is %s plans", n)
}
// From now on a plan is saved to the bucket and the table takes none.
later := aBatch("plan-3", time.Now().UTC())
if err := inv.SavePlan(ctx, later); err != nil {
t.Fatal(err)
}
if n := rowsInTheTable(t, inv, "release_plan"); n != "2" {
t.Fatalf("the table took a plan after the move: %s", n)
}
again, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatal(err)
}
allAlready(t, again)
}
// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a
// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module
// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the
// assignment goes with it.
func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
inv, node := aNodeWithModules(t, "postgres", "web")
ctx := t.Context()
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
start := generationNow(t, inv)
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added {
t.Fatalf("assigned %v: %v", added, err)
}
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added {
t.Fatalf("assigned again %v: %v", added, err)
}
if n := rowsInTheTable(t, inv, "assignment"); n != "0" {
t.Fatalf("the table took %s assignments", n)
}
kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() {
t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err)
}
if generationNow(t, inv) <= start {
t.Fatal("an assignment left the generation where it was")
}
if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) {
t.Fatalf("a module the mesh does not know: %v", err)
}
if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) {
t.Fatalf("assigned on %s: %v %v", node, on, err)
}
if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) {
t.Fatalf("running: %v %v", running, err)
}
entries, err := inv.Catalogued(ctx)
if err != nil {
t.Fatal(err)
}
for _, e := range entries {
switch e.Manifest.Module {
case "postgres":
if !reflect.DeepEqual(e.On, []string{node}) {
t.Errorf("postgres is on %v", e.On)
}
case "web":
if len(e.On) != 0 {
t.Errorf("web is on %v", e.On)
}
}
}
if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) {
t.Fatalf("a module still assigned was forgotten: %v", err)
}
// A seat is held by an assignment, or by nothing.
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil {
t.Fatal("a seat was handed to a module that is not assigned")
}
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil {
t.Fatalf("a seat handed to an assignment: %v", err)
}
before := generationNow(t, inv)
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
t.Fatal(err)
}
if generationNow(t, inv) <= before {
t.Fatal("an unassignment left the generation where it was")
}
// No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history.
gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() {
t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err)
}
if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() ||
history[1].Value.Stands() {
t.Fatalf("the earlier values of the key: %+v", history)
}
if on, _ := inv.Assigned(ctx, node); len(on) != 0 {
t.Fatalf("an ended assignment is still assigned: %v", on)
}
if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 {
t.Fatalf("an ended assignment is running: %v", running)
}
if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 {
t.Fatalf("the seat the assignment held is held by %+v: %v", held, err)
}
if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" {
t.Fatalf("an unassignment of nothing: %v", err)
}
if err := inv.ForgetModule(ctx, "postgres"); err != nil {
t.Fatalf("a module nothing is assigned could not be forgotten: %v", err)
}
// Made again after it ended: a fresh value over the ended one, which stays in history.
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
t.Fatal(added, err)
}
if added, err := inv.Assign(ctx, node, "web"); err != nil || added {
t.Fatal(added, err)
}
if err := inv.Unassign(ctx, node, "web"); err != nil {
t.Fatal(err)
}
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
t.Fatalf("assigned again after it ended: %v %v", added, err)
}
again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web")
if !again.Value.Stands() || !again.Value.Ended.IsZero() {
t.Fatalf("the assignment made again is %+v", again.Value)
}
if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() {
t.Fatalf("the values of the key: %+v", history)
}
}
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
// ends an assignment.
func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) {
inv, node := aNodeWithModules(t, "dunst", "web")
ctx := t.Context()
for _, m := range []string{"dunst", "web"} {
if _, err := inv.Assign(ctx, node, m); err != nil {
t.Fatal(err)
}
}
// A build and a commit put in the value by another write.
cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
cur.Value.Build, cur.Value.Commit = "build-1", "c1"
if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil {
t.Fatal(err)
}
record, _ := inv.NodeByName(ctx, node)
digest := "sha256:" + strings.Repeat("d", 64)
plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest,
Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc",
Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}}
raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`)
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil {
t.Fatal(err)
}
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
written, removed, err := inv.KeepAssignmentPlans(ctx)
if err != nil || written != 2 || removed != 0 {
t.Fatalf("kept: %d written, %d ended: %v", written, removed, err)
}
dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() {
t.Fatalf("dunst's value is %+v", dunst.Value)
}
if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil {
t.Fatalf("web has a plan: %+v", web.Value)
}
notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify")
if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil {
t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found)
}
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
t.Fatalf("a pulled-in module is assigned: %v", on)
}
if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 {
t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err)
}
// Sent again without the pulled-in module: its value ends, the assignments stay.
only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`)
if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil {
t.Fatal(err)
}
if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 {
t.Fatalf("ended %d: %v", removed, err)
}
if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() {
t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value)
}
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
t.Fatalf("assigned: %v", on)
}
}
// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value**
// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build
// already is left as it is.
func TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
for _, name := range []string{"anchor", "laptop"} {
if _, err := inv.AddNode(ctx, name); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"notes", "wiki"} {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
t.Fatal(err)
}
}
for _, a := range [][2]string{{"anchor", "notes"}, {"anchor", "wiki"}, {"laptop", "notes"}} {
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
t.Fatal(err)
}
}
aBuildRow(t, inv, "build-notes", "notes", "c-notes", "")
anchor, _ := inv.NodeByName(ctx, "anchor")
if err := inv.RecordSent(ctx, anchor.ID, "sent-1", map[string]string{"notes": "c-notes"}); err != nil {
t.Fatal(err)
}
// The mirror of the store put a value under one key, with a plan.
if _, err := inv.chain.Assignments.Create(ctx, "anchor.wiki", chain.Assignment{Node: "anchor", Module: "wiki",
Assigned: time.Now().UTC(), Provisioning: json.RawMessage(`{"plan":1}`)}); err != nil {
t.Fatal(err)
}
reports, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatal(err)
}
var moved MoveReport
for _, r := range reports {
if r.Kind == KindAssignments {
moved = r
}
}
if moved.Rows != 3 || moved.Written+moved.Present != 3 {
t.Fatalf("the move of the assignments reported %+v", moved)
}
notes, _, _ := inv.chain.Assignments.Get(ctx, "anchor.notes")
if notes.Value.Commit != "c-notes" || notes.Value.Build != "build-notes" || notes.Value.Since.IsZero() || notes.Value.Assigned.IsZero() {
t.Fatalf("anchor.notes moved as %+v", notes.Value)
}
if notes.Value.Artifact != "sha256:build-notes" {
t.Fatalf("what the build published: %+v", notes.Value)
}
laptop, _, _ := inv.chain.Assignments.Get(ctx, "laptop.notes")
if laptop.Value.Commit != "" || laptop.Value.Build != "" {
t.Fatalf("a machine never sent has a build: %+v", laptop.Value)
}
wiki, _, _ := inv.chain.Assignments.Get(ctx, "anchor.wiki")
if string(wiki.Value.Provisioning) != `{"plan":1}` {
t.Fatalf("the plan the mirror put in the value was lost: %+v", wiki.Value)
}
// From now on the bucket is what is written.
if _, err := inv.Assign(ctx, "laptop", "wiki"); err != nil {
t.Fatal(err)
}
if n := rowsInTheTable(t, inv, "assignment"); n != "3" {
t.Fatalf("the table took an assignment after the move: %s", n)
}
if on, _ := inv.Assigned(ctx, "laptop"); !reflect.DeepEqual(on, []string{"notes", "wiki"}) {
t.Fatalf("assigned on laptop: %v", on)
}
}
// hooked runs a function the first time a key is made in a bucket: the old controller writing the store between the move's
// first pass and its switch.
type hooked struct {
chain.Backend
once *sync.Once
then func()
}
func (h hooked) Create(ctx context.Context, key string, value []byte) (uint64, error) {
rev, err := h.Backend.Create(ctx, key, value)
h.once.Do(h.then)
return rev, err
}
// **A row the old controller changes after the move's first pass is in the bucket when the kind is recorded as moved**
// (ADR 0408 §8, issue 981 note 7 Q2): before the switch the store is the only truth, so under the lock the move reads the
// rows again and writes over every key that differs — a plan's state and a build's failure — and ends an assignment the
// store dropped. The count alone would have kept the older value and refused the next save of the plan.
func TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
for _, n := range []string{"anchor"} {
if _, err := inv.AddNode(ctx, n); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"notes", "wiki"} {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"notes", "wiki"} {
if _, err := inv.Assign(ctx, "anchor", m); err != nil {
t.Fatal(err)
}
}
p := aBatch("plan-1", time.Now().Add(-time.Hour).UTC())
p.State, p.Tiers = PlanBuilding, [][]string{{"notes"}}
if err := inv.SavePlan(ctx, p); err != nil {
t.Fatal(err)
}
// Between a kind's first pass and its switch the old controller saves the plan again, fails the build and unassigns wiki:
// each is done when the first key of its kind is made, so the first pass has read the rows as they were.
after := func(sql string) hooked {
return hooked{chain.NewMemory(), &sync.Once{}, func() { inv.ExecForTest(t, sql) }}
}
inv.UseChain(chain.New(
after(`update build set failed = 'it broke later' where id = 'build-1'`),
after(`update release_plan set state = 'rolling', revision = revision + 1, note = 'the old controller was here' where id = 'plan-1'`),
after(`delete from assignment where module = 'wiki'`),
func(context.Context) error { return nil }))
if _, err := inv.MoveChain(ctx, nil); err != nil {
t.Fatal(err)
}
now, err := inv.PlanByID(ctx, "plan-1")
if err != nil {
t.Fatal(err)
}
if now.State != PlanRolling || now.Note != "the old controller was here" || now.Revision != 2 {
t.Fatalf("the bucket holds a plan from before the change: %s %q revision %d", now.State, now.Note, now.Revision)
}
if b, found, _ := inv.BuildByID(ctx, "build-1"); !found || b.Failed != "it broke later" {
t.Fatalf("the bucket holds the build as it was: %+v", b)
}
if on, _ := inv.Assigned(ctx, "anchor"); !slices.Equal(on, []string{"notes"}) {
t.Fatalf("anchor is assigned %v, though the store dropped wiki", on)
}
// The plan can be saved at the revision the store had.
now.Note = "saved after the move"
if err := inv.SavePlan(ctx, &now); err != nil {
t.Fatalf("the next save of the plan is refused: %v", err)
}
}
// sameEvent says two encodings of the forge's announcement are one: the store keeps it as jsonb, which spells it with its own
// white space, and whoever reads it (the cut, mesh-delivery) decodes it, so the white space is no part of what it says.
func sameEvent(a, b json.RawMessage) bool {
var x, y bytes.Buffer
return json.Compact(&x, a) == nil && json.Compact(&y, b) == nil && x.String() == y.String()
}
internal/inventory/chain_moved_test.go
-45
View File
@@ -1,45 +0,0 @@
package inventory
import (
"strings"
"testing"
)
// **The move of the chain has a switch, made by the last migration and written by no migration** (ADR 0408 §8, issue
// 1089): migration 0104 makes the one-row table chain_moved, and does not drop a table of the chain, which a later
// delivery does once the move is read back.
func TestMigration0104MakesTheSwitchOfTheChainsMoveAndNothingElse(t *testing.T) {
migrations, err := Migrations()
if err != nil {
t.Fatal(err)
}
var found bool
for _, m := range migrations {
if m.Number != 104 {
continue
}
found = true
sql := strings.ToLower(m.SQL)
if !strings.Contains(sql, "create table chain_moved") || !strings.Contains(sql, "check (only_one)") {
t.Errorf("migration 0104 does not make the one-row chain_moved:\n%s", m.SQL)
}
for _, line := range strings.Split(sql, "\n") {
if strings.HasPrefix(strings.TrimSpace(line), "--") {
continue
}
for _, verb := range []string{"drop table", "insert ", "delete ", "update "} {
if strings.Contains(line, verb) {
t.Errorf("migration 0104 does more than make the switch: %q", line)
}
}
// The one other change: a seat's holder is no longer a foreign key to the assignment (issue 1093); the tables of
// the chain are dropped by a later migration (issue 1096).
if strings.Contains(line, "alter ") && !strings.Contains(line, "alter table seat_holding drop constraint") {
t.Errorf("migration 0104 alters more than the seat holdings' foreign key: %q", line)
}
}
}
if !found {
t.Fatal("no migration 0104")
}
}
internal/inventory/chain_pending_test.go
+272
View File
@@ -0,0 +1,272 @@
//go:build integration
package inventory
import (
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/novox/mesh-controller/internal/chain"
)
// **A pending assignment is the value of the key of the assignment it waits for** (ADR 0261, ADR 0408, issue 1094): state
// pending, the wait in its `pending` object under an id that is a string, no table, and one open wait per machine and module.
func TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket(t *testing.T) {
inv, node := aNodeWithModules(t, "web")
ctx := t.Context()
p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-1", Repository: "novox/mesh-catalog",
Path: "/modules/sensors/"})
if err != nil {
t.Fatal(err)
}
if want := fmt.Sprintf("p-%d", p.Since.UnixNano()); p.ID != want || p.State != PendingWaiting || p.Path != "modules/sensors" {
t.Fatalf("recorded %+v, want id %s", p, want)
}
kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.sensors")
if err != nil || !found || kept.Value.State != chain.StatePending || kept.Value.Stands() || !kept.Value.Assigned.IsZero() ||
kept.Value.Pending == nil || kept.Value.Pending.ID != p.ID || kept.Value.Pending.Build != "build-1" {
t.Fatalf("the key holds %+v (found %v, %v)", kept.Value, found, err)
}
rows, err := inv.Pending(ctx, time.Time{})
if err != nil || len(rows) != 1 || rows[0].ID != p.ID || rows[0].Node != "anchor" || rows[0].Build != "build-1" {
t.Fatalf("waiting: %+v %v", rows, err)
}
if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"}); !errors.Is(err, ErrAlreadyPending) {
t.Fatalf("a second open wait: %v", err)
}
if _, err := inv.Assign(ctx, node, "web"); err != nil {
t.Fatal(err)
}
if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "build-3"}); !errors.Is(err, ErrOnTheMachine) {
t.Fatalf("a wait for a module that is on the machine: %v", err)
}
// Written over once it ended: a settled wait is replaced by a new one under a new id.
if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil || !ok {
t.Fatalf("settled %v %v", ok, err)
}
next, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"})
if err != nil || next.ID == p.ID {
t.Fatalf("a wait after one ended: %+v %v", next, err)
}
// The earlier wait ended unmade and nobody answered its condition, so the new wait carries it (issue 1108): both are on record.
if known, _ := inv.PendingKnown(ctx, []string{p.ID, next.ID}); !known[p.ID] || !known[next.ID] {
t.Fatalf("known %v: the key holds the newer one and carries the unanswered earlier one", known)
}
// The first wait, its settling and the new wait: three values, the earlier wait still in the history.
if history, _ := inv.chain.Assignments.History(ctx, "anchor.sensors"); len(history) != 3 {
t.Fatalf("the earlier wait is in the key's history: %d values", len(history))
}
}
// **A pending value is skipped by what reads assignments** (issue 1094): the machine is assigned nothing by it, the module
// runs nowhere, and a module only waiting for a build is not forgotten as assigned.
func TestARecordedPendingIsSkippedByAssignedAndTheJoins(t *testing.T) {
inv, node := aNodeWithModules(t, "web")
ctx := t.Context()
if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"}); err != nil {
t.Fatal(err)
}
if on, _ := inv.Assigned(ctx, node); len(on) != 0 {
t.Fatalf("assigned %v", on)
}
if running, _ := inv.Running(ctx, "web"); len(running) != 0 {
t.Fatalf("running %v", running)
}
if runs, _ := inv.AssignmentRuns(ctx, node); len(runs) != 0 {
t.Fatalf("runs %v", runs)
}
entries, _ := inv.Catalogued(ctx)
for _, e := range entries {
if e.Manifest.Module == "web" && len(e.On) != 0 {
t.Fatalf("web is on %v", e.On)
}
}
if err := inv.ForgetModule(ctx, "web"); errors.Is(err, ErrStillAssigned) {
t.Fatalf("a module only waiting is assigned: %v", err)
}
}
// **Two claims of one wait make one, and the other sees false** (issue 1094): by compare-and-set on the key's revision.
func TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"})
if err != nil {
t.Fatal(err)
}
var won atomic.Int32
var wg sync.WaitGroup
for k := 0; k < 8; k++ {
wg.Add(1)
go func() {
defer wg.Done()
if ok, err := inv.ClaimPending(ctx, p.ID); err == nil && ok {
won.Add(1)
}
}()
}
wg.Wait()
if won.Load() != 1 {
t.Fatalf("%d claims won", won.Load())
}
if again, err := inv.ClaimPending(ctx, p.ID); err != nil || again {
t.Fatalf("a claim of a claimed wait: %v %v", again, err)
}
if rows, _ := inv.Stuck(ctx, time.Now().Add(time.Minute)); len(rows) != 1 || rows[0].State != PendingApplying || rows[0].Settled == nil {
t.Fatalf("stuck %+v", rows)
}
if err := inv.ReleaseClaim(ctx, p.ID); err != nil {
t.Fatal(err)
}
if rows, _ := inv.Pending(ctx, time.Time{}); len(rows) != 1 || rows[0].Settled != nil {
t.Fatalf("released %+v", rows)
}
// Only a wait that waits is withdrawn or expired, only a claim is applied.
if ok, _ := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingApplied, "x"); ok {
t.Fatal("a wait was applied")
}
if ok, _ := inv.SettlePending(ctx, p.ID, PendingApplying, PendingRefused, "x"); ok {
t.Fatal("a wait that is not claimed was refused as a claim")
}
if ok, err := inv.RepointPending(ctx, p.ID, "b-2", "novox/other", "x/"); err != nil || !ok {
t.Fatalf("repoint %v %v", ok, err)
}
if rows, _ := inv.Pending(ctx, time.Time{}); rows[0].Build != "b-2" || rows[0].Path != "x" {
t.Fatalf("repointed %+v", rows)
}
}
// **Assigning over a wait carries the wait into the value that stands** (issue 1094): the claim stays applying for the one
// making it to settle applied, and one still waiting is made by hand, and says so.
func TestAssignOverAPendingKeepsItsRecordAsApplied(t *testing.T) {
inv, node := aNodeWithModules(t, "web", "cache")
ctx := t.Context()
claimed, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"})
if ok, err := inv.ClaimPending(ctx, claimed.ID); err != nil || !ok {
t.Fatal(ok, err)
}
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
t.Fatalf("assigned %v %v", added, err)
}
standing, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web")
if !standing.Value.Stands() || standing.Value.Assigned.IsZero() || standing.Value.Pending == nil ||
standing.Value.Pending.State != PendingApplying {
t.Fatalf("the value that stands: %+v", standing.Value)
}
if ok, err := inv.SettlePending(ctx, claimed.ID, PendingApplying, PendingApplied, "made"); err != nil || !ok {
t.Fatalf("settled %v %v", ok, err)
}
if on, _ := inv.Assigned(ctx, node); len(on) != 1 || on[0] != "web" {
t.Fatalf("assigned %v", on)
}
rows, _ := inv.Pending(ctx, time.Now().Add(-time.Hour))
if len(rows) != 1 || rows[0].State != PendingApplied || rows[0].Note != "made" {
t.Fatalf("what became of the wait: %+v", rows)
}
waiting, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "cache", Build: "b-2"})
if _, err := inv.Assign(ctx, node, "cache"); err != nil {
t.Fatal(err)
}
got, _ := inv.PendingFor(ctx, node, "cache")
if len(got) != 1 || got[0].ID != waiting.ID || got[0].State != PendingApplied || !strings.Contains(got[0].Note, "by hand") {
t.Fatalf("a wait assigned by hand: %+v", got)
}
}
// **Marks are stamped once, and a wait of a machine the store dropped is no wait** (issue 1094).
func TestAPendingIsMarkedOnceAndGoesWithItsMachine(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
p, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"})
if _, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil {
t.Fatal(err)
}
if rows, _ := inv.ToRaise(ctx); len(rows) != 1 {
t.Fatalf("to raise %+v", rows)
}
if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil {
t.Fatal(err)
}
first, _ := inv.Pending(ctx, time.Now().Add(-time.Hour))
time.Sleep(2 * time.Millisecond)
if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil {
t.Fatal(err)
}
second, _ := inv.Pending(ctx, time.Now().Add(-time.Hour))
if !first[0].Raised.Equal(*second[0].Raised) {
t.Fatalf("the first stamp did not stand: %v then %v", first[0].Raised, second[0].Raised)
}
if err := inv.MarkPending(ctx, p.ID, "frobnicated"); err == nil {
t.Fatal("a mark nobody knows")
}
if rows, _ := inv.ToRaise(ctx); len(rows) != 0 {
t.Fatalf("to raise %+v", rows)
}
if rows, _ := inv.ToClear(ctx); len(rows) != 1 {
t.Fatalf("to clear %+v", rows)
}
if _, err := inv.RemoveNodeForTest(ctx, node); err != nil {
t.Fatal(err)
}
if rows, _ := inv.Pending(ctx, time.Unix(0, 0)); len(rows) != 0 {
t.Fatalf("a wait outlived its machine: %+v", rows)
}
}
// **A wait replaced by a newer one never leaves the operator without the condition it raised, or was to raise** (issue
// 1108): the key carries the replaced wait while its condition is unanswered, still known, still to clear, and a newer
// wait takes only the unanswered ones forward.
func TestAReplacedWaitNeverLeavesTheOperatorWithoutItsCondition(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record := func(build string) PendingAssignment {
p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: build})
if err != nil {
t.Fatal(err)
}
return p
}
end := func(p PendingAssignment) {
if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom "+p.Build); err != nil || !ok {
t.Fatal(ok, err)
}
}
first := record("b-1")
end(first)
if err := inv.MarkPending(ctx, first.ID, "raised"); err != nil {
t.Fatal(err)
}
second := record("b-2")
if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID}); !known[first.ID] || !known[second.ID] {
t.Fatalf("known %v: the replaced wait's condition is unanswered", known)
}
if clear, _ := inv.ToClear(ctx); len(clear) != 1 || clear[0].ID != first.ID {
t.Fatalf("to clear %+v: the first's condition is still to clear", clear)
}
rows, _ := inv.PendingFor(ctx, node, "sensors")
if len(rows) != 2 || rows[0].ID != second.ID || rows[1].ID != first.ID {
t.Fatalf("the key says %+v, newest first", rows)
}
// The second ends unmade before the first was answered: both are to be said, the first's condition not cleared by it.
end(second)
if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID {
t.Fatalf("to raise %+v", raise)
}
// A third replaces the second, which was not raised yet: it is carried, with the first (still unanswered).
if err := inv.MarkPending(ctx, first.ID, "acknowledged"); err != nil {
t.Fatal(err)
}
third := record("b-3")
if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID {
t.Fatalf("the second was never raised and is to be: %+v", raise)
}
if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID, third.ID}); known[first.ID] || !known[second.ID] || !known[third.ID] {
t.Fatalf("known %v: the first was answered (taken back) and is let go, the second is not", known)
}
}
internal/inventory/chain_requests.go
+6 -6
View File
@@ -163,8 +163,8 @@ func requestRecords(ctx context.Context, c *chain.Chain) ([]chain.Kept[chain.Bui
// RequestsToAskAgain are the requests the move found never handed over (issue 981 note 7 Q5), not yet asked again: each is
// asked once, and never twice.
func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, error) {
c, on, err := i.ChainReads(ctx, KindBuilds)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
all, err := requestRecords(ctx, c)
@@ -185,8 +185,8 @@ func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, err
// ClaimToAskAgain takes a request to ask again, by compare-and-set: true for the one process that took it. **Claimed
// before it is asked and settled after**, so a crash between loses the ask rather than making it twice.
func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error) {
c, on, err := i.ChainReads(ctx, KindBuilds)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return false, err
}
cur, found, err := c.Builds.Get(ctx, chain.RequestKey(id))
@@ -205,8 +205,8 @@ func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error
// AskedAgainAs records the build id a claimed request was asked as.
func (i *Inventory) AskedAgainAs(ctx context.Context, id, asked string) error {
c, on, err := i.ChainReads(ctx, KindBuilds)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return err
}
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.Reask = asked })
internal/inventory/chain_rollouts.go
+27 -27
View File
@@ -139,17 +139,17 @@ func (v *rolloutView) list() []viewEntry {
// rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole.
func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) {
i.moved.mu.Lock()
v := i.moved.view
i.state.mu.Lock()
v := i.state.view
if v == nil || v.chain != c || v.dead.Load() {
if v != nil {
v.stop()
}
v = newRolloutView(c)
i.moved.view = v
i.state.view = v
v.start()
}
i.moved.mu.Unlock()
i.state.mu.Unlock()
select {
case <-v.ready:
case <-ctx.Done():
@@ -165,11 +165,11 @@ func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutVi
// stopViews ends the view's watch: the inventory is closing.
func (i *Inventory) stopViews() {
i.moved.mu.Lock()
defer i.moved.mu.Unlock()
if i.moved.view != nil {
i.moved.view.stop()
i.moved.view = nil
i.state.mu.Lock()
defer i.state.mu.Unlock()
if i.state.view != nil {
i.state.view.stop()
i.state.view = nil
}
}
@@ -492,12 +492,12 @@ func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m Batched
return false, err
}
if !found {
i.moved.mu.Lock()
if i.moved.pending == nil {
i.moved.pending = map[string]pendingMerge{}
i.state.mu.Lock()
if i.state.pending == nil {
i.state.pending = map[string]pendingMerge{}
}
i.moved.pending[key] = pendingMerge{plan: m.Plan, m: held}
i.moved.mu.Unlock()
i.state.pending[key] = pendingMerge{plan: m.Plan, m: held}
i.state.mu.Unlock()
}
return true, nil
}
@@ -712,12 +712,12 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []
held, has := copies[key]
if !has {
// A merge carried for a plan not saved yet goes where it is moved.
i.moved.mu.Lock()
if pm, pending := i.moved.pending[key]; pending && m.Plan != "" {
i.state.mu.Lock()
if pm, pending := i.state.pending[key]; pending && m.Plan != "" {
pm.plan, pm.m.Alone = m.Plan, m.Alone
i.moved.pending[key] = pm
i.state.pending[key] = pm
}
i.moved.mu.Unlock()
i.state.mu.Unlock()
continue
}
w := winner(held)
@@ -803,14 +803,14 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []
if e := edits[p.ID]; e != nil {
held = e.apply(held)
}
i.moved.mu.Lock()
for key, pm := range i.moved.pending {
i.state.mu.Lock()
for key, pm := range i.state.pending {
if pm.plan == p.ID {
held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held)
_ = key
}
}
i.moved.mu.Unlock()
i.state.mu.Unlock()
record.Held = held
entered, duration := tierLeftOf(old, *p, time.Now())
record.TierEntered, record.Updated = entered, time.Now().UTC()
@@ -859,13 +859,13 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []
}
}
i.moved.mu.Lock()
for key, pm := range i.moved.pending {
i.state.mu.Lock()
for key, pm := range i.state.pending {
if _, saved := done[pm.plan]; saved {
delete(i.moved.pending, key)
delete(i.state.pending, key)
}
}
i.moved.mu.Unlock()
i.state.mu.Unlock()
for _, p := range plans {
w := done[p.ID]
p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held
@@ -908,8 +908,8 @@ func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uin
// RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired
// at the next cut (ADR 0408, issue 1092). It says how many it repaired.
func (i *Inventory) RepairMerges(ctx context.Context) (int, error) {
c, on, err := i.ChainReads(ctx, KindRollouts)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return 0, err
}
v, err := i.rolloutView(ctx, c)
internal/inventory/chain_rollouts_test.go
+6 -9
View File
@@ -15,7 +15,7 @@ import (
func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) {
t.Helper()
c := chain.InMemory(nil)
inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
inv.UseChain(c)
t.Cleanup(inv.stopViews)
return inv, c
@@ -117,10 +117,10 @@ func TestAMergeAddedToAPlanNotSavedYetIsWrittenByItsSave(t *testing.T) {
}
return errors.New("the lease is not held")
})
other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded}
other := &Inventory{state: &chainState{}, recorded: inv.recorded}
other.UseChain(away)
t.Cleanup(other.stopViews)
other.moved.pending = inv.moved.pending
other.state.pending = inv.state.pending
p := aBatch("plan-new", plansAt)
if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); !errors.Is(err, chain.ErrNotHeld) {
t.Fatalf("the save answered %v", err)
@@ -158,7 +158,7 @@ func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *test
var fail atomic.Bool
rollouts := failingOn{Backend: chain.NewMemory(), key: "plan-a", fail: &fail}
c = chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil })
inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
inv.UseChain(c)
t.Cleanup(inv.stopViews)
ctx := context.Background()
@@ -190,9 +190,6 @@ func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *test
if n := len(mergeHolders(v.list())["novox/mesh-catalog|m1"]); n != 2 {
t.Fatalf("the merge is held by %d records, want the two of a stopped move", n)
}
inv.moved.mu.Lock()
inv.moved.kind = map[string]bool{KindRollouts: true}
inv.moved.mu.Unlock()
repaired := 0
for k, copies := range mergeHolders(v.list()) {
if len(copies) < 2 {
@@ -310,7 +307,7 @@ func (c *countingBackend) Keys(ctx context.Context) ([]string, error) {
func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) {
counting := &countingBackend{Backend: chain.NewMemory()}
c := chain.New(chain.NewMemory(), counting, chain.NewMemory(), func(context.Context) error { return nil })
inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
inv.UseChain(c)
t.Cleanup(inv.stopViews)
ctx := context.Background()
@@ -337,7 +334,7 @@ func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) {
t.Fatalf("the bucket was read %d times and its keys %d times for the surveys", counting.gets.Load()-before, counting.keys.Load())
}
// Another process writes; the view follows.
other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded}
other := &Inventory{state: &chainState{}, recorded: inv.recorded}
other.UseChain(c)
t.Cleanup(other.stopViews)
late := aBatch("plan-4", plansAt.Add(time.Hour))
internal/inventory/chain_sent_copy_test.go
+111
View File
@@ -0,0 +1,111 @@
package inventory
import (
"testing"
"time"
"github.com/novox/mesh-controller/internal/chain"
)
// **What delivery 1 recorded in node.sent_builds is copied onto the keys of the assignments, once** (issue 1109, review
// round 1): a module pulled in by an assignment had no commit on its key, so it would fall out of SentBuilds. The copy
// creates a standing pulled-in key where none stands, fills a standing key that names no commit (commit, and the build
// when one is known, no rollout), and overwrites nothing: a key with a commit, a hand send's unknown, a wait for a build.
func TestWhatTheColumnRecordedIsCopiedOntoTheKeysAndOverwritesNothing(t *testing.T) {
inv, c := anInventoryOnTheChain(t)
ctx := t.Context()
at := time.Date(2026, 10, 18, 4, 0, 0, 0, time.UTC)
if err := inv.chainRecordBuild(ctx, c, Build{ID: "b-wg", Module: "mesh-wireguard", Commit: "c-wg"}); err != nil {
t.Fatal(err)
}
put := func(module string, a chain.Assignment) {
a.Node, a.Module = "laptop", module
if _, err := c.Assignments.Create(ctx, assignmentKey("laptop", module), a); err != nil {
t.Fatal(err)
}
}
put("notes", chain.Assignment{Assigned: at}) // assigned, no commit yet
put("web", chain.Assignment{Assigned: at, Commit: "c-web-new", Build: "b1"}) // already has one
put("hand", chain.Assignment{Assigned: at, SentUnknown: at}) // a hand send since
put("waits", chain.Assignment{Assigned: at, State: chain.StatePending}) // a wait for a build
put("ended", chain.Assignment{Assigned: at, Ended: at}) // unassigned
sent := map[string]string{"notes": "c-notes", "web": "c-web-old", "mesh-wireguard": "c-wg", "hand": "c-hand",
"waits": "c-waits", "ended": "c-ended"}
for pass := 0; pass < 2; pass++ { // and the second pass changes nothing
if err := inv.copySentBuildsOf(ctx, c, "laptop", sent, at); err != nil {
t.Fatal(err)
}
get := func(module string) chain.Assignment {
t.Helper()
k, found, err := c.Assignments.Get(ctx, assignmentKey("laptop", module))
if err != nil || !found {
t.Fatalf("%s: %v %v", module, found, err)
}
return k.Value
}
if a := get("notes"); a.Commit != "c-notes" || a.Rollout != "" || !a.Stands() || a.PulledIn {
t.Errorf("notes: %+v", a)
}
if a := get("web"); a.Commit != "c-web-new" || a.Build != "b1" {
t.Errorf("a key with a commit was overwritten: %+v", a)
}
if a := get("hand"); a.Commit != "" || a.SentUnknown.IsZero() {
t.Errorf("a hand send's unknown was overwritten: %+v", a)
}
if a := get("waits"); a.Commit != "" || a.State != chain.StatePending {
t.Errorf("a wait was overwritten: %+v", a)
}
if a := get("mesh-wireguard"); a.Commit != "c-wg" || a.Build != "b-wg" || !a.PulledIn || !a.Stands() || a.Rollout != "" {
t.Errorf("the pulled-in module has no standing key of its own: %+v", a)
}
if a := get("ended"); a.Commit != "" || a.Stands() {
t.Errorf("an ended key was revived or given a commit: %+v", a)
}
}
}
// The copy reads the column of the store, writes the keys, and empties the column it has copied: SentBuilds then answers the
// pulled-in module, and a second copy finds nothing to do.
func TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
if _, err := inv.AddNode(ctx, "laptop"); err != nil {
t.Fatal(err)
}
inv.ExecForTest(t, `update node set sent_builds = '{"mesh-wireguard": "c-wg"}'::jsonb, sent_at = now() where name = 'laptop'`)
if n, err := inv.CopySentBuilds(ctx); err != nil || n != 1 {
t.Fatalf("the copy did %d nodes (%v)", n, err)
}
got, known, err := inv.SentBuilds(ctx, "laptop")
if err != nil || !known || got["mesh-wireguard"] != "c-wg" {
t.Fatalf("SentBuilds answers %v, known %v (%v)", got, known, err)
}
if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" {
t.Fatalf("the column was not emptied: %v", rows)
}
if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 {
t.Fatalf("a second copy did %d nodes (%v)", n, err)
}
}
// A machine delivery 2 has sent has its column emptied by the send itself, so a copy after it leaves a key the send cleared
// without a commit.
func TestASendEmptiesTheColumnSoALaterCopyLeavesAClearedKeyBare(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
record, err := inv.AddNode(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
inv.ExecForTest(t, `update node set sent_builds = '{"notes": "c-old"}'::jsonb where name = 'laptop'`)
if err := inv.RecordSentUnder(ctx, record.ID, "d1", map[string]string{}, 1); err != nil {
t.Fatal(err)
}
if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" {
t.Fatalf("a send left the column: %v", rows)
}
if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 {
t.Fatalf("the copy did %d nodes (%v)", n, err)
}
}
internal/inventory/chain_sent_test.go
+116
View File
@@ -0,0 +1,116 @@
//go:build integration
package inventory
import (
"context"
"errors"
"reflect"
"testing"
"github.com/novox/mesh-controller/internal/chain"
)
// **What a machine was sent is read from the keys of its assignments that name a commit** (ADR 0408, issue 1109): an assigned
// module's, and a module only pulled in, whose key is a standing value marked pulled-in and never a state.
func TestSentBuildsAnswersFromTheAssignmentsKeysOfTheNode(t *testing.T) {
inv, node := aNodeWithModules(t, "web", "cache")
ctx := t.Context()
record, _ := inv.NodeByName(ctx, node)
if _, err := inv.Assign(ctx, node, "web"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil {
t.Fatal(err)
}
if builds, known, err := inv.SentBuilds(ctx, node); err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"web": "c1", "cache": "c2"}) {
t.Fatalf("sent %v %v %v", builds, known, err)
}
pulled, found, _ := inv.AssignmentValue(ctx, node, "cache")
if !found || !pulled.Stands() || pulled.State != "" || !pulled.PulledIn || !pulled.Assigned.IsZero() || pulled.Commit != "c2" {
t.Fatalf("the module only carried has the key %+v", pulled)
}
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"web"}) {
t.Fatalf("a module only pulled in is assigned: %v", on)
}
// Assigned later, it keeps what it was sent and is pulled in no longer.
if _, err := inv.Assign(ctx, node, "cache"); err != nil {
t.Fatal(err)
}
made, _, _ := inv.AssignmentValue(ctx, node, "cache")
if made.PulledIn || made.Assigned.IsZero() || made.Commit != "c2" {
t.Fatalf("assigned over a pulled-in key: %+v", made)
}
}
// **A send replaces the whole map of the node, and a hand send makes it unknown until the next send that names its builds**
// (issue 1109): a module no longer carried loses its commit, the node reads unknown when any key says so.
func TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown(t *testing.T) {
inv, node := aNodeWithModules(t, "web", "cache")
ctx := t.Context()
record, _ := inv.NodeByName(ctx, node)
if _, err := inv.Assign(ctx, node, "web"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, record.ID, "d2", map[string]string{"web": "c3"}); err != nil {
t.Fatal(err)
}
if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c3"}) {
t.Fatalf("the second send did not replace the map: %v %v", builds, known)
}
if old, _, _ := inv.AssignmentValue(ctx, node, "cache"); old.Commit != "" || old.Build != "" {
t.Fatalf("a module no longer carried keeps %+v", old)
}
// By hand: unknown, with a time and no commit anywhere.
if err := inv.RecordSent(ctx, record.ID, "d3", nil); err != nil {
t.Fatal(err)
}
if builds, known, _ := inv.SentBuilds(ctx, node); known || builds != nil {
t.Fatalf("a hand send reads as %v %v", builds, known)
}
web, _, _ := inv.AssignmentValue(ctx, node, "web")
if web.SentUnknown.IsZero() || web.Commit != "" || !web.Stands() {
t.Fatalf("web after a hand send: %+v", web)
}
if sent, _ := inv.Outstanding(ctx, node); sent != "d3" {
t.Fatalf("the digest of the hand send: %q", sent)
}
// A send that names its builds makes it known again.
if err := inv.RecordSent(ctx, record.ID, "d4", map[string]string{"web": "c4"}); err != nil {
t.Fatal(err)
}
if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c4"}) {
t.Fatalf("after a send that names its builds: %v %v", builds, known)
}
if again, _, _ := inv.AssignmentValue(ctx, node, "web"); !again.SentUnknown.IsZero() {
t.Fatalf("web still says unknown: %+v", again)
}
}
// failingToWrite is an assignments bucket that fails every write.
type failingToWrite struct{ chain.Backend }
func (f failingToWrite) Create(context.Context, string, []byte) (uint64, error) {
return 0, errors.New("the bus went away")
}
func (f failingToWrite) Update(context.Context, string, []byte, uint64) (uint64, error) {
return 0, errors.New("the bus went away")
}
// **A links write that fails fails the record of the send, loudly** (issue 1109): what a machine was sent is read from the
// keys, so a send recorded without them would read as one that never happened.
func TestAFailedWriteOfTheLinksFailsTheRecordOfTheSend(t *testing.T) {
inv, node := aNodeWithModules(t, "web")
ctx := t.Context()
record, _ := inv.NodeByName(ctx, node)
inv.UseChain(chain.New(chain.NewMemory(), chain.NewMemory(), failingToWrite{chain.NewMemory()},
func(context.Context) error { return nil }))
err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1"})
if err == nil {
t.Fatal("a send whose links could not be written was recorded")
}
}
internal/inventory/chain_source_test.go
+146
View File
@@ -0,0 +1,146 @@
package inventory
import (
"go/ast"
"go/parser"
"go/token"
"io/fs"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"testing"
)
// The tables of the chain, and the column, that the controller's store no longer holds (novox/hq ADR 0408 §8, issue 1096).
var chainTables = []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment"}
// sqlStatement is what makes a literal a statement of the store: it opens with a verb of SQL.
var sqlStatement = regexp.MustCompile(`(?is)^\s*(select|insert|update|delete|alter|drop|create|with|truncate|from|join|where|and|set|values)\b|\b(select\s.+\sfrom|insert\s+into|delete\s+from)\b`)
// **No code names a table of the chain** (acceptance 12, issue 1096), found by reading the sources: a string literal of any
// non-test file of internal/ or cmd/ that names one after from, into, update, join, delete from, references or table —
// build_gate and port_assignment are other words —. This is the static half of the proof; the
// integration test reads the catalog.
func TestNoCodeNamesATableOfTheChain(t *testing.T) {
names := regexp.MustCompile(`(?i)\b(from|into|update|join|delete\s+from|references|table)\s+(` + strings.Join(chainTables, "|") + `)\b`)
var found []string
for _, root := range []string{"../../internal", "../../cmd"} {
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() && d.Name() == "vendor" {
return filepath.SkipDir
}
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
return nil
}
file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0)
if err != nil {
return err
}
ast.Inspect(file, func(n ast.Node) bool {
lit, ok := n.(*ast.BasicLit)
if !ok || lit.Kind != token.STRING {
return true
}
text, err := strconv.Unquote(lit.Value)
if err != nil {
return true
}
if !sqlStatement.MatchString(text) {
return true // prose that happens to say "from assignment" is no statement
}
if m := names.FindString(text); m != "" {
found = append(found, filepath.ToSlash(path)+": "+strings.Join(strings.Fields(m), " "))
}
return true
})
return nil
})
if err != nil {
t.Fatal(err)
}
}
if len(found) > 0 {
t.Fatalf("code still names a table of the chain, which the store no longer holds:\n %s", strings.Join(found, "\n "))
}
}
// The scan sees what it says it sees: planted, a string naming a table is found, and a comment or a word beside is not.
func TestTheSourceScanFindsATablePlanted(t *testing.T) {
names := regexp.MustCompile(`(?i)\b(from|into|update|join|delete\s+from|references|table)\s+(` + strings.Join(chainTables, "|") + `)\b`)
for text, want := range map[string]bool{
"select id from build where id = $1": true,
"insert into assignment (node, module)": true,
"select 1 from build_gate": false,
"delete from port_assignment": false,
"select a from build_request a join build b": true,
"an assignment from build to build": true,
"the assignment generation": false,
"select count(*) from pending_assignment": true,
} {
if got := names.MatchString(text); got != want {
t.Errorf("%q: %v, want %v", text, got, want)
}
}
}
// **The last migration drops the tables of the chain, behind its guard** (issue 1096): it names each table, the column and
// the function, drops no build_gate, and raises before it drops anything.
func TestTheLastMigrationDropsTheTablesOfTheChain(t *testing.T) {
migrations, err := Migrations()
if err != nil {
t.Fatal(err)
}
last := migrations[len(migrations)-1]
if last.Number < 105 {
t.Fatalf("the last migration is %d", last.Number)
}
raw, err := os.ReadFile(filepath.Join("migrations", migrationFile(t, last.Number)))
if err != nil {
t.Fatal(err)
}
var code []string
for _, line := range strings.Split(string(raw), "\n") {
if !strings.HasPrefix(strings.TrimSpace(line), "--") {
code = append(code, line)
}
}
sql := strings.ToLower(strings.Join(code, "\n"))
for _, table := range append(append([]string{}, chainTables...), "chain_moved") {
if !strings.Contains(sql, "drop table "+table+";") {
t.Errorf("migration %d does not drop %s", last.Number, table)
}
}
for _, want := range []string{"drop function raise_assignment_generation()"} {
if !strings.Contains(sql, want) {
t.Errorf("migration %d lacks %q", last.Number, want)
}
}
if strings.Contains(sql, "sent_builds") {
t.Errorf("migration %d touches node.sent_builds, which stays as the source of the copy (issue 1112 drops it)", last.Number)
}
if strings.Contains(sql, "build_gate") || strings.Contains(sql, "assignment_generation;") {
t.Errorf("migration %d touches a table that stays", last.Number)
}
if raise, drop := strings.Index(sql, "raise exception"), strings.Index(sql, "drop table"); raise < 0 || drop < 0 || raise > drop {
t.Errorf("migration %d does not raise before it drops (%d, %d)", last.Number, raise, drop)
}
for _, kind := range []string{"'builds'", "'rollouts'", "'assignments'", "pending_assignment", "chain_moved"} {
if !strings.Contains(sql[:strings.Index(sql, "drop table")], kind) {
t.Errorf("the guard of migration %d does not look at %s", last.Number, kind)
}
}
}
func migrationFile(t *testing.T, number int) string {
t.Helper()
files, err := filepath.Glob(filepath.Join("migrations", strconv.Itoa(number/1000%10)+strconv.Itoa(number/100%10)+strconv.Itoa(number/10%10)+strconv.Itoa(number%10)+"-*.sql"))
if err != nil || len(files) != 1 {
t.Fatalf("the file of migration %d: %v %v", number, files, err)
}
return filepath.Base(files[0])
}
internal/inventory/chain_store_test.go
+348
View File
@@ -0,0 +1,348 @@
//go:build integration
package inventory
import (
"context"
"errors"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/chain"
)
// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its
// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none.
func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
first := aBuild("build-1", "notes", "")
first.Commit = "c1"
first.Against = []string{"artifact-store://base@sha256:aa"}
first.ReuseKey = "rk-1"
second := aBuild("build-2", "notes", "")
second.Commit = "c2"
failed := aBuild("build-3", "notes", "compile error")
other := aBuild("build-4", "wiki", "")
for _, b := range []Build{first, second, failed, other, first} {
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
kept, found, err := inv.chain.Builds.Get(ctx, chain.BuildKey("notes", "build-1"))
if err != nil || !found || kept.Value.ReuseKey != "rk-1" {
t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err)
}
if history, _ := inv.chain.Builds.History(ctx, chain.BuildKey("notes", "build-1")); len(history) != 1 {
t.Fatalf("a build recorded twice has %d values", len(history))
}
list, err := inv.Builds(ctx, "notes", 10)
if err != nil || len(list) != 3 {
t.Fatalf("notes has %d builds: %v", len(list), err)
}
one, found, err := inv.BuildByID(ctx, "build-2")
if err != nil || !found || one.Commit != "c2" || one.Module != "notes" {
t.Fatalf("build-2 reads as %+v %v %v", one, found, err)
}
if _, found, _ := inv.BuildByID(ctx, "build-nope"); found {
t.Fatal("a build nobody recorded was found")
}
if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" {
t.Fatalf("the build of notes at c1 is %q: %v", id, err)
}
held, err := inv.Held(ctx)
if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" {
t.Fatalf("held %v: %v", held, err)
}
against, err := inv.BuiltAgainst(ctx)
if err != nil || len(against["notes"]) != 0 {
// The newest successful build of notes, build-2, stood on nothing.
t.Fatalf("built against %v: %v", against, err)
}
announced, err := inv.Announceable(ctx)
if err != nil || len(announced) != 3 {
t.Fatalf("announceable %d: %v", len(announced), err)
}
prints, err := inv.Fingerprints(ctx)
if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" {
t.Fatalf("fingerprints %v: %v", prints, err)
}
}
// A build request row as the store held it before the move.
// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue
// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none
// after the outcome is heard. The table of requests takes none.
func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true}
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
t.Fatalf("a request kept twice: %v", err)
}
if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 {
t.Fatalf("a request nobody answered is a build: %+v %v", builds, err)
}
if _, found, _ := inv.BuildByID(ctx, "build-1"); found {
t.Fatal("a request nobody answered has an outcome")
}
got, found, err := inv.BuildRequestByID(ctx, "build-1")
if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() {
t.Fatalf("the request reads back as %+v %v %v", got, found, err)
}
if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil {
t.Fatal(err)
}
asked, err := inv.RequestsNamed(ctx, "notes")
if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" {
t.Fatalf("the request is %+v: %v", asked, err)
}
if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) {
t.Fatalf("requested names %v: %v", names, err)
}
// The outcome comes: the same key, now a build, with the request beside it.
built := aBuild("build-1", "notes", "")
if err := inv.RecordBuild(ctx, built); err != nil {
t.Fatal(err)
}
if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" {
t.Fatalf("the outcome reads as %+v %v %v", one, found, err)
}
answered, err := inv.RequestsNamed(ctx, "notes")
if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" {
t.Fatalf("the request answered is %+v: %v", answered, err)
}
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" {
t.Fatalf("the request was lost with the outcome: %+v", again)
}
// Heard: the marks of an unanswered request change nothing.
if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil {
t.Fatal(err)
}
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" {
t.Fatalf("an answered request was marked not asked: %+v", again)
}
if history, _ := inv.chain.Builds.History(ctx, chain.RequestKey("build-1")); len(history) != 3 {
// asked, marked unknown, answered.
t.Fatalf("the record has %d values", len(history))
}
// An outcome heard before its request is kept, and the request lands beside it.
if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil {
t.Fatal(err)
}
if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" {
t.Fatalf("the request kept after its outcome reads %+v", late)
}
// A request older than the time they are kept is no longer listed.
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old",
At: time.Now().Add(-2 * KeptFor)}); err != nil {
t.Fatal(err)
}
if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") {
t.Fatalf("a request past its time is listed: %v", names)
}
}
// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a
// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it
// is asked and so asked at most once.
// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a
// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module
// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the
// assignment goes with it.
func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
inv, node := aNodeWithModules(t, "postgres", "web")
ctx := t.Context()
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
start := generationNow(t, inv)
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added {
t.Fatalf("assigned %v: %v", added, err)
}
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added {
t.Fatalf("assigned again %v: %v", added, err)
}
kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() {
t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err)
}
if generationNow(t, inv) <= start {
t.Fatal("an assignment left the generation where it was")
}
if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) {
t.Fatalf("a module the mesh does not know: %v", err)
}
if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) {
t.Fatalf("assigned on %s: %v %v", node, on, err)
}
if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) {
t.Fatalf("running: %v %v", running, err)
}
entries, err := inv.Catalogued(ctx)
if err != nil {
t.Fatal(err)
}
for _, e := range entries {
switch e.Manifest.Module {
case "postgres":
if !reflect.DeepEqual(e.On, []string{node}) {
t.Errorf("postgres is on %v", e.On)
}
case "web":
if len(e.On) != 0 {
t.Errorf("web is on %v", e.On)
}
}
}
if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) {
t.Fatalf("a module still assigned was forgotten: %v", err)
}
// A seat is held by an assignment, or by nothing.
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil {
t.Fatal("a seat was handed to a module that is not assigned")
}
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil {
t.Fatalf("a seat handed to an assignment: %v", err)
}
before := generationNow(t, inv)
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
t.Fatal(err)
}
if generationNow(t, inv) <= before {
t.Fatal("an unassignment left the generation where it was")
}
// No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history.
gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() {
t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err)
}
if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() ||
history[1].Value.Stands() {
t.Fatalf("the earlier values of the key: %+v", history)
}
if on, _ := inv.Assigned(ctx, node); len(on) != 0 {
t.Fatalf("an ended assignment is still assigned: %v", on)
}
if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 {
t.Fatalf("an ended assignment is running: %v", running)
}
if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 {
t.Fatalf("the seat the assignment held is held by %+v: %v", held, err)
}
if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" {
t.Fatalf("an unassignment of nothing: %v", err)
}
if err := inv.ForgetModule(ctx, "postgres"); err != nil {
t.Fatalf("a module nothing is assigned could not be forgotten: %v", err)
}
// Made again after it ended: a fresh value over the ended one, which stays in history.
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
t.Fatal(added, err)
}
if added, err := inv.Assign(ctx, node, "web"); err != nil || added {
t.Fatal(added, err)
}
if err := inv.Unassign(ctx, node, "web"); err != nil {
t.Fatal(err)
}
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
t.Fatalf("assigned again after it ended: %v %v", added, err)
}
again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web")
if !again.Value.Stands() || !again.Value.Ended.IsZero() {
t.Fatalf("the assignment made again is %+v", again.Value)
}
if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() {
t.Fatalf("the values of the key: %+v", history)
}
}
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
// ends an assignment.
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
// ends an assignment.
func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) {
inv, node := aNodeWithModules(t, "dunst", "web")
ctx := t.Context()
for _, m := range []string{"dunst", "web"} {
if _, err := inv.Assign(ctx, node, m); err != nil {
t.Fatal(err)
}
}
record, _ := inv.NodeByName(ctx, node)
digest := "sha256:" + strings.Repeat("d", 64)
plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest,
Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc",
Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}}
raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`)
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil {
t.Fatal(err)
}
// A build and a commit put in the value by another write, after the send: a send that carries no builds (a hand send, as this
// one) forgets what each assignment of its machine runs (issue 1109), so they are written once it is recorded.
cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
cur.Value.Build, cur.Value.Commit = "build-1", "c1"
if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil {
t.Fatal(err)
}
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
written, removed, err := inv.KeepAssignmentPlans(ctx)
if err != nil || written != 2 || removed != 0 {
t.Fatalf("kept: %d written, %d ended: %v", written, removed, err)
}
dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() {
t.Fatalf("dunst's value is %+v", dunst.Value)
}
if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil {
t.Fatalf("web has a plan: %+v", web.Value)
}
notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify")
if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil {
t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found)
}
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
t.Fatalf("a pulled-in module is assigned: %v", on)
}
if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 {
t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err)
}
// Sent again without the pulled-in module: its value ends, the assignments stay.
only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`)
if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil {
t.Fatal(err)
}
if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 {
t.Fatalf("ended %d: %v", removed, err)
}
if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() {
t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value)
}
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
t.Fatalf("assigned: %v", on)
}
}
// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value**
// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build
// already is left as it is.
internal/inventory/collection.go
+19 -70
View File
@@ -2,7 +2,6 @@ package inventory
import (
"context"
"encoding/json"
"sort"
"strings"
@@ -327,84 +326,34 @@ type recentBuild struct {
// recentBuilds is, of each module the mesh still holds, its KeptBuilds newest successful builds' artifacts and bases.
func (i *Inventory) recentBuilds(ctx context.Context) ([]recentBuild, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
builds, err := i.chainRecentMade(ctx, c)
if err != nil {
return nil, err
}
out := make([]recentBuild, 0, len(builds))
for _, b := range builds {
out = append(out, recentBuild{made: b.Made, against: b.Against})
}
return out, nil
}
rows, err := i.store.Pool().Query(ctx,
`select made, built_against from (
select b.made, b.built_against,
row_number() over (partition by b.module order by b.at desc, b.id desc) as back
from build b
join module m on m.name = b.module
where b.failed = '' and b.module is not null and b.module <> ''
) ranked where back <= $1`, KeptBuilds)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
var out []recentBuild
for rows.Next() {
var raw, against []byte
if err := rows.Scan(&raw, &against); err != nil {
return nil, err
}
var r recentBuild
if err := json.Unmarshal(raw, &r.made); err != nil {
r.made = nil
}
if err := json.Unmarshal(against, &r.against); err != nil {
r.against = nil
}
out = append(out, r)
builds, err := i.chainRecentMade(ctx, c)
if err != nil {
return nil, err
}
return out, rows.Err()
out := make([]recentBuild, 0, len(builds))
for _, b := range builds {
out = append(out, recentBuild{made: b.Made, against: b.Against})
}
return out, nil
}
// everyReferenceMade is every artifact reference any successful build recorded, oldest build first.
func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) {
var made [][]Artifact
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
builds, err := i.chainEveryReferenceMade(ctx, c)
if err != nil {
return nil, err
}
for _, b := range builds {
made = append(made, b.Made)
}
} else {
rows, err := i.store.Pool().Query(ctx,
`select made from build where failed = '' and module is not null and module <> ''
order by at asc, id asc`)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var raw []byte
if err := rows.Scan(&raw); err != nil {
return nil, err
}
var a []Artifact
if err := json.Unmarshal(raw, &a); err != nil {
continue
}
made = append(made, a)
}
if err := rows.Err(); err != nil {
return nil, err
}
}
builds, err := i.chainEveryReferenceMade(ctx, c)
if err != nil {
return nil, err
}
var made [][]Artifact
for _, b := range builds {
made = append(made, b.Made)
}
seen := map[string]bool{}
var out []string
internal/inventory/durations.go
-42
View File
@@ -2,11 +2,7 @@ package inventory
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
)
// The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066.
@@ -148,41 +144,3 @@ func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) {
}
return tag.RowsAffected(), nil
}
// planTierLeft is the measurement a plan's save makes when it leaves a tier: the tier moved on, or
// the plan ended. Read in the save's own transaction, so two saves cannot both measure one tier.
func planTierLeft(ctx context.Context, tx pgx.Tx, p Plan, now time.Time) (entered time.Time, err error) {
var oldTier int
var oldState string
var since time.Time
err = tx.QueryRow(ctx,
`select tier, state, coalesce(tier_entered, created) from release_plan where id = $1 for update`,
p.ID).Scan(&oldTier, &oldState, &since)
if errors.Is(err, pgx.ErrNoRows) {
return now, nil // a new plan enters its first tier now
}
if err != nil {
return time.Time{}, err
}
if oldState == PlanAssembling || oldState == PlanQueued {
return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276)
}
wasOpen := oldState == PlanBuilding || oldState == PlanRolling
if !wasOpen || (oldTier == p.Tier && p.Open()) {
return since, nil // still in the tier, or already ended
}
var modules []string
if oldTier >= 0 && oldTier < len(p.Tiers) {
modules = p.Tiers[oldTier]
}
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, oldTier, len(p.Tiers), modules, p.State)
if p.Open() {
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, oldTier, len(p.Tiers), modules, p.Tier)
}
_, err = tx.Exec(ctx,
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
values ($1, $2, '', $3, $4, $5, $6) on conflict do nothing`,
DurationPlanTier, p.Repository, fmt.Sprintf("%s/tier-%d", p.ID, oldTier), since,
now.Sub(since).Milliseconds(), detail)
return now, err
}
internal/inventory/fortest.go
+28 -7
View File
@@ -65,16 +65,15 @@ func ForTest(t *testing.T) *Inventory {
if _, err := inv.store.Migrate(t.Context(), migrations); err != nil {
t.Fatal(err)
}
// The chain is on the bus in a mesh; in a test it is in memory, with every kind moved, as a mesh raised after this
// release has them (ADR 0408, issue 1089). The tests of the move itself use ForTestBefore.
// The chain is on the bus in a mesh; in a test it is in memory (ADR 0408).
c := chain.InMemory(nil)
// A fresh test chain has nothing to lay out: it is laid out already, as the controller's start leaves a mesh (issue 1121).
if _, err := c.Builds.CreateRaw(t.Context(), chain.LayoutMarkerKey, []byte(`{}`)); err != nil {
t.Fatal(err)
}
inv.UseChain(c)
testChains.Store(os.Getenv(store.Variable(Name)), c)
t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) })
if _, err := inv.store.Pool().Exec(t.Context(),
`insert into chain_moved (counts, largest) values ($1, $1)`, `{"builds": 0, "builds-layout": 0, "rollouts": 0, "assignments": 0}`); err != nil {
t.Fatal(err)
}
return inv
}
@@ -132,8 +131,12 @@ func ForTestBefore(t *testing.T, number int) (*Inventory, func()) {
if err := inv.Ready(t.Context(), 20*time.Second); err != nil {
t.Fatal(err)
}
// The chain is in memory and nothing is moved: the tests of a move seed the tables and move them.
// The chain is in memory, for what a test of the store before a migration reads of it.
c := chain.InMemory(nil)
// A fresh test chain has nothing to lay out: it is laid out already, as the controller's start leaves a mesh (issue 1121).
if _, err := c.Builds.CreateRaw(t.Context(), chain.LayoutMarkerKey, []byte(`{}`)); err != nil {
t.Fatal(err)
}
inv.UseChain(c)
testChains.Store(os.Getenv(store.Variable(Name)), c)
t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) })
@@ -183,3 +186,21 @@ func (i *Inventory) QueryForTest(t *testing.T, sql string, args ...any) [][]stri
}
return out
}
// PlanValuesForTest runs the pass that keeps the provisioner's plans in the values of the assignments (KeepAssignmentPlans)
// and answers the values that stand, by key: what a reader of the bucket reads.
func (i *Inventory) PlanValuesForTest(t *testing.T) map[string][]byte {
t.Helper()
if _, _, err := i.KeepAssignmentPlans(t.Context()); err != nil {
t.Fatal(err)
}
kept, err := i.KeptAssignments(t.Context())
if err != nil {
t.Fatal(err)
}
out := make(map[string][]byte, len(kept))
for key, v := range kept {
out[key] = v.Value
}
return out
}
internal/inventory/gate.go
+12 -86
View File
@@ -2,8 +2,6 @@ package inventory
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -241,65 +239,20 @@ func (i *Inventory) RestoreModule(ctx context.Context, b Build) error {
// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told
// from one that does (the bus's planned step, ADR 0236).
func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainBuildFingerprints(ctx, c, module)
}
rows, err := i.store.Pool().Query(ctx,
`select commit_hash, made, coalesce(manifest::text, '') from build
where module = $1 and failed = '' and commit_hash <> '' order by at desc limit 50`, module)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]string{}
for rows.Next() {
var commit, manifest string
var made []byte
if err := rows.Scan(&commit, &made, &manifest); err != nil {
return nil, err
}
if _, seen := out[commit]; seen {
continue
}
sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...))
out[commit] = hex.EncodeToString(sum[:])
}
return out, rows.Err()
return i.chainBuildFingerprints(ctx, c, module)
}
// Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint.
func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainFingerprints(ctx, c)
}
rows, err := i.store.Pool().Query(ctx,
`select module, commit_hash, made, coalesce(manifest::text, '') from build
where module is not null and failed = '' and commit_hash <> '' order by at desc`)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]map[string]string{}
for rows.Next() {
var module, commit, manifest string
var made []byte
if err := rows.Scan(&module, &commit, &made, &manifest); err != nil {
return nil, err
}
if out[module] == nil {
out[module] = map[string]string{}
}
if _, seen := out[module][commit]; seen {
continue
}
sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...))
out[module][commit] = hex.EncodeToString(sum[:])
}
return out, rows.Err()
return i.chainFingerprints(ctx, c)
}
// PassedCommits is, per module, the commits a build of which passed its gate on some machine.
@@ -326,19 +279,11 @@ func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bo
// BuildOf is the id of the newest successful build of a module from a commit; empty when none is
// recorded (a manifest handed over by hand).
func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return "", err
} else if on {
return i.chainBuildOf(ctx, c, module, commit)
}
var id string
err := i.store.Pool().QueryRow(ctx,
`select id from build where module = $1 and commit_hash = $2 and failed = '' order by at desc limit 1`,
module, commit).Scan(&id)
if errors.Is(err, pgx.ErrNoRows) {
return "", nil
}
return id, err
return i.chainBuildOf(ctx, c, module, commit)
}
// ManifestAt is the manifest a module was registered with at a commit (or a commit it abbreviates): the
@@ -372,38 +317,19 @@ func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (cata
// manifestOf is the manifest a build was recorded with; nil when it kept none.
func (i *Inventory) manifestOf(ctx context.Context, id string) ([]byte, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainManifestOf(ctx, c, id)
}
var manifest []byte
if err := i.store.Pool().QueryRow(ctx, `select manifest from build where id = $1`, id).Scan(&manifest); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return manifest, nil
return i.chainManifestOf(ctx, c, id)
}
// newestManifestAt is the newest successful build of a module from a commit, or one it abbreviates, that kept a manifest:
// its id and manifest; false when there is none.
func (i *Inventory) newestManifestAt(ctx context.Context, module, commit string) (string, []byte, bool, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return "", nil, false, err
} else if on {
return i.chainManifestBuild(ctx, c, module, commit)
}
var id string
var raw []byte
err := i.store.Pool().QueryRow(ctx,
`select id, manifest from build
where module = $1 and failed = '' and manifest is not null and manifest::text <> 'null'
and (commit_hash = $2 or starts_with(commit_hash, $2))
order by at desc limit 1`, module, commit).Scan(&id, &raw)
if errors.Is(err, pgx.ErrNoRows) {
return "", nil, false, nil
}
c, err := i.theChain(ctx)
if err != nil {
return "", nil, false, err
}
return id, raw, true, nil
return i.chainManifestBuild(ctx, c, module, commit)
}
internal/inventory/given.go
+6 -15
View File
@@ -93,28 +93,19 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s
if declared == "" {
return nil, nil
}
// Only a module assigned there: from the bucket when the assignments are on the bus, else the store's join.
_, onTheBus, err := i.ChainReads(ctx, KindAssignments)
// Only a module assigned there, from the assignments bucket.
assignedHere := map[string]bool{}
modules, err := i.assignedModules(ctx, node)
if err != nil {
return nil, err
}
assignedHere := map[string]bool{}
join := `join assignment a on a.node = s.node and a.module = s.module`
if onTheBus {
join = ``
modules, err := i.assignedModules(ctx, node)
if err != nil {
return nil, err
}
for _, m := range modules {
assignedHere[m] = true
}
for _, m := range modules {
assignedHere[m] = true
}
rows, err := i.store.Pool().Query(ctx,
`select s.node, s.module, s.name, s.replace_after_start
from module_secret s
join node n on n.id = s.node
`+join+`
where n.name = $1 and n.sent = $2 and n.sent_at > s.replace_after_start
and s.origin = 'accepted' and s.replace_after_start is not null
and (not n.adopted or exists (select 1 from taken t where t.node = s.node and t.module = s.module))
@@ -134,7 +125,7 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s
rows.Close()
return nil, err
}
if onTheBus && !assignedHere[d.module] {
if !assignedHere[d.module] {
continue
}
dues = append(dues, d)
internal/inventory/migrations/0105-the-chain-leaves-the-store.sql
+52
View File
@@ -0,0 +1,52 @@
-- The tables of builds, rollouts and assignments leave the store (novox/hq ADR 0408 §8, design 47 milestone 5, issue 1096).
--
-- Delivery 1 wrote their rows into the bus's buckets at the controller's start and recorded each kind it had counted in
-- chain_moved; from then on nothing in the controller reads or writes the tables. This drops them. **It is one-way**: a
-- controller older than this schema is refused by the put-back gate, and the rows are in the buckets only.
--
-- **The guard comes first, in the same transaction as the drops**, because a migration runs when the store is opened, before
-- the controller of this delivery serves and before any connection to the bus: it can prove the move from the store alone.
-- A kind that holds rows and is not named in chain_moved was never moved, so dropping its tables would lose it; a pending
-- assignment still waiting, being made, or ended unmade with its condition unanswered has no mover (it is a value of the
-- assignments bucket from this delivery on), so the table must hold none. A store that never held a row of a kind — a new
-- mesh — passes. A migration that raises leaves nothing applied, `prepare` fails with the reason, and the running controller
-- keeps its tables.
do $$
declare
kind text;
holds boolean;
missing text[] := '{}';
begin
foreach kind in array array['builds', 'rollouts', 'assignments'] loop
holds := case kind
when 'builds' then exists (select 1 from build) or exists (select 1 from build_request)
when 'rollouts' then exists (select 1 from release_plan) or exists (select 1 from batched_merge)
when 'assignments' then exists (select 1 from assignment)
end;
if holds and not coalesce((select counts ? kind from chain_moved), false) then
missing := missing || kind;
end if;
end loop;
if array_length(missing, 1) > 0 then
raise exception 'the controller''s store still holds rows of % that chain_moved does not name as moved to the bus; '
'run the controller of the previous delivery and read chain_moved back first', array_to_string(missing, ', ');
end if;
if exists (select 1 from pending_assignment
where state in ('waiting', 'applying') or (raised_at is not null and acknowledged_at is null and cleared_at is null)) then
raise exception 'a pending assignment is still in the table (waiting, being made, or ended with its condition '
'unanswered); let it settle, then deliver again';
end if;
end
$$;
drop table build_request;
drop table build;
drop table batched_merge;
drop table release_plan;
drop table pending_assignment;
-- Its triggers go with it; the function they called goes here. The counter table stays: the controller raises it.
drop table assignment;
drop function raise_assignment_generation();
-- node.sent_builds STAYS, only as the source of the one-time copy onto the assignments' keys (issue 1109); dropping it is
-- issue 1112. build_gate stays (issue 1097).
drop table chain_moved;
internal/inventory/mirrored_test.go
-35
View File
@@ -5,7 +5,6 @@ package inventory
import (
"context"
"fmt"
"os"
"slices"
"strings"
"testing"
@@ -148,40 +147,6 @@ func TestAPersonRecordsACopyAndTheFirstRecordStands(t *testing.T) {
}
}
// The migration records the copies the build records already name: only a module's own on-<argument>
// repository, and only once, by the earliest build naming it.
func TestTheCopiesTheRecordsAlreadyNameAreRecorded(t *testing.T) {
inv := inTheTables(t)
ctx := context.Background()
copyRef := mirrorRef("route-proxy/on-go_base", 1)
notACopy := ref("go-base", "build", 2)
builtOn(t, inv, "b1", "route-proxy", 1, nil, copyRef, notACopy)
builtOn(t, inv, "b2", "route-proxy", 2, nil, copyRef)
// The rows a mesh had before the table: written by builds that did not say what they copied.
if _, err := inv.store.Pool().Exec(ctx, `truncate artifact_mirrored`); err != nil {
t.Fatal(err)
}
sql, err := os.ReadFile("migrations/0081-a-mirrored-base-is-recorded.sql")
if err != nil {
t.Fatal(err)
}
_, backfill, _ := strings.Cut(string(sql), "insert into artifact_mirrored")
if _, err := inv.store.Pool().Exec(ctx, "insert into artifact_mirrored"+backfill); err != nil {
t.Fatal(err)
}
var reference, build string
var n int
if err := inv.store.Pool().QueryRow(ctx, `select count(*) from artifact_mirrored`).Scan(&n); err != nil || n != 1 {
t.Fatalf("%d copies recorded (%v); want the one", n, err)
}
if err := inv.store.Pool().QueryRow(ctx, `select reference, build from artifact_mirrored`).Scan(&reference, &build); err != nil {
t.Fatal(err)
}
if reference != copyRef || build != "b1" {
t.Fatalf("recorded %s by %s", reference, build)
}
}
// A copy let go of and copied again by a later build is held again: its record is back, and the sweep
// decides about it as before.
func TestACopyLetGoOfAndCopiedAgainIsHeldAgain(t *testing.T) {
internal/inventory/mirrors.go
+2 -34
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"time"
)
@@ -70,39 +69,8 @@ type DeclaredMirrored struct {
var DeclaredMirroredFields = []string{"kind", "default", "why", "type", "description", "enum", "minimum", "maximum",
"pattern", "trusted", "predefined", "required", "runtime"}
// ErrAssignmentsMoved is a write of the store's mirror of the assignments, refused because the assignments are keys of the bus.
var ErrAssignmentsMoved = errors.New("the assignments are keys of the bus, and the store's rows are no longer what they say")
// AssignmentsMirrorToWrite is AssignmentsMirror for the mirror's writer, which lays what it returns over the bucket and
// ends the keys it does not return: **refused once the assignments are moved** (ADR 0408 §4), asked at the moment of the
// write, so a tick that decided before the switch cannot lay mirror-shaped values over the links after it.
func (i *Inventory) AssignmentsMirrorToWrite(ctx context.Context) (map[string][]byte, error) {
if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
return nil, err
} else if on {
return nil, ErrAssignmentsMoved
}
return i.AssignmentsMirror(ctx)
}
// AssignmentsMirror is every assignment's mirrored value, keyed `<node>.<module>`, with the provisioner's plan of
// each assignment of a module handed to it, and a value of its own for each module handed to it on a node that
// nothing assigned there, only pulled in by another (issue 765).
func (i *Inventory) AssignmentsMirror(ctx context.Context) (map[string][]byte, error) {
var plans map[string]Provisioning
if i.provisioningPlans != nil {
var err error
if plans, err = i.provisioningPlans(ctx); err != nil {
return nil, fmt.Errorf("composing the provisioner's plans: %w", err)
}
}
assignments, err := i.assignmentsMirrored(ctx)
if err != nil {
return nil, err
}
return AssignmentValues(assignments, plans)
}
// assignmentsMirrored is every standing assignment as the provisioner's plans are matched to it (SentProvisioningPlans,
// KeepAssignmentPlans).
func (i *Inventory) assignmentsMirrored(ctx context.Context) ([]AssignmentMirrored, error) {
rows, err := i.assignments(ctx)
if err != nil {
internal/inventory/nodes.go
+34 -43
View File
@@ -34,11 +34,9 @@ type Inventory struct {
// configValues reads a module's config values from mesh-config once the move of config has run (ReadsConfigFrom,
// issue 664); nil reads none.
configValues ConfigValues
// chain is the controller's chain on the bus (novox/hq ADR 0408, issue 1089), once attached (UseChain); moved is which
// kinds of it have been moved from the tables, as chain_moved says (ChainReads). A kind not moved is read and written
// in the tables as before.
// chain is the controller's chain on the bus (novox/hq ADR 0408), once attached (UseChain).
chain *chain.Chain
moved *movedKinds
state *chainState
// recorded, when set, takes the durations a save of the chain measures in place of the store: a test with no store.
recorded func(context.Context, Duration) error
}
@@ -49,7 +47,7 @@ func Open(ctx context.Context) (*Inventory, error) {
if err != nil {
return nil, err
}
inv := &Inventory{store: s, moved: &movedKinds{}}
inv := &Inventory{store: s, state: &chainState{}}
// A test's store has the chain a test made for it in memory, shared by every inventory opened on that store.
if c, ok := testChains.Load(os.Getenv(store.Variable(Name))); ok {
inv.chain = c.(*chain.Chain)
@@ -1264,22 +1262,12 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu
e := int64(epoch)
sentEpoch = &e
}
var carried *string
if builds != nil {
raw, err := json.Marshal(builds)
if err != nil {
return err
}
text := string(raw)
carried = &text
if _, err := i.store.Pool().Exec(ctx,
`update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3 where id = $1`, node, digest, sentEpoch); err != nil {
return err
}
_, err := i.store.Pool().Exec(ctx,
`update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4 where id = $1`,
node, digest, carried, sentEpoch)
if err == nil {
saidRunsNot(node, i.recordRuns(ctx, node, builds, time.Now()))
}
return err
// What it was sent of each module is the assignments' (SentBuilds); a write of it that fails fails this record.
return i.recordRuns(ctx, node, builds, time.Now())
}
// AwaitingSince is since when a machine has had something waiting for its next send (novox/hq issue
@@ -1288,40 +1276,43 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu
// read from: every change a push carries happened after the last push, and an assignment is the one
// that says when.
func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time, error) {
if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
return time.Time{}, err
} else if on {
return i.chainAwaitingSince(ctx, name)
}
var since time.Time
err := i.store.Pool().QueryRow(ctx,
`select coalesce(
(select min(a.assigned) from assignment a
where a.node = n.id and (n.sent_at is null or a.assigned > n.sent_at)),
n.sent_at, n.created)
from node n where n.name = $1`, name).Scan(&since)
if errors.Is(err, pgx.ErrNoRows) {
return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
}
return since, err
return i.chainAwaitingSince(ctx, name)
}
// SentBuilds is the build of each module a machine was last sent, by its name: module to the commit
// its build was made from (novox/hq issue 259). Known is false when that was not kept — a machine
// last sent before it was, sent a declaration by hand, or one the mesh does not know.
// its build was made from (novox/hq issue 259), read from the keys of the machine's assignments that name a commit — an
// assigned module and one only pulled in (ADR 0408, issue 1109). Known is false when that was not kept: a machine never sent
// anything, or last sent a declaration by hand, which makes every key of it say its commit unknown.
func (i *Inventory) SentBuilds(ctx context.Context, name string) (builds map[string]string, known bool, err error) {
var raw []byte
err = i.store.Pool().QueryRow(ctx, `select sent_builds from node where name = $1`, name).Scan(&raw)
var sentAt *time.Time
err = i.store.Pool().QueryRow(ctx, `select sent_at from node where name = $1`, name).Scan(&sentAt)
if errors.Is(err, pgx.ErrNoRows) {
return nil, false, nil
}
if err != nil || raw == nil {
if err != nil || sentAt == nil {
return nil, false, err
}
builds = map[string]string{}
if err := json.Unmarshal(raw, &builds); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, false, err
}
all, err := c.Assignments.All(ctx)
if err != nil {
return nil, false, fmt.Errorf("what %s was sent cannot be read from the bus: %w", name, err)
}
builds = map[string]string{}
for _, k := range all {
a := k.Value
if a.Node != name || !a.Stands() {
continue
}
if !a.SentUnknown.IsZero() {
return nil, false, nil
}
if a.Commit != "" {
builds[a.Module] = a.Commit
}
}
return builds, true, nil
}
internal/inventory/pending.go
+293 -203
View File
@@ -3,12 +3,12 @@ package inventory
import (
"context"
"errors"
"fmt"
"path"
"sort"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/novox/mesh-controller/internal/chain"
)
@@ -69,85 +69,40 @@ type RequestOutcome struct {
// RecordBuildRequest keeps one build request, once it is asked. Idempotent on the id. Requests older than
// KeptFor are deleted in the same act.
func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) error {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return err
} else if on {
return i.chainRecordBuildRequest(ctx, c, a)
}
at := a.At
if at.IsZero() {
at = time.Now().UTC()
}
var notAsked *string
if a.NotAsked != "" {
notAsked = &a.NotAsked
}
if _, err := i.store.Pool().Exec(ctx,
`insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at,
at_terminal)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
on conflict (id) do nothing`,
a.ID, a.Repository, a.Seat, strings.Trim(a.Path, "/"), a.Ref, a.Commit, a.For, notAsked, at,
a.AtTerminal); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return err
}
_, err := i.store.Pool().Exec(ctx, `delete from build_request where asked_at < $1`, time.Now().Add(-KeptFor))
return err
return i.chainRecordBuildRequest(ctx, c, a)
}
// BuildRequestByID is the build request kept under this id, and whether one was kept.
func (i *Inventory) BuildRequestByID(ctx context.Context, id string) (BuildRequest, bool, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return BuildRequest{}, false, err
} else if on {
return i.chainBuildRequestByID(ctx, c, id)
}
var a BuildRequest
err := i.store.Pool().QueryRow(ctx,
`select id, repository, seat, source_path, ref, commit_hash, asked_for, at_terminal, asked_at
from build_request where id = $1`, id).Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit,
&a.For, &a.AtTerminal, &a.At)
if errors.Is(err, pgx.ErrNoRows) {
return BuildRequest{}, false, nil
}
return a, err == nil, err
return i.chainBuildRequestByID(ctx, c, id)
}
// MarkNotAsked says a kept build request was never handed over: the words are kept, unless its outcome was
// heard first.
func (i *Inventory) MarkNotAsked(ctx context.Context, id, why string) error {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return err
} else if on {
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.NotAsked = why })
}
if heard, err := i.outcomeHeard(ctx, id); err != nil || heard {
c, err := i.theChain(ctx)
if err != nil {
return err
}
_, err := i.store.Pool().Exec(ctx, `update build_request set not_asked = $2 where id = $1`, id, why)
return err
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.NotAsked = why })
}
// MarkOutcomeUnknown says the asker of a build it handed over stopped waiting for it: asked, outcome unknown.
// The build may still run; its outcome, when heard, is the last word.
func (i *Inventory) MarkOutcomeUnknown(ctx context.Context, id, why string) error {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return err
} else if on {
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.OutcomeUnknown = why })
}
if heard, err := i.outcomeHeard(ctx, id); err != nil || heard {
c, err := i.theChain(ctx)
if err != nil {
return err
}
_, err := i.store.Pool().Exec(ctx, `update build_request set outcome_unknown = $2 where id = $1`, id, why)
return err
}
// outcomeHeard says a build of this id was recorded in the table: its outcome was taken in.
func (i *Inventory) outcomeHeard(ctx context.Context, id string) (bool, error) {
var heard bool
err := i.store.Pool().QueryRow(ctx, `select exists (select 1 from build where id = $1)`, id).Scan(&heard)
return heard, err
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.OutcomeUnknown = why })
}
// RequestsNamed is every build request kept whose directory names the module, or whose build said it built
@@ -185,46 +140,28 @@ func (i *Inventory) RequestedNames(ctx context.Context) ([]string, error) {
}
func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainRequests(ctx, c)
}
rows, err := i.store.Pool().Query(ctx,
`select a.id, a.repository, a.seat, a.source_path, a.ref, a.commit_hash, a.asked_for,
coalesce(a.not_asked, ''), coalesce(a.outcome_unknown, ''), a.asked_at,
b.id is not null, coalesce(b.failed, ''), coalesce(b.module, ''), coalesce(b.at, a.asked_at)
from build_request a left join build b on b.id = a.id
order by a.asked_at desc, a.id desc`)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
var out []RequestOutcome
for rows.Next() {
var a RequestOutcome
if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked,
&a.OutcomeUnknown, &a.At, &a.Heard, &a.Failed, &a.Module, &a.HeardAt); err != nil {
return nil, err
}
out = append(out, a)
}
return out, rows.Err()
return i.chainRequests(ctx, c)
}
// The states of a pending assignment.
const (
PendingWaiting = "waiting"
PendingApplying = "applying"
PendingApplied = "applied"
PendingRefused = "refused"
PendingExpired = "expired"
PendingWithdrawn = "withdrawn"
PendingWaiting = chain.PendingWaiting
PendingApplying = chain.PendingApplying
PendingApplied = chain.PendingApplied
PendingRefused = chain.PendingRefused
PendingExpired = chain.PendingExpired
PendingWithdrawn = chain.PendingWithdrawn
)
// PendingAssignment is an assignment kept until its module is registered (ADR 0261).
// PendingAssignment is an assignment kept until its module is registered (ADR 0261): the value of its key in the
// assignments bucket while it waits (state pending), and the record carried into the value that stands once it is made.
type PendingAssignment struct {
ID int64
// ID is p-<unix nanoseconds>, made when it was recorded.
ID string
Node string
Module string
// Build is the build it waits for; Repository and Path where that build is from.
@@ -247,195 +184,348 @@ func (p PendingAssignment) Open() bool {
// ErrAlreadyPending is a second pending assignment of one module to one machine.
var ErrAlreadyPending = errors.New("that assignment is already waiting for its build")
const pendingColumns = `p.id, n.name, p.module, p.build_id, p.repository, p.source_path, p.since, p.state, p.note,
p.settled_at, p.raised_at, p.acknowledged_at, p.cleared_at`
// ErrOnTheMachine is a pending assignment of a module that is on the machine already.
var ErrOnTheMachine = errors.New("that module is on the machine already")
func scanPending(rows pgx.Rows) ([]PendingAssignment, error) {
defer rows.Close()
var out []PendingAssignment
for rows.Next() {
var p PendingAssignment
if err := rows.Scan(&p.ID, &p.Node, &p.Module, &p.Build, &p.Repository, &p.Path, &p.Since, &p.State,
&p.Note, &p.Settled, &p.Raised, &p.Acknowledged, &p.Cleared); err != nil {
return nil, err
}
out = append(out, p)
}
return out, rows.Err()
func pendingOf(v chain.Assignment) PendingAssignment { return pendingRecord(v, *v.Pending) }
// pendingRecord is one wait of a key, the one it holds or one it replaced.
func pendingRecord(v chain.Assignment, p chain.Pending) PendingAssignment {
return PendingAssignment{ID: p.ID, Node: v.Node, Module: v.Module, Build: p.Build, Repository: p.Repository, Path: p.Path,
Since: p.Since, State: p.State, Note: p.Note, Settled: p.Settled, Raised: p.Raised, Acknowledged: p.Acknowledged,
Cleared: p.Cleared}
}
// RecordPending keeps an assignment until its module is registered. One is open per machine and module.
// pendingValues is every value of the bucket that holds a pending assignment, on a machine the store knows (a removed
// machine's is none, as the node's foreign key made it), with its key and revision.
func (i *Inventory) pendingValues(ctx context.Context) ([]chain.Kept[chain.Assignment], error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
all, err := c.Assignments.All(ctx)
if err != nil {
return nil, fmt.Errorf("the pending assignments cannot be read from the bus: %w", err)
}
nodes, err := i.nodeNames(ctx)
if err != nil {
return nil, err
}
var out []chain.Kept[chain.Assignment]
for key, k := range all {
if k.Value.Pending != nil && nodes[k.Value.Node] {
k.Key = key
out = append(out, k)
}
}
sort.Slice(out, func(a, b int) bool {
x, y := out[a].Value.Pending, out[b].Value.Pending
if !x.Since.Equal(y.Since) {
return x.Since.Before(y.Since)
}
return x.ID < y.ID
})
return out, nil
}
func pendingList(kept []chain.Kept[chain.Assignment], keep func(chain.Pending) bool) []PendingAssignment {
var out []PendingAssignment
for _, k := range kept {
for _, p := range pendingWaits(k.Value) {
if keep(p) {
out = append(out, pendingRecord(k.Value, p))
}
}
}
sort.SliceStable(out, func(a, b int) bool {
if !out[a].Since.Equal(out[b].Since) {
return out[a].Since.Before(out[b].Since)
}
return out[a].ID < out[b].ID
})
return out
}
// pendingWaits is the wait a value holds and the earlier ones it carries, each without the others.
func pendingWaits(v chain.Assignment) []chain.Pending {
if v.Pending == nil {
return nil
}
out := make([]chain.Pending, 0, 1+len(v.Pending.Earlier))
for _, e := range v.Pending.Earlier {
e.Earlier = nil
out = append(out, e)
}
held := *v.Pending
held.Earlier = nil
return append(out, held)
}
// changePending changes the pending assignment of an id — the wait a key holds or one it carries from before — by
// compare-and-set on the key that holds it, read again when it moved: false when no key holds that id now, or change says no.
func (i *Inventory) changePending(ctx context.Context, id string, change func(p *chain.Pending) bool) (bool, error) {
c, err := i.theChain(ctx)
if err != nil {
return false, err
}
for try := 0; try < casTries; try++ {
held, err := i.pendingValues(ctx)
if err != nil {
return false, err
}
var at *chain.Kept[chain.Assignment]
var target *chain.Pending
for k := range held {
v := held[k].Value
p := *v.Pending
p.Earlier = append([]chain.Pending(nil), p.Earlier...)
v.Pending = &p
switch {
case p.ID == id:
at, target = &held[k], v.Pending
default:
for e := range v.Pending.Earlier {
if v.Pending.Earlier[e].ID == id {
at, target = &held[k], &v.Pending.Earlier[e]
}
}
}
if at != nil {
held[k].Value = v
break
}
}
if at == nil {
return false, nil
}
if !change(target) {
return false, nil
}
if _, err := c.Assignments.Update(ctx, at.Key, at.Value, at.Revision); errors.Is(err, chain.ErrMoved) {
continue
} else if err != nil {
return false, err
}
return true, nil
}
return false, fmt.Errorf("the pending assignment %s kept moving under the write", id)
}
// RecordPending keeps an assignment until its module is registered, as the value of its key (state pending). One is open per
// machine and module; a key that holds the module on the machine refuses it, and one that holds an ended assignment or a
// settled wait is written over.
func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (PendingAssignment, error) {
node, err := i.NodeByName(ctx, p.Node)
if err != nil {
return PendingAssignment{}, err
}
err = i.store.Pool().QueryRow(ctx,
`insert into pending_assignment (node, module, build_id, repository, source_path)
values ($1, $2, $3, $4, $5)
on conflict (node, module) where state in ('waiting', 'applying') do nothing
returning id, since`,
node.ID, p.Module, p.Build, p.Repository, strings.Trim(p.Path, "/")).Scan(&p.ID, &p.Since)
if errors.Is(err, pgx.ErrNoRows) {
return PendingAssignment{}, ErrAlreadyPending
}
c, err := i.theChain(ctx)
if err != nil {
return PendingAssignment{}, err
}
p.State = PendingWaiting
return p, nil
key := assignmentKey(node.Name, p.Module)
for try := 0; try < casTries; try++ {
cur, found, err := c.Assignments.Get(ctx, key)
if err != nil {
return PendingAssignment{}, err
}
switch {
case found && cur.Value.Stands():
return PendingAssignment{}, fmt.Errorf("%w: %s on %s", ErrOnTheMachine, p.Module, node.Name)
case found && cur.Value.Pending != nil && cur.Value.State == chain.StatePending && cur.Value.Pending.Open():
return PendingAssignment{}, ErrAlreadyPending
}
since := time.Now().UTC()
p.ID, p.Since, p.State = fmt.Sprintf("p-%d", since.UnixNano()), since, PendingWaiting
p.Path = strings.Trim(p.Path, "/")
value := chain.Assignment{Node: node.Name, Module: p.Module, State: chain.StatePending, Pending: &chain.Pending{
ID: p.ID, State: PendingWaiting, Build: p.Build, Repository: p.Repository, Path: p.Path, Since: since,
Earlier: unansweredWaits(cur.Value, found)}}
if found {
_, err = c.Assignments.Update(ctx, key, value, cur.Revision)
} else {
_, err = c.Assignments.Create(ctx, key, value)
}
if errors.Is(err, chain.ErrMoved) || errors.Is(err, chain.ErrExists) {
continue
}
if err != nil {
return PendingAssignment{}, err
}
return p, nil
}
return PendingAssignment{}, fmt.Errorf("the assignment of %s to %s kept moving under the write", p.Module, node.Name)
}
// RepointPending makes a waiting pending assignment wait for another build. False when it no longer waits.
func (i *Inventory) RepointPending(ctx context.Context, id int64, build, repository, sourcePath string) (bool, error) {
tag, err := i.store.Pool().Exec(ctx,
`update pending_assignment set build_id = $2, repository = $3, source_path = $4
where id = $1 and state = 'waiting'`, id, build, repository, strings.Trim(sourcePath, "/"))
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
func (i *Inventory) RepointPending(ctx context.Context, id string, build, repository, sourcePath string) (bool, error) {
return i.changePending(ctx, id, func(p *chain.Pending) bool {
if p.State != PendingWaiting {
return false
}
p.Build, p.Repository, p.Path = build, repository, strings.Trim(sourcePath, "/")
return true
})
}
// Pending is every pending assignment still waiting, oldest first, when endedSince is zero. Otherwise it is
// every open one and every one ended since then.
func (i *Inventory) Pending(ctx context.Context, endedSince time.Time) ([]PendingAssignment, error) {
if endedSince.IsZero() {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.state = 'waiting' order by p.since, p.id`)
if err != nil {
return nil, err
}
return scanPending(rows)
}
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.state in ('waiting', 'applying') or p.settled_at >= $1
order by p.since, p.id`, endedSince)
held, err := i.pendingValues(ctx)
if err != nil {
return nil, err
}
return scanPending(rows)
if endedSince.IsZero() {
return pendingList(held, func(p chain.Pending) bool { return p.State == PendingWaiting }), nil
}
return pendingList(held, func(p chain.Pending) bool {
return p.Open() || (p.Settled != nil && !p.Settled.Before(endedSince))
}), nil
}
// PendingFor is every pending assignment of one module to one machine, newest first.
// PendingFor is every pending assignment of one module to one machine the key still has to say, newest first: the wait it
// holds now and the unanswered ones it replaced.
func (i *Inventory) PendingFor(ctx context.Context, node, module string) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where n.name = $1 and p.module = $2 order by p.since desc, p.id desc`, node, module)
held, err := i.pendingValues(ctx)
if err != nil {
return nil, err
}
return scanPending(rows)
var out []PendingAssignment
for _, k := range held {
if k.Value.Node == node && k.Value.Module == module {
for _, w := range pendingWaits(k.Value) {
out = append(out, pendingRecord(k.Value, w))
}
}
}
// Newest first.
sort.SliceStable(out, func(a, b int) bool { return out[a].Since.After(out[b].Since) })
return out, nil
}
// Stuck is every pending assignment left applying since before the time given: a controller that claimed
// it and stopped before it said what came of it.
func (i *Inventory) Stuck(ctx context.Context, before time.Time) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.state = 'applying' and p.settled_at < $1 order by p.id`, before)
held, err := i.pendingValues(ctx)
if err != nil {
return nil, err
}
return scanPending(rows)
return pendingList(held, func(p chain.Pending) bool {
return p.State == PendingApplying && p.Settled != nil && p.Settled.Before(before)
}), nil
}
// ToRaise is every pending assignment that expired or was refused and has not been raised as a condition;
// ToClear every one raised and not cleared.
func (i *Inventory) ToRaise(ctx context.Context) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.state in ('expired', 'refused') and p.raised_at is null order by p.id`)
held, err := i.pendingValues(ctx)
if err != nil {
return nil, err
}
return scanPending(rows)
return pendingList(held, func(p chain.Pending) bool {
return (p.State == PendingExpired || p.State == PendingRefused) && p.Raised == nil
}), nil
}
func (i *Inventory) ToClear(ctx context.Context) ([]PendingAssignment, error) {
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
from pending_assignment p join node n on n.id = p.node
where p.raised_at is not null and p.cleared_at is null order by p.id`)
held, err := i.pendingValues(ctx)
if err != nil {
return nil, err
}
return scanPending(rows)
return pendingList(held, func(p chain.Pending) bool { return p.Raised != nil && p.Cleared == nil }), nil
}
// MarkPending stamps a pending assignment raised, acknowledged or cleared, now.
func (i *Inventory) MarkPending(ctx context.Context, id int64, what string) error {
column := map[string]string{"raised": "raised_at", "acknowledged": "acknowledged_at", "cleared": "cleared_at"}[what]
if column == "" {
func (i *Inventory) MarkPending(ctx context.Context, id string, what string) error {
if what != "raised" && what != "acknowledged" && what != "cleared" {
return errors.New("a pending assignment is marked raised, acknowledged or cleared")
}
_, err := i.store.Pool().Exec(ctx,
`update pending_assignment set `+column+` = coalesce(`+column+`, now()) where id = $1`, id)
now := time.Now().UTC()
_, err := i.changePending(ctx, id, func(p *chain.Pending) bool {
mark := map[string]**time.Time{"raised": &p.Raised, "acknowledged": &p.Acknowledged, "cleared": &p.Cleared}[what]
if *mark != nil {
return false // the first stamp stands
}
*mark = &now
return true
})
return err
}
// ClaimPending takes a waiting pending assignment for making, so nothing else settles or withdraws it while
// it is made. False when it no longer waits: another process took it, or it was withdrawn.
func (i *Inventory) ClaimPending(ctx context.Context, id int64) (bool, error) {
tag, err := i.store.Pool().Exec(ctx,
`update pending_assignment set state = 'applying', settled_at = now() where id = $1 and state = 'waiting'`, id)
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
func (i *Inventory) ClaimPending(ctx context.Context, id string) (bool, error) {
now := time.Now().UTC()
return i.changePending(ctx, id, func(p *chain.Pending) bool {
if !chain.CanMove(p.State, PendingApplying) {
return false
}
p.State, p.Settled = PendingApplying, &now
return true
})
}
// ReleaseClaim puts a claimed pending assignment back to waiting.
func (i *Inventory) ReleaseClaim(ctx context.Context, id int64) error {
_, err := i.store.Pool().Exec(ctx,
`update pending_assignment set state = 'waiting', settled_at = null where id = $1 and state = 'applying'`, id)
func (i *Inventory) ReleaseClaim(ctx context.Context, id string) error {
_, err := i.changePending(ctx, id, func(p *chain.Pending) bool {
if p.State != PendingApplying {
return false
}
p.State, p.Settled = PendingWaiting, nil
return true
})
return err
}
// SettlePending ends a pending assignment in the state it is read in (from): waiting for an expiry or a
// withdrawal, applying for the outcome of making it. False when it was no longer in that state.
func (i *Inventory) SettlePending(ctx context.Context, id int64, from, state, note string) (bool, error) {
func (i *Inventory) SettlePending(ctx context.Context, id string, from, state, note string) (bool, error) {
if state == PendingWaiting || state == PendingApplying {
return false, errors.New("a pending assignment is settled to applied, refused, expired or withdrawn")
}
tag, err := i.store.Pool().Exec(ctx,
`update pending_assignment set state = $3, note = $4, settled_at = now()
where id = $1 and state = $2`, id, from, state, note)
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
now := time.Now().UTC()
return i.changePending(ctx, id, func(p *chain.Pending) bool {
if p.State != from || !chain.CanSettle(from, state) {
return false
}
p.State, p.Note, p.Settled = state, note, &now
return true
})
}
// ForgetEndedPending deletes ended pending assignments settled more than KeptFor ago, and says how many. One
// raised as a condition is kept until that condition is cleared: deleting it would leave the condition
// with nothing to answer it.
func (i *Inventory) ForgetEndedPending(ctx context.Context, now time.Time) (int64, error) {
tag, err := i.store.Pool().Exec(ctx,
`delete from pending_assignment where state not in ('waiting', 'applying') and settled_at < $1
and (raised_at is null or cleared_at is not null)`,
now.Add(-KeptFor))
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
// PendingKnown says which of these pending assignments are still on record.
func (i *Inventory) PendingKnown(ctx context.Context, ids []int64) (map[int64]bool, error) {
rows, err := i.store.Pool().Query(ctx, `select id from pending_assignment where id = any($1)`, ids)
// PendingKnown says which of these pending assignments are still on record: a key holds them now, or carries them as
// unanswered earlier waits.
func (i *Inventory) PendingKnown(ctx context.Context, ids []string) (map[string]bool, error) {
held, err := i.pendingValues(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[int64]bool{}
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
}
out[id] = true
want := map[string]bool{}
for _, id := range ids {
want[id] = true
}
return out, rows.Err()
out := map[string]bool{}
for _, k := range held {
for _, w := range pendingWaits(k.Value) {
if want[w.ID] {
out[w.ID] = true
}
}
}
return out, nil
}
// unansweredWaits are the waits a key holds that ended unmade and whose condition nobody has answered yet — taken back or
// cleared — which a new wait replacing them carries, so the operator is not left without it (issue 1108).
func unansweredWaits(v chain.Assignment, found bool) []chain.Pending {
if !found || v.State != chain.StatePending {
return nil
}
var out []chain.Pending
for _, w := range pendingWaits(v) {
if (w.State == PendingExpired || w.State == PendingRefused) && w.Cleared == nil && w.Acknowledged == nil {
out = append(out, w)
}
}
return out
}
internal/inventory/pin_completed_test.go
-88
View File
@@ -1,88 +0,0 @@
//go:build integration
package inventory
import (
"context"
"os"
"testing"
)
// A pin made before it named the module (migration 0051, novox/hq #258): completed where the node it
// names answers once, left for a person where it answers twice.
func legacyPin(t *testing.T, inv *Inventory, node, provision, provider string) {
t.Helper()
_, err := inv.store.Pool().Exec(context.Background(),
`insert into provision_pin (node, name, provider)
select u.id, $2, p.id from node u, node p where u.name = $1 and p.name = $3`,
node, provision, provider)
if err != nil {
t.Fatal(err)
}
}
func completeEarlierPins(t *testing.T, inv *Inventory) {
t.Helper()
sql, err := os.ReadFile("migrations/0051-a-pin-made-before-is-completed.sql")
if err != nil {
t.Fatal(err)
}
if _, err := inv.store.Pool().Exec(context.Background(), string(sql)); err != nil {
t.Fatal(err)
}
}
func TestAPinMadeBeforeIsCompletedWhenTheNodeAnswersOnce(t *testing.T) {
inv := inTheTables(t)
ctx := context.Background()
for _, n := range []string{"user", "provider"} {
if _, err := inv.AddNode(ctx, n); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"postgres", "redis"} {
if err := inv.RegisterModule(ctx, manifest(m, []string{m + "-database"}, nil), Source{}); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "provider", m); err != nil {
t.Fatal(err)
}
}
legacyPin(t, inv, "user", "postgres-database", "provider")
completeEarlierPins(t, inv)
pins, err := inv.PinsFor(ctx, "user")
if err != nil {
t.Fatal(err)
}
if got := pins["postgres-database"]; got.Node != "provider" || got.Module != "postgres" {
t.Fatalf("the record was not completed with the one module that answers: %+v", got)
}
}
func TestAPinMadeBeforeIsLeftOpenWhenTheNodeAnswersTwice(t *testing.T) {
inv := inTheTables(t)
ctx := context.Background()
for _, n := range []string{"user", "provider"} {
if _, err := inv.AddNode(ctx, n); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"public-acme", "step-ca"} {
if err := inv.RegisterModule(ctx, manifest(m, []string{"acme-ca"}, nil), Source{}); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "provider", m); err != nil {
t.Fatal(err)
}
}
legacyPin(t, inv, "user", "acme-ca", "provider")
completeEarlierPins(t, inv)
pins, err := inv.PinsFor(ctx, "user")
if err != nil {
t.Fatal(err)
}
if got := pins["acme-ca"]; got.Node != "provider" || got.Module != "" {
t.Fatalf("a node that answers twice was guessed for: %+v", got)
}
}
internal/inventory/plans.go
+21 -243
View File
@@ -2,15 +2,12 @@ package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/novox/mesh-controller/internal/chain"
)
@@ -680,140 +677,12 @@ type MergeMove struct {
// leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by
// somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once
// the transaction is committed, in order.
func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) (err error) {
if c, on, cerr := i.ChainReads(ctx, KindRollouts); cerr != nil {
return cerr
} else if on {
return i.chainSavePlans(ctx, c, plans, moves)
}
epoch, err := i.actingEpoch(ctx)
if err != nil {
if len(plans) > 0 {
return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err)
}
return err
}
// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
// measure one twice.
tx, err := i.store.Pool().Begin(ctx)
func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error {
c, err := i.theChain(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
type written struct {
revision int64
entered time.Time
}
done := make([]written, len(plans))
// **The save that starts or ends a walk is marked** (novox/hq ADR 0308 decision 1), in the same write, and
// unmarked should the write not be made, so the next save marks it again.
marked := time.Now()
undo := make([]func(), 0, len(plans))
defer func() {
if err != nil {
for _, u := range undo {
u()
}
}
}()
for _, p := range plans {
undo = append(undo, p.markWalk(marked))
}
for n, p := range plans {
revision, entered, err := writePlan(ctx, tx, p, epoch)
if err != nil {
return err
}
done[n] = written{revision, entered}
}
for _, m := range moves {
if _, err := tx.Exec(ctx,
`update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`,
m.Repository, m.Commit, m.Plan, m.Alone); err != nil {
return err
}
}
if err := tx.Commit(ctx); err != nil {
return err
}
for n, p := range plans {
p.Revision, p.TierEntered = done[n].revision, done[n].entered
if epoch != nil {
p.Epoch = uint64(*epoch)
} else {
p.Epoch = 0
}
if PlanSaved != nil {
PlanSaved(i, *p)
}
}
return nil
}
// writePlan writes one plan within a transaction, by compare-and-set on its revision: its new revision, and when
// it entered its tier.
func writePlan(ctx context.Context, tx pgx.Tx, p *Plan, epoch *int64) (int64, time.Time, error) {
tiers, err := json.Marshal(p.Tiers)
if err != nil {
return 0, time.Time{}, err
}
modules, err := json.Marshal(p.Modules)
if err != nil {
return 0, time.Time{}, err
}
var release, delivery, commits, times, backups []byte
if p.Backups != nil {
if backups, err = json.Marshal(p.Backups); err != nil {
return 0, time.Time{}, err
}
}
if p.Times != nil {
if times, err = json.Marshal(p.Times); err != nil {
return 0, time.Time{}, err
}
}
if len(p.Commits) > 0 {
if commits, err = json.Marshal(p.Commits); err != nil {
return 0, time.Time{}, err
}
}
if p.Release != nil {
if release, err = json.Marshal(p.Release); err != nil {
return 0, time.Time{}, err
}
}
if p.Delivery != nil {
if delivery, err = json.Marshal(p.Delivery); err != nil {
return 0, time.Time{}, err
}
}
entered, err := planTierLeft(ctx, tx, *p, time.Now())
if err != nil {
return 0, time.Time{}, err
}
var revision int64
err = tx.QueryRow(ctx,
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note,
branch, tier_entered, revision, epoch, release, delivery, merged_at, commits, times, backups)
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14, $15, $16, $17, $18, $19)
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch,
release = excluded.release, delivery = excluded.delivery, repository = excluded.repository,
commit_hash = excluded.commit_hash, merged_at = excluded.merged_at, commits = excluded.commits,
created = excluded.created, times = excluded.times, backups = excluded.backups
where release_plan.revision = $12
returning revision`,
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered,
p.Revision, epoch, release, delivery, mergedAt(p.Merged), commits, times, backups).Scan(&revision)
if errors.Is(err, pgx.ErrNoRows) {
// The row is there and at another revision — moved since this was read, or there already
// when this one is new: either way not this writer's to overwrite. (A plan saved before plans
// had revisions is at zero, and its first save here is from a read at zero.)
return 0, time.Time{}, fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
}
return revision, entered, err
return i.chainSavePlans(ctx, c, plans, moves)
}
// short is a commit as a person reads it.
@@ -824,145 +693,54 @@ func short(commit string) string {
return commit
}
// mergedAt is a plan's merge time as the store keeps it: null when not known.
func mergedAt(t time.Time) *time.Time {
if t.IsZero() {
return nil
}
return &t
}
// OpenPlans is every plan still being worked, oldest first.
func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
}
return i.plans(ctx, `where state in ('building', 'rolling') order by created`)
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
}
// PlansSince is every plan made after a moment, and every plan still being worked, oldest first.
func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated)
}
return i.plans(ctx, `where created > $1 or state in ('building', 'rolling') order by created`, since)
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated)
}
// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest)
if err == nil && limit >= 0 && len(plans) > limit {
plans = plans[:limit]
}
return plans, err
}
return i.plans(ctx, `order by created desc limit $1`, limit)
plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest)
if err == nil && limit >= 0 && len(plans) > limit {
plans = plans[:limit]
}
return plans, err
}
// NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh
// planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it
// recorded when its merge was made.
func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
return Plan{}, false, err
} else if on {
return i.chainNewestMergeOf(ctx, c, repository, branch)
}
plans, err := i.plans(ctx, `where lower(repository) = lower($1) and branch = $2 and merged_at is not null
and release is null order by merged_at desc, created desc limit 1`, repository, branch)
if err != nil || len(plans) == 0 {
c, err := i.theChain(ctx)
if err != nil {
return Plan{}, false, err
}
return plans[0], true, nil
return i.chainNewestMergeOf(ctx, c, repository, branch)
}
// PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a
// caller of the plans verb.
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
return Plan{}, err
} else if on {
return i.chainPlanByID(ctx, c, id)
}
plans, err := i.plans(ctx, `where id = $1`, id)
c, err := i.theChain(ctx)
if err != nil {
return Plan{}, err
}
if len(plans) == 0 {
return Plan{}, fmt.Errorf("no plan %s", id)
}
return plans[0], nil
}
func (i *Inventory) plans(ctx context.Context, tail string, args ...any) ([]Plan, error) {
rows, err := i.store.Pool().Query(ctx,
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery, merged_at, commits, times, backups
from release_plan `+tail, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Plan
for rows.Next() {
var p Plan
var tiers, modules, release, delivery, commits, times, backups []byte
var epoch int64
var merged *time.Time
if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch, &release,
&delivery, &merged, &commits, &times, &backups); err != nil {
return nil, err
}
if len(backups) > 0 {
if err := json.Unmarshal(backups, &p.Backups); err != nil {
return nil, err
}
}
if len(times) > 0 {
if err := json.Unmarshal(times, &p.Times); err != nil {
return nil, err
}
}
if len(commits) > 0 {
if err := json.Unmarshal(commits, &p.Commits); err != nil {
return nil, err
}
}
if merged != nil {
p.Merged = merged.UTC()
}
if len(release) > 0 {
if err := json.Unmarshal(release, &p.Release); err != nil {
return nil, err
}
}
if len(delivery) > 0 {
if err := json.Unmarshal(delivery, &p.Delivery); err != nil {
return nil, err
}
}
p.Epoch = uint64(epoch)
if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
return nil, err
}
if err := json.Unmarshal(modules, &p.Modules); err != nil {
return nil, err
}
if p.Modules == nil {
p.Modules = map[string]*PlanModule{}
}
out = append(out, p)
}
if errors.Is(rows.Err(), pgx.ErrNoRows) {
return nil, nil
}
return out, rows.Err()
return i.chainPlanByID(ctx, c, id)
}
internal/inventory/sends.go
+8 -15
View File
@@ -2,7 +2,6 @@ package inventory
import (
"context"
"encoding/json"
"fmt"
"time"
@@ -150,32 +149,26 @@ func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, bui
e := int64(epoch)
sentEpoch = &e
}
var carried *string
if builds != nil {
raw, err := json.Marshal(builds)
if err != nil {
return err
}
text := string(raw)
carried = &text
}
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
if _, err := tx.Exec(ctx,
`update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4, sent_generation = $5,
sent_provisioning = $6::jsonb
where id = $1`, node, digest, carried, sentEpoch, nullIfZero(generation), nullIfEmpty(s.Provisioning)); err != nil {
`update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3, sent_generation = $4,
sent_provisioning = $5::jsonb
where id = $1`, node, digest, sentEpoch, nullIfZero(generation), nullIfEmpty(s.Provisioning)); err != nil {
return err
}
recordErr := recordSend(ctx, tx, node, s)
if err := tx.Commit(ctx); err != nil {
return err
}
// And what it put on the machine, in the assignments of the modules it carried a build of (issue 1095).
saidRunsNot(node, i.recordRuns(ctx, node, builds, time.Now()))
// And what it put on the machine, in the assignments (issues 1095, 1109): a write of it that fails fails this record,
// for what a machine was sent is read from there.
if err := i.recordRuns(ctx, node, builds, time.Now()); err != nil {
return fmt.Errorf("%s was sent its declaration, and what each module of it runs there could not be written: %w", node, err)
}
if recordErr != nil {
return &SendNotRecordedError{Err: recordErr}
}
internal/inventory/sent_builds_test.go
+3 -1
View File
@@ -27,8 +27,10 @@ func TestASendKeepsTheBuildsItCarried(t *testing.T) {
if err := inv.RecordSent(ctx, node.ID, "d1", carried); err != nil {
t.Fatal(err)
}
// What it was sent is read from the keys of the machine's assignments that name a commit (issue 1109): a module carried
// with no commit known names none.
builds, known, err := inv.SentBuilds(ctx, "anchor")
if err != nil || !known || !reflect.DeepEqual(builds, carried) {
if err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"resolver": "abc123"}) {
t.Fatalf("the builds sent were not kept: %v %v %v", builds, known, err)
}
internal/inventory/sent_plans_test.go
+3 -6
View File
@@ -48,10 +48,7 @@ func TestASendKeepsThePlansItHandedAndTheMirrorCarriesThem(t *testing.T) {
t.Fatalf("the plans mirrored are %+v (%v)", plans, err)
}
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
mirror, err := inv.AssignmentsMirror(ctx)
if err != nil {
t.Fatal(err)
}
mirror := inv.PlanValuesForTest(t)
var dunst, web AssignmentMirrored
if err := json.Unmarshal(mirror[node+".dunst"], &dunst); err != nil {
t.Fatal(err)
@@ -81,8 +78,8 @@ func TestASendKeepsThePlansItHandedAndTheMirrorCarriesThem(t *testing.T) {
if sent, _, err := inv.SentTo(ctx, node); err != nil || sent.Provisioned != nil {
t.Fatalf("a send handing nothing says it handed %v (%v)", sent.Provisioned, err)
}
if mirror, err := inv.AssignmentsMirror(ctx); err != nil || strings.Contains(string(mirror[node+".dunst"]), "provisioning") {
t.Fatalf("dunst's assignment still carries a plan: %s (%v)", mirror[node+".dunst"], err)
if mirror := inv.PlanValuesForTest(t); strings.Contains(string(mirror[node+".dunst"]), "provisioning") {
t.Fatalf("dunst's assignment still carries a plan: %s", mirror[node+".dunst"])
} else if _, has := mirror[node+".notify"]; has {
t.Fatalf("a module the machine no longer runs is mirrored: %s", mirror[node+".notify"])
}
internal/inventory/source.go
+12 -85
View File
@@ -2,7 +2,6 @@ package inventory
import (
"context"
"encoding/json"
"github.com/novox/mesh-controller/internal/chain"
)
@@ -35,39 +34,11 @@ type sourceRow struct {
// sourceRows is every successful build of each module (of one module, when named), newest request
// first.
func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][]sourceRow, []string, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, nil, err
} else if on {
return i.chainSourceRows(ctx, c, module)
}
query := `select b.module, b.id, b.commit_hash, b.source_fingerprint, b.made, b.manifest,
coalesce(g.verdict = 'failed', false)
from build b left join build_gate g on g.build = b.id
where b.module is not null and b.module <> '' and b.failed = ''`
args := []any{}
if module != "" {
query += ` and b.module = $1`
args = append(args, module)
}
rows, err := i.store.Pool().Query(ctx, query+` order by b.module, coalesce(b.asked, b.at) desc, b.at desc`, args...)
c, err := i.theChain(ctx)
if err != nil {
return nil, nil, err
}
defer rows.Close()
out := map[string][]sourceRow{}
var order []string
for rows.Next() {
var m string
var r sourceRow
if err := rows.Scan(&m, &r.id, &r.commit, &r.fingerprint, &r.made, &r.manifest, &r.failedGate); err != nil {
return nil, nil, err
}
if _, seen := out[m]; !seen {
order = append(order, m)
}
out[m] = append(out[m], r)
}
return out, order, rows.Err()
return i.chainSourceRows(ctx, c, module)
}
// standing is, of a module's builds newest first, the index of the build that stands for the build at
@@ -102,8 +73,8 @@ func (i *Inventory) StandingBuild(ctx context.Context, module, build string) (st
s := builds[standing(builds, k)]
if len(s.manifest) == 0 && s.manifestDigest != "" {
// The manifest is kept apart from the record (issue 1121): one Get.
c, on, err := i.ChainReads(ctx, KindBuilds)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return "", nil, err
}
raw, err := manifestOfRecord(ctx, c, chain.Build{ID: s.id, ManifestDigest: s.manifestDigest})
@@ -118,76 +89,32 @@ func (i *Inventory) StandingBuild(ctx context.Context, module, build string) (st
// SourceFingerprints is, per module, per commit, the source fingerprint of the newest successful build
// from it that has one: module → commit → fingerprint. A commit whose builds carry none is absent.
func (i *Inventory) SourceFingerprints(ctx context.Context) (map[string]map[string]string, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
// Read from the commit pointers, which carry each commit's source fingerprint: no record is read (issue 1121).
return i.chainSourceFingerprints(ctx, c)
}
all, _, err := i.sourceRows(ctx, "")
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
out := map[string]map[string]string{}
for m, builds := range all {
for _, b := range builds {
if b.fingerprint == "" || b.commit == "" {
continue
}
if out[m] == nil {
out[m] = map[string]string{}
}
if _, seen := out[m][b.commit]; !seen {
out[m][b.commit] = b.fingerprint
}
}
}
return out, nil
// Read from the commit pointers, which carry each commit's source fingerprint: no record is read (issue 1121).
return i.chainSourceFingerprints(ctx, c)
}
// BuildSourceFingerprints is SourceFingerprints for one module: commit → fingerprint.
func (i *Inventory) BuildSourceFingerprints(ctx context.Context, module string) (map[string]string, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
// This module's commit pointers only (issue 1121).
return i.chainBuildSourceFingerprints(ctx, c, module)
}
all, err := i.SourceFingerprints(ctx)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
if all[module] == nil {
return map[string]string{}, nil
}
return all[module], nil
// This module's commit pointers only (issue 1121).
return i.chainBuildSourceFingerprints(ctx, c, module)
}
// heldMade is, per module, what the build standing for its newest successful build made — what Held
// answers (novox/hq issue 280).
func (i *Inventory) heldMade(ctx context.Context) (map[string][]Artifact, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainHeldMade(ctx, c)
}
all, _, err := i.sourceRows(ctx, "")
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
out := map[string][]Artifact{}
for m, builds := range all {
if len(builds) == 0 {
continue
}
var made []Artifact
if err := json.Unmarshal(builds[standing(builds, 0)].made, &made); err != nil {
// Skipped rather than fatal, as Held always did: the build that needs it says it is missing.
continue
}
out[m] = made
}
return out, nil
return i.chainHeldMade(ctx, c)
}
// SameSourceCommits is the commits of the builds in a build's run (novox/hq issue 280): the build and
internal/recordmirror/mirror.go
+1 -1
View File
@@ -60,7 +60,7 @@ var (
)
// Mirrors is every mirror this package writes, in the order they are written.
func Mirrors() []Mirror { return []Mirror{Assignments, DeclaredConfig, Predefined} }
func Mirrors() []Mirror { return []Mirror{DeclaredConfig, Predefined} }
// The changes an event says.
const (
internal/recordmirror/mirror_store_test.go
+19 -77
View File
@@ -12,30 +12,19 @@ import (
"github.com/novox/mesh-controller/internal/inventory"
)
// **Design 57's F3, end to end** (novox/hq issue 652): from the controller's store to a real bus. An
// assignment changes exactly one key of the assignments mirror and raises exactly one `assignment-changed`;
// a registration changes exactly one key of the declared-config mirror and raises exactly one
// `declared-config-changed`; and the same records written again change nothing and say nothing — a module
// registered again unchanged included, although the store's registration time moved.
// **Design 57's F3, end to end** (novox/hq issue 652): from the controller's store to a real bus. A registration
// changes exactly one key of the declared-config mirror and raises exactly one `declared-config-changed`; and the same
// records written again change nothing and say nothing — a module registered again unchanged included, although the
// store's registration time moved. (The assignments are keys of the bus, no mirror of the store: ADR 0408.)
func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *testing.T) {
inv := inventory.ForTest(t)
ctx := t.Context()
js, api := aMirrorBus(t)
assignments := aWriter(t, js, api, Assignments)
declared := aWriter(t, js, api, DeclaredConfig)
heardAssignments := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+Assignments.Event)
heardDeclared := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+DeclaredConfig.Event)
pass := func() (Result, Result) {
pass := func() Result {
t.Helper()
a, err := inv.AssignmentsMirror(ctx)
if err != nil {
t.Fatal(err)
}
ra, err := assignments.Sync(ctx, a)
if err != nil {
t.Fatal(err)
}
d, err := inv.DeclaredConfigMirror(ctx)
if err != nil {
t.Fatal(err)
@@ -44,7 +33,7 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes
if err != nil {
t.Fatal(err)
}
return ra, rd
return rd
}
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
@@ -56,72 +45,37 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes
t.Fatal(err)
}
pass()
assignmentWrites, declaredWrites := writes(t, api, Assignments.Bucket), writes(t, api, DeclaredConfig.Bucket)
assignmentsSaid, declaredSaid := len(settle(heardAssignments, 0)), len(settle(heardDeclared, 1))
declaredWrites := writes(t, api, DeclaredConfig.Bucket)
declaredSaid := len(settle(heardDeclared, 1))
// **A registration**: a module new to the catalogue.
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "notes", Version: "1"}, inventory.Source{}); err != nil {
t.Fatal(err)
}
ra, rd := pass()
if ra != (Result{}) || rd != (Result{Written: 1}) {
t.Errorf("a registration wrote assignments %+v and declared config %+v", ra, rd)
if rd := pass(); rd != (Result{Written: 1}) {
t.Errorf("a registration wrote declared config %+v", rd)
}
if got := writes(t, api, DeclaredConfig.Bucket) - declaredWrites; got != 1 {
t.Errorf("a registration changed %d keys of the declared config", got)
}
if got := writes(t, api, Assignments.Bucket) - assignmentWrites; got != 0 {
t.Errorf("a registration changed %d keys of the assignments", got)
}
said := settle(heardDeclared, declaredSaid+1)[declaredSaid:]
if len(said) != 1 || said[0].Key != "notes" || said[0].Change != Added {
t.Errorf("a registration raised %+v", said)
}
declaredWrites, declaredSaid = writes(t, api, DeclaredConfig.Bucket), declaredSaid+len(said)
// **An assignment.**
if _, err := inv.Assign(ctx, "anchor", "ui"); err != nil {
t.Fatal(err)
}
ra, rd = pass()
if ra != (Result{Written: 1}) || rd != (Result{}) {
t.Errorf("an assignment wrote assignments %+v and declared config %+v", ra, rd)
}
if got := writes(t, api, Assignments.Bucket) - assignmentWrites; got != 1 {
t.Errorf("an assignment changed %d keys of the assignments", got)
}
said = settle(heardAssignments, assignmentsSaid+1)[assignmentsSaid:]
if len(said) != 1 || said[0].Key != "anchor.ui" || said[0].Change != Added {
t.Errorf("an assignment raised %+v", said)
}
assignmentWrites, assignmentsSaid = writes(t, api, Assignments.Bucket), assignmentsSaid+len(said)
e, err := assignments.KV.Get(ctx, "anchor.ui")
if err != nil {
t.Fatal(err)
}
var value inventory.AssignmentMirrored
if err := json.Unmarshal(e.Value(), &value); err != nil || value.Node != "anchor" || value.Module != "ui" ||
value.Assigned.IsZero() {
t.Errorf("the assignment is mirrored as %s (%v)", e.Value(), err)
}
// **The same records written again change nothing**: an assignment made again, and a module registered
// again with the manifest it had.
if _, err := inv.Assign(ctx, "anchor", "ui"); err != nil {
t.Fatal(err)
}
// **The same records written again change nothing**: a module registered again with the manifest it had.
if err := inv.RegisterModule(ctx, ui, inventory.Source{}); err != nil {
t.Fatal(err)
}
if ra, rd := pass(); ra != (Result{}) || rd != (Result{}) {
t.Errorf("unchanged records wrote assignments %+v and declared config %+v", ra, rd)
if rd := pass(); rd != (Result{}) {
t.Errorf("unchanged records wrote declared config %+v", rd)
}
if writes(t, api, Assignments.Bucket) != assignmentWrites || writes(t, api, DeclaredConfig.Bucket) != declaredWrites {
if writes(t, api, DeclaredConfig.Bucket) != declaredWrites {
t.Error("unchanged records changed a key")
}
if a, d := settle(heardAssignments, assignmentsSaid+1), settle(heardDeclared, declaredSaid+1); len(a) != assignmentsSaid ||
len(d) != declaredSaid {
t.Errorf("unchanged records raised %+v and %+v", a[assignmentsSaid:], d[declaredSaid:])
if d := settle(heardDeclared, declaredSaid+1); len(d) != declaredSaid {
t.Errorf("unchanged records raised %+v", d[declaredSaid:])
}
// **A declaration changed by a registration**: one key, one event, naming the field that moved.
@@ -131,8 +85,8 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes
if err := inv.RegisterModule(ctx, ui, inventory.Source{}); err != nil {
t.Fatal(err)
}
if ra, rd := pass(); ra != (Result{}) || rd != (Result{Written: 1}) {
t.Errorf("a changed declaration wrote assignments %+v and declared config %+v", ra, rd)
if rd := pass(); rd != (Result{Written: 1}) {
t.Errorf("a changed declaration wrote declared config %+v", rd)
}
said = settle(heardDeclared, declaredSaid+1)[declaredSaid:]
if len(said) != 1 || said[0].Key != "ui" || said[0].Change != Changed || strings.Join(said[0].Fields, ",") != "settings" {
@@ -146,25 +100,13 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes
t.Errorf("the declaration is mirrored as %s, want %s", got.Value(), want)
}
// **An unassignment** removes its key, said once.
if err := inv.Unassign(ctx, "anchor", "ui"); err != nil {
t.Fatal(err)
}
if ra, _ := pass(); ra != (Result{Removed: 1}) {
t.Errorf("an unassignment did %+v", ra)
}
said = settle(heardAssignments, assignmentsSaid+1)[assignmentsSaid:]
if len(said) != 1 || said[0].Key != "anchor.ui" || said[0].Change != Removed {
t.Errorf("an unassignment raised %+v", said)
}
// **A module no longer registered** has its key removed, said once as `removed` (ADR 0356 §4).
if err := inv.ForgetModule(ctx, "notes"); err != nil {
t.Fatal(err)
}
declaredWrites = writes(t, api, DeclaredConfig.Bucket)
declaredSaid = len(heardDeclared.all())
if _, rd := pass(); rd != (Result{Removed: 1}) {
if rd := pass(); rd != (Result{Removed: 1}) {
t.Errorf("a module forgotten did %+v", rd)
}
if got := writes(t, api, DeclaredConfig.Bucket) - declaredWrites; got != 1 {
testdata/store-tests
Vendored
+15 -9
View File
@@ -139,19 +139,25 @@
./cmd/mesh-controller TestADeliveryGroupMovingAModuleMachinesRunPassesTheGate
./cmd/mesh-controller TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt
./cmd/mesh-controller TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheBus
./internal/inventory TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing
./internal/inventory TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing
./internal/inventory TestTheMoveRefusesAValueOverTheLargestMessage
./internal/inventory TestTheMoveRefusesOnACountThatDiffers
./internal/inventory TestTheMoveWithoutTheLeaseWritesNothing
./internal/inventory TestBuildsOnTheBusAreReadBackByEveryReader
./internal/inventory TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace
./internal/inventory TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain
./internal/inventory TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame
./cmd/mesh-controller TestAMergeWhosePlanCannotBeSavedIsHeldAndRecordedWhenHandedOverAgain
./internal/inventory TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt
./internal/inventory TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves
./internal/inventory TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent
./cmd/mesh-controller TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments
./cmd/mesh-controller TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment
./internal/inventory TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved
./internal/inventory TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket
./internal/inventory TestARecordedPendingIsSkippedByAssignedAndTheJoins
./internal/inventory TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse
./internal/inventory TestAssignOverAPendingKeepsItsRecordAsApplied
./internal/inventory TestAPendingIsMarkedOnceAndGoesWithItsMachine
./internal/inventory TestAReplacedWaitNeverLeavesTheOperatorWithoutItsCondition
./cmd/mesh-controller TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket
./internal/inventory TestSentBuildsAnswersFromTheAssignmentsKeysOfTheNode
./internal/inventory TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown
./internal/inventory TestAFailedWriteOfTheLinksFailsTheRecordOfTheSend
./internal/inventory TestMigration0105RefusesAStoreWhoseRowsWereNotMovedAndDropsTheTablesOfAMovedOne
./internal/inventory TestMigration0105PassesAStoreThatNeverHeldARow
./cmd/mesh-controller TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments
./internal/inventory TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied
./internal/inventory TestASendEmptiesTheColumnSoALaterCopyLeavesAClearedKeyBare