From 88ea361bf6765b94ceff45cb96e9041372677ec2 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 05:17:30 +0200 Subject: [PATCH 01/11] Fold a pending assignment into the value of its assignment key (issue 1094) A wait for a build is the value of . with state pending and a pending object (id p-, a string), so every reader that skips a value with a state skips it, and the controller's own lists skip it through Stands. The transitions are compare-and-set on the key's revision (claim, release, settle, repoint, mark); assigning over a wait carries it into the value that stands, applied when it was waiting; a plan pass never overwrites it; one open wait per machine and module, and a key that holds the module refuses a new one. The table is no longer read or written, ForgetEndedPending is gone (no key is removed), and a wait of a machine the store dropped is none. Callers change type only; their conditions are the next issue. --- cmd/mesh-controller/pending.go | 18 +- .../pending_integration_test.go | 5 +- cmd/mesh-controller/pending_test.go | 4 +- internal/chain/pending_test.go | 63 ++++ internal/chain/records.go | 55 ++- internal/inventory/chain.go | 18 + internal/inventory/chain_assignments.go | 20 +- .../inventory/chain_assignments_key_test.go | 17 +- internal/inventory/chain_pending_test.go | 221 ++++++++++++ internal/inventory/pending.go | 341 +++++++++++------- testdata/store-tests | 5 + 11 files changed, 620 insertions(+), 147 deletions(-) create mode 100644 internal/chain/pending_test.go create mode 100644 internal/inventory/chain_pending_test.go diff --git a/cmd/mesh-controller/pending.go b/cmd/mesh-controller/pending.go index 7743f165..977617bb 100644 --- a/cmd/mesh-controller/pending.go +++ b/cmd/mesh-controller/pending.go @@ -491,8 +491,7 @@ func settlingPending(ctx context.Context, open *stores) { // settlePending is one tick: claims left by a controller that stopped are settled from what the mesh holds; // a module registered meanwhile is assigned; a build heard and not registered, or silent past its bound, -// ends its assignment; ended ones are raised as conditions and cleared once answered; ended rows older than -// KeptFor are deleted. Returns what it did. +// ends its assignment; ended ones are raised as conditions and cleared once answered. Returns what it did. func settlePending(ctx context.Context, open *stores, now time.Time) []string { inv := open.inventory var said []string @@ -565,11 +564,6 @@ func settlePending(ctx context.Context, open *stores, now time.Time) []string { } said = append(said, raisePendingEnded(ctx, inv)...) - if n, err := inv.ForgetEndedPending(ctx, now); err != nil { - say("ended pending assignments could not be deleted: %v", err) - } else if n > 0 { - say("deleted %d pending assignment(s) ended more than %s ago", n, inventory.KeptFor) - } return said } @@ -585,7 +579,7 @@ func containsString(xs []string, x string) bool { // pendingObservation is the condition for a pending assignment that was not made: one per pending // assignment, its row's id the last part of its id, so one row's condition is never another's. func pendingObservation(p inventory.PendingAssignment) conditions.Observation { - return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%d", p.Node, p.Module, p.ID), + return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%s", p.Node, p.Module, p.ID), Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Source: "pending assignments", Summary: p.Note} @@ -671,8 +665,8 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor if err != nil { return []string{fmt.Sprintf("the open conditions could not be read to clear orphaned ones: %v", err)} } - byID := map[int64]string{} - var ids []int64 + byID := map[string]string{} + var ids []string for _, c := range open { if c.Kind != kindPendingEnded { continue @@ -682,8 +676,8 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor if len(parts) < 2 { continue } - var id int64 - if _, err := fmt.Sscan(parts[len(parts)-2], &id); err != nil { + id := parts[len(parts)-2] + if !strings.HasPrefix(id, "p-") { continue } byID[id] = c.Key diff --git a/cmd/mesh-controller/pending_integration_test.go b/cmd/mesh-controller/pending_integration_test.go index bbb10749..5bbf1287 100644 --- a/cmd/mesh-controller/pending_integration_test.go +++ b/cmd/mesh-controller/pending_integration_test.go @@ -637,8 +637,9 @@ func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) { } settlePending(ctx, open, time.Now()) settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] { - t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err) + // No key of the chain is removed (ADR 0408 §4): a cleared wait stays in the value of its key, saying it was cleared. + if known, err := inv.PendingKnown(ctx, []string{row.ID}); err != nil || !known[row.ID] || rowOf(t, open, row.ID).Cleared == nil { + t.Fatalf("a cleared row is not kept, saying so: %v", err) } // On anchor, then anchor removed: the row goes, and its condition with it at the next tick. diff --git a/cmd/mesh-controller/pending_test.go b/cmd/mesh-controller/pending_test.go index d949c02c..c5171bca 100644 --- a/cmd/mesh-controller/pending_test.go +++ b/cmd/mesh-controller/pending_test.go @@ -23,7 +23,7 @@ func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool { return found } -func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment { +func rowOf(t *testing.T, open *stores, id string) inventory.PendingAssignment { t.Helper() rows, err := open.inventory.Pending(t.Context(), time.Unix(0, 0)) if err != nil { @@ -34,7 +34,7 @@ func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment { return r } } - t.Fatalf("pending assignment %d is not on record", id) + t.Fatalf("pending assignment %s is not on record", id) return inventory.PendingAssignment{} } diff --git a/internal/chain/pending_test.go b/internal/chain/pending_test.go new file mode 100644 index 00000000..c0e73f9d --- /dev/null +++ b/internal/chain/pending_test.go @@ -0,0 +1,63 @@ +package chain + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +// **A pending value is not a standing assignment** (ADR 0261, ADR 0408, issue 1094): it names its node and module, holds no +// time assigned, says `state: pending`, and every reader that skips a value with a state skips it. +func TestAPendingValueIsNotAStandingAssignment(t *testing.T) { + since := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) + a := Assignment{Node: "anchor", Module: "notes", State: StatePending, + Pending: &Pending{ID: "p-1", State: PendingWaiting, Build: "build-1", Since: since}} + if a.Stands() { + t.Fatal("a pending value stands") + } + raw, _ := json.Marshal(a) + for _, want := range []string{`"node":"anchor"`, `"module":"notes"`, `"state":"pending"`, `"pending":{"id":"p-1","state":"waiting"`} { + if !strings.Contains(string(raw), want) { + t.Errorf("the value lacks %s: %s", want, raw) + } + } + if strings.Contains(string(raw), `"assigned"`) { + t.Errorf("a pending value names a time assigned: %s", raw) + } + // The wait carried into the value that stands: no state, so it stands, and the record stays. + made := Assignment{Node: "anchor", Module: "notes", Assigned: since, Pending: &Pending{ID: "p-1", State: PendingApplied}} + if !made.Stands() || !a.Pending.Open() || made.Pending.Open() { + t.Fatalf("stands %v, open before %v, open after %v", made.Stands(), a.Pending.Open(), made.Pending.Open()) + } + var back Assignment + if err := json.Unmarshal(raw, &back); err != nil || back.Pending == nil || back.Pending.Build != "build-1" || !back.Pending.Since.Equal(since) { + t.Fatalf("read back %+v: %v", back, err) + } + now := since + raised := Pending{ID: "p-2", State: PendingExpired, Raised: &now, Acknowledged: &now, Cleared: &now} + body, _ := json.Marshal(raised) + if !strings.Contains(string(body), `"raised"`) || !strings.Contains(string(body), `"acknowledged"`) || !strings.Contains(string(body), `"cleared"`) { + t.Errorf("the marks of a wait: %s", body) + } +} + +func TestTheTransitionsOfAPendingAssignment(t *testing.T) { + ok := [][2]string{{PendingWaiting, PendingExpired}, {PendingWaiting, PendingWithdrawn}, {PendingApplying, PendingApplied}, + {PendingApplying, PendingRefused}} + for _, p := range ok { + if !CanSettle(p[0], p[1]) { + t.Errorf("%s may end as %s", p[0], p[1]) + } + } + for _, p := range [][2]string{{PendingWaiting, PendingApplied}, {PendingWaiting, PendingRefused}, {PendingApplying, PendingExpired}, + {PendingApplying, PendingWithdrawn}, {PendingApplied, PendingExpired}, {PendingExpired, PendingWithdrawn}, + {PendingWaiting, PendingWaiting}} { + if CanSettle(p[0], p[1]) { + t.Errorf("%s may not end as %s", p[0], p[1]) + } + } + if !CanMove(PendingWaiting, PendingApplying) || !CanMove(PendingApplying, PendingWaiting) || CanMove(PendingApplied, PendingWaiting) { + t.Error("a claim moves a wait to applying and back") + } +} diff --git a/internal/chain/records.go b/internal/chain/records.go index 709aa833..54facfc1 100644 --- a/internal/chain/records.go +++ b/internal/chain/records.go @@ -189,9 +189,62 @@ type Assignment struct { // Ended is when the assignment ended (an unassignment): absent while it stands. **No key of the chain is removed** // (ADR 0408 §4), so an unassigned module keeps its key, saying so; every reader treats a value with Ended, or with a - // State, as no assignment. State is, in a later delivery, the pending assignment waiting for its build (issue 1094). + // State, as no assignment. State is StatePending while the key holds an assignment waiting for its build, or one that + // was never made (issue 1094). Ended time.Time `json:"ended,omitzero"` State string `json:"state,omitempty"` + // Pending is the assignment's wait for its build (novox/hq ADR 0261), kept in the value of the key it waits for and, when + // it is made, carried into the value that stands: the record of what became of the wait. + Pending *Pending `json:"pending,omitempty"` +} + +// StatePending is the State of a value that holds no assignment but a pending one. +const StatePending = "pending" + +// The states of a pending assignment (novox/hq ADR 0261). +const ( + PendingWaiting = "waiting" + PendingApplying = "applying" + PendingApplied = "applied" + PendingRefused = "refused" + PendingExpired = "expired" + PendingWithdrawn = "withdrawn" +) + +// Pending is an assignment kept until its module is registered: the build it waits for, where that build is from, and the +// times of its claim and its end. Its id is p-, unique for a key, which holds one at a time. +type Pending struct { + ID string `json:"id"` + State string `json:"state"` + Build string `json:"build,omitempty"` + Repository string `json:"repository,omitempty"` + Path string `json:"path,omitempty"` + Since time.Time `json:"since"` + Note string `json:"note,omitempty"` + Settled *time.Time `json:"settled,omitempty"` + Raised *time.Time `json:"raised,omitempty"` + Acknowledged *time.Time `json:"acknowledged,omitempty"` + Cleared *time.Time `json:"cleared,omitempty"` +} + +// Open says the assignment is still to be made. +func (p Pending) Open() bool { return p.State == PendingWaiting || p.State == PendingApplying } + +// CanSettle says a pending assignment in state from may end in state to: a wait expires or is withdrawn, a claim becomes +// applied or refused, and nothing else. +func CanSettle(from, to string) bool { + switch from { + case PendingWaiting: + return to == PendingExpired || to == PendingWithdrawn + case PendingApplying: + return to == PendingApplied || to == PendingRefused + } + return false +} + +// CanMove says a pending assignment in state from may go to state to without ending: claimed, or released again. +func CanMove(from, to string) bool { + return (from == PendingWaiting && to == PendingApplying) || (from == PendingApplying && to == PendingWaiting) } // Stands says the value is an assignment that holds: it has not ended and is not waiting in a state of its own. diff --git a/internal/inventory/chain.go b/internal/inventory/chain.go index 148d22da..a56e784a 100644 --- a/internal/inventory/chain.go +++ b/internal/inventory/chain.go @@ -147,3 +147,21 @@ func (i *Inventory) BeginSwitchForTest(kind string) { i.moved.switching.Store(true) } } + +// theChain is the chain this inventory writes and reads, opened if a command has not yet: for what has no table of its own +// to be read from while it is moved (a pending assignment is a value of the assignments bucket from the start). +func (i *Inventory) theChain(ctx context.Context) (*chain.Chain, error) { + i.moved.mu.Lock() + defer i.moved.mu.Unlock() + if i.chain == nil && i.moved.open != nil { + c, err := i.moved.open(ctx) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrNoChain, err) + } + i.chain = c + } + if i.chain == nil { + return nil, ErrNoChain + } + return i.chain, nil +} diff --git a/internal/inventory/chain_assignments.go b/internal/inventory/chain_assignments.go index 958924e0..6ca7e82c 100644 --- a/internal/inventory/chain_assignments.go +++ b/internal/inventory/chain_assignments.go @@ -175,7 +175,9 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, cur.Value.Assigned = now _, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision) case found: - // One that ended is assigned again: a fresh value over it, the ended one staying in history. + // One that ended is assigned again, and one that was pending is made: a fresh value over it, the earlier one + // staying in history. The wait is carried into the value that stands (issue 1094). + fresh.Pending = carriedPending(cur.Value, now) _, err = c.Assignments.Update(ctx, key, fresh, cur.Revision) default: _, err = c.Assignments.Create(ctx, key, fresh) @@ -384,6 +386,8 @@ func keepStep(has bool, cur, next chain.Assignment) string { return keepSkip // assigned since the bucket was read: its own write is not here yet case !has: return keepCreate + case cur.State == chain.StatePending: + return keepSkip // a wait for a build is not pulled in by a plan case !cur.Stands(): if !cur.Assigned.IsZero() || !next.Assigned.IsZero() { return keepSkip // an assignment that ended is not pulled in again by a plan @@ -394,3 +398,17 @@ func keepStep(has bool, cur, next chain.Assignment) string { } return keepField } + +// carriedPending is the wait a value that does not stand carries into the assignment made over it: a wait that was applying +// stays applying, for the one making it settles it applied; one still waiting is made by hand, and says so. Nothing for a +// value that held no wait, or whose key is not one that was pending. +func carriedPending(was chain.Assignment, now time.Time) *chain.Pending { + if was.Pending == nil || was.State != chain.StatePending { + return nil + } + p := *was.Pending + if p.State == chain.PendingWaiting { + p.State, p.Note, p.Settled = chain.PendingApplied, "assigned by hand", &now + } + return &p +} diff --git a/internal/inventory/chain_assignments_key_test.go b/internal/inventory/chain_assignments_key_test.go index 62e87cc7..839404f1 100644 --- a/internal/inventory/chain_assignments_key_test.go +++ b/internal/inventory/chain_assignments_key_test.go @@ -51,18 +51,21 @@ func TestAPassThatKeepsThePlansNeverRevivesAnEndedAssignment(t *testing.T) { ended := chain.Assignment{Node: "a", Module: "m", Assigned: at, Ended: at.Add(time.Minute)} pulledIn := chain.Assignment{Node: "a", Module: "p", RequiredBy: "m", Provisioning: plan} pulledInEnded := chain.Assignment{Node: "a", Module: "p", Ended: at} + waiting := chain.Assignment{Node: "a", Module: "p", State: chain.StatePending, + Pending: &chain.Pending{ID: "p-1", State: chain.PendingWaiting}} for name, tc := range map[string]struct { has bool cur, next chain.Assignment want string }{ - "the interleaving: wanted assigned, ended since": {true, ended, assigned, keepSkip}, - "wanted assigned, not in the bucket yet": {false, chain.Assignment{}, assigned, keepSkip}, - "a pulled-in module, not in the bucket": {false, chain.Assignment{}, pulledIn, keepCreate}, - "a pulled-in module that ended, pulled again": {true, pulledInEnded, pulledIn, keepFresh}, - "an assignment ended, a plan for it": {true, ended, chain.Assignment{Node: "a", Module: "m", RequiredBy: "x", Provisioning: plan}, keepSkip}, - "the plan is the same": {true, assigned, assigned, keepSkip}, - "the plan changed": {true, chain.Assignment{Node: "a", Module: "m", Assigned: at}, assigned, keepField}, + "the interleaving: wanted assigned, ended since": {true, ended, assigned, keepSkip}, + "wanted assigned, not in the bucket yet": {false, chain.Assignment{}, assigned, keepSkip}, + "a pulled-in module, not in the bucket": {false, chain.Assignment{}, pulledIn, keepCreate}, + "a pulled-in module that ended, pulled again": {true, pulledInEnded, pulledIn, keepFresh}, + "an assignment ended, a plan for it": {true, ended, chain.Assignment{Node: "a", Module: "m", RequiredBy: "x", Provisioning: plan}, keepSkip}, + "a wait for a build, a plan for the module pulled in": {true, waiting, pulledIn, keepSkip}, + "the plan is the same": {true, assigned, assigned, keepSkip}, + "the plan changed": {true, chain.Assignment{Node: "a", Module: "m", Assigned: at}, assigned, keepField}, } { if got := keepStep(tc.has, tc.cur, tc.next); got != tc.want { t.Errorf("%s: %s, want %s", name, got, tc.want) diff --git a/internal/inventory/chain_pending_test.go b/internal/inventory/chain_pending_test.go new file mode 100644 index 00000000..1e996403 --- /dev/null +++ b/internal/inventory/chain_pending_test.go @@ -0,0 +1,221 @@ +//go:build integration + +package inventory + +import ( + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/chain" +) + +// **A pending assignment is the value of the key of the assignment it waits for** (ADR 0261, ADR 0408, issue 1094): state +// pending, the wait in its `pending` object under an id that is a string, no table, and one open wait per machine and module. +func TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket(t *testing.T) { + inv, node := aNodeWithModules(t, "web") + ctx := t.Context() + p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-1", Repository: "novox/mesh-catalog", + Path: "/modules/sensors/"}) + if err != nil { + t.Fatal(err) + } + if want := fmt.Sprintf("p-%d", p.Since.UnixNano()); p.ID != want || p.State != PendingWaiting || p.Path != "modules/sensors" { + t.Fatalf("recorded %+v, want id %s", p, want) + } + kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.sensors") + if err != nil || !found || kept.Value.State != chain.StatePending || kept.Value.Stands() || !kept.Value.Assigned.IsZero() || + kept.Value.Pending == nil || kept.Value.Pending.ID != p.ID || kept.Value.Pending.Build != "build-1" { + t.Fatalf("the key holds %+v (found %v, %v)", kept.Value, found, err) + } + rows, err := inv.Pending(ctx, time.Time{}) + if err != nil || len(rows) != 1 || rows[0].ID != p.ID || rows[0].Node != "anchor" || rows[0].Build != "build-1" { + t.Fatalf("waiting: %+v %v", rows, err) + } + if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"}); !errors.Is(err, ErrAlreadyPending) { + t.Fatalf("a second open wait: %v", err) + } + if _, err := inv.Assign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "build-3"}); !errors.Is(err, ErrOnTheMachine) { + t.Fatalf("a wait for a module that is on the machine: %v", err) + } + // Written over once it ended: a settled wait is replaced by a new one under a new id. + if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil || !ok { + t.Fatalf("settled %v %v", ok, err) + } + next, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "build-2"}) + if err != nil || next.ID == p.ID { + t.Fatalf("a wait after one ended: %+v %v", next, err) + } + if known, _ := inv.PendingKnown(ctx, []string{p.ID, next.ID}); known[p.ID] || !known[next.ID] { + t.Fatalf("known %v: the key holds the newer one now", known) + } + if history, _ := inv.chain.Assignments.History(ctx, "anchor.sensors"); len(history) != 2 { + t.Fatalf("the earlier wait is in the key's history: %d values", len(history)) + } + if n := rowsInTheTable(t, inv, "pending_assignment"); n != "0" { + t.Fatalf("the table took %s pending assignments", n) + } +} + +// **A pending value is skipped by what reads assignments** (issue 1094): the machine is assigned nothing by it, the module +// runs nowhere, and a module only waiting for a build is not forgotten as assigned. +func TestARecordedPendingIsSkippedByAssignedAndTheJoins(t *testing.T) { + inv, node := aNodeWithModules(t, "web") + ctx := t.Context() + if _, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"}); err != nil { + t.Fatal(err) + } + if on, _ := inv.Assigned(ctx, node); len(on) != 0 { + t.Fatalf("assigned %v", on) + } + if running, _ := inv.Running(ctx, "web"); len(running) != 0 { + t.Fatalf("running %v", running) + } + if runs, _ := inv.AssignmentRuns(ctx, node); len(runs) != 0 { + t.Fatalf("runs %v", runs) + } + entries, _ := inv.Catalogued(ctx) + for _, e := range entries { + if e.Manifest.Module == "web" && len(e.On) != 0 { + t.Fatalf("web is on %v", e.On) + } + } + if err := inv.ForgetModule(ctx, "web"); errors.Is(err, ErrStillAssigned) { + t.Fatalf("a module only waiting is assigned: %v", err) + } +} + +// **Two claims of one wait make one, and the other sees false** (issue 1094): by compare-and-set on the key's revision. +func TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"}) + if err != nil { + t.Fatal(err) + } + var won atomic.Int32 + var wg sync.WaitGroup + for k := 0; k < 8; k++ { + wg.Add(1) + go func() { + defer wg.Done() + if ok, err := inv.ClaimPending(ctx, p.ID); err == nil && ok { + won.Add(1) + } + }() + } + wg.Wait() + if won.Load() != 1 { + t.Fatalf("%d claims won", won.Load()) + } + if again, err := inv.ClaimPending(ctx, p.ID); err != nil || again { + t.Fatalf("a claim of a claimed wait: %v %v", again, err) + } + if rows, _ := inv.Stuck(ctx, time.Now().Add(time.Minute)); len(rows) != 1 || rows[0].State != PendingApplying || rows[0].Settled == nil { + t.Fatalf("stuck %+v", rows) + } + if err := inv.ReleaseClaim(ctx, p.ID); err != nil { + t.Fatal(err) + } + if rows, _ := inv.Pending(ctx, time.Time{}); len(rows) != 1 || rows[0].Settled != nil { + t.Fatalf("released %+v", rows) + } + // Only a wait that waits is withdrawn or expired, only a claim is applied. + if ok, _ := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingApplied, "x"); ok { + t.Fatal("a wait was applied") + } + if ok, _ := inv.SettlePending(ctx, p.ID, PendingApplying, PendingRefused, "x"); ok { + t.Fatal("a wait that is not claimed was refused as a claim") + } + if ok, err := inv.RepointPending(ctx, p.ID, "b-2", "novox/other", "x/"); err != nil || !ok { + t.Fatalf("repoint %v %v", ok, err) + } + if rows, _ := inv.Pending(ctx, time.Time{}); rows[0].Build != "b-2" || rows[0].Path != "x" { + t.Fatalf("repointed %+v", rows) + } +} + +// **Assigning over a wait carries the wait into the value that stands** (issue 1094): the claim stays applying for the one +// making it to settle applied, and one still waiting is made by hand, and says so. +func TestAssignOverAPendingKeepsItsRecordAsApplied(t *testing.T) { + inv, node := aNodeWithModules(t, "web", "cache") + ctx := t.Context() + claimed, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "web", Build: "b-1"}) + if ok, err := inv.ClaimPending(ctx, claimed.ID); err != nil || !ok { + t.Fatal(ok, err) + } + if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { + t.Fatalf("assigned %v %v", added, err) + } + standing, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web") + if !standing.Value.Stands() || standing.Value.Assigned.IsZero() || standing.Value.Pending == nil || + standing.Value.Pending.State != PendingApplying { + t.Fatalf("the value that stands: %+v", standing.Value) + } + if ok, err := inv.SettlePending(ctx, claimed.ID, PendingApplying, PendingApplied, "made"); err != nil || !ok { + t.Fatalf("settled %v %v", ok, err) + } + if on, _ := inv.Assigned(ctx, node); len(on) != 1 || on[0] != "web" { + t.Fatalf("assigned %v", on) + } + rows, _ := inv.Pending(ctx, time.Now().Add(-time.Hour)) + if len(rows) != 1 || rows[0].State != PendingApplied || rows[0].Note != "made" { + t.Fatalf("what became of the wait: %+v", rows) + } + + waiting, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "cache", Build: "b-2"}) + if _, err := inv.Assign(ctx, node, "cache"); err != nil { + t.Fatal(err) + } + got, _ := inv.PendingFor(ctx, node, "cache") + if len(got) != 1 || got[0].ID != waiting.ID || got[0].State != PendingApplied || !strings.Contains(got[0].Note, "by hand") { + t.Fatalf("a wait assigned by hand: %+v", got) + } +} + +// **Marks are stamped once, and a wait of a machine the store dropped is no wait** (issue 1094). +func TestAPendingIsMarkedOnceAndGoesWithItsMachine(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + p, _ := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: "b-1"}) + if _, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom"); err != nil { + t.Fatal(err) + } + if rows, _ := inv.ToRaise(ctx); len(rows) != 1 { + t.Fatalf("to raise %+v", rows) + } + if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil { + t.Fatal(err) + } + first, _ := inv.Pending(ctx, time.Now().Add(-time.Hour)) + time.Sleep(2 * time.Millisecond) + if err := inv.MarkPending(ctx, p.ID, "raised"); err != nil { + t.Fatal(err) + } + second, _ := inv.Pending(ctx, time.Now().Add(-time.Hour)) + if !first[0].Raised.Equal(*second[0].Raised) { + t.Fatalf("the first stamp did not stand: %v then %v", first[0].Raised, second[0].Raised) + } + if err := inv.MarkPending(ctx, p.ID, "frobnicated"); err == nil { + t.Fatal("a mark nobody knows") + } + if rows, _ := inv.ToRaise(ctx); len(rows) != 0 { + t.Fatalf("to raise %+v", rows) + } + if rows, _ := inv.ToClear(ctx); len(rows) != 1 { + t.Fatalf("to clear %+v", rows) + } + if _, err := inv.RemoveNodeForTest(ctx, node); err != nil { + t.Fatal(err) + } + if rows, _ := inv.Pending(ctx, time.Unix(0, 0)); len(rows) != 0 { + t.Fatalf("a wait outlived its machine: %+v", rows) + } +} diff --git a/internal/inventory/pending.go b/internal/inventory/pending.go index 7f8258d5..ff41e0f6 100644 --- a/internal/inventory/pending.go +++ b/internal/inventory/pending.go @@ -3,7 +3,9 @@ package inventory import ( "context" "errors" + "fmt" "path" + "sort" "strings" "time" @@ -214,17 +216,19 @@ func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) { // The states of a pending assignment. const ( - PendingWaiting = "waiting" - PendingApplying = "applying" - PendingApplied = "applied" - PendingRefused = "refused" - PendingExpired = "expired" - PendingWithdrawn = "withdrawn" + PendingWaiting = chain.PendingWaiting + PendingApplying = chain.PendingApplying + PendingApplied = chain.PendingApplied + PendingRefused = chain.PendingRefused + PendingExpired = chain.PendingExpired + PendingWithdrawn = chain.PendingWithdrawn ) -// PendingAssignment is an assignment kept until its module is registered (ADR 0261). +// PendingAssignment is an assignment kept until its module is registered (ADR 0261): the value of its key in the +// assignments bucket while it waits (state pending), and the record carried into the value that stands once it is made. type PendingAssignment struct { - ID int64 + // ID is p-, made when it was recorded. + ID string Node string Module string // Build is the build it waits for; Repository and Path where that build is from. @@ -247,195 +251,288 @@ func (p PendingAssignment) Open() bool { // ErrAlreadyPending is a second pending assignment of one module to one machine. var ErrAlreadyPending = errors.New("that assignment is already waiting for its build") -const pendingColumns = `p.id, n.name, p.module, p.build_id, p.repository, p.source_path, p.since, p.state, p.note, - p.settled_at, p.raised_at, p.acknowledged_at, p.cleared_at` +// ErrOnTheMachine is a pending assignment of a module that is on the machine already. +var ErrOnTheMachine = errors.New("that module is on the machine already") -func scanPending(rows pgx.Rows) ([]PendingAssignment, error) { - defer rows.Close() - var out []PendingAssignment - for rows.Next() { - var p PendingAssignment - if err := rows.Scan(&p.ID, &p.Node, &p.Module, &p.Build, &p.Repository, &p.Path, &p.Since, &p.State, - &p.Note, &p.Settled, &p.Raised, &p.Acknowledged, &p.Cleared); err != nil { - return nil, err - } - out = append(out, p) - } - return out, rows.Err() +func pendingOf(v chain.Assignment) PendingAssignment { + p := v.Pending + return PendingAssignment{ID: p.ID, Node: v.Node, Module: v.Module, Build: p.Build, Repository: p.Repository, Path: p.Path, + Since: p.Since, State: p.State, Note: p.Note, Settled: p.Settled, Raised: p.Raised, Acknowledged: p.Acknowledged, + Cleared: p.Cleared} } -// RecordPending keeps an assignment until its module is registered. One is open per machine and module. +// pendingValues is every value of the bucket that holds a pending assignment, on a machine the store knows (a removed +// machine's is none, as the node's foreign key made it), with its key and revision. +func (i *Inventory) pendingValues(ctx context.Context) ([]chain.Kept[chain.Assignment], error) { + c, err := i.theChain(ctx) + if err != nil { + return nil, err + } + all, err := c.Assignments.All(ctx) + if err != nil { + return nil, fmt.Errorf("the pending assignments cannot be read from the bus: %w", err) + } + nodes, err := i.nodeNames(ctx) + if err != nil { + return nil, err + } + var out []chain.Kept[chain.Assignment] + for key, k := range all { + if k.Value.Pending != nil && nodes[k.Value.Node] { + k.Key = key + out = append(out, k) + } + } + sort.Slice(out, func(a, b int) bool { + x, y := out[a].Value.Pending, out[b].Value.Pending + if !x.Since.Equal(y.Since) { + return x.Since.Before(y.Since) + } + return x.ID < y.ID + }) + return out, nil +} + +func pendingList(kept []chain.Kept[chain.Assignment], keep func(chain.Pending) bool) []PendingAssignment { + var out []PendingAssignment + for _, k := range kept { + if keep(*k.Value.Pending) { + out = append(out, pendingOf(k.Value)) + } + } + return out +} + +// changePending changes the pending assignment of an id by compare-and-set on the key that holds it, read again when it +// moved: false when no key holds that id now, or change says no. +func (i *Inventory) changePending(ctx context.Context, id string, change func(v *chain.Assignment) bool) (bool, error) { + c, err := i.theChain(ctx) + if err != nil { + return false, err + } + for try := 0; try < casTries; try++ { + held, err := i.pendingValues(ctx) + if err != nil { + return false, err + } + var at *chain.Kept[chain.Assignment] + for k := range held { + if held[k].Value.Pending.ID == id { + at = &held[k] + } + } + if at == nil { + return false, nil + } + v := at.Value + p := *v.Pending + v.Pending = &p + if !change(&v) { + return false, nil + } + if _, err := c.Assignments.Update(ctx, at.Key, v, at.Revision); errors.Is(err, chain.ErrMoved) { + continue + } else if err != nil { + return false, err + } + return true, nil + } + return false, fmt.Errorf("the pending assignment %s kept moving under the write", id) +} + +// RecordPending keeps an assignment until its module is registered, as the value of its key (state pending). One is open per +// machine and module; a key that holds the module on the machine refuses it, and one that holds an ended assignment or a +// settled wait is written over. func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (PendingAssignment, error) { node, err := i.NodeByName(ctx, p.Node) if err != nil { return PendingAssignment{}, err } - err = i.store.Pool().QueryRow(ctx, - `insert into pending_assignment (node, module, build_id, repository, source_path) - values ($1, $2, $3, $4, $5) - on conflict (node, module) where state in ('waiting', 'applying') do nothing - returning id, since`, - node.ID, p.Module, p.Build, p.Repository, strings.Trim(p.Path, "/")).Scan(&p.ID, &p.Since) - if errors.Is(err, pgx.ErrNoRows) { - return PendingAssignment{}, ErrAlreadyPending - } + c, err := i.theChain(ctx) if err != nil { return PendingAssignment{}, err } - p.State = PendingWaiting - return p, nil + key := assignmentKey(node.Name, p.Module) + for try := 0; try < casTries; try++ { + cur, found, err := c.Assignments.Get(ctx, key) + if err != nil { + return PendingAssignment{}, err + } + switch { + case found && cur.Value.Stands(): + return PendingAssignment{}, fmt.Errorf("%w: %s on %s", ErrOnTheMachine, p.Module, node.Name) + case found && cur.Value.Pending != nil && cur.Value.State == chain.StatePending && cur.Value.Pending.Open(): + return PendingAssignment{}, ErrAlreadyPending + } + since := time.Now().UTC() + p.ID, p.Since, p.State = fmt.Sprintf("p-%d", since.UnixNano()), since, PendingWaiting + p.Path = strings.Trim(p.Path, "/") + value := chain.Assignment{Node: node.Name, Module: p.Module, State: chain.StatePending, Pending: &chain.Pending{ + ID: p.ID, State: PendingWaiting, Build: p.Build, Repository: p.Repository, Path: p.Path, Since: since}} + if found { + _, err = c.Assignments.Update(ctx, key, value, cur.Revision) + } else { + _, err = c.Assignments.Create(ctx, key, value) + } + if errors.Is(err, chain.ErrMoved) || errors.Is(err, chain.ErrExists) { + continue + } + if err != nil { + return PendingAssignment{}, err + } + return p, nil + } + return PendingAssignment{}, fmt.Errorf("the assignment of %s to %s kept moving under the write", p.Module, node.Name) } // RepointPending makes a waiting pending assignment wait for another build. False when it no longer waits. -func (i *Inventory) RepointPending(ctx context.Context, id int64, build, repository, sourcePath string) (bool, error) { - tag, err := i.store.Pool().Exec(ctx, - `update pending_assignment set build_id = $2, repository = $3, source_path = $4 - where id = $1 and state = 'waiting'`, id, build, repository, strings.Trim(sourcePath, "/")) - if err != nil { - return false, err - } - return tag.RowsAffected() == 1, nil +func (i *Inventory) RepointPending(ctx context.Context, id string, build, repository, sourcePath string) (bool, error) { + return i.changePending(ctx, id, func(v *chain.Assignment) bool { + if v.Pending.State != PendingWaiting { + return false + } + v.Pending.Build, v.Pending.Repository, v.Pending.Path = build, repository, strings.Trim(sourcePath, "/") + return true + }) } // Pending is every pending assignment still waiting, oldest first, when endedSince is zero. Otherwise it is // every open one and every one ended since then. func (i *Inventory) Pending(ctx context.Context, endedSince time.Time) ([]PendingAssignment, error) { - if endedSince.IsZero() { - rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` - from pending_assignment p join node n on n.id = p.node - where p.state = 'waiting' order by p.since, p.id`) - if err != nil { - return nil, err - } - return scanPending(rows) - } - rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` - from pending_assignment p join node n on n.id = p.node - where p.state in ('waiting', 'applying') or p.settled_at >= $1 - order by p.since, p.id`, endedSince) + held, err := i.pendingValues(ctx) if err != nil { return nil, err } - return scanPending(rows) + if endedSince.IsZero() { + return pendingList(held, func(p chain.Pending) bool { return p.State == PendingWaiting }), nil + } + return pendingList(held, func(p chain.Pending) bool { + return p.Open() || (p.Settled != nil && !p.Settled.Before(endedSince)) + }), nil } -// PendingFor is every pending assignment of one module to one machine, newest first. +// PendingFor is the pending assignment of one module to one machine: the one its key holds now, which is the newest +// (an earlier one is read from the key's history, which nothing here needs). func (i *Inventory) PendingFor(ctx context.Context, node, module string) ([]PendingAssignment, error) { - rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` - from pending_assignment p join node n on n.id = p.node - where n.name = $1 and p.module = $2 order by p.since desc, p.id desc`, node, module) + held, err := i.pendingValues(ctx) if err != nil { return nil, err } - return scanPending(rows) + var out []PendingAssignment + for _, k := range held { + if k.Value.Node == node && k.Value.Module == module { + out = append(out, pendingOf(k.Value)) + } + } + return out, nil } // Stuck is every pending assignment left applying since before the time given: a controller that claimed // it and stopped before it said what came of it. func (i *Inventory) Stuck(ctx context.Context, before time.Time) ([]PendingAssignment, error) { - rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` - from pending_assignment p join node n on n.id = p.node - where p.state = 'applying' and p.settled_at < $1 order by p.id`, before) + held, err := i.pendingValues(ctx) if err != nil { return nil, err } - return scanPending(rows) + return pendingList(held, func(p chain.Pending) bool { + return p.State == PendingApplying && p.Settled != nil && p.Settled.Before(before) + }), nil } // ToRaise is every pending assignment that expired or was refused and has not been raised as a condition; // ToClear every one raised and not cleared. func (i *Inventory) ToRaise(ctx context.Context) ([]PendingAssignment, error) { - rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` - from pending_assignment p join node n on n.id = p.node - where p.state in ('expired', 'refused') and p.raised_at is null order by p.id`) + held, err := i.pendingValues(ctx) if err != nil { return nil, err } - return scanPending(rows) + return pendingList(held, func(p chain.Pending) bool { + return (p.State == PendingExpired || p.State == PendingRefused) && p.Raised == nil + }), nil } func (i *Inventory) ToClear(ctx context.Context) ([]PendingAssignment, error) { - rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+` - from pending_assignment p join node n on n.id = p.node - where p.raised_at is not null and p.cleared_at is null order by p.id`) + held, err := i.pendingValues(ctx) if err != nil { return nil, err } - return scanPending(rows) + return pendingList(held, func(p chain.Pending) bool { return p.Raised != nil && p.Cleared == nil }), nil } // MarkPending stamps a pending assignment raised, acknowledged or cleared, now. -func (i *Inventory) MarkPending(ctx context.Context, id int64, what string) error { - column := map[string]string{"raised": "raised_at", "acknowledged": "acknowledged_at", "cleared": "cleared_at"}[what] - if column == "" { +func (i *Inventory) MarkPending(ctx context.Context, id string, what string) error { + if what != "raised" && what != "acknowledged" && what != "cleared" { return errors.New("a pending assignment is marked raised, acknowledged or cleared") } - _, err := i.store.Pool().Exec(ctx, - `update pending_assignment set `+column+` = coalesce(`+column+`, now()) where id = $1`, id) + now := time.Now().UTC() + _, err := i.changePending(ctx, id, func(v *chain.Assignment) bool { + mark := map[string]**time.Time{"raised": &v.Pending.Raised, "acknowledged": &v.Pending.Acknowledged, "cleared": &v.Pending.Cleared}[what] + if *mark != nil { + return false // the first stamp stands + } + *mark = &now + return true + }) return err } // ClaimPending takes a waiting pending assignment for making, so nothing else settles or withdraws it while // it is made. False when it no longer waits: another process took it, or it was withdrawn. -func (i *Inventory) ClaimPending(ctx context.Context, id int64) (bool, error) { - tag, err := i.store.Pool().Exec(ctx, - `update pending_assignment set state = 'applying', settled_at = now() where id = $1 and state = 'waiting'`, id) - if err != nil { - return false, err - } - return tag.RowsAffected() == 1, nil +func (i *Inventory) ClaimPending(ctx context.Context, id string) (bool, error) { + now := time.Now().UTC() + return i.changePending(ctx, id, func(v *chain.Assignment) bool { + if !chain.CanMove(v.Pending.State, PendingApplying) { + return false + } + v.Pending.State, v.Pending.Settled = PendingApplying, &now + return true + }) } // ReleaseClaim puts a claimed pending assignment back to waiting. -func (i *Inventory) ReleaseClaim(ctx context.Context, id int64) error { - _, err := i.store.Pool().Exec(ctx, - `update pending_assignment set state = 'waiting', settled_at = null where id = $1 and state = 'applying'`, id) +func (i *Inventory) ReleaseClaim(ctx context.Context, id string) error { + _, err := i.changePending(ctx, id, func(v *chain.Assignment) bool { + if v.Pending.State != PendingApplying { + return false + } + v.Pending.State, v.Pending.Settled = PendingWaiting, nil + return true + }) return err } // SettlePending ends a pending assignment in the state it is read in (from): waiting for an expiry or a // withdrawal, applying for the outcome of making it. False when it was no longer in that state. -func (i *Inventory) SettlePending(ctx context.Context, id int64, from, state, note string) (bool, error) { +func (i *Inventory) SettlePending(ctx context.Context, id string, from, state, note string) (bool, error) { if state == PendingWaiting || state == PendingApplying { return false, errors.New("a pending assignment is settled to applied, refused, expired or withdrawn") } - tag, err := i.store.Pool().Exec(ctx, - `update pending_assignment set state = $3, note = $4, settled_at = now() - where id = $1 and state = $2`, id, from, state, note) - if err != nil { - return false, err - } - return tag.RowsAffected() == 1, nil + now := time.Now().UTC() + return i.changePending(ctx, id, func(v *chain.Assignment) bool { + if v.Pending.State != from || !chain.CanSettle(from, state) { + return false + } + v.Pending.State, v.Pending.Note, v.Pending.Settled = state, note, &now + return true + }) } -// ForgetEndedPending deletes ended pending assignments settled more than KeptFor ago, and says how many. One -// raised as a condition is kept until that condition is cleared: deleting it would leave the condition -// with nothing to answer it. -func (i *Inventory) ForgetEndedPending(ctx context.Context, now time.Time) (int64, error) { - tag, err := i.store.Pool().Exec(ctx, - `delete from pending_assignment where state not in ('waiting', 'applying') and settled_at < $1 - and (raised_at is null or cleared_at is not null)`, - now.Add(-KeptFor)) - if err != nil { - return 0, err - } - return tag.RowsAffected(), nil -} - -// PendingKnown says which of these pending assignments are still on record. -func (i *Inventory) PendingKnown(ctx context.Context, ids []int64) (map[int64]bool, error) { - rows, err := i.store.Pool().Query(ctx, `select id from pending_assignment where id = any($1)`, ids) +// PendingKnown says which of these pending assignments are still on record: a key holds them now. One a newer pending +// assignment of its key replaced is not. +func (i *Inventory) PendingKnown(ctx context.Context, ids []string) (map[string]bool, error) { + held, err := i.pendingValues(ctx) if err != nil { return nil, err } - defer rows.Close() - out := map[int64]bool{} - for rows.Next() { - var id int64 - if err := rows.Scan(&id); err != nil { - return nil, err - } - out[id] = true + want := map[string]bool{} + for _, id := range ids { + want[id] = true } - return out, rows.Err() + out := map[string]bool{} + for _, k := range held { + if want[k.Value.Pending.ID] { + out[k.Value.Pending.ID] = true + } + } + return out, nil } diff --git a/testdata/store-tests b/testdata/store-tests index 91c80206..8b3ec1c9 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -155,3 +155,8 @@ ./cmd/mesh-controller TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments ./cmd/mesh-controller TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment ./internal/inventory TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved +./internal/inventory TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket +./internal/inventory TestARecordedPendingIsSkippedByAssignedAndTheJoins +./internal/inventory TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse +./internal/inventory TestAssignOverAPendingKeepsItsRecordAsApplied +./internal/inventory TestAPendingIsMarkedOnceAndGoesWithItsMachine From 3b989d8bb3fa69d5a4965ce15e5c4a96bd94b62e Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 05:19:32 +0200 Subject: [PATCH 02/11] Callers of the pending assignments on the bucket: a replaced wait keeps its condition; one read of the key through assign (issue 1108) A wait that ended unmade and whose condition nobody answered is carried by the wait that replaces it, so the operator is never left without the condition for a module still not on its machine, and taking it back, its condition and the orphan sweep read it there. The condition id stays node.module. with the string id. clearOrphaned says a taken-back wait too. TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket enters through the assign verb and reads the key before and after the build's outcome. --- cmd/mesh-controller/chain_integration_test.go | 39 ++++++ cmd/mesh-controller/pending.go | 2 +- internal/chain/records.go | 4 + internal/inventory/chain_assignments.go | 10 ++ internal/inventory/chain_pending_test.go | 52 +++++++ internal/inventory/pending.go | 132 +++++++++++++----- testdata/store-tests | 2 + 7 files changed, 204 insertions(+), 37 deletions(-) diff --git a/cmd/mesh-controller/chain_integration_test.go b/cmd/mesh-controller/chain_integration_test.go index c6a98e1c..565ca164 100644 --- a/cmd/mesh-controller/chain_integration_test.go +++ b/cmd/mesh-controller/chain_integration_test.go @@ -207,3 +207,42 @@ func runOf(on []chain.Assignment, module string) chain.Assignment { a, _ := assignmentOf(on, module) return a } + +// **An assignment waiting for its build is one key of the assignments bucket** (ADR 0261, ADR 0408, issue 1094): entered +// through the `assign` verb for a module whose build is in flight, one read of `.` says state pending with the +// build it waits for, the build's outcome through the build seat's path makes the key stand with the wait carried as applied, +// and the table of pending assignments is read by nobody. No key is written by hand. +func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatalf("an assignment while its build runs was refused: %v", err) + } + waiting, found, err := inv.AssignmentValue(ctx, "laptop", "sensors") + if err != nil || !found || waiting.State != chain.StatePending || waiting.Stands() || !waiting.Assigned.IsZero() || + waiting.Pending == nil || waiting.Pending.State != chain.PendingWaiting || waiting.Pending.Build != "b-modules/sensors" || + !strings.HasPrefix(waiting.Pending.ID, "p-") { + t.Fatalf("one read of the key says %+v (found %v, %v)", waiting, found, err) + } + if got := assignedTo(t, open, "laptop"); slices.Contains(got, "sensors") { + t.Fatalf("a module only waiting is assigned: %v", got) + } + + if err := (builds{inv: inv, open: open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil { + t.Fatal(err) + } + made, found, err := inv.AssignmentValue(ctx, "laptop", "sensors") + if err != nil || !found || !made.Stands() || made.State != "" || made.Assigned.IsZero() || made.Pending == nil || + made.Pending.State != chain.PendingApplied || made.Pending.ID != waiting.Pending.ID { + t.Fatalf("one read of the key after the build says %+v (found %v, %v)", made, found, err) + } + if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("the build registered sensors and it was not assigned") + } + if rows := inv.QueryForTest(t, `select count(*) from pending_assignment`); rows[0][0] != "0" { + t.Fatalf("the table of pending assignments took %s rows", rows[0][0]) + } +} diff --git a/cmd/mesh-controller/pending.go b/cmd/mesh-controller/pending.go index 977617bb..a5ec7359 100644 --- a/cmd/mesh-controller/pending.go +++ b/cmd/mesh-controller/pending.go @@ -695,7 +695,7 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor if known[id] { continue } - if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed"); err != nil { + if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed, or the module was assigned and the wait taken back"); err != nil { said = append(said, fmt.Sprintf("the condition %s could not be cleared: %v", key, err)) } } diff --git a/internal/chain/records.go b/internal/chain/records.go index 54facfc1..ceb23533 100644 --- a/internal/chain/records.go +++ b/internal/chain/records.go @@ -225,6 +225,10 @@ type Pending struct { Raised *time.Time `json:"raised,omitempty"` Acknowledged *time.Time `json:"acknowledged,omitempty"` Cleared *time.Time `json:"cleared,omitempty"` + // Earlier are the waits of this key that ended unmade and were replaced by this one before their condition was answered + // (taken back or cleared): each stays, with its own marks, so a replaced wait never leaves the operator without the + // condition it raised or was to raise. Dropped once answered. + Earlier []Pending `json:"earlier,omitempty"` } // Open says the assignment is still to be made. diff --git a/internal/inventory/chain_assignments.go b/internal/inventory/chain_assignments.go index 6ca7e82c..3a4d058f 100644 --- a/internal/inventory/chain_assignments.go +++ b/internal/inventory/chain_assignments.go @@ -412,3 +412,13 @@ func carriedPending(was chain.Assignment, now time.Time) *chain.Pending { } return &p } + +// AssignmentValue is one key of the assignments bucket as it is held, standing or not: one read of an assignment. +func (i *Inventory) AssignmentValue(ctx context.Context, node, module string) (chain.Assignment, bool, error) { + c, err := i.theChain(ctx) + if err != nil { + return chain.Assignment{}, false, err + } + k, found, err := c.Assignments.Get(ctx, assignmentKey(node, module)) + return k.Value, found, err +} diff --git a/internal/inventory/chain_pending_test.go b/internal/inventory/chain_pending_test.go index 1e996403..2f62ee2c 100644 --- a/internal/inventory/chain_pending_test.go +++ b/internal/inventory/chain_pending_test.go @@ -219,3 +219,55 @@ func TestAPendingIsMarkedOnceAndGoesWithItsMachine(t *testing.T) { t.Fatalf("a wait outlived its machine: %+v", rows) } } + +// **A wait replaced by a newer one never leaves the operator without the condition it raised, or was to raise** (issue +// 1108): the key carries the replaced wait while its condition is unanswered, still known, still to clear, and a newer +// wait takes only the unanswered ones forward. +func TestAReplacedWaitNeverLeavesTheOperatorWithoutItsCondition(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record := func(build string) PendingAssignment { + p, err := inv.RecordPending(ctx, PendingAssignment{Node: node, Module: "sensors", Build: build}) + if err != nil { + t.Fatal(err) + } + return p + } + end := func(p PendingAssignment) { + if ok, err := inv.SettlePending(ctx, p.ID, PendingWaiting, PendingExpired, "boom "+p.Build); err != nil || !ok { + t.Fatal(ok, err) + } + } + first := record("b-1") + end(first) + if err := inv.MarkPending(ctx, first.ID, "raised"); err != nil { + t.Fatal(err) + } + second := record("b-2") + if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID}); !known[first.ID] || !known[second.ID] { + t.Fatalf("known %v: the replaced wait's condition is unanswered", known) + } + if clear, _ := inv.ToClear(ctx); len(clear) != 1 || clear[0].ID != first.ID { + t.Fatalf("to clear %+v: the first's condition is still to clear", clear) + } + rows, _ := inv.PendingFor(ctx, node, "sensors") + if len(rows) != 2 || rows[0].ID != second.ID || rows[1].ID != first.ID { + t.Fatalf("the key says %+v, newest first", rows) + } + // The second ends unmade before the first was answered: both are to be said, the first's condition not cleared by it. + end(second) + if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID { + t.Fatalf("to raise %+v", raise) + } + // A third replaces the second, which was not raised yet: it is carried, with the first (still unanswered). + if err := inv.MarkPending(ctx, first.ID, "acknowledged"); err != nil { + t.Fatal(err) + } + third := record("b-3") + if raise, _ := inv.ToRaise(ctx); len(raise) != 1 || raise[0].ID != second.ID { + t.Fatalf("the second was never raised and is to be: %+v", raise) + } + if known, _ := inv.PendingKnown(ctx, []string{first.ID, second.ID, third.ID}); known[first.ID] || !known[second.ID] || !known[third.ID] { + t.Fatalf("known %v: the first was answered (taken back) and is let go, the second is not", known) + } +} diff --git a/internal/inventory/pending.go b/internal/inventory/pending.go index ff41e0f6..4d163eef 100644 --- a/internal/inventory/pending.go +++ b/internal/inventory/pending.go @@ -254,8 +254,10 @@ var ErrAlreadyPending = errors.New("that assignment is already waiting for its b // ErrOnTheMachine is a pending assignment of a module that is on the machine already. var ErrOnTheMachine = errors.New("that module is on the machine already") -func pendingOf(v chain.Assignment) PendingAssignment { - p := v.Pending +func pendingOf(v chain.Assignment) PendingAssignment { return pendingRecord(v, *v.Pending) } + +// pendingRecord is one wait of a key, the one it holds or one it replaced. +func pendingRecord(v chain.Assignment, p chain.Pending) PendingAssignment { return PendingAssignment{ID: p.ID, Node: v.Node, Module: v.Module, Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, State: p.State, Note: p.Note, Settled: p.Settled, Raised: p.Raised, Acknowledged: p.Acknowledged, Cleared: p.Cleared} @@ -296,16 +298,39 @@ func (i *Inventory) pendingValues(ctx context.Context) ([]chain.Kept[chain.Assig func pendingList(kept []chain.Kept[chain.Assignment], keep func(chain.Pending) bool) []PendingAssignment { var out []PendingAssignment for _, k := range kept { - if keep(*k.Value.Pending) { - out = append(out, pendingOf(k.Value)) + for _, p := range pendingWaits(k.Value) { + if keep(p) { + out = append(out, pendingRecord(k.Value, p)) + } } } + sort.SliceStable(out, func(a, b int) bool { + if !out[a].Since.Equal(out[b].Since) { + return out[a].Since.Before(out[b].Since) + } + return out[a].ID < out[b].ID + }) return out } -// changePending changes the pending assignment of an id by compare-and-set on the key that holds it, read again when it -// moved: false when no key holds that id now, or change says no. -func (i *Inventory) changePending(ctx context.Context, id string, change func(v *chain.Assignment) bool) (bool, error) { +// pendingWaits is the wait a value holds and the earlier ones it carries, each without the others. +func pendingWaits(v chain.Assignment) []chain.Pending { + if v.Pending == nil { + return nil + } + out := make([]chain.Pending, 0, 1+len(v.Pending.Earlier)) + for _, e := range v.Pending.Earlier { + e.Earlier = nil + out = append(out, e) + } + held := *v.Pending + held.Earlier = nil + return append(out, held) +} + +// changePending changes the pending assignment of an id — the wait a key holds or one it carries from before — by +// compare-and-set on the key that holds it, read again when it moved: false when no key holds that id now, or change says no. +func (i *Inventory) changePending(ctx context.Context, id string, change func(p *chain.Pending) bool) (bool, error) { c, err := i.theChain(ctx) if err != nil { return false, err @@ -316,21 +341,34 @@ func (i *Inventory) changePending(ctx context.Context, id string, change func(v return false, err } var at *chain.Kept[chain.Assignment] + var target *chain.Pending for k := range held { - if held[k].Value.Pending.ID == id { - at = &held[k] + v := held[k].Value + p := *v.Pending + p.Earlier = append([]chain.Pending(nil), p.Earlier...) + v.Pending = &p + switch { + case p.ID == id: + at, target = &held[k], v.Pending + default: + for e := range v.Pending.Earlier { + if v.Pending.Earlier[e].ID == id { + at, target = &held[k], &v.Pending.Earlier[e] + } + } + } + if at != nil { + held[k].Value = v + break } } if at == nil { return false, nil } - v := at.Value - p := *v.Pending - v.Pending = &p - if !change(&v) { + if !change(target) { return false, nil } - if _, err := c.Assignments.Update(ctx, at.Key, v, at.Revision); errors.Is(err, chain.ErrMoved) { + if _, err := c.Assignments.Update(ctx, at.Key, at.Value, at.Revision); errors.Is(err, chain.ErrMoved) { continue } else if err != nil { return false, err @@ -368,7 +406,8 @@ func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (Pen p.ID, p.Since, p.State = fmt.Sprintf("p-%d", since.UnixNano()), since, PendingWaiting p.Path = strings.Trim(p.Path, "/") value := chain.Assignment{Node: node.Name, Module: p.Module, State: chain.StatePending, Pending: &chain.Pending{ - ID: p.ID, State: PendingWaiting, Build: p.Build, Repository: p.Repository, Path: p.Path, Since: since}} + ID: p.ID, State: PendingWaiting, Build: p.Build, Repository: p.Repository, Path: p.Path, Since: since, + Earlier: unansweredWaits(cur.Value, found)}} if found { _, err = c.Assignments.Update(ctx, key, value, cur.Revision) } else { @@ -387,11 +426,11 @@ func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (Pen // RepointPending makes a waiting pending assignment wait for another build. False when it no longer waits. func (i *Inventory) RepointPending(ctx context.Context, id string, build, repository, sourcePath string) (bool, error) { - return i.changePending(ctx, id, func(v *chain.Assignment) bool { - if v.Pending.State != PendingWaiting { + return i.changePending(ctx, id, func(p *chain.Pending) bool { + if p.State != PendingWaiting { return false } - v.Pending.Build, v.Pending.Repository, v.Pending.Path = build, repository, strings.Trim(sourcePath, "/") + p.Build, p.Repository, p.Path = build, repository, strings.Trim(sourcePath, "/") return true }) } @@ -411,8 +450,8 @@ func (i *Inventory) Pending(ctx context.Context, endedSince time.Time) ([]Pendin }), nil } -// PendingFor is the pending assignment of one module to one machine: the one its key holds now, which is the newest -// (an earlier one is read from the key's history, which nothing here needs). +// PendingFor is every pending assignment of one module to one machine the key still has to say, newest first: the wait it +// holds now and the unanswered ones it replaced. func (i *Inventory) PendingFor(ctx context.Context, node, module string) ([]PendingAssignment, error) { held, err := i.pendingValues(ctx) if err != nil { @@ -421,9 +460,13 @@ func (i *Inventory) PendingFor(ctx context.Context, node, module string) ([]Pend var out []PendingAssignment for _, k := range held { if k.Value.Node == node && k.Value.Module == module { - out = append(out, pendingOf(k.Value)) + for _, w := range pendingWaits(k.Value) { + out = append(out, pendingRecord(k.Value, w)) + } } } + // Newest first. + sort.SliceStable(out, func(a, b int) bool { return out[a].Since.After(out[b].Since) }) return out, nil } @@ -465,8 +508,8 @@ func (i *Inventory) MarkPending(ctx context.Context, id string, what string) err return errors.New("a pending assignment is marked raised, acknowledged or cleared") } now := time.Now().UTC() - _, err := i.changePending(ctx, id, func(v *chain.Assignment) bool { - mark := map[string]**time.Time{"raised": &v.Pending.Raised, "acknowledged": &v.Pending.Acknowledged, "cleared": &v.Pending.Cleared}[what] + _, err := i.changePending(ctx, id, func(p *chain.Pending) bool { + mark := map[string]**time.Time{"raised": &p.Raised, "acknowledged": &p.Acknowledged, "cleared": &p.Cleared}[what] if *mark != nil { return false // the first stamp stands } @@ -480,22 +523,22 @@ func (i *Inventory) MarkPending(ctx context.Context, id string, what string) err // it is made. False when it no longer waits: another process took it, or it was withdrawn. func (i *Inventory) ClaimPending(ctx context.Context, id string) (bool, error) { now := time.Now().UTC() - return i.changePending(ctx, id, func(v *chain.Assignment) bool { - if !chain.CanMove(v.Pending.State, PendingApplying) { + return i.changePending(ctx, id, func(p *chain.Pending) bool { + if !chain.CanMove(p.State, PendingApplying) { return false } - v.Pending.State, v.Pending.Settled = PendingApplying, &now + p.State, p.Settled = PendingApplying, &now return true }) } // ReleaseClaim puts a claimed pending assignment back to waiting. func (i *Inventory) ReleaseClaim(ctx context.Context, id string) error { - _, err := i.changePending(ctx, id, func(v *chain.Assignment) bool { - if v.Pending.State != PendingApplying { + _, err := i.changePending(ctx, id, func(p *chain.Pending) bool { + if p.State != PendingApplying { return false } - v.Pending.State, v.Pending.Settled = PendingWaiting, nil + p.State, p.Settled = PendingWaiting, nil return true }) return err @@ -508,17 +551,17 @@ func (i *Inventory) SettlePending(ctx context.Context, id string, from, state, n return false, errors.New("a pending assignment is settled to applied, refused, expired or withdrawn") } now := time.Now().UTC() - return i.changePending(ctx, id, func(v *chain.Assignment) bool { - if v.Pending.State != from || !chain.CanSettle(from, state) { + return i.changePending(ctx, id, func(p *chain.Pending) bool { + if p.State != from || !chain.CanSettle(from, state) { return false } - v.Pending.State, v.Pending.Note, v.Pending.Settled = state, note, &now + p.State, p.Note, p.Settled = state, note, &now return true }) } -// PendingKnown says which of these pending assignments are still on record: a key holds them now. One a newer pending -// assignment of its key replaced is not. +// PendingKnown says which of these pending assignments are still on record: a key holds them now, or carries them as +// unanswered earlier waits. func (i *Inventory) PendingKnown(ctx context.Context, ids []string) (map[string]bool, error) { held, err := i.pendingValues(ctx) if err != nil { @@ -530,9 +573,26 @@ func (i *Inventory) PendingKnown(ctx context.Context, ids []string) (map[string] } out := map[string]bool{} for _, k := range held { - if want[k.Value.Pending.ID] { - out[k.Value.Pending.ID] = true + for _, w := range pendingWaits(k.Value) { + if want[w.ID] { + out[w.ID] = true + } } } return out, nil } + +// unansweredWaits are the waits a key holds that ended unmade and whose condition nobody has answered yet — taken back or +// cleared — which a new wait replacing them carries, so the operator is not left without it (issue 1108). +func unansweredWaits(v chain.Assignment, found bool) []chain.Pending { + if !found || v.State != chain.StatePending { + return nil + } + var out []chain.Pending + for _, w := range pendingWaits(v) { + if (w.State == PendingExpired || w.State == PendingRefused) && w.Cleared == nil && w.Acknowledged == nil { + out = append(out, w) + } + } + return out +} diff --git a/testdata/store-tests b/testdata/store-tests index 8b3ec1c9..1c1c3e58 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -160,3 +160,5 @@ ./internal/inventory TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse ./internal/inventory TestAssignOverAPendingKeepsItsRecordAsApplied ./internal/inventory TestAPendingIsMarkedOnceAndGoesWithItsMachine +./internal/inventory TestAReplacedWaitNeverLeavesTheOperatorWithoutItsCondition +./cmd/mesh-controller TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket From fdab58ae4f1e10f166adfce16bf0a9d44b1b0b67 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 05:22:28 +0200 Subject: [PATCH 03/11] Read what a node was sent from its assignment keys; a pulled-in module keeps a standing key; a hand send is a time (issue 1109) SentBuilds answers from the keys of the node's assignments that name a commit, known when the node was sent anything and no key says its commit unknown. A send that names its builds writes the commit, build and rollout on each carried module's key (a module nothing assigns gets a standing key marked pulled-in, never a state) and clears what every other key of the node named; a send by hand writes sent-unknown, a time, on every key and clears their commits. A failed write of these fails the record of the send. node.sent_builds is no longer written. The plan pass ends a pulled-in key only when no send carries it. --- cmd/mesh-controller/chain_integration_test.go | 6 +- internal/chain/pending_test.go | 24 ++++ internal/chain/records.go | 8 ++ internal/inventory/chain_assignments.go | 10 +- internal/inventory/chain_links.go | 126 +++++++++++++----- internal/inventory/chain_sent_test.go | 120 +++++++++++++++++ internal/inventory/nodes.go | 52 +++++--- internal/inventory/sends.go | 23 ++-- internal/inventory/sent_builds_test.go | 4 +- testdata/store-tests | 3 + 10 files changed, 300 insertions(+), 76 deletions(-) create mode 100644 internal/inventory/chain_sent_test.go diff --git a/cmd/mesh-controller/chain_integration_test.go b/cmd/mesh-controller/chain_integration_test.go index 565ca164..e5c3610a 100644 --- a/cmd/mesh-controller/chain_integration_test.go +++ b/cmd/mesh-controller/chain_integration_test.go @@ -176,12 +176,12 @@ func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" { t.Fatalf("the assignment names %+v", on) } - // A declaration sent by hand with no builds known says nothing of them. + // A declaration sent by hand with no builds known makes what the machine runs unknown: a time on the key, no commit. if err := inv.RecordSent(ctx, node, "d3", nil); err != nil { t.Fatal(err) } - if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c1" { - t.Fatalf("a send of unknown builds changed %+v", on) + if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "" || runOf(on, "notes").SentUnknown.IsZero() { + t.Fatalf("a send of unknown builds left %+v", on) } // A module sent a commit no build is kept of is named by its commit alone. if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil { diff --git a/internal/chain/pending_test.go b/internal/chain/pending_test.go index c0e73f9d..655aae40 100644 --- a/internal/chain/pending_test.go +++ b/internal/chain/pending_test.go @@ -61,3 +61,27 @@ func TestTheTransitionsOfAPendingAssignment(t *testing.T) { t.Error("a claim moves a wait to applying and back") } } + +// **A module only pulled in is a standing value marked pulled-in, never a state, and a hand send makes a key say its commit +// unknown with a time** (issue 1109): every reader treats a non-empty state as no assignment, so a pulled-in key must have +// none; and unknown is never written in `commit`. +func TestAPulledInValueStandsAndAHandSendIsATimeNotACommit(t *testing.T) { + at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) + pulled := Assignment{Node: "anchor", Module: "mesh-wireguard", PulledIn: true, Commit: "c1", Build: "b1", Since: at} + if !pulled.Stands() || pulled.State != "" { + t.Fatalf("a pulled-in value does not stand: %+v", pulled) + } + raw, _ := json.Marshal(pulled) + if !strings.Contains(string(raw), `"pulled-in":true`) || strings.Contains(string(raw), `"state"`) || strings.Contains(string(raw), `"assigned"`) { + t.Fatalf("the pulled-in value is %s", raw) + } + unknown := Assignment{Node: "anchor", Module: "notes", Assigned: at, SentUnknown: at} + raw, _ = json.Marshal(unknown) + if !strings.Contains(string(raw), `"sent-unknown":"2026-10-18T03:00:00Z"`) || strings.Contains(string(raw), `"commit"`) || !unknown.Stands() { + t.Fatalf("the hand-sent value is %s", raw) + } + plain, _ := json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at}) + if strings.Contains(string(plain), "pulled-in") || strings.Contains(string(plain), "sent-unknown") { + t.Fatalf("a value that is neither says so: %s", plain) + } +} diff --git a/internal/chain/records.go b/internal/chain/records.go index ceb23533..337f4c3b 100644 --- a/internal/chain/records.go +++ b/internal/chain/records.go @@ -196,6 +196,14 @@ type Assignment struct { // Pending is the assignment's wait for its build (novox/hq ADR 0261), kept in the value of the key it waits for and, when // it is made, carried into the value that stands: the record of what became of the wait. Pending *Pending `json:"pending,omitempty"` + + // PulledIn marks a standing value of a module that is on the machine only because another requires it, or because a + // send carried it: no time assigned, but a key all the same, so what the machine was sent of it is kept and read + // (issue 1109). Never a state: every reader treats a non-empty state as no assignment, and would take it off the machine. + PulledIn bool `json:"pulled-in,omitempty"` + // SentUnknown is when a send by hand made what the machine runs unknown (a declaration sent with no builds known): the + // commit is cleared with it. Absent normally; a send that names its builds clears it. + SentUnknown time.Time `json:"sent-unknown,omitzero"` } // StatePending is the State of a value that holds no assignment but a pending one. diff --git a/internal/inventory/chain_assignments.go b/internal/inventory/chain_assignments.go index 3a4d058f..767580ab 100644 --- a/internal/inventory/chain_assignments.go +++ b/internal/inventory/chain_assignments.go @@ -172,7 +172,7 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, switch { case found && cur.Value.Stands(): // A module only pulled in by another becomes an assignment, keeping what it holds. - cur.Value.Assigned = now + cur.Value.Assigned, cur.Value.PulledIn = now, false _, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision) case found: // One that ended is assigned again, and one that was pending is made: a fresh value over it, the earlier one @@ -291,11 +291,11 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i case keepSkip: continue case keepCreate: - next.Assigned = time.Time{} + next.Assigned, next.PulledIn = time.Time{}, true _, err = c.Assignments.Create(ctx, key, next) case keepFresh: // A value that ended, pulled in again: a fresh one over it, the ended one staying in history. - next.Assigned = time.Time{} + next.Assigned, next.PulledIn = time.Time{}, true _, err = c.Assignments.Update(ctx, key, next, cur.Revision) default: cur.Value.RequiredBy, cur.Value.Provisioning = next.RequiredBy, next.Provisioning @@ -310,8 +310,8 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i written++ } for key, cur := range have { - if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() { - continue + if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() || cur.Value.Commit != "" { + continue // one a send still carries is on the machine, and ends when a send does not carry it any more } // A value of a module no longer pulled in, which no assignment names: it ends, and stays. cur.Value.Ended = time.Now().UTC() diff --git a/internal/inventory/chain_links.go b/internal/inventory/chain_links.go index c671c93f..cb989951 100644 --- a/internal/inventory/chain_links.go +++ b/internal/inventory/chain_links.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "os" "sort" "time" @@ -50,7 +49,7 @@ func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string out := map[string]chain.Ref{} for _, k := range all { a := k.Value - if a.Node == node && !a.Assigned.IsZero() && a.Stands() && (a.Build != "" || a.Commit != "") { + if a.Node == node && a.Stands() && (a.Build != "" || a.Commit != "") { out[a.Module] = chain.Ref{Build: a.Build, Commit: a.Commit} } } @@ -77,14 +76,18 @@ func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Ass return out, nil } -// recordRuns writes what a send put on a machine into the assignments, once the send is recorded: for each module it carried -// a build of — by the commit — the build, what it published, the commit, the rollout and since when. **A module that runs the -// commit already keeps what it names**, filled where it names nothing and never blanked, so a send that carries it again, or a -// push by hand, changes nothing it knows. A module the machine is not assigned is no assignment to write. +// recordRuns writes what a send put on a machine into the assignments, once the send is recorded, and it is how the mesh +// knows what a machine was sent (SentBuilds). **A send that names its builds** writes, for each module it carried a build of +// — by the commit — the build, what it published, the commit, the rollout and since when, on the module's key: filled where +// the key names nothing, replaced when the commit moved, never blanked by a push by hand; a module nothing assigns, carried +// because another requires it, gets a standing key marked pulled-in; and every other key of the machine loses what it named, +// for the machine is not sent it any more. **A send that names none** (a declaration by hand) makes what the machine runs +// unknown: every key of it says so (sent-unknown) and names no commit. **An error here fails the record of the send**, for +// what a machine was sent is read from these keys and from nowhere else. func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[string]string, at time.Time) error { - c, on, err := i.ChainReads(ctx, KindAssignments) - if err != nil || !on || len(builds) == 0 { - return err + c, err := i.theChain(ctx) + if err != nil { + return fmt.Errorf("what the machine runs cannot be written: %w", err) } var node string if err := i.store.Pool().QueryRow(ctx, `select name from node where id = $1`, nodeID).Scan(&node); errors.Is(err, pgx.ErrNoRows) { @@ -93,24 +96,81 @@ func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[st return err } rollout, _ := ctx.Value(rolloutKey{}).(rolloutOf) + all, err := c.Assignments.All(ctx) + if err != nil { + return fmt.Errorf("the assignments of %s cannot be read: %w", node, err) + } + var failed []error + for _, k := range sortedValues(all) { + a := k.Value + if a.Node != node || !a.Stands() { + continue + } + if builds[a.Module] != "" { + continue // carried, and its commit is written below + } + if err := i.forgetRun(ctx, c, k, builds == nil, at); err != nil { + failed = append(failed, fmt.Errorf("%s on %s: %w", a.Module, node, err)) + } + } modules := make([]string, 0, len(builds)) for m := range builds { modules = append(modules, m) } sort.Strings(modules) - var failed []error for _, m := range modules { - commit := builds[m] - if commit == "" { + if builds[m] == "" { continue } - if err := i.recordRun(ctx, c, node, m, commit, rollout, at); err != nil { + if err := i.recordRun(ctx, c, node, m, builds[m], rollout, at); err != nil { failed = append(failed, fmt.Errorf("%s on %s: %w", m, node, err)) } } return errors.Join(failed...) } +func sortedValues(all map[string]chain.Kept[chain.Assignment]) []chain.Kept[chain.Assignment] { + out := make([]chain.Kept[chain.Assignment], 0, len(all)) + for key, k := range all { + k.Key = key + out = append(out, k) + } + sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key }) + return out +} + +// forgetRun writes that a machine is no longer sent what a key named: its links are cleared, and for a send by hand the key +// says what the machine runs is unknown. +func (i *Inventory) forgetRun(ctx context.Context, c *chain.Chain, k chain.Kept[chain.Assignment], hand bool, at time.Time) error { + for try := 0; try < casTries; try++ { + next := k.Value + next.Build, next.Artifact, next.Package, next.Commit, next.Rollout, next.Since = "", "", nil, "", "", time.Time{} + next.SentUnknown = time.Time{} + if hand { + next.SentUnknown = at.UTC() + } + if next.Build == k.Value.Build && next.Commit == k.Value.Commit && next.Rollout == k.Value.Rollout && + next.Artifact == k.Value.Artifact && next.Package == k.Value.Package && next.Since.Equal(k.Value.Since) && + next.SentUnknown.Equal(k.Value.SentUnknown) { + return nil + } + if _, err := c.Assignments.Update(ctx, k.Key, next, k.Revision); errors.Is(err, chain.ErrMoved) { + cur, found, gerr := c.Assignments.Get(ctx, k.Key) + if gerr != nil { + return gerr + } + if !found || !cur.Value.Stands() { + return nil + } + k = chain.Kept[chain.Assignment]{Key: k.Key, Value: cur.Value, Revision: cur.Revision} + continue + } else { + return err + } + } + return fmt.Errorf("the assignment kept moving under the write") +} + func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, commit string, rollout rolloutOf, at time.Time) error { key := assignmentKey(node, module) var build, rolloutID string @@ -124,11 +184,12 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, if err != nil { return err } - if !found || cur.Value.Assigned.IsZero() || !cur.Value.Stands() { - return nil // nothing assigned there to say what it runs + if found && cur.Value.State != "" { + return nil // a wait for a build: the module is not on the machine by it } - same := cur.Value.Commit == commit - if same && cur.Value.Build != "" && (rolloutID == "" || cur.Value.Rollout != "") { + standing := found && cur.Value.Stands() + same := standing && cur.Value.Commit == commit + if same && cur.Value.Build != "" && cur.Value.SentUnknown.IsZero() && (rolloutID == "" || cur.Value.Rollout != "") { return nil // a send that carries what the machine runs already says nothing new } if !resolved { @@ -148,7 +209,13 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, resolved = true } next := cur.Value - if same { + switch { + case !standing: + // Carried and assigned to nobody here: a standing key, no time assigned, marked pulled in. An ended one is + // written over, the ended one staying in history. + next = chain.Assignment{Node: node, Module: module, PulledIn: true, Commit: commit, Build: build, Rollout: rolloutID, + Since: at.UTC()} + case same: // Already running it: fill what is missing, blank nothing. if next.Build == "" { next.Build = build @@ -159,18 +226,26 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, if next.Since.IsZero() { next.Since = at.UTC() } - } else { + default: next.Commit, next.Build, next.Rollout, next.Since = commit, build, rolloutID, at.UTC() next.Artifact, next.Package = "", nil } + next.SentUnknown = time.Time{} if published != nil && next.Artifact == "" && next.Package == nil && next.Build == build { next.Artifact, next.Package = published.Artifact, published.Package } - if next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout && + if standing && next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout && next.Artifact == cur.Value.Artifact && (next.Package == nil) == (cur.Value.Package == nil) && - next.Since.Equal(cur.Value.Since) { + next.Since.Equal(cur.Value.Since) && next.SentUnknown.Equal(cur.Value.SentUnknown) { return nil } + if !found { + if _, err := c.Assignments.Create(ctx, key, next); errors.Is(err, chain.ErrExists) { + continue + } else { + return err + } + } if _, err := c.Assignments.Update(ctx, key, next, cur.Revision); errors.Is(err, chain.ErrMoved) { continue } else { @@ -179,12 +254,3 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, } return fmt.Errorf("the assignment kept moving under the write") } - -// saidRunsNot says a send's links could not be written, to the log of whoever sent: the send is away and is recorded, and -// a record of what it put where that could not be written is said, never a reason to call the send failed. -func saidRunsNot(node string, err error) { - if err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: %s was sent its declaration, and the build, commit and rollout of each module "+ - "could not be written in its assignments: %v\n", node, err) - } -} diff --git a/internal/inventory/chain_sent_test.go b/internal/inventory/chain_sent_test.go new file mode 100644 index 00000000..d65614f4 --- /dev/null +++ b/internal/inventory/chain_sent_test.go @@ -0,0 +1,120 @@ +//go:build integration + +package inventory + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/novox/mesh-controller/internal/chain" +) + +// **What a machine was sent is read from the keys of its assignments that name a commit** (ADR 0408, issue 1109): an assigned +// module's, and a module only pulled in, whose key is a standing value marked pulled-in and never a state. +func TestSentBuildsAnswersFromTheAssignmentsKeysOfTheNode(t *testing.T) { + inv, node := aNodeWithModules(t, "web", "cache") + ctx := t.Context() + record, _ := inv.NodeByName(ctx, node) + if _, err := inv.Assign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil { + t.Fatal(err) + } + if builds, known, err := inv.SentBuilds(ctx, node); err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"web": "c1", "cache": "c2"}) { + t.Fatalf("sent %v %v %v", builds, known, err) + } + pulled, found, _ := inv.AssignmentValue(ctx, node, "cache") + if !found || !pulled.Stands() || pulled.State != "" || !pulled.PulledIn || !pulled.Assigned.IsZero() || pulled.Commit != "c2" { + t.Fatalf("the module only carried has the key %+v", pulled) + } + if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"web"}) { + t.Fatalf("a module only pulled in is assigned: %v", on) + } + // Assigned later, it keeps what it was sent and is pulled in no longer. + if _, err := inv.Assign(ctx, node, "cache"); err != nil { + t.Fatal(err) + } + made, _, _ := inv.AssignmentValue(ctx, node, "cache") + if made.PulledIn || made.Assigned.IsZero() || made.Commit != "c2" { + t.Fatalf("assigned over a pulled-in key: %+v", made) + } +} + +// **A send replaces the whole map of the node, and a hand send makes it unknown until the next send that names its builds** +// (issue 1109): a module no longer carried loses its commit, the node reads unknown when any key says so. +func TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown(t *testing.T) { + inv, node := aNodeWithModules(t, "web", "cache") + ctx := t.Context() + record, _ := inv.NodeByName(ctx, node) + if _, err := inv.Assign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, record.ID, "d2", map[string]string{"web": "c3"}); err != nil { + t.Fatal(err) + } + if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c3"}) { + t.Fatalf("the second send did not replace the map: %v %v", builds, known) + } + if old, _, _ := inv.AssignmentValue(ctx, node, "cache"); old.Commit != "" || old.Build != "" { + t.Fatalf("a module no longer carried keeps %+v", old) + } + // By hand: unknown, with a time and no commit anywhere. + if err := inv.RecordSent(ctx, record.ID, "d3", nil); err != nil { + t.Fatal(err) + } + if builds, known, _ := inv.SentBuilds(ctx, node); known || builds != nil { + t.Fatalf("a hand send reads as %v %v", builds, known) + } + web, _, _ := inv.AssignmentValue(ctx, node, "web") + if web.SentUnknown.IsZero() || web.Commit != "" || !web.Stands() { + t.Fatalf("web after a hand send: %+v", web) + } + if sent, _ := inv.Outstanding(ctx, node); sent != "d3" { + t.Fatalf("the digest of the hand send: %q", sent) + } + // A send that names its builds makes it known again. + if err := inv.RecordSent(ctx, record.ID, "d4", map[string]string{"web": "c4"}); err != nil { + t.Fatal(err) + } + if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c4"}) { + t.Fatalf("after a send that names its builds: %v %v", builds, known) + } + if again, _, _ := inv.AssignmentValue(ctx, node, "web"); !again.SentUnknown.IsZero() { + t.Fatalf("web still says unknown: %+v", again) + } + // The column is not written any more. + if rows := inv.QueryForTest(t, `select sent_builds is null from node where name = $1`, node); rows[0][0] != "true" { + t.Fatalf("node.sent_builds was written: %v", rows) + } +} + +// failingToWrite is an assignments bucket that fails every write. +type failingToWrite struct{ chain.Backend } + +func (f failingToWrite) Create(context.Context, string, []byte) (uint64, error) { + return 0, errors.New("the bus went away") +} + +func (f failingToWrite) Update(context.Context, string, []byte, uint64) (uint64, error) { + return 0, errors.New("the bus went away") +} + +// **A links write that fails fails the record of the send, loudly** (issue 1109): what a machine was sent is read from the +// keys, so a send recorded without them would read as one that never happened. +func TestAFailedWriteOfTheLinksFailsTheRecordOfTheSend(t *testing.T) { + inv, node := aNodeWithModules(t, "web") + ctx := t.Context() + record, _ := inv.NodeByName(ctx, node) + inv.UseChain(chain.New(chain.NewMemory(), chain.NewMemory(), failingToWrite{chain.NewMemory()}, + func(context.Context) error { return nil })) + err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1"}) + if err == nil { + t.Fatal("a send whose links could not be written was recorded") + } +} diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 3f6c6dd2..597fe925 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -1264,22 +1264,12 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu e := int64(epoch) sentEpoch = &e } - var carried *string - if builds != nil { - raw, err := json.Marshal(builds) - if err != nil { - return err - } - text := string(raw) - carried = &text + if _, err := i.store.Pool().Exec(ctx, + `update node set sent = $2, sent_at = now(), sent_epoch = $3 where id = $1`, node, digest, sentEpoch); err != nil { + return err } - _, err := i.store.Pool().Exec(ctx, - `update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4 where id = $1`, - node, digest, carried, sentEpoch) - if err == nil { - saidRunsNot(node, i.recordRuns(ctx, node, builds, time.Now())) - } - return err + // What it was sent of each module is the assignments' (SentBuilds); a write of it that fails fails this record. + return i.recordRuns(ctx, node, builds, time.Now()) } // AwaitingSince is since when a machine has had something waiting for its next send (novox/hq issue @@ -1307,21 +1297,39 @@ func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time, } // SentBuilds is the build of each module a machine was last sent, by its name: module to the commit -// its build was made from (novox/hq issue 259). Known is false when that was not kept — a machine -// last sent before it was, sent a declaration by hand, or one the mesh does not know. +// its build was made from (novox/hq issue 259), read from the keys of the machine's assignments that name a commit — an +// assigned module and one only pulled in (ADR 0408, issue 1109). Known is false when that was not kept: a machine never sent +// anything, or last sent a declaration by hand, which makes every key of it say its commit unknown. func (i *Inventory) SentBuilds(ctx context.Context, name string) (builds map[string]string, known bool, err error) { - var raw []byte - err = i.store.Pool().QueryRow(ctx, `select sent_builds from node where name = $1`, name).Scan(&raw) + var sentAt *time.Time + err = i.store.Pool().QueryRow(ctx, `select sent_at from node where name = $1`, name).Scan(&sentAt) if errors.Is(err, pgx.ErrNoRows) { return nil, false, nil } - if err != nil || raw == nil { + if err != nil || sentAt == nil { return nil, false, err } - builds = map[string]string{} - if err := json.Unmarshal(raw, &builds); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, false, err } + all, err := c.Assignments.All(ctx) + if err != nil { + return nil, false, fmt.Errorf("what %s was sent cannot be read from the bus: %w", name, err) + } + builds = map[string]string{} + for _, k := range all { + a := k.Value + if a.Node != name || !a.Stands() { + continue + } + if !a.SentUnknown.IsZero() { + return nil, false, nil + } + if a.Commit != "" { + builds[a.Module] = a.Commit + } + } return builds, true, nil } diff --git a/internal/inventory/sends.go b/internal/inventory/sends.go index f9fdcdf4..4b28e33e 100644 --- a/internal/inventory/sends.go +++ b/internal/inventory/sends.go @@ -2,7 +2,6 @@ package inventory import ( "context" - "encoding/json" "fmt" "time" @@ -150,32 +149,26 @@ func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, bui e := int64(epoch) sentEpoch = &e } - var carried *string - if builds != nil { - raw, err := json.Marshal(builds) - if err != nil { - return err - } - text := string(raw) - carried = &text - } tx, err := i.store.Pool().Begin(ctx) if err != nil { return err } defer func() { _ = tx.Rollback(ctx) }() if _, err := tx.Exec(ctx, - `update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4, sent_generation = $5, - sent_provisioning = $6::jsonb - where id = $1`, node, digest, carried, sentEpoch, nullIfZero(generation), nullIfEmpty(s.Provisioning)); err != nil { + `update node set sent = $2, sent_at = now(), sent_epoch = $3, sent_generation = $4, + sent_provisioning = $5::jsonb + where id = $1`, node, digest, sentEpoch, nullIfZero(generation), nullIfEmpty(s.Provisioning)); err != nil { return err } recordErr := recordSend(ctx, tx, node, s) if err := tx.Commit(ctx); err != nil { return err } - // And what it put on the machine, in the assignments of the modules it carried a build of (issue 1095). - saidRunsNot(node, i.recordRuns(ctx, node, builds, time.Now())) + // And what it put on the machine, in the assignments (issues 1095, 1109): a write of it that fails fails this record, + // for what a machine was sent is read from there. + if err := i.recordRuns(ctx, node, builds, time.Now()); err != nil { + return fmt.Errorf("%s was sent its declaration, and what each module of it runs there could not be written: %w", node, err) + } if recordErr != nil { return &SendNotRecordedError{Err: recordErr} } diff --git a/internal/inventory/sent_builds_test.go b/internal/inventory/sent_builds_test.go index f28ab693..cde4b6ff 100644 --- a/internal/inventory/sent_builds_test.go +++ b/internal/inventory/sent_builds_test.go @@ -27,8 +27,10 @@ func TestASendKeepsTheBuildsItCarried(t *testing.T) { if err := inv.RecordSent(ctx, node.ID, "d1", carried); err != nil { t.Fatal(err) } + // What it was sent is read from the keys of the machine's assignments that name a commit (issue 1109): a module carried + // with no commit known names none. builds, known, err := inv.SentBuilds(ctx, "anchor") - if err != nil || !known || !reflect.DeepEqual(builds, carried) { + if err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"resolver": "abc123"}) { t.Fatalf("the builds sent were not kept: %v %v %v", builds, known, err) } diff --git a/testdata/store-tests b/testdata/store-tests index 1c1c3e58..3c07590e 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -162,3 +162,6 @@ ./internal/inventory TestAPendingIsMarkedOnceAndGoesWithItsMachine ./internal/inventory TestAReplacedWaitNeverLeavesTheOperatorWithoutItsCondition ./cmd/mesh-controller TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket +./internal/inventory TestSentBuildsAnswersFromTheAssignmentsKeysOfTheNode +./internal/inventory TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown +./internal/inventory TestAFailedWriteOfTheLinksFailsTheRecordOfTheSend From 9859923664495b8975c5176e168a83647aae2e68 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 05:30:19 +0200 Subject: [PATCH 04/11] Delete the dead table paths, the move, the store's assignments mirror and its tick (issue 1110) Every method of builds, build requests, rollouts, merges and assignments reads and writes the chain alone: the else branches that read the tables, the movers and MoveChain with their tests, chain_moved reads, ChainReads and the flags that switched a kind, the store's assignments mirror (KeepAssignmentPlans needs none of it: it uses only the AssignmentMirrored type, mirrors.go, and AssignmentValues, provisioning.go) and its tick, the move code of the controller's start, and ForTest's chain_moved row. The listed store tests of the move go with their functions; the tests of the readers, requests, assignments and plans that lived beside them stay in chain_store_test.go. Tests that replayed old data migrations over the build, assignment and pin tables are removed with the tables. The static test that no plan is written but through SavePlansMoving now reads the plan's chain path. --- cmd/mesh-controller/chain.go | 100 +- .../plan_moved_every_writer_test.go | 39 +- cmd/mesh-controller/record_mirrors.go | 43 +- .../record_mirrors_switch_test.go | 29 - .../sent_required_by_integration_test.go | 5 +- internal/inventory/batches.go | 107 +- internal/inventory/builds.go | 295 +----- internal/inventory/catalogue.go | 102 +- internal/inventory/chain.go | 132 +-- internal/inventory/chain_assignments.go | 45 +- .../inventory/chain_assignments_key_test.go | 12 - internal/inventory/chain_links.go | 8 +- internal/inventory/chain_move.go | 663 ------------ internal/inventory/chain_move_test.go | 949 ------------------ internal/inventory/chain_moved_test.go | 45 - internal/inventory/chain_pending_test.go | 3 - internal/inventory/chain_requests.go | 12 +- internal/inventory/chain_rollouts.go | 54 +- internal/inventory/chain_rollouts_test.go | 15 +- internal/inventory/chain_store_test.go | 346 +++++++ internal/inventory/collection.go | 89 +- internal/inventory/durations.go | 42 - internal/inventory/fortest.go | 27 +- internal/inventory/gate.go | 98 +- internal/inventory/given.go | 21 +- internal/inventory/mirrored_test.go | 35 - internal/inventory/mirrors.go | 36 +- internal/inventory/nodes.go | 25 +- internal/inventory/pending.go | 95 +- internal/inventory/pin_completed_test.go | 88 -- internal/inventory/plans.go | 264 +---- internal/inventory/sent_plans_test.go | 9 +- internal/inventory/source.go | 32 +- internal/recordmirror/mirror.go | 2 +- internal/recordmirror/mirror_store_test.go | 96 +- testdata/store-tests | 9 - 36 files changed, 641 insertions(+), 3331 deletions(-) delete mode 100644 cmd/mesh-controller/record_mirrors_switch_test.go delete mode 100644 internal/inventory/chain_move.go delete mode 100644 internal/inventory/chain_move_test.go delete mode 100644 internal/inventory/chain_moved_test.go create mode 100644 internal/inventory/chain_store_test.go delete mode 100644 internal/inventory/pin_completed_test.go diff --git a/cmd/mesh-controller/chain.go b/cmd/mesh-controller/chain.go index 18436e8b..f54e6258 100644 --- a/cmd/mesh-controller/chain.go +++ b/cmd/mesh-controller/chain.go @@ -7,15 +7,12 @@ import ( "time" "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the -// bus's key-value store, written by the controller that holds the lease. The tables of the controller's store are -// copied into them once, by the serving controller at its start, and a kind is read from its bucket only once the move -// has recorded it (inventory.ChainReads). +// bus's key-value store, written by the controller that holds the lease, and the controller's store keeps none of them. // chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes. func chainHolds(ctx context.Context) error { @@ -33,97 +30,7 @@ func openTheChain(ctx context.Context) (*chain.Chain, error) { return chain.OnTheBus(ctx, js.Conn(), chainHolds) } -// kindChainMoveRefused is the condition of a move that stopped without switching a kind. -const kindChainMoveRefused = "chain-move-refused" - -// chainMoveRetry is how long a move that failed for want of the bus waits before it tries again. -const chainMoveRetry = 30 * time.Second - -// movingTheChain copies the tables into the buckets and records it, at the serving controller's start: after the buckets -// exist and the lease is held, before anything reads a kind that has not moved. **The first try is made here, before the -// controller serves**, so what it serves is read from one place. A move that fails for want of the bus is tried again -// every half minute while the controller serves from the tables; one that is refused — a count that differs, a value -// over the bus's largest — is raised as an urgent condition, and the tables stay what is read: nothing is half-switched. -func movingTheChain(ctx context.Context, inv *inventory.Inventory) { - say := func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) } - // The first try is made here, before the controller serves; the tries after it, while it serves. - if movedOrStopped(ctx, inv, say) { - return - } - go func() { - for { - select { - case <-ctx.Done(): - return - case <-time.After(chainMoveRetry): - } - if movedOrStopped(ctx, inv, say) { - return - } - } - }() -} - -// movedOrStopped makes one try at the move and says whether there is nothing more to try: moved, refused, or the controller -// ending. False for a failure of the kind that is tried again (the bus away, a write not granted yet, the lease not held). -func movedOrStopped(ctx context.Context, inv *inventory.Inventory, say func(string, ...any)) bool { - _, err := inv.MoveChain(ctx, say) - var refusal *inventory.Refusal - switch { - case err == nil: - go clearChainMove(ctx) - go askAgainSoon(ctx, inv) - return true - case errors.As(err, &refusal): - say("%v", err) - go raiseChainMove(ctx, refusal) - return true - case ctx.Err() != nil: - return true - } - say("the move to the bus did not finish, and the tables are read until it does: %v", err) - return false -} - -// chainMoveObservation is the urgent condition for a refused move. -func chainMoveObservation(r *inventory.Refusal) conditions.Observation { - return conditions.Observation{Scope: conditions.ScopeBus, ID: "chain." + r.Kind, Token: kindChainMoveRefused, - Kind: kindChainMoveRefused, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, - Summary: "the controller's " + r.Kind + " were not moved to the bus: " + r.Why, Source: "chain move"} -} - -// whenKept runs f once the serving controller keeps conditions: the move runs before they are kept. -func whenKept(ctx context.Context, f func(k *conditions.Keeper)) { - for conditionsFrom == nil { - select { - case <-ctx.Done(): - return - case <-time.After(time.Second): - } - } - f(conditionsFrom) -} - -func raiseChainMove(ctx context.Context, r *inventory.Refusal) { - whenKept(ctx, func(k *conditions.Keeper) { - if _, err := k.Observe(ctx, chainMoveObservation(r)); err != nil { - fmt.Printf("chain: the condition for the refused move could not be raised: %v\n", err) - } - }) -} - -func clearChainMove(ctx context.Context) { - whenKept(ctx, func(k *conditions.Keeper) { - for _, kind := range []string{inventory.KindBuilds, inventory.KindRollouts, inventory.KindAssignments} { - key := chainMoveObservation(&inventory.Refusal{Kind: kind}).Key() - if _, err := k.Clear(ctx, key, "the "+kind+" are on the bus"); err != nil { - fmt.Printf("chain: the condition for the move of the %s could not be cleared: %v\n", kind, err) - } - } - }) -} - -// attachTheChain gives the inventory of the serving controller the chain on its connection, then moves what has not moved. +// attachTheChain gives the inventory of the serving controller the chain on its connection. func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error { bus, ok := server.Bus().(link.OverNATS) if !ok { @@ -134,7 +41,8 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link. return err } inv.UseChain(c) - movingTheChain(ctx, inv) + // And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving. + go askAgainSoon(ctx, inv) return nil } diff --git a/cmd/mesh-controller/plan_moved_every_writer_test.go b/cmd/mesh-controller/plan_moved_every_writer_test.go index 0973ae79..be7f712b 100644 --- a/cmd/mesh-controller/plan_moved_every_writer_test.go +++ b/cmd/mesh-controller/plan_moved_every_writer_test.go @@ -12,9 +12,9 @@ import ( ) // **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan -// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there -// after the commit; so no statement that writes the plan table may stand anywhere else, and writePlan, the one that -// does, may be called from nowhere but SavePlansMoving. Checked by reading the sources, no store needed. +// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there after the +// write; so no code but chainSavePlans (which SavePlansMoving calls, and no other function) writes a plan to the rollouts +// bucket by anything but a change of the merges it holds. Checked by reading the sources, no store needed. func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) { write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`) var found []string @@ -31,11 +31,13 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) { return err } text := string(raw) - if write.MatchString(text) && filepath.ToSlash(path) != "../../internal/inventory/plans.go" { + if write.MatchString(text) { found = append(found, path+" writes the plan table") } - if strings.Contains(text, "writePlan(") && filepath.ToSlash(path) != "../../internal/inventory/plans.go" { - found = append(found, path+" calls writePlan") + // A plan record is written by the bucket's raw create and update, which only the plans' own file calls. + if filepath.ToSlash(path) != "../../internal/inventory/chain_rollouts.go" && filepath.ToSlash(path) != "../../internal/chain/chain.go" && + strings.Contains(text, "chain.RolloutsOf") { + found = append(found, path+" reaches the rollouts bucket") } return nil }) @@ -51,17 +53,26 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) { t.Fatal(err) } text := string(raw) - if n := len(write.FindAllString(text, -1)); n != 1 { - t.Errorf("plans.go has %d statements that write the plan table; want the one in writePlan", n) - } - calls := regexp.MustCompile(`writePlan\(ctx, tx`).FindAllStringIndex(text, -1) save := strings.Index(text, "func (i *Inventory) SavePlansMoving") next := strings.Index(text[save:], "\nfunc ") - if len(calls) != 1 || calls[0][0] < save || calls[0][0] > save+next { - t.Errorf("writePlan is called %d times, and not only from SavePlansMoving", len(calls)) + if next < 0 { + next = len(text) - save } - if !strings.Contains(text[save:save+next], "PlanSaved(i, *p)") { - t.Error("SavePlansMoving does not say each plan it wrote") + if !strings.Contains(text[save:save+next], "chainSavePlans(") { + t.Error("SavePlansMoving does not write through chainSavePlans") + } + chainRaw, err := os.ReadFile("../../internal/inventory/chain_rollouts.go") + if err != nil { + t.Fatal(err) + } + chainText := string(chainRaw) + at := strings.Index(chainText, "func (i *Inventory) chainSavePlans") + end := strings.Index(chainText[at:], "\nfunc ") + if !strings.Contains(chainText[at:at+end], "PlanSaved(i, *p)") { + t.Error("chainSavePlans does not say each plan it wrote") + } + if n := strings.Count(chainText, "createRaw(ctx, c, p.ID"); n != 1 { + t.Errorf("a plan is made from %d places of chain_rollouts.go, want the one in chainSavePlans", n) } } diff --git a/cmd/mesh-controller/record_mirrors.go b/cmd/mesh-controller/record_mirrors.go index 970956e8..25fd9d15 100644 --- a/cmd/mesh-controller/record_mirrors.go +++ b/cmd/mesh-controller/record_mirrors.go @@ -44,7 +44,6 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats. return e } sources := map[string]func(context.Context) (map[string][]byte, error){ - recordmirror.Assignments.Name: inv.AssignmentsMirrorToWrite, recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror, recordmirror.Predefined.Name: inv.PredefinedMirror, } @@ -55,15 +54,11 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats. events := &assignmentEvents{} for { if holds() { - // The assignments, once they are keys of the bus, are written by the verbs that change them (novox/hq ADR - // 0408): this tick keeps the provisioner's plan in their values and says what changed; the mirror of the store - // writes them only until the move. - onTheBus := keepingAssignments(ctx, inv, announce, epoch, events, failing) + // The assignments are keys of the bus, written by the verbs that change them (novox/hq ADR 0408): this tick + // keeps the provisioner's plan in their values and says what changed. They are no mirror of the store. + keepingAssignments(ctx, inv, announce, epoch, events, failing) for _, m := range recordmirror.Mirrors() { - if m.Name == recordmirror.Assignments.Name && onTheBus { - continue - } - why := mirrorOnce(ctx, inv, api, m, writers, sources[m.Name], announce, epoch, holds) + why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds) if why != failing[m.Name] { if why != "" { fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why) @@ -83,7 +78,7 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats. } // mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could. -func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer, +func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer, source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64, holds func() bool) string { reading, cancel := context.WithTimeout(ctx, 30*time.Second) @@ -95,9 +90,6 @@ func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.Jet return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err) } w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds} - if m.Name == recordmirror.Assignments.Name { - w.Holds = assignmentsHolds(inv, holds) - } writers[m.Name] = w } want, err := source(reading) @@ -128,21 +120,10 @@ type assignmentEvents struct { // the values it belongs to, and one `assignment-changed` event for each key that is new, changed or ended since the last // look — whoever wrote it, a verb in this process or a command in another. False when they are not on the bus yet. func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce recordmirror.Announcer, epoch func() uint64, - events *assignmentEvents, failing map[string]string) bool { + events *assignmentEvents, failing map[string]string) { reading, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() const name = "assignments" - if _, on, err := inv.KeptAssignments(reading); err != nil || !on { - if err != nil { - // Not known to be off the bus: the store's mirror is not written over what the bus may hold. - if failing[name] == "" { - failing[name] = err.Error() - fmt.Printf("the assignments cannot be read from the bus: %v\n", err) - } - return true - } - return false - } if _, _, err := inv.KeepAssignmentPlans(reading); err != nil { if failing[name] != err.Error() { failing[name] = err.Error() @@ -152,9 +133,9 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce fmt.Printf("the provisioner's plans are kept in the assignments again\n") failing[name] = "" } - now, _, err := inv.KeptAssignments(reading) + now, err := inv.KeptAssignments(reading) if err != nil { - return true + return } if events.known != nil { keys := make([]string, 0, len(now)+len(events.known)) @@ -194,12 +175,4 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce } } events.known = now - return true -} - -// assignmentsHolds is the assignments mirror's leave to write: the lease's, and the assignments neither being switched to -// the bus nor on it. Asked by the writer before every write and purge, so a pass that read its source before the switch -// writes nothing after it (ADR 0408 §4). -func assignmentsHolds(inv *inventory.Inventory, holds func() bool) func() bool { - return func() bool { return holds() && inv.AssignmentsMirrorMayWrite() } } diff --git a/cmd/mesh-controller/record_mirrors_switch_test.go b/cmd/mesh-controller/record_mirrors_switch_test.go deleted file mode 100644 index 88ed29a1..00000000 --- a/cmd/mesh-controller/record_mirrors_switch_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "errors" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/recordmirror" -) - -// **A writer of the store's mirror of the assignments, made before the assignments are switched to the bus, writes and -// purges nothing after** (ADR 0408 §4, issue 1093): its leave to write is asked before every write, and false once they are -// switching or moved. -func TestAMirrorWriterMadeBeforeTheSwitchWritesAndPurgesNothingAfter(t *testing.T) { - inv := inventory.NewForTest() - w := &recordmirror.Writer{Mirror: recordmirror.Assignments, Holds: assignmentsHolds(inv, func() bool { return true })} - if !w.Holds() { - t.Fatal("the writer may not write before the switch") - } - inv.BeginSwitchForTest(inventory.KindAssignments) - if w.Holds() { - t.Fatal("the writer may write while the assignments are switching") - } - // Refused before it reads the bucket (it has none here) or lays or ends a key. - r, err := w.Sync(t.Context(), map[string][]byte{"anchor.notes": []byte(`{}`)}) - if !errors.Is(err, recordmirror.ErrNotHeld) || r.Written != 0 || r.Removed != 0 { - t.Fatalf("the writer wrote after the switch: %+v %v", r, err) - } -} diff --git a/cmd/mesh-controller/sent_required_by_integration_test.go b/cmd/mesh-controller/sent_required_by_integration_test.go index 5cef69b8..077a4192 100644 --- a/cmd/mesh-controller/sent_required_by_integration_test.go +++ b/cmd/mesh-controller/sent_required_by_integration_test.go @@ -85,10 +85,7 @@ func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) t.Fatal(err) } inv.SetProvisioningPlans(inv.SentProvisioningPlans) - mirror, err := inv.AssignmentsMirror(ctx) - if err != nil { - t.Fatal(err) - } + mirror := inv.PlanValuesForTest(t) var value inventory.AssignmentMirrored if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil { t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err) diff --git a/internal/inventory/batches.go b/internal/inventory/batches.go index c0cd6ab2..294c5f00 100644 --- a/internal/inventory/batches.go +++ b/internal/inventory/batches.go @@ -3,11 +3,8 @@ package inventory import ( "context" "encoding/json" - "errors" "strings" "time" - - "github.com/jackc/pgx/v5" ) // The merges a batch or walk answers (novox/hq ADR 0276): every merge the controller heard into a branch a @@ -33,138 +30,78 @@ type BatchedMerge struct { // AddMerge keeps a merge for a plan; false when the merge was kept already, for whatever plan. func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { - return false, err - } else if on { - return i.chainAddMerge(ctx, c, m) - } - tag, err := i.store.Pool().Exec(ctx, - `insert into batched_merge (repository, branch, commit_hash, merged_at, heard_at, event, plan_id, alone) - values (lower($1), $2, $3, $4, $5, $6, nullif($7, ''), $8) on conflict do nothing`, - m.Repository, m.Branch, m.Commit, mergedAt(m.Merged), m.Heard, []byte(m.Event), m.Plan, m.Alone) + c, err := i.theChain(ctx) if err != nil { return false, err } - return tag.RowsAffected() == 1, nil + return i.chainAddMerge(ctx, c, m) } // MergeOf is one kept merge; false when it was never heard. func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (BatchedMerge, bool, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { - return BatchedMerge{}, false, err - } else if on { - return i.chainMergeOf(ctx, c, repository, commit) - } - ms, err := i.merges(ctx, `where repository = lower($1) and commit_hash = $2`, repository, commit) - if err != nil || len(ms) == 0 { + c, err := i.theChain(ctx) + if err != nil { return BatchedMerge{}, false, err } - return ms[0], true, nil + return i.chainMergeOf(ctx, c, repository, commit) } // MergesOf is every merge a plan answers, oldest merge first. func (i *Inventory) MergesOf(ctx context.Context, plan string) ([]BatchedMerge, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - return i.chainMergesOf(ctx, c, plan) } - return i.merges(ctx, `where plan_id = $1 order by merged_at nulls last, heard_at`, plan) + return i.chainMergesOf(ctx, c, plan) } // LaterMergesOf is every merge of a repository's branch made after a moment and answered by a plan, the // newest first. func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string, after time.Time) ([]BatchedMerge, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - return i.chainLaterMergesOf(ctx, c, repository, branch, after) } - return i.merges(ctx, `where repository = lower($1) and branch = $2 and merged_at > $3 and plan_id is not null - order by merged_at desc`, repository, branch, after) + return i.chainLaterMergesOf(ctx, c, repository, branch, after) } // KeptOnBranch says the controller keeps a merge of a repository's branch, and when it heard the branch's first: a merge of a branch it keeps merges of, made since then, is the record's to answer, whatever a // module's last look says (novox/hq ADR 0276). func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string) (bool, time.Time, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { - return false, time.Time{}, err - } else if on { - return i.chainKeptOnBranch(ctx, c, repository, branch) - } - var first *time.Time - var kept bool - err := i.store.Pool().QueryRow(ctx, - `select min(heard_at), count(*) > 0 from batched_merge where repository = lower($1) and branch = $2`, - repository, branch).Scan(&first, &kept) - if err != nil || first == nil { + c, err := i.theChain(ctx) + if err != nil { return false, time.Time{}, err } - return kept, first.UTC(), nil + return i.chainKeptOnBranch(ctx, c, repository, branch) } // AloneMerges is every merge waiting to be walked on its own commit, the newest first (ADR 0276 decision 3). func (i *Inventory) AloneMerges(ctx context.Context) ([]BatchedMerge, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - return i.chainAloneMerges(ctx, c) } - return i.merges(ctx, `where plan_id is null and alone order by merged_at desc nulls last, heard_at desc`) + return i.chainAloneMerges(ctx, c) } // AnswerMerge moves a merge to the plan that answers it now; an empty plan with alone leaves it waiting to // be walked on its own commit. func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan string, alone bool) error { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return err - } else if on { - return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}}) } - _, err := i.store.Pool().Exec(ctx, - `update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`, - repository, commit, plan, alone) - return err + return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}}) } // Batches is every batch not yet cut: assembling or queued, oldest first. One at a time is the rule; more // are read so a fault is seen. func (i *Inventory) Batches(ctx context.Context) ([]Plan, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { - return nil, err - } else if on { - return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated) - } - return i.plans(ctx, `where state in ('assembling', 'queued') order by created`) -} - -func (i *Inventory) merges(ctx context.Context, tail string, args ...any) ([]BatchedMerge, error) { - rows, err := i.store.Pool().Query(ctx, - `select repository, branch, commit_hash, merged_at, heard_at, event, coalesce(plan_id, ''), alone - from batched_merge `+tail, args...) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - var out []BatchedMerge - for rows.Next() { - var m BatchedMerge - var merged *time.Time - var event []byte - if err := rows.Scan(&m.Repository, &m.Branch, &m.Commit, &merged, &m.Heard, &event, &m.Plan, &m.Alone); err != nil { - return nil, err - } - if merged != nil { - m.Merged = merged.UTC() - } - m.Heard = m.Heard.UTC() - m.Event = event - out = append(out, m) - } - if errors.Is(rows.Err(), pgx.ErrNoRows) { - return nil, nil - } - return out, rows.Err() + return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated) } // SameRepository says two spellings of a repository are one. diff --git a/internal/inventory/builds.go b/internal/inventory/builds.go index 1fa1c78d..3226d029 100644 --- a/internal/inventory/builds.go +++ b/internal/inventory/builds.go @@ -2,14 +2,9 @@ package inventory import ( "context" - "encoding/json" - "errors" - "sort" "strings" "time" - "github.com/jackc/pgx/v5" - "github.com/novox/mesh-controller/internal/chain" ) @@ -134,51 +129,11 @@ func (b Build) Worked() bool { return b.Failed == "" } // whoever asked and once on the exchange when nobody was. Recording both would show one build as // two, and which of the two is real is not a question anybody could answer afterwards. func (i *Inventory) RecordBuild(ctx context.Context, b Build) error { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return err - } else if on { - return i.chainRecordBuild(ctx, c, b) - } - made, err := json.Marshal(b.Made) + c, err := i.theChain(ctx) if err != nil { return err } - against, err := json.Marshal(b.Against) - if err != nil { - return err - } - read, err := json.Marshal(b.Read) - if err != nil { - return err - } - var sources any - if len(b.Sources) > 0 { - raw, err := json.Marshal(b.Sources) - if err != nil { - return err - } - sources = raw - } - var module *string - if b.Module != "" { - module = &b.Module - } - var asked *time.Time - if !b.Asked.IsZero() { - asked = &b.Asked - } - _, err = i.store.Pool().Exec(ctx, - `insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, - source_path, manifest, built_against, built_contexts, asked, source_fingerprint, - build_sources) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) - on conflict (id) do nothing`, - b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made, - b.Path, manifestOrNil(b.Manifest), against, read, asked, b.SourceFingerprint, sources) - if err != nil { - return err - } - return i.RecordMirrored(ctx, b.ID, "", b.Mirrored) + return i.chainRecordBuild(ctx, c, b) } // RecordMirrored keeps that the mesh copied these bases into the artifact store (novox/hq ADR 0257): @@ -221,79 +176,22 @@ func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Bui if limit <= 0 { limit = 20 } - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainBuildList(ctx, c, module, limit) - } - if limit <= 0 { - limit = 20 - } - query := `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at - from build order by at desc limit $1` - args := []any{limit} - if module != "" { - query = `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at - from build where module = $2 order by at desc limit $1` - args = append(args, module) - } - - rows, err := i.store.Pool().Query(ctx, query, args...) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - - var out []Build - for rows.Next() { - var b Build - var made []byte - var asked *time.Time - if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, - &b.On, &b.Failed, &made, &asked, &b.At); err != nil { - return nil, err - } - if asked != nil { - b.Asked = *asked - } - if err := json.Unmarshal(made, &b.Made); err != nil { - return nil, err - } - out = append(out, b) - } - return out, rows.Err() + return i.chainBuildList(ctx, c, module, limit) } // BuildByID is one build's record, by the id its request carried — what a plan matches its outcome // by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the // repository, path, ref and commit from. False when no outcome with that id was recorded. func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return Build{}, false, err - } else if on { - return i.chainBuildByID(ctx, c, id) - } - var b Build - var made []byte - var asked *time.Time - err := i.store.Pool().QueryRow(ctx, - `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, - coalesce(source_path,''), asked, at - from build where id = $1`, id).Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, - &b.On, &b.Failed, &made, &b.Path, &asked, &b.At) - if errors.Is(err, pgx.ErrNoRows) { - return Build{}, false, nil - } + c, err := i.theChain(ctx) if err != nil { return Build{}, false, err } - if asked != nil { - b.Asked = *asked - } - if err := json.Unmarshal(made, &b.Made); err != nil { - return Build{}, false, err - } - return b, true, nil + return i.chainBuildByID(ctx, c, id) } // Held is every artifact this mesh has built, keyed "/". @@ -375,40 +273,11 @@ func registeredOnly(held, registered map[string]string) map[string]string { // edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a // module standing on nothing looks like: an edge the mesh has not derived is not an edge. func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainBuiltAgainst(ctx, c) - } - rows, err := i.store.Pool().Query(ctx, - `select distinct on (module) module, built_against - from build - where module is not null and module <> '' and failed = '' - order by module, `+newestRequestFirst) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - - against := map[string][]string{} - for rows.Next() { - var module string - var raw []byte - if err := rows.Scan(&module, &raw); err != nil { - return nil, err - } - if len(raw) == 0 { - continue - } - var refs []string - if err := json.Unmarshal(raw, &refs); err != nil { - continue - } - if len(refs) > 0 { - against[module] = refs - } - } - return against, rows.Err() + return i.chainBuiltAgainst(ctx, c) } // ReadRepositories is what each module's newest successful build read source from besides its own @@ -423,95 +292,11 @@ func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, erro // none — off the trunk, from a builder that predates it, or of a source not known — leaves its module read // as before: every file of a context, and its own directory or root. func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainReadRepositories(ctx, c) - } - // The newest build of each module whatever its outcome: a failed one newer than the newest that - // worked leaves that one's build source stale — the merge it was asked for may have changed the closure - // (novox/hq ADR 0267), so its module is read whole until a build works again. - newest := map[string]struct { - at time.Time - failed bool - }{} - tried, err := i.store.Pool().Query(ctx, - `select distinct on (module) module, coalesce(asked, at), failed <> '' - from build - where module is not null and module <> '' - order by module, `+newestRequestFirst) + c, err := i.theChain(ctx) if err != nil { return nil, err } - for tried.Next() { - var module string - var at time.Time - var failed bool - if err := tried.Scan(&module, &at, &failed); err != nil { - tried.Close() - return nil, err - } - newest[module] = struct { - at time.Time - failed bool - }{at, failed} - } - tried.Close() - if err := tried.Err(); err != nil { - return nil, err - } - - rows, err := i.store.Pool().Query(ctx, - `select distinct on (module) module, built_contexts, build_sources, coalesce(asked, at), - (select source from module where module.name = build.module) - from build - where module is not null and module <> '' and failed = '' - order by module, `+newestRequestFirst) - if err != nil { - return nil, err - } - defer rows.Close() - - read := map[string][]ReadRepository{} - for rows.Next() { - var module string - var raw, rawSources []byte - var built time.Time - var own *string - if err := rows.Scan(&module, &raw, &rawSources, &built, &own); err != nil { - return nil, err - } - var of []ReadRepository - if len(raw) > 0 { - if err := json.Unmarshal(raw, &of); err != nil { - of = nil - } - } - var sources []BuildSource - if len(rawSources) > 0 { - if err := json.Unmarshal(rawSources, &sources); err != nil { - sources = nil - } - } - whole := "" - if n, known := newest[module]; known && n.failed && n.at.After(built) && len(sources) > 0 { - sources, whole = nil, "a newer build of it failed" - } - of = WithBuildSources(of, sources) - if own != nil { - of = OwnRepositoryAsOwn(of, *own) - } - if whole != "" { - of = append(of, ReadRepository{Own: true, Whole: whole}) - } - if len(of) > 0 { - for k := range of { - of[k].Built, of[k].Looked = built, newest[module].at - } - read[module] = of - } - } - return read, rows.Err() + return i.chainReadRepositories(ctx, c) } // WithBuildSources lays a build's build sources over the repositories it read: a context's paths on its @@ -588,18 +373,6 @@ func namesRepository(registered, named string) bool { return reg == name || strings.HasSuffix(reg, "/"+name) } -// manifestOrNil keeps the difference between "declared nothing" and "predates this being kept". -// -// A build recorded before the mesh kept manifests has no manifest, and that is not the same as one -// whose manifest was empty. A replay can then say which it is holding instead of inventing an -// empty declaration for a module that certainly had one. -func manifestOrNil(raw []byte) any { - if len(raw) == 0 { - return nil - } - return raw -} - // Announceable is every build worth telling a catalogue about, oldest first. // // **Oldest first, because a graph is built in the order things happened.** Registering a module @@ -614,53 +387,9 @@ func manifestOrNil(raw []byte) any { // One row per module and commit: a module built twice at the same commit is one fact, and the // row asked last is the one whose artifacts are current (novox/hq 04-ISSUES/219). func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainAnnounceable(ctx, c) - } - rows, err := i.store.Pool().Query(ctx, - `select distinct on (module, commit_hash) - id, repository, ref, module, commit_hash, built_on, failed, made, - source_path, manifest, built_against, at - from build - where failed = '' and module is not null and module <> '' and commit_hash <> '' - order by module, commit_hash, `+newestRequestFirst) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - - var out []Build - for rows.Next() { - var b Build - var made []byte - var manifest, against []byte - if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, - &b.On, &b.Failed, &made, &b.Path, &manifest, &against, &b.At); err != nil { - return nil, err - } - if err := json.Unmarshal(made, &b.Made); err != nil { - return nil, err - } - // Null rather than empty is a build recorded before the mesh kept these, and saying so is - // the point of keeping them nullable: the replay carries nothing rather than an empty - // declaration for a module that certainly had one. - if len(manifest) > 0 { - b.Manifest = manifest - } - if len(against) > 0 { - if err := json.Unmarshal(against, &b.Against); err != nil { - return nil, err - } - } - out = append(out, b) - } - if err := rows.Err(); err != nil { - return nil, err - } - // Sorted here rather than in the query, because `distinct on` fixes the ordering it needs and - // the order that matters to a catalogue is a different one. - sort.Slice(out, func(a, b int) bool { return out[a].At.Before(out[b].At) }) - return out, nil + return i.chainAnnounceable(ctx, c) } diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 56a11d17..adc40bd3 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -258,31 +258,17 @@ func (i *Inventory) SourceOf(ctx context.Context, module string) (Source, error) // machines are running last week's version" is the question somebody actually has, and it is the // one novox/hq ADR 0010 names as the thing that must not be lost. func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) { - query := `select m.name, coalesce(n.name, '') - from module m - left join assignment a on a.module = m.name - left join node n on n.id = a.node - where m.source is not null - and m.source_head is not null - and coalesce(m.built_from, '') is distinct from m.source_head - order by m.name, n.name` - _, onTheBus, err := i.ChainReads(ctx, KindAssignments) + assigned, err := i.assignments(ctx) if err != nil { return nil, err } - var assigned []assignRow - if onTheBus { - query = `select m.name, '' + rows, err := i.store.Pool().Query(ctx, + `select m.name from module m where m.source is not null and m.source_head is not null and coalesce(m.built_from, '') is distinct from m.source_head - order by m.name` - if assigned, err = i.assignments(ctx); err != nil { - return nil, err - } - } - rows, err := i.store.Pool().Query(ctx, query) + order by m.name`) if err != nil { return nil, err } @@ -290,16 +276,11 @@ func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) { out := map[string][]string{} for rows.Next() { - var module, node string - if err := rows.Scan(&module, &node); err != nil { + var module string + if err := rows.Scan(&module); err != nil { return nil, err } - if _, seen := out[module]; !seen { - out[module] = nil - } - if node != "" { - out[module] = append(out[module], node) - } + out[module] = nil } if err := rows.Err(); err != nil { return nil, err @@ -654,21 +635,11 @@ func (i *Inventory) Assign(ctx context.Context, nodeName, module string) (bool, if err := i.runsSomewhere(ctx, module); err != nil { return false, err } - if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil { - return false, err - } else if on { - return i.chainAssign(ctx, c, node, module) - } - tag, err := i.store.Pool().Exec(ctx, - `insert into assignment (node, module) values ($1, $2) on conflict do nothing`, - node.ID, module) - if err != nil && strings.Contains(err.Error(), "assignment_module_fkey") { - return false, fmt.Errorf("%w: %s", ErrNoSuchModule, module) - } + c, err := i.theChain(ctx) if err != nil { return false, err } - return tag.RowsAffected() > 0, nil + return i.chainAssign(ctx, c, node, module) } // Unassign takes a module off a node. @@ -677,21 +648,12 @@ func (i *Inventory) Unassign(ctx context.Context, nodeName, module string) error if err != nil { return err } - if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil { + c, err := i.theChain(ctx) + if err != nil { + return err + } + if err := i.chainUnassign(ctx, c, node, module); err != nil { return err - } else if on { - if err := i.chainUnassign(ctx, c, node, module); err != nil { - return err - } - } else { - tag, err := i.store.Pool().Exec(ctx, - `delete from assignment where node = $1 and module = $2`, node.ID, module) - if err != nil { - return err - } - if tag.RowsAffected() == 0 { - return fmt.Errorf("%s is not assigned to %s", module, nodeName) - } } // The seat it held falls back to derivation rather than pointing at nothing: what the store's foreign key did. if _, err := i.store.Pool().Exec(ctx, `delete from seat_holding where node = $1 and module = $2`, node.ID, module); err != nil { @@ -1362,39 +1324,21 @@ type Entry struct { // catalogue nobody lists, and the questions here — what is this, where did it come from, who is // running it — are asked together every time. func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) { - query := `select m.name, m.manifest, - coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''), - coalesce(m.built_from, ''), coalesce(m.source_head, ''), - coalesce(m.source_seen, to_timestamp(0)), - coalesce(array_agg(n.name order by n.name) filter (where n.name is not null), '{}') - from module m - left join assignment a on a.module = m.name - left join node n on n.id = a.node - group by m.name, m.manifest, m.source, m.source_path, m.source_seat, m.ref, m.built_from, - m.source_head, m.source_seen - order by m.name` - // On the bus the assignments are read from the bucket, and the machines of a module put beside it here. - _, onTheBus, err := i.ChainReads(ctx, KindAssignments) + // The machines of a module are read from the assignments bucket and put beside it here. + assigned, err := i.assignments(ctx) if err != nil { return nil, err } - var assignedBy map[string][]string - if onTheBus { - query = `select m.name, m.manifest, + assignedBy := map[string][]string{} + for _, a := range assigned { + assignedBy[a.Module] = append(assignedBy[a.Module], a.Node) + } + const query = `select m.name, m.manifest, coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''), coalesce(m.built_from, ''), coalesce(m.source_head, ''), coalesce(m.source_seen, to_timestamp(0)), '{}'::text[] from module m order by m.name` - rows, err := i.assignments(ctx) - if err != nil { - return nil, err - } - assignedBy = map[string][]string{} - for _, a := range rows { - assignedBy[a.Module] = append(assignedBy[a.Module], a.Node) - } - } rows, err := i.store.Pool().Query(ctx, query) if err != nil { return nil, err @@ -1414,9 +1358,7 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) { if source.Seen.Unix() == 0 { source.Seen = time.Time{} } - if onTheBus { - on = assignedBy[name] - } + on = assignedBy[name] var m catalogue.Manifest if err := json.Unmarshal(raw, &m); err != nil { return nil, err diff --git a/internal/inventory/chain.go b/internal/inventory/chain.go index a56e784a..38e8e389 100644 --- a/internal/inventory/chain.go +++ b/internal/inventory/chain.go @@ -5,55 +5,30 @@ import ( "errors" "fmt" "sync" - "sync/atomic" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" "github.com/novox/mesh-controller/internal/chain" ) -// **The chain moves to the bus kind by kind** (novox/hq ADR 0408 §8, issue 981). The tables of builds, rollouts and -// assignments are copied once into the chain's buckets by the serving controller at its start (MoveChain), which then -// records each kind it moved in chain_moved (migration 0104). **Until a kind is recorded there nothing reads its bucket; -// once it is, nothing reads its tables**: ChainReads answers which, and every method of a kind asks it first. - -// The kinds of the chain, as chain_moved names them. -const ( - KindBuilds = "builds" - KindRollouts = "rollouts" -) +// **The chain lives on the bus** (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the +// bus's key-value store, the controller's store keeps none of them, and every method of a kind reads and writes the chain this +// inventory is given (theChain). // testChains are the chains in memory a test made for its store, by the store's address: every inventory opened on that // store is given the same chain, as every process of a mesh is on the same bus. Empty outside a test. var testChains sync.Map -// ErrNoChain is a kind moved to the bus read by an inventory that has no chain attached: a process that did not -// open the bus. Refused loudly, never answered from the tables the move left behind. +// ErrNoChain is a read or write of the chain by an inventory that has none attached: a process that did not open the bus. +// Refused loudly. var ErrNoChain = errors.New("this kind of record lives on the bus, and this process has not opened the bus") -// movedKinds is which kinds this process has read as moved: once moved, always moved, so it is read from the store -// until it is true and not after. -type movedKinds struct { +// chainState is what an inventory keeps of the chain it is given. +type chainState struct { mu sync.Mutex - kind map[string]bool open ChainOpener // view is the rollouts bucket kept in memory by one watch (issue 1092); pending are the merges added to a plan that // is not saved yet. view *rolloutView pending map[string]pendingMerge - // switching is set when the assignments begin to be switched to the bus and kept once they are: the store's mirror of - // them writes nothing from then on (AssignmentsMirrorMayWrite). Cleared only if the switch fails. - switching atomic.Bool -} - -// AssignmentsMirrorMayWrite says the store's mirror of the assignments may still write the bucket: the assignments are -// neither being switched to the bus nor on it. Asked before every write of the mirror's writer, not once a pass. -func (i *Inventory) AssignmentsMirrorMayWrite() bool { - i.moved.mu.Lock() - moved := i.moved.kind[KindAssignments] - i.moved.mu.Unlock() - return !moved && !i.moved.switching.Load() } // ChainOpener opens the chain on the bus when a process first needs it: a command that reads and never moved anything has @@ -62,99 +37,36 @@ type ChainOpener func(ctx context.Context) (*chain.Chain, error) // UseChainOpener gives the inventory the means to open the chain when it needs it and none is attached. func (i *Inventory) UseChainOpener(open ChainOpener) { - i.moved.mu.Lock() - i.moved.open = open - i.moved.mu.Unlock() + i.state.mu.Lock() + i.state.open = open + i.state.mu.Unlock() } // UseChain attaches the chain the moved kinds are read and written through. The serving controller attaches the bus's, // holding the lease for its writes; a test attaches one in memory. func (i *Inventory) UseChain(c *chain.Chain) { - i.moved.mu.Lock() - defer i.moved.mu.Unlock() + i.state.mu.Lock() + defer i.state.mu.Unlock() i.chain = c - if i.moved.view != nil { - i.moved.view.stop() - i.moved.view = nil + if i.state.view != nil { + i.state.view.stop() + i.state.view = nil } } // attached is the chain this inventory has, nil when none. func (i *Inventory) attached() *chain.Chain { - i.moved.mu.Lock() - defer i.moved.mu.Unlock() + i.state.mu.Lock() + defer i.state.mu.Unlock() return i.chain } -// ChainReads says whether a kind of the chain is read from its bucket: chain_moved names it. It answers the chain to read -// through, and ErrNoChain when the kind is moved and none is attached. -func (i *Inventory) ChainReads(ctx context.Context, kind string) (*chain.Chain, bool, error) { - i.moved.mu.Lock() - known := i.moved.kind[kind] - i.moved.mu.Unlock() - if !known { - var moved bool - err := i.store.Pool().QueryRow(ctx, `select counts ? $1 from chain_moved`, kind).Scan(&moved) - var pgErr *pgconn.PgError - if errors.Is(err, pgx.ErrNoRows) || (errors.As(err, &pgErr) && pgErr.Code == "42P01") { - // No row, or a store older than the migration that makes the table: nothing moved. - return nil, false, nil - } - if err != nil { - return nil, false, fmt.Errorf("whether %s moved to the bus cannot be read: %w", kind, err) - } - if !moved { - return nil, false, nil - } - i.moved.mu.Lock() - if i.moved.kind == nil { - i.moved.kind = map[string]bool{} - } - i.moved.kind[kind] = true - i.moved.mu.Unlock() - } - i.moved.mu.Lock() - defer i.moved.mu.Unlock() - if i.chain == nil && i.moved.open != nil { - c, err := i.moved.open(ctx) - if err != nil { - return nil, false, fmt.Errorf("%w: %s: %v", ErrNoChain, kind, err) - } - i.chain = c - } - if i.chain == nil { - return nil, false, fmt.Errorf("%w: %s", ErrNoChain, kind) - } - return i.chain, true, nil -} - -// ChainMoved is, per kind moved, how many records the move wrote; empty before any is. -func (i *Inventory) ChainMoved(ctx context.Context) (map[string]int, error) { - var counts map[string]int - err := i.store.Pool().QueryRow(ctx, `select counts from chain_moved`).Scan(&counts) - if errors.Is(err, pgx.ErrNoRows) { - return map[string]int{}, nil - } - return counts, err -} - -// NewForTest is an inventory with no store, for a test of what needs none. -func NewForTest() *Inventory { return &Inventory{moved: &movedKinds{}} } - -// BeginSwitchForTest sets the flag a kind's switch sets, as a switch under way does. -func (i *Inventory) BeginSwitchForTest(kind string) { - if kind == KindAssignments { - i.moved.switching.Store(true) - } -} - -// theChain is the chain this inventory writes and reads, opened if a command has not yet: for what has no table of its own -// to be read from while it is moved (a pending assignment is a value of the assignments bucket from the start). +// theChain is the chain this inventory writes and reads, opened if a command has not yet. func (i *Inventory) theChain(ctx context.Context) (*chain.Chain, error) { - i.moved.mu.Lock() - defer i.moved.mu.Unlock() - if i.chain == nil && i.moved.open != nil { - c, err := i.moved.open(ctx) + i.state.mu.Lock() + defer i.state.mu.Unlock() + if i.chain == nil && i.state.open != nil { + c, err := i.state.open(ctx) if err != nil { return nil, fmt.Errorf("%w: %v", ErrNoChain, err) } diff --git a/internal/inventory/chain_assignments.go b/internal/inventory/chain_assignments.go index 767580ab..ff7b8a02 100644 --- a/internal/inventory/chain_assignments.go +++ b/internal/inventory/chain_assignments.go @@ -21,7 +21,6 @@ import ( // here (chain.Assignment.Stands). No `pending` state is written yet (issue 1094). // KindAssignments is the kind of the chain that is each module assigned to a node, with the build it runs there. -const KindAssignments = "assignments" func assignmentKey(node, module string) string { return node + "." + module } @@ -31,32 +30,15 @@ type assignRow struct { Assigned time.Time } -// assignments is every assignment, by node then module: from the bucket when the kind is moved, else from the store. +// assignments is every assignment, by node then module, read from the assignments bucket. // A key whose value names no time assigned is a module only pulled in by another (issue 765), and one that has ended or // waits in a state is gone: no assignment. One on a machine the store does not know is none either, as the store took a // removed machine's. func (i *Inventory) assignments(ctx context.Context) ([]assignRow, error) { - c, on, err := i.ChainReads(ctx, KindAssignments) + c, err := i.theChain(ctx) if err != nil { return nil, err } - if !on { - rows, err := i.store.Pool().Query(ctx, - `select n.name, a.module, a.assigned from assignment a join node n on n.id = a.node order by n.name, a.module`) - if err != nil { - return nil, err - } - defer rows.Close() - var out []assignRow - for rows.Next() { - var a assignRow - if err := rows.Scan(&a.Node, &a.Module, &a.Assigned); err != nil { - return nil, err - } - out = append(out, a) - } - return out, rows.Err() - } all, err := c.Assignments.All(ctx) if err != nil { return nil, fmt.Errorf("the assignments cannot be read from the bus: %w", err) @@ -255,8 +237,8 @@ func (i *Inventory) chainAwaitingSince(ctx context.Context, name string) (time.T // and rollout another write put in the same value, and never ending an assignment. A plan the provisioner's rules refuse is // refused by name and nothing is written (AssignmentValues). func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed int, err error) { - c, on, err := i.ChainReads(ctx, KindAssignments) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return 0, 0, err } var plans map[string]Provisioning @@ -342,17 +324,16 @@ type KeptValue struct { Revision uint64 } -// KeptAssignments is every key of the assignments bucket that holds a value that stands, and false when the assignments are -// not on the bus. -// What the serving controller compares from one look to the next to say a key changed (`assignment-changed`). -func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, bool, error) { - c, on, err := i.ChainReads(ctx, KindAssignments) - if err != nil || !on { - return nil, false, err +// KeptAssignments is every key of the assignments bucket that holds a value that stands. What the serving controller +// compares from one look to the next to say a key changed (`assignment-changed`). +func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, error) { + c, err := i.theChain(ctx) + if err != nil { + return nil, err } all, err := c.Assignments.All(ctx) if err != nil { - return nil, true, err + return nil, err } out := make(map[string]KeptValue, len(all)) for key, k := range all { @@ -361,11 +342,11 @@ func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, } body, err := json.Marshal(k.Value) if err != nil { - return nil, true, err + return nil, err } out[key] = KeptValue{Value: body, Revision: k.Revision} } - return out, true, nil + return out, nil } // What a pass that keeps the plans does with one key. diff --git a/internal/inventory/chain_assignments_key_test.go b/internal/inventory/chain_assignments_key_test.go index 839404f1..aa7c3efb 100644 --- a/internal/inventory/chain_assignments_key_test.go +++ b/internal/inventory/chain_assignments_key_test.go @@ -1,9 +1,7 @@ package inventory import ( - "context" "encoding/json" - "errors" "testing" "time" @@ -72,13 +70,3 @@ func TestAPassThatKeepsThePlansNeverRevivesAnEndedAssignment(t *testing.T) { } } } - -// **The store's mirror of the assignments is refused to its writer once the assignments are moved** (ADR 0408 §4, issue -// 1093): asked at the moment of the write, so a tick that decided before the switch lays nothing after it. -func TestTheStoresMirrorOfTheAssignmentsIsRefusedToItsWriterOnceTheyAreMoved(t *testing.T) { - inv := &Inventory{moved: &movedKinds{kind: map[string]bool{KindAssignments: true}}} - inv.UseChain(chain.InMemory(nil)) - if _, err := inv.AssignmentsMirrorToWrite(context.Background()); !errors.Is(err, ErrAssignmentsMoved) { - t.Fatalf("the writer was given the store's rows: %v", err) - } -} diff --git a/internal/inventory/chain_links.go b/internal/inventory/chain_links.go index cb989951..69458990 100644 --- a/internal/inventory/chain_links.go +++ b/internal/inventory/chain_links.go @@ -38,8 +38,8 @@ func WithRollout(ctx context.Context, id string, modules map[string]RolloutModul // AssignmentRuns is what each module assigned to a machine runs there, as its assignment names it: the build and the // commit. Empty when the assignments are not on the bus, and for a module not sent yet. func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string]chain.Ref, error) { - c, on, err := i.ChainReads(ctx, KindAssignments) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return nil, err } all, err := c.Assignments.All(ctx) @@ -58,8 +58,8 @@ func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string // AssignmentsOn are the assignments of a machine as the chain names them, by module: the standing ones, with their links. func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Assignment, error) { - c, on, err := i.ChainReads(ctx, KindAssignments) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return nil, err } all, err := c.Assignments.All(ctx) diff --git a/internal/inventory/chain_move.go b/internal/inventory/chain_move.go deleted file mode 100644 index 281ce98f..00000000 --- a/internal/inventory/chain_move.go +++ /dev/null @@ -1,663 +0,0 @@ -package inventory - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "time" - - "github.com/jackc/pgx/v5" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/chain" -) - -// **The move** (novox/hq ADR 0408 §8, issue 981 note 7 Q2): the serving controller writes the rows of a kind into its -// bucket at its start, holding the lease and after the buckets exist, and records the kind in chain_moved. Idempotent: a -// key that exists is already moved and is never overwritten, so a bucket value written since is not laid over by the row -// it came from. It counts the rows against the keys, logs the largest value, and **refuses** (a Refusal, which the -// controller raises as an urgent condition) on a count that differs or a value over the bus's largest, with nothing -// switched: a kind is recorded moved only once all of it is in the bucket. - -// Refusal is a move that stopped without switching a kind, and why. It is permanent until the cause is put right; any -// other error of a move (the bus away, a write not granted) is tried again. -type Refusal struct { - Kind, Why string -} - -func (r *Refusal) Error() string { - return fmt.Sprintf("the move of %s to the bus is refused: %s", r.Kind, r.Why) -} - -// MoveReport is what a move of one kind did. -type MoveReport struct { - Kind string - // Rows is how many records the tables held, Written how many keys this move made, and Present how many already - // held a value (moved by an earlier start). - Rows, Written, Present int - // TableRows is how many rows of the tables those records came from: a request and its outcome are two rows and one - // record. - TableRows int - // Largest is the biggest value, in bytes. - Largest int - // Already is a kind an earlier start moved: nothing was done. - Already bool -} - -// mover is one kind of the chain: its rows, read from its tables, as the keys and values they become. -type mover struct { - kind string - // rows reads the tables: each record's key and its value, whole, and how many rows of the tables they came from. - rows func(ctx context.Context) ([]moveRow, int, error) - // create writes one record under its key; chain.ErrExists when the key holds a value. - create func(ctx context.Context, c *chain.Chain, row moveRow) error - // keys are the keys the bucket holds. - keys func(ctx context.Context, c *chain.Chain) ([]string, error) - // count is how many rows the tables hold now, counted as rows reports them. - count func(ctx context.Context, tx pgx.Tx) (int, error) - // lock keeps the tables from taking a row while the switch is recorded. - lock string - // settle runs under that lock, with the rows read again: it makes the bucket hold what the rows say now. A row the old - // controller changed after the first pass is in the bucket before the kind is recorded as moved, for before the switch - // the table is the only truth. - settle func(ctx context.Context, c *chain.Chain, rows []moveRow) error -} - -type moveRow struct { - key string - value any - size int -} - -func (i *Inventory) movers() []mover { - return []mover{{ - kind: KindBuilds, - rows: func(ctx context.Context) ([]moveRow, int, error) { - builds, err := i.allBuildRows(ctx) - if err != nil { - return nil, 0, err - } - requests, err := i.allRequestRows(ctx) - if err != nil { - return nil, 0, err - } - // A build request is the build's record in state asked, and the request stays beside the outcome that - // answered it: one record under one id (issue 1091). - byID := make(map[string]*chain.Build, len(builds)) - var order []string - for _, b := range builds { - v := toChainBuild(b) - byID[b.ID] = &v - order = append(order, b.ID) - } - for _, r := range requests { - if v, answered := byID[r.ID]; answered { - v.Request = toRequest(r) - continue - } - v := askedRecord(r) - if r.NotAsked != "" { - // Never handed over: asked again once, after the move (askAgain). - v.Request.Reask = chain.ReaskPending - } - byID[r.ID] = &v - order = append(order, r.ID) - } - out := make([]moveRow, 0, len(order)) - for _, id := range order { - body, err := json.Marshal(byID[id]) - if err != nil { - return nil, 0, err - } - out = append(out, moveRow{key: chainBuildKey(id), value: *byID[id], size: len(body)}) - } - return out, len(builds) + len(requests), nil - }, - create: func(ctx context.Context, c *chain.Chain, row moveRow) error { - _, err := c.Builds.Create(ctx, row.key, row.value.(chain.Build)) - return err - }, - keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Builds.Keys(ctx) }, - count: func(ctx context.Context, tx pgx.Tx) (int, error) { - var n int - err := tx.QueryRow(ctx, `select (select count(*) from build) + (select count(*) from build_request)`).Scan(&n) - return n, err - }, - lock: `lock table build, build_request in share mode`, - settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error { - return settleBucket(ctx, c.Builds, rows) - }, - }, { - kind: KindRollouts, - rows: func(ctx context.Context) ([]moveRow, int, error) { - plans, err := i.plans(ctx, `order by created`) - if err != nil { - return nil, 0, err - } - merges, err := i.merges(ctx, ``) - if err != nil { - return nil, 0, err - } - tablePlans := len(plans) - if plans, err = holdMerges(plans, merges); err != nil { - return nil, 0, err - } - out := make([]moveRow, 0, len(plans)) - for _, p := range plans { - body, err := json.Marshal(p) - if err != nil { - return nil, 0, err - } - out = append(out, moveRow{key: p.ID, value: p, size: len(body)}) - } - return out, tablePlans + len(merges), nil - }, - create: func(ctx context.Context, c *chain.Chain, row moveRow) error { - _, err := chain.RolloutsOf[Plan](c).Create(ctx, row.key, row.value.(Plan)) - return err - }, - keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { - return chain.RolloutsOf[Plan](c).Keys(ctx) - }, - count: func(ctx context.Context, tx pgx.Tx) (int, error) { - var n int - err := tx.QueryRow(ctx, `select (select count(*) from release_plan) + (select count(*) from batched_merge)`).Scan(&n) - return n, err - }, - lock: `lock table release_plan, batched_merge in share mode`, - settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error { - return settleBucket(ctx, chain.RolloutsOf[Plan](c), rows) - }, - }, { - kind: KindAssignments, - rows: func(ctx context.Context) ([]moveRow, int, error) { - assigned, err := i.assignedWithSends(ctx) - if err != nil { - return nil, 0, err - } - out := make([]moveRow, 0, len(assigned)) - for _, a := range assigned { - body, err := json.Marshal(a) - if err != nil { - return nil, 0, err - } - out = append(out, moveRow{key: assignmentKey(a.Node, a.Module), value: a, size: len(body)}) - } - return out, len(out), nil - }, - create: createAssignment, - keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Assignments.Keys(ctx) }, - count: func(ctx context.Context, tx pgx.Tx) (int, error) { - var n int - err := tx.QueryRow(ctx, `select count(*) from assignment`).Scan(&n) - return n, err - }, - lock: `lock table assignment in share mode`, - settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error { - have := make(map[string]bool, len(rows)) - for _, r := range rows { - have[r.key] = true - if err := settleAssignment(ctx, c, r); err != nil { - return err - } - } - // An assignment the store no longer holds ended since the first pass: the bucket says so. - all, err := c.Assignments.All(ctx) - if err != nil { - return err - } - for key, k := range all { - if have[key] || k.Value.Assigned.IsZero() || !k.Value.Stands() { - continue - } - k.Value.Ended = time.Now().UTC() - if _, err := c.Assignments.Update(ctx, key, k.Value, k.Revision); err != nil && !errors.Is(err, chain.ErrMoved) { - return err - } - } - return nil - }, - }} -} - -// assignedWithSends is every assignment of the store as the chain keeps it: with the commit its machine was last sent for -// the module (node.sent_builds), the build that is of it and what the build published, and since when (when the machine was -// last sent). An assignment of a module the machine was never sent has none of those. -func (i *Inventory) assignedWithSends(ctx context.Context) ([]chain.Assignment, error) { - rows, err := i.store.Pool().Query(ctx, - `select n.name, a.module, a.assigned, n.sent_at, n.sent_builds - from assignment a join node n on n.id = a.node order by n.name, a.module`) - if err != nil { - return nil, err - } - var out []chain.Assignment - for rows.Next() { - var a chain.Assignment - var sentAt *time.Time - var raw []byte - if err := rows.Scan(&a.Node, &a.Module, &a.Assigned, &sentAt, &raw); err != nil { - rows.Close() - return nil, err - } - a.Assigned = a.Assigned.UTC() - var sent map[string]string - if len(raw) > 0 { - _ = json.Unmarshal(raw, &sent) - } - if commit := sent[a.Module]; commit != "" && sentAt != nil { - a.Commit, a.Since = commit, sentAt.UTC() - } - out = append(out, a) - } - rows.Close() - if err := rows.Err(); err != nil { - return nil, err - } - // The build of each commit, read once: the newest successful build of the module at it. - type at struct{ module, commit string } - var all []Build - if _, moved, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if moved { - if all, err = chainBuilds(ctx, i.attached()); err != nil { - return nil, err - } - } else if all, err = i.allBuildRows(ctx); err != nil { - return nil, err - } - newest := map[at]Build{} - for _, b := range all { - k := at{b.Module, b.Commit} - if b.Failed != "" || b.Module == "" { - continue - } - if had, has := newest[k]; !has || b.At.After(had.At) || (b.At.Equal(had.At) && b.ID > had.ID) { - newest[k] = b - } - } - for k := range out { - if out[k].Commit == "" { - continue - } - b, found := newest[at{out[k].Module, out[k].Commit}] - if !found { - continue - } - out[k].Build = b.ID - if published := publishedOf(b.Made); published != nil { - out[k].Artifact, out[k].Package = published.Artifact, published.Package - } - } - return out, nil -} - -// holdMerges puts each batched merge into the record of the plan that answers it. A merge no plan answers — the plan_id -// of a merge a failed walk carried is null, or names a plan the store never kept — is held, flagged waiting, by the -// newest plan that lists it among its merges, else the record kept for the merges no plan answers: the record that held it -// last. -func holdMerges(plans []Plan, merges []BatchedMerge) ([]Plan, error) { - waitingAt := -1 - at := make(map[string]int, len(plans)) - for k, p := range plans { - at[p.ID] = k - } - newest := func(match func(Plan) bool) int { - best := -1 - for k, p := range plans { - if match(p) && (best < 0 || p.Created.After(plans[best].Created)) { - best = k - } - } - return best - } - for _, m := range merges { - held := heldOf(m, m.Heard) - holder, answered := at[m.Plan] - if m.Plan == "" || !answered { - held.Waiting = true - holder = newest(func(p Plan) bool { - if p.Delivery == nil { - return false - } - for _, listed := range p.Delivery.Merges { - if strings.EqualFold(listed.Repository, m.Repository) && listed.Commit == m.Commit { - return true - } - } - return false - }) - if holder < 0 { - // Held by the record kept for the merges no plan answers, made with the others. - if waitingAt < 0 { - plans = append(plans, Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1, - Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"}) - waitingAt = len(plans) - 1 - } - holder = waitingAt - } - } - plans[holder].Held = append(plans[holder].Held, held) - } - return plans, nil -} - -// MoveChain moves every kind not moved yet, in order, and says what it did. It needs the chain attached and the lease -// held (the chain's writes ask). It stops at the first kind that fails. -func (i *Inventory) MoveChain(ctx context.Context, say func(format string, args ...any)) ([]MoveReport, error) { - if say == nil { - say = func(string, ...any) {} - } - if i.attached() == nil { - return nil, ErrNoChain - } - var out []MoveReport - for _, m := range i.movers() { - r, err := i.moveKind(ctx, m, say) - if err != nil { - return out, err - } - out = append(out, r) - } - return out, nil -} - -// movePasses is how often a move reads the tables again when they took a row while it wrote: the controller before this -// one may still have been writing as the lease changed hands. -const movePasses = 5 - -func (i *Inventory) moveKind(ctx context.Context, m mover, say func(string, ...any)) (MoveReport, error) { - report := MoveReport{Kind: m.kind} - moved, err := i.ChainMoved(ctx) - if err != nil { - return report, err - } - if _, done := moved[m.kind]; done { - report.Already = true - return report, nil - } - for pass := 1; pass <= movePasses; pass++ { - rows, tableRows, err := m.rows(ctx) - if err != nil { - return report, fmt.Errorf("the %s cannot be read from the tables: %w", m.kind, err) - } - report.Rows, report.TableRows, report.Largest = len(rows), tableRows, 0 - // Every value measured before any is written: a value over the bus's largest stops the move with nothing - // written (ADR 0408 §6), never cut. - for _, r := range rows { - report.Largest = max(report.Largest, r.size) - if r.size > broker.ChainValueBytes { - return report, &Refusal{m.kind, fmt.Sprintf("the value of %s is %d bytes and the bus holds at most %d; "+ - "nothing is cut, so the record must be made smaller first", r.key, r.size, broker.ChainValueBytes)} - } - } - report.Written, report.Present = 0, 0 - for _, r := range rows { - switch err := m.create(ctx, i.attached(), r); { - case err == nil: - report.Written++ - case errors.Is(err, chain.ErrExists): - report.Present++ - case errors.Is(err, chain.ErrBadKey): - return report, &Refusal{m.kind, fmt.Sprintf("%s cannot be a key of the bus: %v", r.key, err)} - default: - return report, fmt.Errorf("%s %s is not written to the bus: %w", m.kind, r.key, err) - } - } - // Rows against keys: every row's key is in the bucket. - keys, err := m.keys(ctx, i.attached()) - if err != nil { - return report, fmt.Errorf("the keys of the %s cannot be read back: %w", m.kind, err) - } - have := make(map[string]bool, len(keys)) - for _, k := range keys { - have[k] = true - } - for _, r := range rows { - if !have[r.key] { - return report, &Refusal{m.kind, fmt.Sprintf("the tables hold %d records and the bucket does not hold %s", len(rows), r.key)} - } - } - settled, err := i.switchKind(ctx, m, report) - if err != nil { - return report, err - } - if settled { - say("moved the %s to the bus: %d records, %d written now and %d already there, the largest %d bytes", - m.kind, report.Rows, report.Written, report.Present, report.Largest) - return report, nil - } - say("the %s took a row while they were moved, so they are read again (pass %d of %d)", m.kind, pass, movePasses) - } - return report, &Refusal{m.kind, fmt.Sprintf("the tables kept taking rows through %d passes; another controller is still writing them", movePasses)} -} - -// switchKind records the kind as moved, in one transaction that holds the tables against writes and finds their count -// as the move read it: false when they took a row since. -func (i *Inventory) switchKind(ctx context.Context, m mover, r MoveReport) (bool, error) { - committed := false - tx, err := i.store.Pool().Begin(ctx) - if err != nil { - return false, err - } - defer func() { _ = tx.Rollback(ctx) }() - if _, err := tx.Exec(ctx, m.lock); err != nil { - return false, err - } - n, err := m.count(ctx, tx) - if err != nil { - return false, err - } - if n != r.TableRows { - return false, nil - } - // The store's mirror of the assignments writes nothing from here: it would lay values without the links over the keys - // the settle writes and end the ones it ended. - if m.kind == KindAssignments { - i.moved.switching.Store(true) - defer func() { - if !committed { - i.moved.switching.Store(false) - } - }() - } - // The rows read again under the lock: nothing writes them now, so what the bucket holds is made what they say. - rows, tableRows, err := m.rows(ctx) - if err != nil { - return false, err - } - if tableRows != r.TableRows { - return false, nil - } - if err := m.settle(ctx, i.attached(), rows); err != nil { - return false, fmt.Errorf("the %s changed since they were first written, and the bucket cannot be brought to them: %w", m.kind, err) - } - counts, _ := json.Marshal(map[string]int{m.kind: r.Rows}) - largest, _ := json.Marshal(map[string]int{m.kind: r.Largest}) - if _, err := tx.Exec(ctx, - `insert into chain_moved (counts, largest) values ($1, $2) - on conflict (only_one) do update - set counts = chain_moved.counts || excluded.counts, largest = chain_moved.largest || excluded.largest, - moved_at = now()`, counts, largest); err != nil { - return false, err - } - if err := tx.Commit(ctx); err != nil { - return false, err - } - committed = true - i.moved.mu.Lock() - if i.moved.kind == nil { - i.moved.kind = map[string]bool{} - } - i.moved.kind[m.kind] = true - i.moved.mu.Unlock() - return true, nil -} - -// allBuildRows is every build of the table, whole. -func (i *Inventory) allBuildRows(ctx context.Context) ([]Build, error) { - rows, err := i.store.Pool().Query(ctx, - `select id, repository, ref, coalesce(module, ''), commit_hash, built_on, failed, made, source_path, manifest, - built_against, built_contexts, build_sources, source_fingerprint, asked, at - from build order by at, id`) - if err != nil { - return nil, err - } - defer rows.Close() - var out []Build - for rows.Next() { - var b Build - var made, manifest, against, read, sources []byte - var asked *time.Time - if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, &b.On, &b.Failed, &made, &b.Path, - &manifest, &against, &read, &sources, &b.SourceFingerprint, &asked, &b.At); err != nil { - return nil, err - } - if asked != nil { - b.Asked = *asked - } - b.Manifest = manifest - for _, part := range []struct { - raw []byte - to any - who string - }{{made, &b.Made, "made"}, {against, &b.Against, "built_against"}, {read, &b.Read, "built_contexts"}, - {sources, &b.Sources, "build_sources"}} { - if len(part.raw) == 0 { - continue - } - if err := json.Unmarshal(part.raw, part.to); err != nil { - return nil, fmt.Errorf("build %s: %s is not readable: %w", b.ID, part.who, err) - } - } - out = append(out, b) - } - return out, rows.Err() -} - -// allRequestRows is every build request of the table. -func (i *Inventory) allRequestRows(ctx context.Context) ([]BuildRequest, error) { - rows, err := i.store.Pool().Query(ctx, - `select id, repository, seat, source_path, ref, commit_hash, asked_for, coalesce(not_asked, ''), - coalesce(outcome_unknown, ''), asked_at, at_terminal - from build_request order by asked_at, id`) - if err != nil { - return nil, err - } - defer rows.Close() - var out []BuildRequest - for rows.Next() { - var a BuildRequest - if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked, - &a.OutcomeUnknown, &a.At, &a.AtTerminal); err != nil { - return nil, err - } - out = append(out, a) - } - return out, rows.Err() -} - -// createAssignment writes an assignment's row under its key; chain.ErrExists when the key holds a value that stays. -func createAssignment(ctx context.Context, c *chain.Chain, row moveRow) error { - want := row.value.(chain.Assignment) - // The mirror of the store kept a value under this key until now, with the provisioner's plan beside it: the - // plan stays, the record's links are laid in. A value that names its build already is a later word, and stays. - for try := 0; try < casTries; try++ { - cur, found, err := c.Assignments.Get(ctx, row.key) - if err != nil { - return err - } - if !found { - if _, err := c.Assignments.Create(ctx, row.key, want); !errors.Is(err, chain.ErrExists) { - return err - } - continue - } - if cur.Value.Build != "" || cur.Value.Commit != "" || !cur.Value.Since.IsZero() || (want.Build == "" && want.Commit == "") { - return chain.ErrExists - } - merged := want - merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning - if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) { - continue - } else { - return err - } - } - return fmt.Errorf("the assignment %s kept moving under the write", row.key) -} - -// settleBucket makes a bucket of whole values hold each row's value, by compare-and-set: a key that is absent is made, one -// whose bytes differ from the row's is written over, and one that is the same is left. -func settleBucket[T any](ctx context.Context, b chain.Bucket[T], rows []moveRow) error { - for _, r := range rows { - want, err := json.Marshal(r.value) - if err != nil { - return err - } - settled := false - for try := 0; try < casTries && !settled; try++ { - e, found, err := b.GetRaw(ctx, r.key) - if err != nil { - return err - } - switch { - case !found: - _, err = b.CreateRaw(ctx, r.key, want) - if errors.Is(err, chain.ErrExists) { - continue - } - case bytes.Equal(e.Value, want): - settled = true - continue - default: - _, err = b.UpdateRaw(ctx, r.key, want, e.Revision) - if errors.Is(err, chain.ErrMoved) { - continue - } - } - if err != nil { - return err - } - settled = true - } - if !settled { - return fmt.Errorf("%s kept moving under the write", r.key) - } - } - return nil -} - -// settleAssignment makes the value under an assignment's key what its row says, keeping the plan the mirror put beside it: -// before the switch the store is the only truth of the links, so a commit it changed since the first pass is written over. -func settleAssignment(ctx context.Context, c *chain.Chain, row moveRow) error { - want := row.value.(chain.Assignment) - for try := 0; try < casTries; try++ { - cur, found, err := c.Assignments.Get(ctx, row.key) - if err != nil { - return err - } - if !found { - if _, err := c.Assignments.Create(ctx, row.key, want); errors.Is(err, chain.ErrExists) { - continue - } else { - return err - } - } - merged := want - merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning - a, _ := json.Marshal(merged) - b, _ := json.Marshal(cur.Value) - if bytes.Equal(a, b) { - return nil - } - if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) { - continue - } else { - return err - } - } - return fmt.Errorf("the assignment %s kept moving under the write", row.key) -} diff --git a/internal/inventory/chain_move_test.go b/internal/inventory/chain_move_test.go deleted file mode 100644 index e72a07b9..00000000 --- a/internal/inventory/chain_move_test.go +++ /dev/null @@ -1,949 +0,0 @@ -//go:build integration - -package inventory - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "reflect" - "slices" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/chain" -) - -// A build row as the store held it before the move: written by SQL, the only hand-written state these tests have, for -// it stands for what the mesh held before this release. -func aBuildRow(t *testing.T, inv *Inventory, id, module, commit, failed string) { - t.Helper() - inv.ExecForTest(t, - `insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, source_path, manifest, - built_against, built_contexts, asked, source_fingerprint, build_sources) - values ($1::text, 'novox/' || $2::text, 'main', $2::text, $3::text, 'ace', $4::text, - case when $4::text = '' - then ('[{"name":"app","kind":"image","reference":"artifact-store://' || $2::text || '@sha256:' || $1::text || '"}]')::jsonb - else '[]'::jsonb end, - 'modules/' || $2::text, ('{"module":"' || $2::text || '"}')::jsonb, - '["artifact-store://base@sha256:aa"]'::jsonb, '[{"repository":"novox/other","ref":"main"}]'::jsonb, - now() - interval '1 hour', 'fp-' || $1::text, - ('[{"paths":["modules/' || $2::text || '/module.json"]}]')::jsonb)`, - id, module, commit, failed) -} - -// inTheTables is an inventory before the move: its builds and assignments are rows of the store, which a test of a data -// migration of the store (that reads those rows) needs. -func inTheTables(t *testing.T) *Inventory { - t.Helper() - inv, finish := ForTestBefore(t, 105) - finish() - return inv -} - -// reportOf is the report of one kind among those a move made, which are one for each kind in the order they are moved. -func reportOf(t *testing.T, reports []MoveReport, kind string) MoveReport { - t.Helper() - for _, r := range reports { - if r.Kind == kind { - return r - } - } - t.Fatalf("no report of %s among %+v", kind, reports) - return MoveReport{} -} - -// allAlready says every kind was moved by an earlier start: the second run writes nothing, and reports one for each kind. -func allAlready(t *testing.T, reports []MoveReport) { - t.Helper() - if len(reports) != 3 { - t.Fatalf("the move reported %d kinds, want 3: %+v", len(reports), reports) - } - for _, r := range reports { - if !r.Already || r.Written != 0 { - t.Fatalf("the second move of %s %+v", r.Kind, r) - } - } -} - -func rowsInTheTable(t *testing.T, inv *Inventory, table string) string { - t.Helper() - rows := inv.QueryForTest(t, `select count(*) from `+table) - return rows[0][0] -} - -// **The move copies every build row into the builds bucket, and a second run writes nothing** (ADR 0408 §8, issue -// 1090): each row under its id, whole; the kind recorded in chain_moved; and from then on a build is read from the -// bucket and written to it, the table taking no more. -func TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - aBuildRow(t, inv, "build-2", "notes", "c2", "") - aBuildRow(t, inv, "build-3", "wiki", "", "compile error") - - // Before the move the tables are read, and the bucket is empty. - if before, err := inv.Builds(ctx, "", 10); err != nil || len(before) != 3 { - t.Fatalf("before the move the tables hold %d builds: %v", len(before), err) - } - if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { - t.Fatalf("moved before the move: %v", moved) - } - - reports, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatal(err) - } - if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.Written != 3 || b.Largest == 0 { - t.Fatalf("the move of the builds reported %+v", reports) - } - moved, err := inv.ChainMoved(ctx) - if err != nil || moved[KindBuilds] != 3 { - t.Fatalf("chain_moved says %v: %v", moved, err) - } - kept, found, err := inv.chain.Builds.Get(ctx, "build-2") - if err != nil || !found { - t.Fatalf("build-2 is not in the bucket: %v %v", found, err) - } - if v := kept.Value; v.Module != "notes" || v.Commit != "c2" || v.Repository != "novox/notes" || v.Path != "modules/notes" || - v.SourceFingerprint != "fp-build-2" || len(v.Made) != 1 || len(v.Against) != 1 || len(v.Read) != 1 || len(v.Sources) != 1 || - v.Published == nil || v.Published.Artifact != "sha256:build-2" || v.Asked.IsZero() || v.At.IsZero() || len(v.Manifest) == 0 { - t.Fatalf("build-2 was moved as %+v", v) - } - - // From now on the bucket is what is read and written. - if err := inv.RecordBuild(ctx, aBuild("build-4", "notes", "")); err != nil { - t.Fatal(err) - } - if n := rowsInTheTable(t, inv, "build"); n != "3" { - t.Fatalf("the table took a build after the move: %s rows", n) - } - after, err := inv.Builds(ctx, "notes", 10) - if err != nil || len(after) != 3 { - t.Fatalf("notes has %d builds on the bus: %v", len(after), err) - } - - // A second run finds the kind moved and writes nothing: every key still has the one value it was made with. - again, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatal(err) - } - allAlready(t, again) - for _, id := range []string{"build-1", "build-2", "build-3", "build-4"} { - if history, err := inv.chain.Builds.History(ctx, chainBuildKey(id)); err != nil || len(history) != 1 { - t.Fatalf("%s has %d values: %v", id, len(history), err) - } - } -} - -// failingToCreate is a bucket that fails to make one key while told to: the bus going away half way through a move. -type failingToCreate struct { - chain.Backend - key string - fail *atomic.Bool -} - -func (f failingToCreate) Create(ctx context.Context, key string, value []byte) (uint64, error) { - if key == f.key && f.fail.Load() { - return 0, errors.New("the bus went away") - } - return f.Backend.Create(ctx, key, value) -} - -// **A controller restarted half way through the move finishes it: the bucket equals the table, with no key made twice and -// no refusal** (ADR 0408 §8). Before a kind is recorded as moved nothing but the move writes its keys, and every write of -// the kind goes to its table (each method of a kind asks ChainReads first and writes the store until it says moved), so the -// table is the only truth: a key the first start made is the row it was, and the second start settles it to the row it is. -func TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - aBuildRow(t, inv, "build-2", "notes", "c2", "") - aBuildRow(t, inv, "build-3", "wiki", "c3", "") - var away atomic.Bool - away.Store(true) - builds := chain.NewMemory() - inv.UseChain(chain.New(failingToCreate{builds, "build-3", &away}, chain.NewMemory(), chain.NewMemory(), - func(context.Context) error { return nil })) - // The first start made two of the three keys and the bus went away. - _, err := inv.MoveChain(ctx, nil) - var refusal *Refusal - if err == nil || errors.As(err, &refusal) { - t.Fatalf("the first start answered %v: it should fail to be tried again, not be refused", err) - } - if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { - t.Fatalf("a move that did not finish recorded %v", moved) - } - if keys, _ := builds.Keys(ctx); len(keys) != 2 { - t.Fatalf("the first start made %v", keys) - } - // The controller is started again, the bus is back, and the move finishes. - away.Store(false) - reports, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatalf("the second start was refused or failed: %v", err) - } - if b := reportOf(t, reports, KindBuilds); b.Written != 1 || b.Present != 2 || b.Rows != 3 { - t.Fatalf("the move reported %+v", b) - } - if moved, _ := inv.ChainMoved(ctx); moved[KindBuilds] != 3 { - t.Fatalf("chain_moved says %v", moved) - } - // The bucket is the table: three keys, each made once and not again, each the row's own. - for _, id := range []string{"build-1", "build-2", "build-3"} { - history, err := builds.History(ctx, id) - if err != nil || len(history) != 1 { - t.Fatalf("%s has %d values: %v", id, len(history), err) - } - } - rows, err := inv.allBuildRows(ctx) - if err != nil || len(rows) != 3 { - t.Fatalf("the table holds %d builds: %v", len(rows), err) - } - for _, row := range rows { - got, found, err := inv.chain.Builds.Get(ctx, row.ID) - want, _ := json.Marshal(toChainBuild(row)) - have, _ := json.Marshal(got.Value) - if err != nil || !found || string(want) != string(have) { - t.Fatalf("%s in the bucket is %s, the row says %s (%v)", row.ID, have, want, err) - } - } -} - -// **A value over the bus's largest message stops the move with nothing written and nothing switched** (ADR 0408 §6): -// a Refusal, no key in the bucket, no kind recorded, and the tables still what is read. -func TestTheMoveRefusesAValueOverTheLargestMessage(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - inv.ExecForTest(t, `insert into build (id, repository, module, commit_hash, manifest) - values ('build-big', 'novox/wiki', 'wiki', 'c9', ('{"x":"' || repeat('a', 1100000) || '"}')::jsonb)`) - _, err := inv.MoveChain(ctx, nil) - var refusal *Refusal - if !errors.As(err, &refusal) || refusal.Kind != KindBuilds { - t.Fatalf("the move answered %v", err) - } - if keys, _ := inv.chain.Builds.Keys(ctx); len(keys) != 0 { - t.Fatalf("the bucket holds %v after a refused move", keys) - } - if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { - t.Fatalf("a refused move recorded %v", moved) - } - if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 2 { - t.Fatalf("the tables are no longer read: %d builds, %v", len(builds), err) - } -} - -// losing is a bucket that says it wrote a key and kept nothing: the move's count of keys against rows must see it. -type losing struct { - chain.Backend - lose string -} - -func (l losing) Create(ctx context.Context, key string, value []byte) (uint64, error) { - if key == l.lose { - return 1, nil - } - return l.Backend.Create(ctx, key, value) -} - -// **The move refuses on a count that differs** (ADR 0408 §8): a row whose key is not in the bucket afterwards stops it, -// nothing is recorded, and the tables are still read. -func TestTheMoveRefusesOnACountThatDiffers(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - aBuildRow(t, inv, "build-2", "notes", "c2", "") - inv.UseChain(chain.New(losing{chain.NewMemory(), "build-2"}, chain.NewMemory(), chain.NewMemory(), - func(context.Context) error { return nil })) - _, err := inv.MoveChain(ctx, nil) - var refusal *Refusal - if !errors.As(err, &refusal) { - t.Fatalf("the move answered %v", err) - } - if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { - t.Fatalf("a refused move recorded %v", moved) - } -} - -// **Without the lease the move writes nothing**, and fails the way the bus being away does — tried again, not refused: -// the tables are still read. -func TestTheMoveWithoutTheLeaseWritesNothing(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - mem := chain.NewMemory() - inv.UseChain(chain.New(mem, chain.NewMemory(), chain.NewMemory(), - func(context.Context) error { return errors.New("another controller holds the lease") })) - _, err := inv.MoveChain(ctx, nil) - var refusal *Refusal - if err == nil || errors.As(err, &refusal) || !errors.Is(err, chain.ErrNotHeld) { - t.Fatalf("the move answered %v", err) - } - if mem.Writes() != 0 { - t.Fatalf("the bucket took %d writes without the lease", mem.Writes()) - } - if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { - t.Fatalf("recorded %v", moved) - } -} - -// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its -// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none. -func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) { - inv := fresh(t) - ctx := context.Background() - first := aBuild("build-1", "notes", "") - first.Commit = "c1" - first.Against = []string{"artifact-store://base@sha256:aa"} - first.ReuseKey = "rk-1" - second := aBuild("build-2", "notes", "") - second.Commit = "c2" - failed := aBuild("build-3", "notes", "compile error") - other := aBuild("build-4", "wiki", "") - for _, b := range []Build{first, second, failed, other, first} { - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - if n := rowsInTheTable(t, inv, "build"); n != "0" { - t.Fatalf("the table took %s builds", n) - } - kept, found, err := inv.chain.Builds.Get(ctx, "build-1") - if err != nil || !found || kept.Value.ReuseKey != "rk-1" { - t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err) - } - if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 1 { - t.Fatalf("a build recorded twice has %d values", len(history)) - } - list, err := inv.Builds(ctx, "notes", 10) - if err != nil || len(list) != 3 { - t.Fatalf("notes has %d builds: %v", len(list), err) - } - one, found, err := inv.BuildByID(ctx, "build-2") - if err != nil || !found || one.Commit != "c2" || one.Module != "notes" { - t.Fatalf("build-2 reads as %+v %v %v", one, found, err) - } - if _, found, _ := inv.BuildByID(ctx, "build-nope"); found { - t.Fatal("a build nobody recorded was found") - } - if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" { - t.Fatalf("the build of notes at c1 is %q: %v", id, err) - } - held, err := inv.Held(ctx) - if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" { - t.Fatalf("held %v: %v", held, err) - } - against, err := inv.BuiltAgainst(ctx) - if err != nil || len(against["notes"]) != 0 { - // The newest successful build of notes, build-2, stood on nothing. - t.Fatalf("built against %v: %v", against, err) - } - announced, err := inv.Announceable(ctx) - if err != nil || len(announced) != 3 { - t.Fatalf("announceable %d: %v", len(announced), err) - } - prints, err := inv.Fingerprints(ctx) - if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" { - t.Fatalf("fingerprints %v: %v", prints, err) - } -} - -// A build request row as the store held it before the move. -func aRequestRow(t *testing.T, inv *Inventory, id, notAsked string) { - t.Helper() - inv.ExecForTest(t, - `insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at) - values ($1::text, 'novox/notes', '', 'modules/notes', 'main', 'c1', 'merge', nullif($2::text, ''), now() - interval '1 hour')`, - id, notAsked) -} - -// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue -// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none -// after the outcome is heard. The table of requests takes none. -func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) { - inv := fresh(t) - ctx := context.Background() - ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true} - if err := inv.RecordBuildRequest(ctx, ask); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuildRequest(ctx, ask); err != nil { - t.Fatalf("a request kept twice: %v", err) - } - if n := rowsInTheTable(t, inv, "build_request"); n != "0" { - t.Fatalf("the table took %s requests", n) - } - if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 { - t.Fatalf("a request nobody answered is a build: %+v %v", builds, err) - } - if _, found, _ := inv.BuildByID(ctx, "build-1"); found { - t.Fatal("a request nobody answered has an outcome") - } - got, found, err := inv.BuildRequestByID(ctx, "build-1") - if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() { - t.Fatalf("the request reads back as %+v %v %v", got, found, err) - } - if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil { - t.Fatal(err) - } - asked, err := inv.RequestsNamed(ctx, "notes") - if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" { - t.Fatalf("the request is %+v: %v", asked, err) - } - if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) { - t.Fatalf("requested names %v: %v", names, err) - } - - // The outcome comes: the same key, now a build, with the request beside it. - built := aBuild("build-1", "notes", "") - if err := inv.RecordBuild(ctx, built); err != nil { - t.Fatal(err) - } - if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" { - t.Fatalf("the outcome reads as %+v %v %v", one, found, err) - } - answered, err := inv.RequestsNamed(ctx, "notes") - if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" { - t.Fatalf("the request answered is %+v: %v", answered, err) - } - if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" { - t.Fatalf("the request was lost with the outcome: %+v", again) - } - // Heard: the marks of an unanswered request change nothing. - if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil { - t.Fatal(err) - } - if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" { - t.Fatalf("an answered request was marked not asked: %+v", again) - } - if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 3 { - // asked, marked unknown, answered. - t.Fatalf("the record has %d values", len(history)) - } - // An outcome heard before its request is kept, and the request lands beside it. - if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil { - t.Fatal(err) - } - if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" { - t.Fatalf("the request kept after its outcome reads %+v", late) - } - // A request older than the time they are kept is no longer listed. - if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old", - At: time.Now().Add(-2 * KeptFor)}); err != nil { - t.Fatal(err) - } - if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") { - t.Fatalf("a request past its time is listed: %v", names) - } -} - -// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a -// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it -// is asked and so asked at most once. -func TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - aRequestRow(t, inv, "build-1", "") - aRequestRow(t, inv, "build-2", "") - aRequestRow(t, inv, "not-asked-c1-modules-notes", "no seat holder") - - if requests, err := inv.RequestsNamed(ctx, "notes"); err != nil || len(requests) != 3 { - t.Fatalf("before the move the table holds %d requests: %v", len(requests), err) - } - reports, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatal(err) - } - if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.TableRows != 4 { - t.Fatalf("the move reported %+v", b) - } - one, _, _ := inv.chain.Builds.Get(ctx, "build-1") - if one.Value.State != "" || one.Value.Request == nil || one.Value.Request.For != "merge" || one.Value.Module != "notes" { - t.Fatalf("an answered request moved as %+v", one.Value) - } - two, _, _ := inv.chain.Builds.Get(ctx, "build-2") - if two.Value.State != chain.BuildAsked || two.Value.Request.Reask != "" { - t.Fatalf("an unanswered request moved as %+v", two.Value) - } - never, _, _ := inv.chain.Builds.Get(ctx, "not-asked-c1-modules-notes") - if never.Value.State != chain.BuildAsked || never.Value.Request.NotAsked != "no seat holder" || never.Value.Request.Reask != chain.ReaskPending { - t.Fatalf("a request never handed over moved as %+v", never.Value) - } - requests, err := inv.RequestsNamed(ctx, "notes") - if err != nil || len(requests) != 3 { - t.Fatalf("after the move the bucket holds %d requests: %v", len(requests), err) - } - toAsk, err := inv.RequestsToAskAgain(ctx) - if err != nil || len(toAsk) != 1 || toAsk[0].ID != "not-asked-c1-modules-notes" { - t.Fatalf("to ask again: %+v %v", toAsk, err) - } - first, err := inv.ClaimToAskAgain(ctx, toAsk[0].ID) - second, err2 := inv.ClaimToAskAgain(ctx, toAsk[0].ID) - if err != nil || err2 != nil || !first || second { - t.Fatalf("claimed %v then %v: %v %v", first, second, err, err2) - } - if left, _ := inv.RequestsToAskAgain(ctx); len(left) != 0 { - t.Fatalf("a claimed request is still to be asked: %+v", left) - } - if err := inv.AskedAgainAs(ctx, toAsk[0].ID, "build-9"); err != nil { - t.Fatal(err) - } - if done, _, _ := inv.BuildRequestByID(ctx, toAsk[0].ID); done.NotAsked != "no seat holder" { - t.Fatalf("the request lost its words: %+v", done) - } -} - -// **The move carries every plan with the merges it answers, and the plans read the same from the bucket as from the -// table** (ADR 0408 §8, ADR 0276, issue 1092): a batch with a merge, a walk that was built and rolled, a merge waiting -// for a walk of its own, and a merge whose plan the store never kept — each in the record of the plan that held it last. -func TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - batch := aBatch("plan-1", time.Now().Add(-3*time.Hour).UTC()) - walk := aBatch("plan-2", time.Now().Add(-2*time.Hour).UTC()) - walk.State, walk.Tier, walk.Tiers = PlanRolling, 1, [][]string{{"a"}, {"b"}} - walk.Modules = map[string]*PlanModule{"a": {State: "built"}, "b": {}} - walk.Delivery = &PlanDelivery{Awaits: "mesh-delivery", Merges: []PlanMerge{{Repository: "novox/mesh-catalog", Commit: "m-waiting"}}} - for _, p := range []*Plan{batch, walk} { - if err := inv.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - } - walk.Note = "second save" - if err := inv.SavePlan(ctx, walk); err != nil { - t.Fatal(err) - } - for _, m := range []BatchedMerge{aMerge("m-batch", "plan-1"), aMerge("m-walk", "plan-2"), aMerge("m-dangling", "plan-never-kept")} { - if added, err := inv.AddMerge(ctx, m); err != nil || !added { - t.Fatalf("added %v: %v", added, err) - } - } - waiting := aMerge("m-waiting", "plan-2") - if _, err := inv.AddMerge(ctx, waiting); err != nil { - t.Fatal(err) - } - if err := inv.AnswerMerge(ctx, waiting.Repository, waiting.Commit, "", true); err != nil { - t.Fatal(err) - } - wasPlans, err := inv.RecentPlans(ctx, 10) - if err != nil || len(wasPlans) != 2 { - t.Fatalf("the table holds %d plans: %v", len(wasPlans), err) - } - wasMerge, _, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk") - - reports, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatal(err) - } - var moved MoveReport - for _, r := range reports { - if r.Kind == KindRollouts { - moved = r - } - } - // Two plans, and the record of the merge whose plan the store never kept. - if moved.Rows != 3 || moved.TableRows != 6 || moved.Written != 3 { - t.Fatalf("the move of the rollouts reported %+v", moved) - } - nowPlans, err := inv.RecentPlans(ctx, 10) - if err != nil || len(nowPlans) != 2 { - t.Fatalf("the bucket holds %d plans: %v", len(nowPlans), err) - } - for k := range wasPlans { - was, now := wasPlans[k], nowPlans[k] - if was.ID != now.ID || was.State != now.State || was.Tier != now.Tier || was.Revision != now.Revision || was.Note != now.Note || - !was.Created.Equal(now.Created) || !reflect.DeepEqual(was.Tiers, now.Tiers) || len(was.Modules) != len(now.Modules) { - t.Errorf("plan %s was %+v and is %+v", was.ID, was, now) - } - } - if open, _ := inv.OpenPlans(ctx); len(open) != 1 || open[0].ID != "plan-2" { - t.Fatalf("open plans %+v", open) - } - if batches, _ := inv.Batches(ctx); len(batches) != 1 || batches[0].ID != "plan-1" { - t.Fatalf("batches %+v", batches) - } - if got, _ := inv.MergesOf(ctx, "plan-1"); len(got) != 1 || got[0].Commit != "m-batch" { - t.Fatalf("the batch answers %+v", got) - } - nowMerge, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk") - if !known || nowMerge.Plan != "plan-2" || nowMerge.Branch != wasMerge.Branch || !nowMerge.Merged.Equal(wasMerge.Merged) || - !nowMerge.Heard.Equal(wasMerge.Heard) || !sameEvent(nowMerge.Event, wasMerge.Event) { - t.Fatalf("a merge was %+v and is %+v", wasMerge, nowMerge) - } - alone, _ := inv.AloneMerges(ctx) - if len(alone) != 1 || alone[0].Commit != "m-waiting" || alone[0].Plan != "" { - t.Fatalf("waiting: %+v", alone) - } - if dangling, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-dangling"); !known || dangling.Plan != "" || dangling.Alone { - t.Fatalf("a merge whose plan was never kept reads as %+v %v", dangling, known) - } - if n := rowsInTheTable(t, inv, "release_plan"); n != "2" { - t.Fatalf("the table is %s plans", n) - } - // From now on a plan is saved to the bucket and the table takes none. - later := aBatch("plan-3", time.Now().UTC()) - if err := inv.SavePlan(ctx, later); err != nil { - t.Fatal(err) - } - if n := rowsInTheTable(t, inv, "release_plan"); n != "2" { - t.Fatalf("the table took a plan after the move: %s", n) - } - again, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatal(err) - } - allAlready(t, again) -} - -// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a -// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module -// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the -// assignment goes with it. -func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { - inv, node := aNodeWithModules(t, "postgres", "web") - ctx := t.Context() - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - start := generationNow(t, inv) - if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added { - t.Fatalf("assigned %v: %v", added, err) - } - if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added { - t.Fatalf("assigned again %v: %v", added, err) - } - if n := rowsInTheTable(t, inv, "assignment"); n != "0" { - t.Fatalf("the table took %s assignments", n) - } - kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres") - if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() { - t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err) - } - if generationNow(t, inv) <= start { - t.Fatal("an assignment left the generation where it was") - } - if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) { - t.Fatalf("a module the mesh does not know: %v", err) - } - if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) { - t.Fatalf("assigned on %s: %v %v", node, on, err) - } - if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) { - t.Fatalf("running: %v %v", running, err) - } - entries, err := inv.Catalogued(ctx) - if err != nil { - t.Fatal(err) - } - for _, e := range entries { - switch e.Manifest.Module { - case "postgres": - if !reflect.DeepEqual(e.On, []string{node}) { - t.Errorf("postgres is on %v", e.On) - } - case "web": - if len(e.On) != 0 { - t.Errorf("web is on %v", e.On) - } - } - } - if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) { - t.Fatalf("a module still assigned was forgotten: %v", err) - } - // A seat is held by an assignment, or by nothing. - if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil { - t.Fatal("a seat was handed to a module that is not assigned") - } - if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil { - t.Fatalf("a seat handed to an assignment: %v", err) - } - - before := generationNow(t, inv) - if err := inv.Unassign(ctx, node, "postgres"); err != nil { - t.Fatal(err) - } - if generationNow(t, inv) <= before { - t.Fatal("an unassignment left the generation where it was") - } - // No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history. - gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres") - if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() { - t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err) - } - if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() || - history[1].Value.Stands() { - t.Fatalf("the earlier values of the key: %+v", history) - } - if on, _ := inv.Assigned(ctx, node); len(on) != 0 { - t.Fatalf("an ended assignment is still assigned: %v", on) - } - if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 { - t.Fatalf("an ended assignment is running: %v", running) - } - if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 { - t.Fatalf("the seat the assignment held is held by %+v: %v", held, err) - } - if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" { - t.Fatalf("an unassignment of nothing: %v", err) - } - if err := inv.ForgetModule(ctx, "postgres"); err != nil { - t.Fatalf("a module nothing is assigned could not be forgotten: %v", err) - } - // Made again after it ended: a fresh value over the ended one, which stays in history. - if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { - t.Fatal(added, err) - } - if added, err := inv.Assign(ctx, node, "web"); err != nil || added { - t.Fatal(added, err) - } - if err := inv.Unassign(ctx, node, "web"); err != nil { - t.Fatal(err) - } - if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { - t.Fatalf("assigned again after it ended: %v %v", added, err) - } - again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web") - if !again.Value.Stands() || !again.Value.Ended.IsZero() { - t.Fatalf("the assignment made again is %+v", again.Value) - } - if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() { - t.Fatalf("the values of the key: %+v", history) - } -} - -// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the -// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay, -// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here -// ends an assignment. -func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) { - inv, node := aNodeWithModules(t, "dunst", "web") - ctx := t.Context() - for _, m := range []string{"dunst", "web"} { - if _, err := inv.Assign(ctx, node, m); err != nil { - t.Fatal(err) - } - } - // A build and a commit put in the value by another write. - cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") - cur.Value.Build, cur.Value.Commit = "build-1", "c1" - if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil { - t.Fatal(err) - } - record, _ := inv.NodeByName(ctx, node) - digest := "sha256:" + strings.Repeat("d", 64) - plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest, - Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc", - Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}} - raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`) - if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil { - t.Fatal(err) - } - inv.SetProvisioningPlans(inv.SentProvisioningPlans) - written, removed, err := inv.KeepAssignmentPlans(ctx) - if err != nil || written != 2 || removed != 0 { - t.Fatalf("kept: %d written, %d ended: %v", written, removed, err) - } - dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") - if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() { - t.Fatalf("dunst's value is %+v", dunst.Value) - } - if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil { - t.Fatalf("web has a plan: %+v", web.Value) - } - notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify") - if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil { - t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found) - } - if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) { - t.Fatalf("a pulled-in module is assigned: %v", on) - } - if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 { - t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err) - } - // Sent again without the pulled-in module: its value ends, the assignments stay. - only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`) - if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil { - t.Fatal(err) - } - if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 { - t.Fatalf("ended %d: %v", removed, err) - } - if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() { - t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value) - } - if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) { - t.Fatalf("assigned: %v", on) - } -} - -// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value** -// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build -// already is left as it is. -func TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - for _, name := range []string{"anchor", "laptop"} { - if _, err := inv.AddNode(ctx, name); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"notes", "wiki"} { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil { - t.Fatal(err) - } - } - for _, a := range [][2]string{{"anchor", "notes"}, {"anchor", "wiki"}, {"laptop", "notes"}} { - if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { - t.Fatal(err) - } - } - aBuildRow(t, inv, "build-notes", "notes", "c-notes", "") - anchor, _ := inv.NodeByName(ctx, "anchor") - if err := inv.RecordSent(ctx, anchor.ID, "sent-1", map[string]string{"notes": "c-notes"}); err != nil { - t.Fatal(err) - } - // The mirror of the store put a value under one key, with a plan. - if _, err := inv.chain.Assignments.Create(ctx, "anchor.wiki", chain.Assignment{Node: "anchor", Module: "wiki", - Assigned: time.Now().UTC(), Provisioning: json.RawMessage(`{"plan":1}`)}); err != nil { - t.Fatal(err) - } - reports, err := inv.MoveChain(ctx, nil) - if err != nil { - t.Fatal(err) - } - var moved MoveReport - for _, r := range reports { - if r.Kind == KindAssignments { - moved = r - } - } - if moved.Rows != 3 || moved.Written+moved.Present != 3 { - t.Fatalf("the move of the assignments reported %+v", moved) - } - notes, _, _ := inv.chain.Assignments.Get(ctx, "anchor.notes") - if notes.Value.Commit != "c-notes" || notes.Value.Build != "build-notes" || notes.Value.Since.IsZero() || notes.Value.Assigned.IsZero() { - t.Fatalf("anchor.notes moved as %+v", notes.Value) - } - if notes.Value.Artifact != "sha256:build-notes" { - t.Fatalf("what the build published: %+v", notes.Value) - } - laptop, _, _ := inv.chain.Assignments.Get(ctx, "laptop.notes") - if laptop.Value.Commit != "" || laptop.Value.Build != "" { - t.Fatalf("a machine never sent has a build: %+v", laptop.Value) - } - wiki, _, _ := inv.chain.Assignments.Get(ctx, "anchor.wiki") - if string(wiki.Value.Provisioning) != `{"plan":1}` { - t.Fatalf("the plan the mirror put in the value was lost: %+v", wiki.Value) - } - // From now on the bucket is what is written. - if _, err := inv.Assign(ctx, "laptop", "wiki"); err != nil { - t.Fatal(err) - } - if n := rowsInTheTable(t, inv, "assignment"); n != "3" { - t.Fatalf("the table took an assignment after the move: %s", n) - } - if on, _ := inv.Assigned(ctx, "laptop"); !reflect.DeepEqual(on, []string{"notes", "wiki"}) { - t.Fatalf("assigned on laptop: %v", on) - } -} - -// hooked runs a function the first time a key is made in a bucket: the old controller writing the store between the move's -// first pass and its switch. -type hooked struct { - chain.Backend - once *sync.Once - then func() -} - -func (h hooked) Create(ctx context.Context, key string, value []byte) (uint64, error) { - rev, err := h.Backend.Create(ctx, key, value) - h.once.Do(h.then) - return rev, err -} - -// **A row the old controller changes after the move's first pass is in the bucket when the kind is recorded as moved** -// (ADR 0408 §8, issue 981 note 7 Q2): before the switch the store is the only truth, so under the lock the move reads the -// rows again and writes over every key that differs — a plan's state and a build's failure — and ends an assignment the -// store dropped. The count alone would have kept the older value and refused the next save of the plan. -func TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved(t *testing.T) { - inv, finish := ForTestBefore(t, 105) - finish() - ctx := context.Background() - aBuildRow(t, inv, "build-1", "notes", "c1", "") - for _, n := range []string{"anchor"} { - if _, err := inv.AddNode(ctx, n); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"notes", "wiki"} { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"notes", "wiki"} { - if _, err := inv.Assign(ctx, "anchor", m); err != nil { - t.Fatal(err) - } - } - p := aBatch("plan-1", time.Now().Add(-time.Hour).UTC()) - p.State, p.Tiers = PlanBuilding, [][]string{{"notes"}} - if err := inv.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - // Between a kind's first pass and its switch the old controller saves the plan again, fails the build and unassigns wiki: - // each is done when the first key of its kind is made, so the first pass has read the rows as they were. - after := func(sql string) hooked { - return hooked{chain.NewMemory(), &sync.Once{}, func() { inv.ExecForTest(t, sql) }} - } - inv.UseChain(chain.New( - after(`update build set failed = 'it broke later' where id = 'build-1'`), - after(`update release_plan set state = 'rolling', revision = revision + 1, note = 'the old controller was here' where id = 'plan-1'`), - after(`delete from assignment where module = 'wiki'`), - func(context.Context) error { return nil })) - if _, err := inv.MoveChain(ctx, nil); err != nil { - t.Fatal(err) - } - now, err := inv.PlanByID(ctx, "plan-1") - if err != nil { - t.Fatal(err) - } - if now.State != PlanRolling || now.Note != "the old controller was here" || now.Revision != 2 { - t.Fatalf("the bucket holds a plan from before the change: %s %q revision %d", now.State, now.Note, now.Revision) - } - if b, found, _ := inv.BuildByID(ctx, "build-1"); !found || b.Failed != "it broke later" { - t.Fatalf("the bucket holds the build as it was: %+v", b) - } - if on, _ := inv.Assigned(ctx, "anchor"); !slices.Equal(on, []string{"notes"}) { - t.Fatalf("anchor is assigned %v, though the store dropped wiki", on) - } - // The plan can be saved at the revision the store had. - now.Note = "saved after the move" - if err := inv.SavePlan(ctx, &now); err != nil { - t.Fatalf("the next save of the plan is refused: %v", err) - } -} - -// sameEvent says two encodings of the forge's announcement are one: the store keeps it as jsonb, which spells it with its own -// white space, and whoever reads it (the cut, mesh-delivery) decodes it, so the white space is no part of what it says. -func sameEvent(a, b json.RawMessage) bool { - var x, y bytes.Buffer - return json.Compact(&x, a) == nil && json.Compact(&y, b) == nil && x.String() == y.String() -} diff --git a/internal/inventory/chain_moved_test.go b/internal/inventory/chain_moved_test.go deleted file mode 100644 index 3ea88b25..00000000 --- a/internal/inventory/chain_moved_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package inventory - -import ( - "strings" - "testing" -) - -// **The move of the chain has a switch, made by the last migration and written by no migration** (ADR 0408 §8, issue -// 1089): migration 0104 makes the one-row table chain_moved, and does not drop a table of the chain, which a later -// delivery does once the move is read back. -func TestMigration0104MakesTheSwitchOfTheChainsMoveAndNothingElse(t *testing.T) { - migrations, err := Migrations() - if err != nil { - t.Fatal(err) - } - var found bool - for _, m := range migrations { - if m.Number != 104 { - continue - } - found = true - sql := strings.ToLower(m.SQL) - if !strings.Contains(sql, "create table chain_moved") || !strings.Contains(sql, "check (only_one)") { - t.Errorf("migration 0104 does not make the one-row chain_moved:\n%s", m.SQL) - } - for _, line := range strings.Split(sql, "\n") { - if strings.HasPrefix(strings.TrimSpace(line), "--") { - continue - } - for _, verb := range []string{"drop table", "insert ", "delete ", "update "} { - if strings.Contains(line, verb) { - t.Errorf("migration 0104 does more than make the switch: %q", line) - } - } - // The one other change: a seat's holder is no longer a foreign key to the assignment (issue 1093); the tables of - // the chain are dropped by a later migration (issue 1096). - if strings.Contains(line, "alter ") && !strings.Contains(line, "alter table seat_holding drop constraint") { - t.Errorf("migration 0104 alters more than the seat holdings' foreign key: %q", line) - } - } - } - if !found { - t.Fatal("no migration 0104") - } -} diff --git a/internal/inventory/chain_pending_test.go b/internal/inventory/chain_pending_test.go index 2f62ee2c..88ee416e 100644 --- a/internal/inventory/chain_pending_test.go +++ b/internal/inventory/chain_pending_test.go @@ -59,9 +59,6 @@ func TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket(t *testing.T) { if history, _ := inv.chain.Assignments.History(ctx, "anchor.sensors"); len(history) != 2 { t.Fatalf("the earlier wait is in the key's history: %d values", len(history)) } - if n := rowsInTheTable(t, inv, "pending_assignment"); n != "0" { - t.Fatalf("the table took %s pending assignments", n) - } } // **A pending value is skipped by what reads assignments** (issue 1094): the machine is assigned nothing by it, the module diff --git a/internal/inventory/chain_requests.go b/internal/inventory/chain_requests.go index 230ba9c3..c14e974a 100644 --- a/internal/inventory/chain_requests.go +++ b/internal/inventory/chain_requests.go @@ -123,8 +123,8 @@ func (i *Inventory) chainRequests(ctx context.Context, c *chain.Chain) ([]Reques // RequestsToAskAgain are the requests the move found never handed over (issue 981 note 7 Q5), not yet asked again: each is // asked once, and never twice. func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, error) { - c, on, err := i.ChainReads(ctx, KindBuilds) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return nil, err } all, err := c.Builds.All(ctx) @@ -144,8 +144,8 @@ func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, err // ClaimToAskAgain takes a request to ask again, by compare-and-set: true for the one process that took it. **Claimed // before it is asked and settled after**, so a crash between loses the ask rather than making it twice. func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error) { - c, on, err := i.ChainReads(ctx, KindBuilds) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return false, err } cur, found, err := c.Builds.Get(ctx, chainBuildKey(id)) @@ -164,8 +164,8 @@ func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error // AskedAgainAs records the build id a claimed request was asked as. func (i *Inventory) AskedAgainAs(ctx context.Context, id, asked string) error { - c, on, err := i.ChainReads(ctx, KindBuilds) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return err } return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.Reask = asked }) diff --git a/internal/inventory/chain_rollouts.go b/internal/inventory/chain_rollouts.go index 98a49f86..b7a9e839 100644 --- a/internal/inventory/chain_rollouts.go +++ b/internal/inventory/chain_rollouts.go @@ -139,17 +139,17 @@ func (v *rolloutView) list() []viewEntry { // rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole. func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) { - i.moved.mu.Lock() - v := i.moved.view + i.state.mu.Lock() + v := i.state.view if v == nil || v.chain != c || v.dead.Load() { if v != nil { v.stop() } v = newRolloutView(c) - i.moved.view = v + i.state.view = v v.start() } - i.moved.mu.Unlock() + i.state.mu.Unlock() select { case <-v.ready: case <-ctx.Done(): @@ -165,11 +165,11 @@ func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutVi // stopViews ends the view's watch: the inventory is closing. func (i *Inventory) stopViews() { - i.moved.mu.Lock() - defer i.moved.mu.Unlock() - if i.moved.view != nil { - i.moved.view.stop() - i.moved.view = nil + i.state.mu.Lock() + defer i.state.mu.Unlock() + if i.state.view != nil { + i.state.view.stop() + i.state.view = nil } } @@ -492,12 +492,12 @@ func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m Batched return false, err } if !found { - i.moved.mu.Lock() - if i.moved.pending == nil { - i.moved.pending = map[string]pendingMerge{} + i.state.mu.Lock() + if i.state.pending == nil { + i.state.pending = map[string]pendingMerge{} } - i.moved.pending[key] = pendingMerge{plan: m.Plan, m: held} - i.moved.mu.Unlock() + i.state.pending[key] = pendingMerge{plan: m.Plan, m: held} + i.state.mu.Unlock() } return true, nil } @@ -712,12 +712,12 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans [] held, has := copies[key] if !has { // A merge carried for a plan not saved yet goes where it is moved. - i.moved.mu.Lock() - if pm, pending := i.moved.pending[key]; pending && m.Plan != "" { + i.state.mu.Lock() + if pm, pending := i.state.pending[key]; pending && m.Plan != "" { pm.plan, pm.m.Alone = m.Plan, m.Alone - i.moved.pending[key] = pm + i.state.pending[key] = pm } - i.moved.mu.Unlock() + i.state.mu.Unlock() continue } w := winner(held) @@ -803,14 +803,14 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans [] if e := edits[p.ID]; e != nil { held = e.apply(held) } - i.moved.mu.Lock() - for key, pm := range i.moved.pending { + i.state.mu.Lock() + for key, pm := range i.state.pending { if pm.plan == p.ID { held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held) _ = key } } - i.moved.mu.Unlock() + i.state.mu.Unlock() record.Held = held entered, duration := tierLeftOf(old, *p, time.Now()) record.TierEntered, record.Updated = entered, time.Now().UTC() @@ -859,13 +859,13 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans [] } } - i.moved.mu.Lock() - for key, pm := range i.moved.pending { + i.state.mu.Lock() + for key, pm := range i.state.pending { if _, saved := done[pm.plan]; saved { - delete(i.moved.pending, key) + delete(i.state.pending, key) } } - i.moved.mu.Unlock() + i.state.mu.Unlock() for _, p := range plans { w := done[p.ID] p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held @@ -908,8 +908,8 @@ func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uin // RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired // at the next cut (ADR 0408, issue 1092). It says how many it repaired. func (i *Inventory) RepairMerges(ctx context.Context) (int, error) { - c, on, err := i.ChainReads(ctx, KindRollouts) - if err != nil || !on { + c, err := i.theChain(ctx) + if err != nil { return 0, err } v, err := i.rolloutView(ctx, c) diff --git a/internal/inventory/chain_rollouts_test.go b/internal/inventory/chain_rollouts_test.go index dffeeb17..fffc8576 100644 --- a/internal/inventory/chain_rollouts_test.go +++ b/internal/inventory/chain_rollouts_test.go @@ -15,7 +15,7 @@ import ( func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) { t.Helper() c := chain.InMemory(nil) - inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }} + inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }} inv.UseChain(c) t.Cleanup(inv.stopViews) return inv, c @@ -117,10 +117,10 @@ func TestAMergeAddedToAPlanNotSavedYetIsWrittenByItsSave(t *testing.T) { } return errors.New("the lease is not held") }) - other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded} + other := &Inventory{state: &chainState{}, recorded: inv.recorded} other.UseChain(away) t.Cleanup(other.stopViews) - other.moved.pending = inv.moved.pending + other.state.pending = inv.state.pending p := aBatch("plan-new", plansAt) if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); !errors.Is(err, chain.ErrNotHeld) { t.Fatalf("the save answered %v", err) @@ -158,7 +158,7 @@ func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *test var fail atomic.Bool rollouts := failingOn{Backend: chain.NewMemory(), key: "plan-a", fail: &fail} c = chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil }) - inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }} + inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }} inv.UseChain(c) t.Cleanup(inv.stopViews) ctx := context.Background() @@ -190,9 +190,6 @@ func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *test if n := len(mergeHolders(v.list())["novox/mesh-catalog|m1"]); n != 2 { t.Fatalf("the merge is held by %d records, want the two of a stopped move", n) } - inv.moved.mu.Lock() - inv.moved.kind = map[string]bool{KindRollouts: true} - inv.moved.mu.Unlock() repaired := 0 for k, copies := range mergeHolders(v.list()) { if len(copies) < 2 { @@ -310,7 +307,7 @@ func (c *countingBackend) Keys(ctx context.Context) ([]string, error) { func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) { counting := &countingBackend{Backend: chain.NewMemory()} c := chain.New(chain.NewMemory(), counting, chain.NewMemory(), func(context.Context) error { return nil }) - inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }} + inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }} inv.UseChain(c) t.Cleanup(inv.stopViews) ctx := context.Background() @@ -337,7 +334,7 @@ func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) { t.Fatalf("the bucket was read %d times and its keys %d times for the surveys", counting.gets.Load()-before, counting.keys.Load()) } // Another process writes; the view follows. - other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded} + other := &Inventory{state: &chainState{}, recorded: inv.recorded} other.UseChain(c) t.Cleanup(other.stopViews) late := aBatch("plan-4", plansAt.Add(time.Hour)) diff --git a/internal/inventory/chain_store_test.go b/internal/inventory/chain_store_test.go new file mode 100644 index 00000000..6547bdef --- /dev/null +++ b/internal/inventory/chain_store_test.go @@ -0,0 +1,346 @@ +//go:build integration + +package inventory + +import ( + "context" + "errors" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its +// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none. +func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) { + inv := fresh(t) + ctx := context.Background() + first := aBuild("build-1", "notes", "") + first.Commit = "c1" + first.Against = []string{"artifact-store://base@sha256:aa"} + first.ReuseKey = "rk-1" + second := aBuild("build-2", "notes", "") + second.Commit = "c2" + failed := aBuild("build-3", "notes", "compile error") + other := aBuild("build-4", "wiki", "") + for _, b := range []Build{first, second, failed, other, first} { + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + kept, found, err := inv.chain.Builds.Get(ctx, "build-1") + if err != nil || !found || kept.Value.ReuseKey != "rk-1" { + t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err) + } + if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 1 { + t.Fatalf("a build recorded twice has %d values", len(history)) + } + list, err := inv.Builds(ctx, "notes", 10) + if err != nil || len(list) != 3 { + t.Fatalf("notes has %d builds: %v", len(list), err) + } + one, found, err := inv.BuildByID(ctx, "build-2") + if err != nil || !found || one.Commit != "c2" || one.Module != "notes" { + t.Fatalf("build-2 reads as %+v %v %v", one, found, err) + } + if _, found, _ := inv.BuildByID(ctx, "build-nope"); found { + t.Fatal("a build nobody recorded was found") + } + if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" { + t.Fatalf("the build of notes at c1 is %q: %v", id, err) + } + held, err := inv.Held(ctx) + if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" { + t.Fatalf("held %v: %v", held, err) + } + against, err := inv.BuiltAgainst(ctx) + if err != nil || len(against["notes"]) != 0 { + // The newest successful build of notes, build-2, stood on nothing. + t.Fatalf("built against %v: %v", against, err) + } + announced, err := inv.Announceable(ctx) + if err != nil || len(announced) != 3 { + t.Fatalf("announceable %d: %v", len(announced), err) + } + prints, err := inv.Fingerprints(ctx) + if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" { + t.Fatalf("fingerprints %v: %v", prints, err) + } +} + +// A build request row as the store held it before the move. + +// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue +// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none +// after the outcome is heard. The table of requests takes none. +func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) { + inv := fresh(t) + ctx := context.Background() + ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true} + if err := inv.RecordBuildRequest(ctx, ask); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuildRequest(ctx, ask); err != nil { + t.Fatalf("a request kept twice: %v", err) + } + if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 { + t.Fatalf("a request nobody answered is a build: %+v %v", builds, err) + } + if _, found, _ := inv.BuildByID(ctx, "build-1"); found { + t.Fatal("a request nobody answered has an outcome") + } + got, found, err := inv.BuildRequestByID(ctx, "build-1") + if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() { + t.Fatalf("the request reads back as %+v %v %v", got, found, err) + } + if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil { + t.Fatal(err) + } + asked, err := inv.RequestsNamed(ctx, "notes") + if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" { + t.Fatalf("the request is %+v: %v", asked, err) + } + if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) { + t.Fatalf("requested names %v: %v", names, err) + } + + // The outcome comes: the same key, now a build, with the request beside it. + built := aBuild("build-1", "notes", "") + if err := inv.RecordBuild(ctx, built); err != nil { + t.Fatal(err) + } + if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" { + t.Fatalf("the outcome reads as %+v %v %v", one, found, err) + } + answered, err := inv.RequestsNamed(ctx, "notes") + if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" { + t.Fatalf("the request answered is %+v: %v", answered, err) + } + if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" { + t.Fatalf("the request was lost with the outcome: %+v", again) + } + // Heard: the marks of an unanswered request change nothing. + if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil { + t.Fatal(err) + } + if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" { + t.Fatalf("an answered request was marked not asked: %+v", again) + } + if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 3 { + // asked, marked unknown, answered. + t.Fatalf("the record has %d values", len(history)) + } + // An outcome heard before its request is kept, and the request lands beside it. + if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil { + t.Fatal(err) + } + if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" { + t.Fatalf("the request kept after its outcome reads %+v", late) + } + // A request older than the time they are kept is no longer listed. + if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old", + At: time.Now().Add(-2 * KeptFor)}); err != nil { + t.Fatal(err) + } + if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") { + t.Fatalf("a request past its time is listed: %v", names) + } +} + +// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a +// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it +// is asked and so asked at most once. + +// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a +// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module +// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the +// assignment goes with it. +func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres", "web") + ctx := t.Context() + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + start := generationNow(t, inv) + if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added { + t.Fatalf("assigned %v: %v", added, err) + } + if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added { + t.Fatalf("assigned again %v: %v", added, err) + } + kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres") + if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() { + t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err) + } + if generationNow(t, inv) <= start { + t.Fatal("an assignment left the generation where it was") + } + if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) { + t.Fatalf("a module the mesh does not know: %v", err) + } + if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) { + t.Fatalf("assigned on %s: %v %v", node, on, err) + } + if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) { + t.Fatalf("running: %v %v", running, err) + } + entries, err := inv.Catalogued(ctx) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + switch e.Manifest.Module { + case "postgres": + if !reflect.DeepEqual(e.On, []string{node}) { + t.Errorf("postgres is on %v", e.On) + } + case "web": + if len(e.On) != 0 { + t.Errorf("web is on %v", e.On) + } + } + } + if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) { + t.Fatalf("a module still assigned was forgotten: %v", err) + } + // A seat is held by an assignment, or by nothing. + if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil { + t.Fatal("a seat was handed to a module that is not assigned") + } + if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil { + t.Fatalf("a seat handed to an assignment: %v", err) + } + + before := generationNow(t, inv) + if err := inv.Unassign(ctx, node, "postgres"); err != nil { + t.Fatal(err) + } + if generationNow(t, inv) <= before { + t.Fatal("an unassignment left the generation where it was") + } + // No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history. + gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres") + if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() { + t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err) + } + if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() || + history[1].Value.Stands() { + t.Fatalf("the earlier values of the key: %+v", history) + } + if on, _ := inv.Assigned(ctx, node); len(on) != 0 { + t.Fatalf("an ended assignment is still assigned: %v", on) + } + if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 { + t.Fatalf("an ended assignment is running: %v", running) + } + if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 { + t.Fatalf("the seat the assignment held is held by %+v: %v", held, err) + } + if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" { + t.Fatalf("an unassignment of nothing: %v", err) + } + if err := inv.ForgetModule(ctx, "postgres"); err != nil { + t.Fatalf("a module nothing is assigned could not be forgotten: %v", err) + } + // Made again after it ended: a fresh value over the ended one, which stays in history. + if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { + t.Fatal(added, err) + } + if added, err := inv.Assign(ctx, node, "web"); err != nil || added { + t.Fatal(added, err) + } + if err := inv.Unassign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + if added, err := inv.Assign(ctx, node, "web"); err != nil || !added { + t.Fatalf("assigned again after it ended: %v %v", added, err) + } + again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web") + if !again.Value.Stands() || !again.Value.Ended.IsZero() { + t.Fatalf("the assignment made again is %+v", again.Value) + } + if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() { + t.Fatalf("the values of the key: %+v", history) + } +} + +// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the +// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay, +// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here +// ends an assignment. + +// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the +// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay, +// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here +// ends an assignment. +func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) { + inv, node := aNodeWithModules(t, "dunst", "web") + ctx := t.Context() + for _, m := range []string{"dunst", "web"} { + if _, err := inv.Assign(ctx, node, m); err != nil { + t.Fatal(err) + } + } + // A build and a commit put in the value by another write. + cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") + cur.Value.Build, cur.Value.Commit = "build-1", "c1" + if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil { + t.Fatal(err) + } + record, _ := inv.NodeByName(ctx, node) + digest := "sha256:" + strings.Repeat("d", 64) + plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest, + Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc", + Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}} + raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`) + if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil { + t.Fatal(err) + } + inv.SetProvisioningPlans(inv.SentProvisioningPlans) + written, removed, err := inv.KeepAssignmentPlans(ctx) + if err != nil || written != 2 || removed != 0 { + t.Fatalf("kept: %d written, %d ended: %v", written, removed, err) + } + dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") + if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() { + t.Fatalf("dunst's value is %+v", dunst.Value) + } + if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil { + t.Fatalf("web has a plan: %+v", web.Value) + } + notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify") + if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil { + t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found) + } + if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) { + t.Fatalf("a pulled-in module is assigned: %v", on) + } + if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 { + t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err) + } + // Sent again without the pulled-in module: its value ends, the assignments stay. + only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`) + if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil { + t.Fatal(err) + } + if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 { + t.Fatalf("ended %d: %v", removed, err) + } + if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() { + t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value) + } + if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) { + t.Fatalf("assigned: %v", on) + } +} + +// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value** +// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build +// already is left as it is. diff --git a/internal/inventory/collection.go b/internal/inventory/collection.go index 6c0ba092..ebb011c4 100644 --- a/internal/inventory/collection.go +++ b/internal/inventory/collection.go @@ -2,7 +2,6 @@ package inventory import ( "context" - "encoding/json" "sort" "strings" @@ -327,84 +326,34 @@ type recentBuild struct { // recentBuilds is, of each module the mesh still holds, its KeptBuilds newest successful builds' artifacts and bases. func (i *Inventory) recentBuilds(ctx context.Context) ([]recentBuild, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - builds, err := i.chainRecentMade(ctx, c) - if err != nil { - return nil, err - } - out := make([]recentBuild, 0, len(builds)) - for _, b := range builds { - out = append(out, recentBuild{made: b.Made, against: b.Against}) - } - return out, nil - } - rows, err := i.store.Pool().Query(ctx, - `select made, built_against from ( - select b.made, b.built_against, - row_number() over (partition by b.module order by b.at desc, b.id desc) as back - from build b - join module m on m.name = b.module - where b.failed = '' and b.module is not null and b.module <> '' - ) ranked where back <= $1`, KeptBuilds) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - var out []recentBuild - for rows.Next() { - var raw, against []byte - if err := rows.Scan(&raw, &against); err != nil { - return nil, err - } - var r recentBuild - if err := json.Unmarshal(raw, &r.made); err != nil { - r.made = nil - } - if err := json.Unmarshal(against, &r.against); err != nil { - r.against = nil - } - out = append(out, r) + builds, err := i.chainRecentMade(ctx, c) + if err != nil { + return nil, err } - return out, rows.Err() + out := make([]recentBuild, 0, len(builds)) + for _, b := range builds { + out = append(out, recentBuild{made: b.Made, against: b.Against}) + } + return out, nil } // everyReferenceMade is every artifact reference any successful build recorded, oldest build first. func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) { - var made [][]Artifact - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - builds, err := i.chainEveryReferenceMade(ctx, c) - if err != nil { - return nil, err - } - for _, b := range builds { - made = append(made, b.Made) - } - } else { - rows, err := i.store.Pool().Query(ctx, - `select made from build where failed = '' and module is not null and module <> '' - order by at asc, id asc`) - if err != nil { - return nil, err - } - defer rows.Close() - for rows.Next() { - var raw []byte - if err := rows.Scan(&raw); err != nil { - return nil, err - } - var a []Artifact - if err := json.Unmarshal(raw, &a); err != nil { - continue - } - made = append(made, a) - } - if err := rows.Err(); err != nil { - return nil, err - } + } + builds, err := i.chainEveryReferenceMade(ctx, c) + if err != nil { + return nil, err + } + var made [][]Artifact + for _, b := range builds { + made = append(made, b.Made) } seen := map[string]bool{} var out []string diff --git a/internal/inventory/durations.go b/internal/inventory/durations.go index 73edbd73..5be4b96f 100644 --- a/internal/inventory/durations.go +++ b/internal/inventory/durations.go @@ -2,11 +2,7 @@ package inventory import ( "context" - "errors" - "fmt" "time" - - "github.com/jackc/pgx/v5" ) // The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066. @@ -148,41 +144,3 @@ func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) { } return tag.RowsAffected(), nil } - -// planTierLeft is the measurement a plan's save makes when it leaves a tier: the tier moved on, or -// the plan ended. Read in the save's own transaction, so two saves cannot both measure one tier. -func planTierLeft(ctx context.Context, tx pgx.Tx, p Plan, now time.Time) (entered time.Time, err error) { - var oldTier int - var oldState string - var since time.Time - err = tx.QueryRow(ctx, - `select tier, state, coalesce(tier_entered, created) from release_plan where id = $1 for update`, - p.ID).Scan(&oldTier, &oldState, &since) - if errors.Is(err, pgx.ErrNoRows) { - return now, nil // a new plan enters its first tier now - } - if err != nil { - return time.Time{}, err - } - if oldState == PlanAssembling || oldState == PlanQueued { - return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276) - } - wasOpen := oldState == PlanBuilding || oldState == PlanRolling - if !wasOpen || (oldTier == p.Tier && p.Open()) { - return since, nil // still in the tier, or already ended - } - var modules []string - if oldTier >= 0 && oldTier < len(p.Tiers) { - modules = p.Tiers[oldTier] - } - detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, oldTier, len(p.Tiers), modules, p.State) - if p.Open() { - detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, oldTier, len(p.Tiers), modules, p.Tier) - } - _, err = tx.Exec(ctx, - `insert into duration (kind, subject, node, ref, started, took_ms, detail) - values ($1, $2, '', $3, $4, $5, $6) on conflict do nothing`, - DurationPlanTier, p.Repository, fmt.Sprintf("%s/tier-%d", p.ID, oldTier), since, - now.Sub(since).Milliseconds(), detail) - return now, err -} diff --git a/internal/inventory/fortest.go b/internal/inventory/fortest.go index 5a2067d0..790401c4 100644 --- a/internal/inventory/fortest.go +++ b/internal/inventory/fortest.go @@ -65,16 +65,11 @@ func ForTest(t *testing.T) *Inventory { if _, err := inv.store.Migrate(t.Context(), migrations); err != nil { t.Fatal(err) } - // The chain is on the bus in a mesh; in a test it is in memory, with every kind moved, as a mesh raised after this - // release has them (ADR 0408, issue 1089). The tests of the move itself use ForTestBefore. + // The chain is on the bus in a mesh; in a test it is in memory (ADR 0408). c := chain.InMemory(nil) inv.UseChain(c) testChains.Store(os.Getenv(store.Variable(Name)), c) t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) }) - if _, err := inv.store.Pool().Exec(t.Context(), - `insert into chain_moved (counts, largest) values ($1, $1)`, `{"builds": 0, "rollouts": 0, "assignments": 0}`); err != nil { - t.Fatal(err) - } return inv } @@ -132,7 +127,7 @@ func ForTestBefore(t *testing.T, number int) (*Inventory, func()) { if err := inv.Ready(t.Context(), 20*time.Second); err != nil { t.Fatal(err) } - // The chain is in memory and nothing is moved: the tests of a move seed the tables and move them. + // The chain is in memory, for what a test of the store before a migration reads of it. c := chain.InMemory(nil) inv.UseChain(c) testChains.Store(os.Getenv(store.Variable(Name)), c) @@ -183,3 +178,21 @@ func (i *Inventory) QueryForTest(t *testing.T, sql string, args ...any) [][]stri } return out } + +// PlanValuesForTest runs the pass that keeps the provisioner's plans in the values of the assignments (KeepAssignmentPlans) +// and answers the values that stand, by key: what a reader of the bucket reads. +func (i *Inventory) PlanValuesForTest(t *testing.T) map[string][]byte { + t.Helper() + if _, _, err := i.KeepAssignmentPlans(t.Context()); err != nil { + t.Fatal(err) + } + kept, err := i.KeptAssignments(t.Context()) + if err != nil { + t.Fatal(err) + } + out := make(map[string][]byte, len(kept)) + for key, v := range kept { + out[key] = v.Value + } + return out +} diff --git a/internal/inventory/gate.go b/internal/inventory/gate.go index 9932b559..08cc9439 100644 --- a/internal/inventory/gate.go +++ b/internal/inventory/gate.go @@ -2,8 +2,6 @@ package inventory import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -241,65 +239,20 @@ func (i *Inventory) RestoreModule(ctx context.Context, b Build) error { // machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told // from one that does (the bus's planned step, ADR 0236). func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainBuildFingerprints(ctx, c, module) - } - rows, err := i.store.Pool().Query(ctx, - `select commit_hash, made, coalesce(manifest::text, '') from build - where module = $1 and failed = '' and commit_hash <> '' order by at desc limit 50`, module) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - out := map[string]string{} - for rows.Next() { - var commit, manifest string - var made []byte - if err := rows.Scan(&commit, &made, &manifest); err != nil { - return nil, err - } - if _, seen := out[commit]; seen { - continue - } - sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...)) - out[commit] = hex.EncodeToString(sum[:]) - } - return out, rows.Err() + return i.chainBuildFingerprints(ctx, c, module) } // Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint. func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainFingerprints(ctx, c) - } - rows, err := i.store.Pool().Query(ctx, - `select module, commit_hash, made, coalesce(manifest::text, '') from build - where module is not null and failed = '' and commit_hash <> '' order by at desc`) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - out := map[string]map[string]string{} - for rows.Next() { - var module, commit, manifest string - var made []byte - if err := rows.Scan(&module, &commit, &made, &manifest); err != nil { - return nil, err - } - if out[module] == nil { - out[module] = map[string]string{} - } - if _, seen := out[module][commit]; seen { - continue - } - sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...)) - out[module][commit] = hex.EncodeToString(sum[:]) - } - return out, rows.Err() + return i.chainFingerprints(ctx, c) } // PassedCommits is, per module, the commits a build of which passed its gate on some machine. @@ -326,19 +279,11 @@ func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bo // BuildOf is the id of the newest successful build of a module from a commit; empty when none is // recorded (a manifest handed over by hand). func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + c, err := i.theChain(ctx) + if err != nil { return "", err - } else if on { - return i.chainBuildOf(ctx, c, module, commit) } - var id string - err := i.store.Pool().QueryRow(ctx, - `select id from build where module = $1 and commit_hash = $2 and failed = '' order by at desc limit 1`, - module, commit).Scan(&id) - if errors.Is(err, pgx.ErrNoRows) { - return "", nil - } - return id, err + return i.chainBuildOf(ctx, c, module, commit) } // ManifestAt is the manifest a module was registered with at a commit (or a commit it abbreviates): the @@ -372,38 +317,19 @@ func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (cata // manifestOf is the manifest a build was recorded with; nil when it kept none. func (i *Inventory) manifestOf(ctx context.Context, id string) ([]byte, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainManifestOf(ctx, c, id) - } - var manifest []byte - if err := i.store.Pool().QueryRow(ctx, `select manifest from build where id = $1`, id).Scan(&manifest); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err } - return manifest, nil + return i.chainManifestOf(ctx, c, id) } // newestManifestAt is the newest successful build of a module from a commit, or one it abbreviates, that kept a manifest: // its id and manifest; false when there is none. func (i *Inventory) newestManifestAt(ctx context.Context, module, commit string) (string, []byte, bool, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return "", nil, false, err - } else if on { - return i.chainManifestBuild(ctx, c, module, commit) - } - var id string - var raw []byte - err := i.store.Pool().QueryRow(ctx, - `select id, manifest from build - where module = $1 and failed = '' and manifest is not null and manifest::text <> 'null' - and (commit_hash = $2 or starts_with(commit_hash, $2)) - order by at desc limit 1`, module, commit).Scan(&id, &raw) - if errors.Is(err, pgx.ErrNoRows) { - return "", nil, false, nil - } + c, err := i.theChain(ctx) if err != nil { return "", nil, false, err } - return id, raw, true, nil + return i.chainManifestBuild(ctx, c, module, commit) } diff --git a/internal/inventory/given.go b/internal/inventory/given.go index 20507be1..fd930fea 100644 --- a/internal/inventory/given.go +++ b/internal/inventory/given.go @@ -93,28 +93,19 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s if declared == "" { return nil, nil } - // Only a module assigned there: from the bucket when the assignments are on the bus, else the store's join. - _, onTheBus, err := i.ChainReads(ctx, KindAssignments) + // Only a module assigned there, from the assignments bucket. + assignedHere := map[string]bool{} + modules, err := i.assignedModules(ctx, node) if err != nil { return nil, err } - assignedHere := map[string]bool{} - join := `join assignment a on a.node = s.node and a.module = s.module` - if onTheBus { - join = `` - modules, err := i.assignedModules(ctx, node) - if err != nil { - return nil, err - } - for _, m := range modules { - assignedHere[m] = true - } + for _, m := range modules { + assignedHere[m] = true } rows, err := i.store.Pool().Query(ctx, `select s.node, s.module, s.name, s.replace_after_start from module_secret s join node n on n.id = s.node - `+join+` where n.name = $1 and n.sent = $2 and n.sent_at > s.replace_after_start and s.origin = 'accepted' and s.replace_after_start is not null and (not n.adopted or exists (select 1 from taken t where t.node = s.node and t.module = s.module)) @@ -134,7 +125,7 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s rows.Close() return nil, err } - if onTheBus && !assignedHere[d.module] { + if !assignedHere[d.module] { continue } dues = append(dues, d) diff --git a/internal/inventory/mirrored_test.go b/internal/inventory/mirrored_test.go index d258ce91..9f56d9b7 100644 --- a/internal/inventory/mirrored_test.go +++ b/internal/inventory/mirrored_test.go @@ -5,7 +5,6 @@ package inventory import ( "context" "fmt" - "os" "slices" "strings" "testing" @@ -148,40 +147,6 @@ func TestAPersonRecordsACopyAndTheFirstRecordStands(t *testing.T) { } } -// The migration records the copies the build records already name: only a module's own on- -// repository, and only once, by the earliest build naming it. -func TestTheCopiesTheRecordsAlreadyNameAreRecorded(t *testing.T) { - inv := inTheTables(t) - ctx := context.Background() - copyRef := mirrorRef("route-proxy/on-go_base", 1) - notACopy := ref("go-base", "build", 2) - builtOn(t, inv, "b1", "route-proxy", 1, nil, copyRef, notACopy) - builtOn(t, inv, "b2", "route-proxy", 2, nil, copyRef) - // The rows a mesh had before the table: written by builds that did not say what they copied. - if _, err := inv.store.Pool().Exec(ctx, `truncate artifact_mirrored`); err != nil { - t.Fatal(err) - } - sql, err := os.ReadFile("migrations/0081-a-mirrored-base-is-recorded.sql") - if err != nil { - t.Fatal(err) - } - _, backfill, _ := strings.Cut(string(sql), "insert into artifact_mirrored") - if _, err := inv.store.Pool().Exec(ctx, "insert into artifact_mirrored"+backfill); err != nil { - t.Fatal(err) - } - var reference, build string - var n int - if err := inv.store.Pool().QueryRow(ctx, `select count(*) from artifact_mirrored`).Scan(&n); err != nil || n != 1 { - t.Fatalf("%d copies recorded (%v); want the one", n, err) - } - if err := inv.store.Pool().QueryRow(ctx, `select reference, build from artifact_mirrored`).Scan(&reference, &build); err != nil { - t.Fatal(err) - } - if reference != copyRef || build != "b1" { - t.Fatalf("recorded %s by %s", reference, build) - } -} - // A copy let go of and copied again by a later build is held again: its record is back, and the sweep // decides about it as before. func TestACopyLetGoOfAndCopiedAgainIsHeldAgain(t *testing.T) { diff --git a/internal/inventory/mirrors.go b/internal/inventory/mirrors.go index fadab025..4359b2be 100644 --- a/internal/inventory/mirrors.go +++ b/internal/inventory/mirrors.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "time" ) @@ -70,39 +69,8 @@ type DeclaredMirrored struct { var DeclaredMirroredFields = []string{"kind", "default", "why", "type", "description", "enum", "minimum", "maximum", "pattern", "trusted", "predefined", "required", "runtime"} -// ErrAssignmentsMoved is a write of the store's mirror of the assignments, refused because the assignments are keys of the bus. -var ErrAssignmentsMoved = errors.New("the assignments are keys of the bus, and the store's rows are no longer what they say") - -// AssignmentsMirrorToWrite is AssignmentsMirror for the mirror's writer, which lays what it returns over the bucket and -// ends the keys it does not return: **refused once the assignments are moved** (ADR 0408 §4), asked at the moment of the -// write, so a tick that decided before the switch cannot lay mirror-shaped values over the links after it. -func (i *Inventory) AssignmentsMirrorToWrite(ctx context.Context) (map[string][]byte, error) { - if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil { - return nil, err - } else if on { - return nil, ErrAssignmentsMoved - } - return i.AssignmentsMirror(ctx) -} - -// AssignmentsMirror is every assignment's mirrored value, keyed `.`, with the provisioner's plan of -// each assignment of a module handed to it, and a value of its own for each module handed to it on a node that -// nothing assigned there, only pulled in by another (issue 765). -func (i *Inventory) AssignmentsMirror(ctx context.Context) (map[string][]byte, error) { - var plans map[string]Provisioning - if i.provisioningPlans != nil { - var err error - if plans, err = i.provisioningPlans(ctx); err != nil { - return nil, fmt.Errorf("composing the provisioner's plans: %w", err) - } - } - assignments, err := i.assignmentsMirrored(ctx) - if err != nil { - return nil, err - } - return AssignmentValues(assignments, plans) -} - +// assignmentsMirrored is every standing assignment as the provisioner's plans are matched to it (SentProvisioningPlans, +// KeepAssignmentPlans). func (i *Inventory) assignmentsMirrored(ctx context.Context) ([]AssignmentMirrored, error) { rows, err := i.assignments(ctx) if err != nil { diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 597fe925..bf98a1a7 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -34,11 +34,9 @@ type Inventory struct { // configValues reads a module's config values from mesh-config once the move of config has run (ReadsConfigFrom, // issue 664); nil reads none. configValues ConfigValues - // chain is the controller's chain on the bus (novox/hq ADR 0408, issue 1089), once attached (UseChain); moved is which - // kinds of it have been moved from the tables, as chain_moved says (ChainReads). A kind not moved is read and written - // in the tables as before. + // chain is the controller's chain on the bus (novox/hq ADR 0408), once attached (UseChain). chain *chain.Chain - moved *movedKinds + state *chainState // recorded, when set, takes the durations a save of the chain measures in place of the store: a test with no store. recorded func(context.Context, Duration) error } @@ -49,7 +47,7 @@ func Open(ctx context.Context) (*Inventory, error) { if err != nil { return nil, err } - inv := &Inventory{store: s, moved: &movedKinds{}} + inv := &Inventory{store: s, state: &chainState{}} // A test's store has the chain a test made for it in memory, shared by every inventory opened on that store. if c, ok := testChains.Load(os.Getenv(store.Variable(Name))); ok { inv.chain = c.(*chain.Chain) @@ -1278,22 +1276,7 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu // read from: every change a push carries happened after the last push, and an assignment is the one // that says when. func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time, error) { - if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil { - return time.Time{}, err - } else if on { - return i.chainAwaitingSince(ctx, name) - } - var since time.Time - err := i.store.Pool().QueryRow(ctx, - `select coalesce( - (select min(a.assigned) from assignment a - where a.node = n.id and (n.sent_at is null or a.assigned > n.sent_at)), - n.sent_at, n.created) - from node n where n.name = $1`, name).Scan(&since) - if errors.Is(err, pgx.ErrNoRows) { - return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name) - } - return since, err + return i.chainAwaitingSince(ctx, name) } // SentBuilds is the build of each module a machine was last sent, by its name: module to the commit diff --git a/internal/inventory/pending.go b/internal/inventory/pending.go index 4d163eef..89eb8d22 100644 --- a/internal/inventory/pending.go +++ b/internal/inventory/pending.go @@ -9,8 +9,6 @@ import ( "strings" "time" - "github.com/jackc/pgx/v5" - "github.com/novox/mesh-controller/internal/chain" ) @@ -71,85 +69,40 @@ type RequestOutcome struct { // RecordBuildRequest keeps one build request, once it is asked. Idempotent on the id. Requests older than // KeptFor are deleted in the same act. func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) error { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return err - } else if on { - return i.chainRecordBuildRequest(ctx, c, a) - } - at := a.At - if at.IsZero() { - at = time.Now().UTC() - } - var notAsked *string - if a.NotAsked != "" { - notAsked = &a.NotAsked - } - if _, err := i.store.Pool().Exec(ctx, - `insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at, - at_terminal) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - on conflict (id) do nothing`, - a.ID, a.Repository, a.Seat, strings.Trim(a.Path, "/"), a.Ref, a.Commit, a.For, notAsked, at, - a.AtTerminal); err != nil { + c, err := i.theChain(ctx) + if err != nil { return err } - _, err := i.store.Pool().Exec(ctx, `delete from build_request where asked_at < $1`, time.Now().Add(-KeptFor)) - return err + return i.chainRecordBuildRequest(ctx, c, a) } // BuildRequestByID is the build request kept under this id, and whether one was kept. func (i *Inventory) BuildRequestByID(ctx context.Context, id string) (BuildRequest, bool, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { + c, err := i.theChain(ctx) + if err != nil { return BuildRequest{}, false, err - } else if on { - return i.chainBuildRequestByID(ctx, c, id) } - var a BuildRequest - err := i.store.Pool().QueryRow(ctx, - `select id, repository, seat, source_path, ref, commit_hash, asked_for, at_terminal, asked_at - from build_request where id = $1`, id).Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, - &a.For, &a.AtTerminal, &a.At) - if errors.Is(err, pgx.ErrNoRows) { - return BuildRequest{}, false, nil - } - return a, err == nil, err + return i.chainBuildRequestByID(ctx, c, id) } // MarkNotAsked says a kept build request was never handed over: the words are kept, unless its outcome was // heard first. func (i *Inventory) MarkNotAsked(ctx context.Context, id, why string) error { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return err - } else if on { - return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.NotAsked = why }) - } - if heard, err := i.outcomeHeard(ctx, id); err != nil || heard { + c, err := i.theChain(ctx) + if err != nil { return err } - _, err := i.store.Pool().Exec(ctx, `update build_request set not_asked = $2 where id = $1`, id, why) - return err + return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.NotAsked = why }) } // MarkOutcomeUnknown says the asker of a build it handed over stopped waiting for it: asked, outcome unknown. // The build may still run; its outcome, when heard, is the last word. func (i *Inventory) MarkOutcomeUnknown(ctx context.Context, id, why string) error { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return err - } else if on { - return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.OutcomeUnknown = why }) - } - if heard, err := i.outcomeHeard(ctx, id); err != nil || heard { + c, err := i.theChain(ctx) + if err != nil { return err } - _, err := i.store.Pool().Exec(ctx, `update build_request set outcome_unknown = $2 where id = $1`, id, why) - return err -} - -// outcomeHeard says a build of this id was recorded in the table: its outcome was taken in. -func (i *Inventory) outcomeHeard(ctx context.Context, id string) (bool, error) { - var heard bool - err := i.store.Pool().QueryRow(ctx, `select exists (select 1 from build where id = $1)`, id).Scan(&heard) - return heard, err + return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.OutcomeUnknown = why }) } // RequestsNamed is every build request kept whose directory names the module, or whose build said it built @@ -187,31 +140,11 @@ func (i *Inventory) RequestedNames(ctx context.Context) ([]string, error) { } func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, err - } else if on { - return i.chainRequests(ctx, c) - } - rows, err := i.store.Pool().Query(ctx, - `select a.id, a.repository, a.seat, a.source_path, a.ref, a.commit_hash, a.asked_for, - coalesce(a.not_asked, ''), coalesce(a.outcome_unknown, ''), a.asked_at, - b.id is not null, coalesce(b.failed, ''), coalesce(b.module, ''), coalesce(b.at, a.asked_at) - from build_request a left join build b on b.id = a.id - order by a.asked_at desc, a.id desc`) + c, err := i.theChain(ctx) if err != nil { return nil, err } - defer rows.Close() - var out []RequestOutcome - for rows.Next() { - var a RequestOutcome - if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked, - &a.OutcomeUnknown, &a.At, &a.Heard, &a.Failed, &a.Module, &a.HeardAt); err != nil { - return nil, err - } - out = append(out, a) - } - return out, rows.Err() + return i.chainRequests(ctx, c) } // The states of a pending assignment. diff --git a/internal/inventory/pin_completed_test.go b/internal/inventory/pin_completed_test.go deleted file mode 100644 index 2cc7d37c..00000000 --- a/internal/inventory/pin_completed_test.go +++ /dev/null @@ -1,88 +0,0 @@ -//go:build integration - -package inventory - -import ( - "context" - "os" - "testing" -) - -// A pin made before it named the module (migration 0051, novox/hq #258): completed where the node it -// names answers once, left for a person where it answers twice. - -func legacyPin(t *testing.T, inv *Inventory, node, provision, provider string) { - t.Helper() - _, err := inv.store.Pool().Exec(context.Background(), - `insert into provision_pin (node, name, provider) - select u.id, $2, p.id from node u, node p where u.name = $1 and p.name = $3`, - node, provision, provider) - if err != nil { - t.Fatal(err) - } -} - -func completeEarlierPins(t *testing.T, inv *Inventory) { - t.Helper() - sql, err := os.ReadFile("migrations/0051-a-pin-made-before-is-completed.sql") - if err != nil { - t.Fatal(err) - } - if _, err := inv.store.Pool().Exec(context.Background(), string(sql)); err != nil { - t.Fatal(err) - } -} - -func TestAPinMadeBeforeIsCompletedWhenTheNodeAnswersOnce(t *testing.T) { - inv := inTheTables(t) - ctx := context.Background() - for _, n := range []string{"user", "provider"} { - if _, err := inv.AddNode(ctx, n); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"postgres", "redis"} { - if err := inv.RegisterModule(ctx, manifest(m, []string{m + "-database"}, nil), Source{}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "provider", m); err != nil { - t.Fatal(err) - } - } - legacyPin(t, inv, "user", "postgres-database", "provider") - completeEarlierPins(t, inv) - pins, err := inv.PinsFor(ctx, "user") - if err != nil { - t.Fatal(err) - } - if got := pins["postgres-database"]; got.Node != "provider" || got.Module != "postgres" { - t.Fatalf("the record was not completed with the one module that answers: %+v", got) - } -} - -func TestAPinMadeBeforeIsLeftOpenWhenTheNodeAnswersTwice(t *testing.T) { - inv := inTheTables(t) - ctx := context.Background() - for _, n := range []string{"user", "provider"} { - if _, err := inv.AddNode(ctx, n); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"public-acme", "step-ca"} { - if err := inv.RegisterModule(ctx, manifest(m, []string{"acme-ca"}, nil), Source{}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "provider", m); err != nil { - t.Fatal(err) - } - } - legacyPin(t, inv, "user", "acme-ca", "provider") - completeEarlierPins(t, inv) - pins, err := inv.PinsFor(ctx, "user") - if err != nil { - t.Fatal(err) - } - if got := pins["acme-ca"]; got.Node != "provider" || got.Module != "" { - t.Fatalf("a node that answers twice was guessed for: %+v", got) - } -} diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index 8ad7a0d1..cecc0d70 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -2,15 +2,12 @@ package inventory import ( "context" - "encoding/json" "errors" "fmt" "slices" "strings" "time" - "github.com/jackc/pgx/v5" - "github.com/novox/mesh-controller/internal/chain" ) @@ -680,140 +677,12 @@ type MergeMove struct { // leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by // somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once // the transaction is committed, in order. -func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) (err error) { - if c, on, cerr := i.ChainReads(ctx, KindRollouts); cerr != nil { - return cerr - } else if on { - return i.chainSavePlans(ctx, c, plans, moves) - } - epoch, err := i.actingEpoch(ctx) - if err != nil { - if len(plans) > 0 { - return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err) - } - return err - } - // **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the - // plan moves, in the same transaction as the move, so no save can move a tier unmeasured or - // measure one twice. - tx, err := i.store.Pool().Begin(ctx) +func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error { + c, err := i.theChain(ctx) if err != nil { return err } - defer func() { _ = tx.Rollback(ctx) }() - type written struct { - revision int64 - entered time.Time - } - done := make([]written, len(plans)) - // **The save that starts or ends a walk is marked** (novox/hq ADR 0308 decision 1), in the same write, and - // unmarked should the write not be made, so the next save marks it again. - marked := time.Now() - undo := make([]func(), 0, len(plans)) - defer func() { - if err != nil { - for _, u := range undo { - u() - } - } - }() - for _, p := range plans { - undo = append(undo, p.markWalk(marked)) - } - for n, p := range plans { - revision, entered, err := writePlan(ctx, tx, p, epoch) - if err != nil { - return err - } - done[n] = written{revision, entered} - } - for _, m := range moves { - if _, err := tx.Exec(ctx, - `update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`, - m.Repository, m.Commit, m.Plan, m.Alone); err != nil { - return err - } - } - if err := tx.Commit(ctx); err != nil { - return err - } - for n, p := range plans { - p.Revision, p.TierEntered = done[n].revision, done[n].entered - if epoch != nil { - p.Epoch = uint64(*epoch) - } else { - p.Epoch = 0 - } - if PlanSaved != nil { - PlanSaved(i, *p) - } - } - return nil -} - -// writePlan writes one plan within a transaction, by compare-and-set on its revision: its new revision, and when -// it entered its tier. -func writePlan(ctx context.Context, tx pgx.Tx, p *Plan, epoch *int64) (int64, time.Time, error) { - tiers, err := json.Marshal(p.Tiers) - if err != nil { - return 0, time.Time{}, err - } - modules, err := json.Marshal(p.Modules) - if err != nil { - return 0, time.Time{}, err - } - var release, delivery, commits, times, backups []byte - if p.Backups != nil { - if backups, err = json.Marshal(p.Backups); err != nil { - return 0, time.Time{}, err - } - } - if p.Times != nil { - if times, err = json.Marshal(p.Times); err != nil { - return 0, time.Time{}, err - } - } - if len(p.Commits) > 0 { - if commits, err = json.Marshal(p.Commits); err != nil { - return 0, time.Time{}, err - } - } - if p.Release != nil { - if release, err = json.Marshal(p.Release); err != nil { - return 0, time.Time{}, err - } - } - if p.Delivery != nil { - if delivery, err = json.Marshal(p.Delivery); err != nil { - return 0, time.Time{}, err - } - } - entered, err := planTierLeft(ctx, tx, *p, time.Now()) - if err != nil { - return 0, time.Time{}, err - } - var revision int64 - err = tx.QueryRow(ctx, - `insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, - branch, tier_entered, revision, epoch, release, delivery, merged_at, commits, times, backups) - values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14, $15, $16, $17, $18, $19) - on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier, - tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch, - tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch, - release = excluded.release, delivery = excluded.delivery, repository = excluded.repository, - commit_hash = excluded.commit_hash, merged_at = excluded.merged_at, commits = excluded.commits, - created = excluded.created, times = excluded.times, backups = excluded.backups - where release_plan.revision = $12 - returning revision`, - p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered, - p.Revision, epoch, release, delivery, mergedAt(p.Merged), commits, times, backups).Scan(&revision) - if errors.Is(err, pgx.ErrNoRows) { - // The row is there and at another revision — moved since this was read, or there already - // when this one is new: either way not this writer's to overwrite. (A plan saved before plans - // had revisions is at zero, and its first save here is from a read at zero.) - return 0, time.Time{}, fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) - } - return revision, entered, err + return i.chainSavePlans(ctx, c, plans, moves) } // short is a commit as a person reads it. @@ -824,145 +693,54 @@ func short(commit string) string { return commit } -// mergedAt is a plan's merge time as the store keeps it: null when not known. -func mergedAt(t time.Time) *time.Time { - if t.IsZero() { - return nil - } - return &t -} - // OpenPlans is every plan still being worked, oldest first. func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) } - return i.plans(ctx, `where state in ('building', 'rolling') order by created`) + return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) } // PlansSince is every plan made after a moment, and every plan still being worked, oldest first. func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated) } - return i.plans(ctx, `where created > $1 or state in ('building', 'rolling') order by created`, since) + return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated) } // RecentPlans is the last few plans, newest first, open or not — what the overview shows. func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { + c, err := i.theChain(ctx) + if err != nil { return nil, err - } else if on { - plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest) - if err == nil && limit >= 0 && len(plans) > limit { - plans = plans[:limit] - } - return plans, err } - return i.plans(ctx, `order by created desc limit $1`, limit) + plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest) + if err == nil && limit >= 0 && len(plans) > limit { + plans = plans[:limit] + } + return plans, err } // NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh // planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it // recorded when its merge was made. func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { - return Plan{}, false, err - } else if on { - return i.chainNewestMergeOf(ctx, c, repository, branch) - } - plans, err := i.plans(ctx, `where lower(repository) = lower($1) and branch = $2 and merged_at is not null - and release is null order by merged_at desc, created desc limit 1`, repository, branch) - if err != nil || len(plans) == 0 { + c, err := i.theChain(ctx) + if err != nil { return Plan{}, false, err } - return plans[0], true, nil + return i.chainNewestMergeOf(ctx, c, repository, branch) } // PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a // caller of the plans verb. func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) { - if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil { - return Plan{}, err - } else if on { - return i.chainPlanByID(ctx, c, id) - } - plans, err := i.plans(ctx, `where id = $1`, id) + c, err := i.theChain(ctx) if err != nil { return Plan{}, err } - if len(plans) == 0 { - return Plan{}, fmt.Errorf("no plan %s", id) - } - return plans[0], nil -} - -func (i *Inventory) plans(ctx context.Context, tail string, args ...any) ([]Plan, error) { - rows, err := i.store.Pool().Query(ctx, - `select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch, - coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery, merged_at, commits, times, backups - from release_plan `+tail, args...) - if err != nil { - return nil, err - } - defer rows.Close() - var out []Plan - for rows.Next() { - var p Plan - var tiers, modules, release, delivery, commits, times, backups []byte - var epoch int64 - var merged *time.Time - if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State, - &p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch, &release, - &delivery, &merged, &commits, ×, &backups); err != nil { - return nil, err - } - if len(backups) > 0 { - if err := json.Unmarshal(backups, &p.Backups); err != nil { - return nil, err - } - } - if len(times) > 0 { - if err := json.Unmarshal(times, &p.Times); err != nil { - return nil, err - } - } - if len(commits) > 0 { - if err := json.Unmarshal(commits, &p.Commits); err != nil { - return nil, err - } - } - if merged != nil { - p.Merged = merged.UTC() - } - if len(release) > 0 { - if err := json.Unmarshal(release, &p.Release); err != nil { - return nil, err - } - } - if len(delivery) > 0 { - if err := json.Unmarshal(delivery, &p.Delivery); err != nil { - return nil, err - } - } - p.Epoch = uint64(epoch) - if err := json.Unmarshal(tiers, &p.Tiers); err != nil { - return nil, err - } - if err := json.Unmarshal(modules, &p.Modules); err != nil { - return nil, err - } - if p.Modules == nil { - p.Modules = map[string]*PlanModule{} - } - out = append(out, p) - } - if errors.Is(rows.Err(), pgx.ErrNoRows) { - return nil, nil - } - return out, rows.Err() + return i.chainPlanByID(ctx, c, id) } diff --git a/internal/inventory/sent_plans_test.go b/internal/inventory/sent_plans_test.go index 63a82deb..fb1a081c 100644 --- a/internal/inventory/sent_plans_test.go +++ b/internal/inventory/sent_plans_test.go @@ -48,10 +48,7 @@ func TestASendKeepsThePlansItHandedAndTheMirrorCarriesThem(t *testing.T) { t.Fatalf("the plans mirrored are %+v (%v)", plans, err) } inv.SetProvisioningPlans(inv.SentProvisioningPlans) - mirror, err := inv.AssignmentsMirror(ctx) - if err != nil { - t.Fatal(err) - } + mirror := inv.PlanValuesForTest(t) var dunst, web AssignmentMirrored if err := json.Unmarshal(mirror[node+".dunst"], &dunst); err != nil { t.Fatal(err) @@ -81,8 +78,8 @@ func TestASendKeepsThePlansItHandedAndTheMirrorCarriesThem(t *testing.T) { if sent, _, err := inv.SentTo(ctx, node); err != nil || sent.Provisioned != nil { t.Fatalf("a send handing nothing says it handed %v (%v)", sent.Provisioned, err) } - if mirror, err := inv.AssignmentsMirror(ctx); err != nil || strings.Contains(string(mirror[node+".dunst"]), "provisioning") { - t.Fatalf("dunst's assignment still carries a plan: %s (%v)", mirror[node+".dunst"], err) + if mirror := inv.PlanValuesForTest(t); strings.Contains(string(mirror[node+".dunst"]), "provisioning") { + t.Fatalf("dunst's assignment still carries a plan: %s", mirror[node+".dunst"]) } else if _, has := mirror[node+".notify"]; has { t.Fatalf("a module the machine no longer runs is mirrored: %s", mirror[node+".notify"]) } diff --git a/internal/inventory/source.go b/internal/inventory/source.go index 63bb9dbc..ca0de05d 100644 --- a/internal/inventory/source.go +++ b/internal/inventory/source.go @@ -32,39 +32,11 @@ type sourceRow struct { // sourceRows is every successful build of each module (of one module, when named), newest request // first. func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][]sourceRow, []string, error) { - if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil { - return nil, nil, err - } else if on { - return i.chainSourceRows(ctx, c, module) - } - query := `select b.module, b.id, b.commit_hash, b.source_fingerprint, b.made, b.manifest, - coalesce(g.verdict = 'failed', false) - from build b left join build_gate g on g.build = b.id - where b.module is not null and b.module <> '' and b.failed = ''` - args := []any{} - if module != "" { - query += ` and b.module = $1` - args = append(args, module) - } - rows, err := i.store.Pool().Query(ctx, query+` order by b.module, coalesce(b.asked, b.at) desc, b.at desc`, args...) + c, err := i.theChain(ctx) if err != nil { return nil, nil, err } - defer rows.Close() - out := map[string][]sourceRow{} - var order []string - for rows.Next() { - var m string - var r sourceRow - if err := rows.Scan(&m, &r.id, &r.commit, &r.fingerprint, &r.made, &r.manifest, &r.failedGate); err != nil { - return nil, nil, err - } - if _, seen := out[m]; !seen { - order = append(order, m) - } - out[m] = append(out[m], r) - } - return out, order, rows.Err() + return i.chainSourceRows(ctx, c, module) } // standing is, of a module's builds newest first, the index of the build that stands for the build at diff --git a/internal/recordmirror/mirror.go b/internal/recordmirror/mirror.go index a751f73c..f330ed4e 100644 --- a/internal/recordmirror/mirror.go +++ b/internal/recordmirror/mirror.go @@ -60,7 +60,7 @@ var ( ) // Mirrors is every mirror this package writes, in the order they are written. -func Mirrors() []Mirror { return []Mirror{Assignments, DeclaredConfig, Predefined} } +func Mirrors() []Mirror { return []Mirror{DeclaredConfig, Predefined} } // The changes an event says. const ( diff --git a/internal/recordmirror/mirror_store_test.go b/internal/recordmirror/mirror_store_test.go index 34d196df..7e4cbdda 100644 --- a/internal/recordmirror/mirror_store_test.go +++ b/internal/recordmirror/mirror_store_test.go @@ -12,30 +12,19 @@ import ( "github.com/novox/mesh-controller/internal/inventory" ) -// **Design 57's F3, end to end** (novox/hq issue 652): from the controller's store to a real bus. An -// assignment changes exactly one key of the assignments mirror and raises exactly one `assignment-changed`; -// a registration changes exactly one key of the declared-config mirror and raises exactly one -// `declared-config-changed`; and the same records written again change nothing and say nothing — a module -// registered again unchanged included, although the store's registration time moved. +// **Design 57's F3, end to end** (novox/hq issue 652): from the controller's store to a real bus. A registration +// changes exactly one key of the declared-config mirror and raises exactly one `declared-config-changed`; and the same +// records written again change nothing and say nothing — a module registered again unchanged included, although the +// store's registration time moved. (The assignments are keys of the bus, no mirror of the store: ADR 0408.) func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *testing.T) { inv := inventory.ForTest(t) ctx := t.Context() js, api := aMirrorBus(t) - assignments := aWriter(t, js, api, Assignments) declared := aWriter(t, js, api, DeclaredConfig) - heardAssignments := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+Assignments.Event) heardDeclared := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+DeclaredConfig.Event) - pass := func() (Result, Result) { + pass := func() Result { t.Helper() - a, err := inv.AssignmentsMirror(ctx) - if err != nil { - t.Fatal(err) - } - ra, err := assignments.Sync(ctx, a) - if err != nil { - t.Fatal(err) - } d, err := inv.DeclaredConfigMirror(ctx) if err != nil { t.Fatal(err) @@ -44,7 +33,7 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes if err != nil { t.Fatal(err) } - return ra, rd + return rd } if _, err := inv.AddNode(ctx, "anchor"); err != nil { @@ -56,72 +45,37 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes t.Fatal(err) } pass() - assignmentWrites, declaredWrites := writes(t, api, Assignments.Bucket), writes(t, api, DeclaredConfig.Bucket) - assignmentsSaid, declaredSaid := len(settle(heardAssignments, 0)), len(settle(heardDeclared, 1)) + declaredWrites := writes(t, api, DeclaredConfig.Bucket) + declaredSaid := len(settle(heardDeclared, 1)) // **A registration**: a module new to the catalogue. if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "notes", Version: "1"}, inventory.Source{}); err != nil { t.Fatal(err) } - ra, rd := pass() - if ra != (Result{}) || rd != (Result{Written: 1}) { - t.Errorf("a registration wrote assignments %+v and declared config %+v", ra, rd) + if rd := pass(); rd != (Result{Written: 1}) { + t.Errorf("a registration wrote declared config %+v", rd) } if got := writes(t, api, DeclaredConfig.Bucket) - declaredWrites; got != 1 { t.Errorf("a registration changed %d keys of the declared config", got) } - if got := writes(t, api, Assignments.Bucket) - assignmentWrites; got != 0 { - t.Errorf("a registration changed %d keys of the assignments", got) - } said := settle(heardDeclared, declaredSaid+1)[declaredSaid:] if len(said) != 1 || said[0].Key != "notes" || said[0].Change != Added { t.Errorf("a registration raised %+v", said) } declaredWrites, declaredSaid = writes(t, api, DeclaredConfig.Bucket), declaredSaid+len(said) - // **An assignment.** - if _, err := inv.Assign(ctx, "anchor", "ui"); err != nil { - t.Fatal(err) - } - ra, rd = pass() - if ra != (Result{Written: 1}) || rd != (Result{}) { - t.Errorf("an assignment wrote assignments %+v and declared config %+v", ra, rd) - } - if got := writes(t, api, Assignments.Bucket) - assignmentWrites; got != 1 { - t.Errorf("an assignment changed %d keys of the assignments", got) - } - said = settle(heardAssignments, assignmentsSaid+1)[assignmentsSaid:] - if len(said) != 1 || said[0].Key != "anchor.ui" || said[0].Change != Added { - t.Errorf("an assignment raised %+v", said) - } - assignmentWrites, assignmentsSaid = writes(t, api, Assignments.Bucket), assignmentsSaid+len(said) - e, err := assignments.KV.Get(ctx, "anchor.ui") - if err != nil { - t.Fatal(err) - } - var value inventory.AssignmentMirrored - if err := json.Unmarshal(e.Value(), &value); err != nil || value.Node != "anchor" || value.Module != "ui" || - value.Assigned.IsZero() { - t.Errorf("the assignment is mirrored as %s (%v)", e.Value(), err) - } - - // **The same records written again change nothing**: an assignment made again, and a module registered - // again with the manifest it had. - if _, err := inv.Assign(ctx, "anchor", "ui"); err != nil { - t.Fatal(err) - } + // **The same records written again change nothing**: a module registered again with the manifest it had. if err := inv.RegisterModule(ctx, ui, inventory.Source{}); err != nil { t.Fatal(err) } - if ra, rd := pass(); ra != (Result{}) || rd != (Result{}) { - t.Errorf("unchanged records wrote assignments %+v and declared config %+v", ra, rd) + if rd := pass(); rd != (Result{}) { + t.Errorf("unchanged records wrote declared config %+v", rd) } - if writes(t, api, Assignments.Bucket) != assignmentWrites || writes(t, api, DeclaredConfig.Bucket) != declaredWrites { + if writes(t, api, DeclaredConfig.Bucket) != declaredWrites { t.Error("unchanged records changed a key") } - if a, d := settle(heardAssignments, assignmentsSaid+1), settle(heardDeclared, declaredSaid+1); len(a) != assignmentsSaid || - len(d) != declaredSaid { - t.Errorf("unchanged records raised %+v and %+v", a[assignmentsSaid:], d[declaredSaid:]) + if d := settle(heardDeclared, declaredSaid+1); len(d) != declaredSaid { + t.Errorf("unchanged records raised %+v", d[declaredSaid:]) } // **A declaration changed by a registration**: one key, one event, naming the field that moved. @@ -131,8 +85,8 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes if err := inv.RegisterModule(ctx, ui, inventory.Source{}); err != nil { t.Fatal(err) } - if ra, rd := pass(); ra != (Result{}) || rd != (Result{Written: 1}) { - t.Errorf("a changed declaration wrote assignments %+v and declared config %+v", ra, rd) + if rd := pass(); rd != (Result{Written: 1}) { + t.Errorf("a changed declaration wrote declared config %+v", rd) } said = settle(heardDeclared, declaredSaid+1)[declaredSaid:] if len(said) != 1 || said[0].Key != "ui" || said[0].Change != Changed || strings.Join(said[0].Fields, ",") != "settings" { @@ -146,25 +100,13 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes t.Errorf("the declaration is mirrored as %s, want %s", got.Value(), want) } - // **An unassignment** removes its key, said once. - if err := inv.Unassign(ctx, "anchor", "ui"); err != nil { - t.Fatal(err) - } - if ra, _ := pass(); ra != (Result{Removed: 1}) { - t.Errorf("an unassignment did %+v", ra) - } - said = settle(heardAssignments, assignmentsSaid+1)[assignmentsSaid:] - if len(said) != 1 || said[0].Key != "anchor.ui" || said[0].Change != Removed { - t.Errorf("an unassignment raised %+v", said) - } - // **A module no longer registered** has its key removed, said once as `removed` (ADR 0356 §4). if err := inv.ForgetModule(ctx, "notes"); err != nil { t.Fatal(err) } declaredWrites = writes(t, api, DeclaredConfig.Bucket) declaredSaid = len(heardDeclared.all()) - if _, rd := pass(); rd != (Result{Removed: 1}) { + if rd := pass(); rd != (Result{Removed: 1}) { t.Errorf("a module forgotten did %+v", rd) } if got := writes(t, api, DeclaredConfig.Bucket) - declaredWrites; got != 1 { diff --git a/testdata/store-tests b/testdata/store-tests index 3c07590e..71222538 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -139,22 +139,13 @@ ./cmd/mesh-controller TestADeliveryGroupMovingAModuleMachinesRunPassesTheGate ./cmd/mesh-controller TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt ./cmd/mesh-controller TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheBus -./internal/inventory TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing -./internal/inventory TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing -./internal/inventory TestTheMoveRefusesAValueOverTheLargestMessage -./internal/inventory TestTheMoveRefusesOnACountThatDiffers -./internal/inventory TestTheMoveWithoutTheLeaseWritesNothing ./internal/inventory TestBuildsOnTheBusAreReadBackByEveryReader ./internal/inventory TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace -./internal/inventory TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain -./internal/inventory TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame ./cmd/mesh-controller TestAMergeWhosePlanCannotBeSavedIsHeldAndRecordedWhenHandedOverAgain ./internal/inventory TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt ./internal/inventory TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves -./internal/inventory TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent ./cmd/mesh-controller TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments ./cmd/mesh-controller TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment -./internal/inventory TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved ./internal/inventory TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket ./internal/inventory TestARecordedPendingIsSkippedByAssignedAndTheJoins ./internal/inventory TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse From 472744bbaaff7541284c46fe370af9ce3093ba72 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 05:32:14 +0200 Subject: [PATCH 05/11] Migration 0105 drops the tables of the chain behind a guard; a source scan proves no code names them (issue 1096) The migration raises first unless chain_moved names builds, rollouts and assignments for every kind that holds rows, and unless pending_assignment holds no open or unanswered row; then it drops build_request, build, batched_merge, release_plan, pending_assignment, assignment, raise_assignment_generation(), node.sent_builds and chain_moved. build_gate stays. --- cmd/mesh-controller/chain_integration_test.go | 3 - internal/inventory/chain_sent_test.go | 4 - internal/inventory/chain_source_test.go | 144 ++++++++++++++++++ .../0105-the-chain-leaves-the-store.sql | 52 +++++++ 4 files changed, 196 insertions(+), 7 deletions(-) create mode 100644 internal/inventory/chain_source_test.go create mode 100644 internal/inventory/migrations/0105-the-chain-leaves-the-store.sql diff --git a/cmd/mesh-controller/chain_integration_test.go b/cmd/mesh-controller/chain_integration_test.go index e5c3610a..81866f2b 100644 --- a/cmd/mesh-controller/chain_integration_test.go +++ b/cmd/mesh-controller/chain_integration_test.go @@ -242,7 +242,4 @@ func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { t.Fatal("the build registered sensors and it was not assigned") } - if rows := inv.QueryForTest(t, `select count(*) from pending_assignment`); rows[0][0] != "0" { - t.Fatalf("the table of pending assignments took %s rows", rows[0][0]) - } } diff --git a/internal/inventory/chain_sent_test.go b/internal/inventory/chain_sent_test.go index d65614f4..a62ea017 100644 --- a/internal/inventory/chain_sent_test.go +++ b/internal/inventory/chain_sent_test.go @@ -88,10 +88,6 @@ func TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown(t *testing. if again, _, _ := inv.AssignmentValue(ctx, node, "web"); !again.SentUnknown.IsZero() { t.Fatalf("web still says unknown: %+v", again) } - // The column is not written any more. - if rows := inv.QueryForTest(t, `select sent_builds is null from node where name = $1`, node); rows[0][0] != "true" { - t.Fatalf("node.sent_builds was written: %v", rows) - } } // failingToWrite is an assignments bucket that fails every write. diff --git a/internal/inventory/chain_source_test.go b/internal/inventory/chain_source_test.go new file mode 100644 index 00000000..99f4c76c --- /dev/null +++ b/internal/inventory/chain_source_test.go @@ -0,0 +1,144 @@ +package inventory + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" +) + +// The tables of the chain, and the column, that the controller's store no longer holds (novox/hq ADR 0408 §8, issue 1096). +var chainTables = []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment"} + +// sqlStatement is what makes a literal a statement of the store: it opens with a verb of SQL. +var sqlStatement = regexp.MustCompile(`(?is)^\s*(select|insert|update|delete|alter|drop|create|with|truncate|from|join|where|and|set|values)\b|\b(select\s.+\sfrom|insert\s+into|delete\s+from)\b`) + +// **No code names a table of the chain** (acceptance 12, issue 1096), found by reading the sources: a string literal of any +// non-test file of internal/ or cmd/ that names one after from, into, update, join, delete from, references or table — +// build_gate and port_assignment are other words — or the column sent_builds. This is the static half of the proof; the +// integration test reads the catalog. +func TestNoCodeNamesATableOfTheChain(t *testing.T) { + names := regexp.MustCompile(`(?i)\b(from|into|update|join|delete\s+from|references|table)\s+(` + strings.Join(chainTables, "|") + `)\b|\bsent_builds\b`) + var found []string + for _, root := range []string{"../../internal", "../../cmd"} { + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() && d.Name() == "vendor" { + return filepath.SkipDir + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + return err + } + ast.Inspect(file, func(n ast.Node) bool { + lit, ok := n.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return true + } + text, err := strconv.Unquote(lit.Value) + if err != nil { + return true + } + if !sqlStatement.MatchString(text) { + return true // prose that happens to say "from assignment" is no statement + } + if m := names.FindString(text); m != "" { + found = append(found, filepath.ToSlash(path)+": "+strings.Join(strings.Fields(m), " ")) + } + return true + }) + return nil + }) + if err != nil { + t.Fatal(err) + } + } + if len(found) > 0 { + t.Fatalf("code still names a table of the chain, which the store no longer holds:\n %s", strings.Join(found, "\n ")) + } +} + +// The scan sees what it says it sees: planted, a string naming a table is found, and a comment or a word beside is not. +func TestTheSourceScanFindsATablePlanted(t *testing.T) { + names := regexp.MustCompile(`(?i)\b(from|into|update|join|delete\s+from|references|table)\s+(` + strings.Join(chainTables, "|") + `)\b|\bsent_builds\b`) + for text, want := range map[string]bool{ + "select id from build where id = $1": true, + "insert into assignment (node, module)": true, + "update node set sent_builds = $1": true, + "select 1 from build_gate": false, + "delete from port_assignment": false, + "select a from build_request a join build b": true, + "an assignment from build to build": true, + "the assignment generation": false, + "select count(*) from pending_assignment": true, + } { + if got := names.MatchString(text); got != want { + t.Errorf("%q: %v, want %v", text, got, want) + } + } +} + +// **The last migration drops the tables of the chain, behind its guard** (issue 1096): it names each table, the column and +// the function, drops no build_gate, and raises before it drops anything. +func TestTheLastMigrationDropsTheTablesOfTheChain(t *testing.T) { + migrations, err := Migrations() + if err != nil { + t.Fatal(err) + } + last := migrations[len(migrations)-1] + if last.Number < 105 { + t.Fatalf("the last migration is %d", last.Number) + } + raw, err := os.ReadFile(filepath.Join("migrations", migrationFile(t, last.Number))) + if err != nil { + t.Fatal(err) + } + var code []string + for _, line := range strings.Split(string(raw), "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "--") { + code = append(code, line) + } + } + sql := strings.ToLower(strings.Join(code, "\n")) + for _, table := range append(append([]string{}, chainTables...), "chain_moved") { + if !strings.Contains(sql, "drop table "+table+";") { + t.Errorf("migration %d does not drop %s", last.Number, table) + } + } + for _, want := range []string{"drop function raise_assignment_generation()", "alter table node drop column sent_builds"} { + if !strings.Contains(sql, want) { + t.Errorf("migration %d lacks %q", last.Number, want) + } + } + if strings.Contains(sql, "build_gate") || strings.Contains(sql, "assignment_generation;") { + t.Errorf("migration %d touches a table that stays", last.Number) + } + if raise, drop := strings.Index(sql, "raise exception"), strings.Index(sql, "drop table"); raise < 0 || drop < 0 || raise > drop { + t.Errorf("migration %d does not raise before it drops (%d, %d)", last.Number, raise, drop) + } + for _, kind := range []string{"'builds'", "'rollouts'", "'assignments'", "pending_assignment", "chain_moved"} { + if !strings.Contains(sql[:strings.Index(sql, "drop table")], kind) { + t.Errorf("the guard of migration %d does not look at %s", last.Number, kind) + } + } +} + +func migrationFile(t *testing.T, number int) string { + t.Helper() + files, err := filepath.Glob(filepath.Join("migrations", strconv.Itoa(number/1000%10)+strconv.Itoa(number/100%10)+strconv.Itoa(number/10%10)+strconv.Itoa(number%10)+"-*.sql")) + if err != nil || len(files) != 1 { + t.Fatalf("the file of migration %d: %v %v", number, files, err) + } + return filepath.Base(files[0]) +} diff --git a/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql b/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql new file mode 100644 index 00000000..e867f97d --- /dev/null +++ b/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql @@ -0,0 +1,52 @@ +-- The tables of builds, rollouts and assignments leave the store (novox/hq ADR 0408 §8, design 47 milestone 5, issue 1096). +-- +-- Delivery 1 wrote their rows into the bus's buckets at the controller's start and recorded each kind it had counted in +-- chain_moved; from then on nothing in the controller reads or writes the tables. This drops them. **It is one-way**: a +-- controller older than this schema is refused by the put-back gate, and the rows are in the buckets only. +-- +-- **The guard comes first, in the same transaction as the drops**, because a migration runs when the store is opened, before +-- the controller of this delivery serves and before any connection to the bus: it can prove the move from the store alone. +-- A kind that holds rows and is not named in chain_moved was never moved, so dropping its tables would lose it; a pending +-- assignment still waiting, being made, or ended unmade with its condition unanswered has no mover (it is a value of the +-- assignments bucket from this delivery on), so the table must hold none. A store that never held a row of a kind — a new +-- mesh — passes. A migration that raises leaves nothing applied, `prepare` fails with the reason, and the running controller +-- keeps its tables. +do $$ +declare + kind text; + holds boolean; + missing text[] := '{}'; +begin + foreach kind in array array['builds', 'rollouts', 'assignments'] loop + holds := case kind + when 'builds' then exists (select 1 from build) or exists (select 1 from build_request) + when 'rollouts' then exists (select 1 from release_plan) or exists (select 1 from batched_merge) + when 'assignments' then exists (select 1 from assignment) + end; + if holds and not coalesce((select counts ? kind from chain_moved), false) then + missing := missing || kind; + end if; + end loop; + if array_length(missing, 1) > 0 then + raise exception 'the controller''s store still holds rows of % that chain_moved does not name as moved to the bus; ' + 'run the controller of the previous delivery and read chain_moved back first', array_to_string(missing, ', '); + end if; + if exists (select 1 from pending_assignment + where state in ('waiting', 'applying') or (raised_at is not null and cleared_at is null)) then + raise exception 'a pending assignment is still in the table (waiting, being made, or ended with its condition ' + 'unanswered); let it settle, then deliver again'; + end if; +end +$$; + +drop table build_request; +drop table build; +drop table batched_merge; +drop table release_plan; +drop table pending_assignment; +-- Its triggers go with it; the function they called goes here. The counter table stays: the controller raises it. +drop table assignment; +drop function raise_assignment_generation(); +alter table node drop column sent_builds; +-- No reader of it remains. build_gate stays (issue 1097). +drop table chain_moved; From 5b08eb09beee0146001563eaf8eedabfd32a6262 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 05:33:42 +0200 Subject: [PATCH 06/11] Integration tests: no table of the chain in the store after a live delivery; migration 0105 refuses unmoved rows and drops the moved (issue 1111) --- .../chain_no_table_integration_test.go | 89 +++++++++++++++++ internal/inventory/chain_drop_test.go | 98 +++++++++++++++++++ .../0105-the-chain-leaves-the-store.sql | 2 +- 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 cmd/mesh-controller/chain_no_table_integration_test.go create mode 100644 internal/inventory/chain_drop_test.go diff --git a/cmd/mesh-controller/chain_no_table_integration_test.go b/cmd/mesh-controller/chain_no_table_integration_test.go new file mode 100644 index 00000000..0bb3a10a --- /dev/null +++ b/cmd/mesh-controller/chain_no_table_integration_test.go @@ -0,0 +1,89 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/link" +) + +// **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR +// 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's +// build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the +// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the column that held what +// a machine was sent, nor the function that raised the generation from the assignment table; the table that stays, stays. +func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asked := asksRecorded(t) + ctx := t.Context() + inv := open.inventory + if _, err := assign(ctx, open, "anchor", "notes"); err != nil { + t.Fatal(err) + } + hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second)) + cutAt(t, open, t0.Add(91*time.Second)) + ws, _ := walks(t, open) + if len(ws) != 1 { + t.Fatalf("%d walk(s) after the cut", len(ws)) + } + rollout := ws[0].ID + asking, err := inv.PlanByID(ctx, rollout) + if err != nil { + t.Fatal(err) + } + for try := 0; try < 3 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == ""); try++ { + advancePlans(ctx, open) + if asking, err = inv.PlanByID(ctx, rollout); err != nil { + t.Fatal(err) + } + } + buildID := asking.Modules["notes"].Build + if buildID == "" || len(*asked) != 1 { + t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID) + } + digest := "sha256:" + strings.Repeat("b", 64) + result := outcome(buildID, "modules/notes", "c-notes", "notes", "") + result.Made = []link.MadeArtifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes/notes@" + digest}} + if err := (builds{inv: inv, open: open}).Built(ctx, result); err != nil { + t.Fatal(err) + } + + // What the live path made is in the chain. + if built, found, err := inv.BuildByID(ctx, buildID); err != nil || !found || built.Commit != "c-notes" { + t.Fatalf("the build is not read from the chain: %+v %v %v", built, found, err) + } + if p, err := inv.PlanByID(ctx, rollout); err != nil || p.ID != rollout { + t.Fatalf("the rollout is not read from the chain: %+v %v", p, err) + } + if on, err := inv.AssignmentsOn(ctx, "anchor"); err != nil { + t.Fatal(err) + } else if _, ok := assignmentOf(on, "notes"); !ok { + t.Fatalf("the assignment is not read from the chain: %+v", on) + } + + // And the catalog of the store holds none of them. + have := map[string]bool{} + for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) { + have[strings.Trim(row[0], `"`)] = true + } + for _, table := range []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment", "chain_moved"} { + if have[table] { + t.Errorf("the table %s is in the controller's store", table) + } + } + for _, table := range []string{"build_gate", "assignment_generation"} { + if !have[table] { + t.Errorf("the table %s, which stays, is gone", table) + } + } + if rows := inv.QueryForTest(t, `select count(*) from information_schema.columns where table_schema = 'public' and column_name = 'sent_builds'`); rows[0][0] != "0" { + t.Errorf("a column sent_builds is in the store: %v", rows) + } + if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" { + t.Errorf("the function raise_assignment_generation is in the store: %v", rows) + } +} diff --git a/internal/inventory/chain_drop_test.go b/internal/inventory/chain_drop_test.go new file mode 100644 index 00000000..80c20517 --- /dev/null +++ b/internal/inventory/chain_drop_test.go @@ -0,0 +1,98 @@ +//go:build integration + +package inventory + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// **Migration 0105 refuses a store whose rows were not moved, and drops the tables of a moved one** (novox/hq ADR 0408 §8, +// issue 1111). The store is built as the mesh holds it before this migration, with an assignment row in the table and no +// record of the move: the migration raises, and nothing it did is kept. Then the move is recorded, a pending assignment +// still open refuses it again, and once it has settled the tables go and build_gate stays. +func TestMigration0105RefusesAStoreWhoseRowsWereNotMovedAndDropsTheTablesOfAMovedOne(t *testing.T) { + inv, _ := ForTestBefore(t, 105) + ctx := t.Context() + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "ui", Version: "1"}, Source{}); err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `insert into assignment (node, module) select id, 'ui' from node where name = 'anchor'`) + all, err := Migrations() + if err != nil { + t.Fatal(err) + } + tables := func() map[string]bool { + t.Helper() + out := map[string]bool{} + for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) { + out[strings.Trim(row[0], `"`)] = true + } + return out + } + refused := func(why string) { + t.Helper() + _, err := inv.store.Migrate(ctx, all) + if err == nil || !strings.Contains(err.Error(), why) { + t.Fatalf("the migration was not refused for %q: %v", why, err) + } + if !tables()["assignment"] || !tables()["chain_moved"] { + t.Fatalf("a refused migration kept a drop") + } + if rows := inv.QueryForTest(t, `select count(*) from assignment`); rows[0][0] != "1" { + t.Fatalf("a refused migration lost the assignment: %v", rows) + } + } + + // No move recorded: the rows of assignments were never copied. + refused("assignments") + + // The move recorded for every kind, and a pending assignment still waiting for its build: no mover carries it. + inv.ExecForTest(t, `insert into chain_moved (counts, largest) values ($1::jsonb, $1::jsonb)`, + `{"builds": 0, "rollouts": 0, "assignments": 1}`) + inv.ExecForTest(t, `insert into pending_assignment (node, module, build_id) + select id, 'ui', 'b1' from node where name = 'anchor'`) + refused("pending assignment") + + // Expired and raised as a condition, and the condition neither answered nor cleared: still refused. + inv.ExecForTest(t, `update pending_assignment set state = 'expired', raised_at = now()`) + refused("pending assignment") + + // Its condition cleared: the guard lets the migration through. + inv.ExecForTest(t, `update pending_assignment set cleared_at = now()`) + done, err := inv.store.Migrate(ctx, all) + if err != nil || len(done) != 1 || done[0].Number != 105 { + t.Fatalf("the migration did %v (%v)", done, err) + } + left := tables() + for _, gone := range append(append([]string{}, chainTables...), "chain_moved") { + if left[gone] { + t.Errorf("the table %s is still in the store", gone) + } + } + for _, stays := range []string{"build_gate", "assignment_generation", "node", "module"} { + if !left[stays] { + t.Errorf("the table %s is gone", stays) + } + } + if rows := inv.QueryForTest(t, `select count(*) from information_schema.columns where table_name = 'node' and column_name = 'sent_builds'`); rows[0][0] != "0" { + t.Errorf("node.sent_builds is still a column") + } + if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" { + t.Errorf("the function raise_assignment_generation is still in the store") + } +} + +// A store that never held a row of the chain — a mesh raised after the move — passes the guard with no record of a move. +func TestMigration0105PassesAStoreThatNeverHeldARow(t *testing.T) { + inv, migrate := ForTestBefore(t, 105) + migrate() + if rows := inv.QueryForTest(t, `select count(*) from information_schema.tables where table_name in ('build', 'release_plan', 'assignment', 'chain_moved')`); rows[0][0] != "0" { + t.Fatalf("tables of the chain remain: %v", rows) + } +} diff --git a/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql b/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql index e867f97d..61c129f8 100644 --- a/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql +++ b/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql @@ -32,7 +32,7 @@ begin 'run the controller of the previous delivery and read chain_moved back first', array_to_string(missing, ', '); end if; if exists (select 1 from pending_assignment - where state in ('waiting', 'applying') or (raised_at is not null and cleared_at is null)) then + where state in ('waiting', 'applying') or (raised_at is not null and acknowledged_at is null and cleared_at is null)) then raise exception 'a pending assignment is still in the table (waiting, being made, or ended with its condition ' 'unanswered); let it settle, then deliver again'; end if; From d095a3dcef658d6e5fc41094e4002e72e83d5db8 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 05:39:28 +0200 Subject: [PATCH 07/11] Copy node.sent_builds onto the assignments' keys once, and keep the column; list the new store tests (issues 1109, 1096) Delivery 1 wrote no commit on the key of a pulled-in module, so after the upgrade those modules fell out of SentBuilds. At the start of the lease holder a copy fills every key that names no commit (creating a standing pulled-in key where none stands), overwrites nothing, empties each column it copied, and raises an urgent condition while it fails. Migration 0105 no longer drops node.sent_builds (issue 1112 does). --- cmd/mesh-controller/chain.go | 43 +++++++ .../chain_no_table_integration_test.go | 6 +- internal/inventory/chain_drop_test.go | 4 +- internal/inventory/chain_links.go | 106 ++++++++++++++++++ internal/inventory/chain_sent_copy_test.go | 90 +++++++++++++++ internal/inventory/chain_source_test.go | 12 +- .../0105-the-chain-leaves-the-store.sql | 4 +- testdata/store-tests | 4 + 8 files changed, 255 insertions(+), 14 deletions(-) create mode 100644 internal/inventory/chain_sent_copy_test.go diff --git a/cmd/mesh-controller/chain.go b/cmd/mesh-controller/chain.go index f54e6258..83239d20 100644 --- a/cmd/mesh-controller/chain.go +++ b/cmd/mesh-controller/chain.go @@ -7,6 +7,7 @@ import ( "time" "github.com/novox/mesh-controller/internal/chain" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -43,6 +44,8 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link. inv.UseChain(c) // And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving. go askAgainSoon(ctx, inv) + // And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds). + go copySentBuildsSoon(ctx, inv) return nil } @@ -87,3 +90,43 @@ func askAgainSoon(ctx context.Context, inv *inventory.Inventory) { askAgain(ctx, inv) } } + +// kindSentBuildsNotCopied is the condition of the one-time copy of node.sent_builds onto the assignments' keys that has not +// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run. +const kindSentBuildsNotCopied = "sent-builds-not-copied" + +// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon +// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails, +// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key +// that names a commit and empties each column it has copied. +func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) { + failed := false + for { + _, err := inv.CopySentBuilds(ctx) + o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied, + Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, + Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "} + if err != nil { + fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err) + if keeper := conditionsFrom; keeper != nil && !failed { + o.Summary += err.Error() + if _, oerr := keeper.Observe(ctx, o); oerr == nil { + failed = true + } + } + } else { + if keeper := conditionsFrom; keeper != nil && failed { + _, _ = keeper.Clear(ctx, o.Key(), "copied") + } + return + } + select { + case <-ctx.Done(): + return + case <-time.After(chainMoveRetry): + } + } +} + +// chainMoveRetry is how long the copy waits before it tries again. +const chainMoveRetry = 30 * time.Second diff --git a/cmd/mesh-controller/chain_no_table_integration_test.go b/cmd/mesh-controller/chain_no_table_integration_test.go index 0bb3a10a..20181348 100644 --- a/cmd/mesh-controller/chain_no_table_integration_test.go +++ b/cmd/mesh-controller/chain_no_table_integration_test.go @@ -13,8 +13,7 @@ import ( // **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR // 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's // build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the -// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the column that held what -// a machine was sent, nor the function that raised the generation from the assignment table; the table that stays, stays. +// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays. func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) { open := windowed(t) everyHeadWaits(t) @@ -80,9 +79,6 @@ func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing t.Errorf("the table %s, which stays, is gone", table) } } - if rows := inv.QueryForTest(t, `select count(*) from information_schema.columns where table_schema = 'public' and column_name = 'sent_builds'`); rows[0][0] != "0" { - t.Errorf("a column sent_builds is in the store: %v", rows) - } if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" { t.Errorf("the function raise_assignment_generation is in the store: %v", rows) } diff --git a/internal/inventory/chain_drop_test.go b/internal/inventory/chain_drop_test.go index 80c20517..12a9b3ea 100644 --- a/internal/inventory/chain_drop_test.go +++ b/internal/inventory/chain_drop_test.go @@ -80,8 +80,8 @@ func TestMigration0105RefusesAStoreWhoseRowsWereNotMovedAndDropsTheTablesOfAMove t.Errorf("the table %s is gone", stays) } } - if rows := inv.QueryForTest(t, `select count(*) from information_schema.columns where table_name = 'node' and column_name = 'sent_builds'`); rows[0][0] != "0" { - t.Errorf("node.sent_builds is still a column") + if rows := inv.QueryForTest(t, `select count(*) from information_schema.columns where table_name = 'node' and column_name = 'sent_builds'`); rows[0][0] != "1" { + t.Errorf("node.sent_builds, which stays as the source of the copy, is gone") } if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" { t.Errorf("the function raise_assignment_generation is still in the store") diff --git a/internal/inventory/chain_links.go b/internal/inventory/chain_links.go index 69458990..56ec03b4 100644 --- a/internal/inventory/chain_links.go +++ b/internal/inventory/chain_links.go @@ -2,8 +2,11 @@ package inventory import ( "context" + "encoding/json" "errors" "fmt" + "maps" + "slices" "sort" "time" @@ -254,3 +257,106 @@ func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, } return fmt.Errorf("the assignment kept moving under the write") } + +// CopySentBuilds copies what delivery 1 recorded in node.sent_builds onto the keys of the assignments, once, at the start of +// the controller that holds the lease and has attached the chain (issue 1109, review round 1): delivery 1 wrote no commit onto +// the key of a module an assignment pulled in, so without it those modules would be missing from SentBuilds although the +// machine is known to have been sent. A key that names a commit is never overwritten, so it is idempotent; once a node's keys +// are written its column is emptied, so a later restart cannot lay a stale commit over a key a later send cleared or ended. +// It answers how many keys it wrote; an error is raised by the caller as a condition. +func (i *Inventory) CopySentBuilds(ctx context.Context) (int, error) { + c, err := i.theChain(ctx) + if err != nil { + return 0, err + } + rows, err := i.store.Pool().Query(ctx, `select name, sent_builds from node where sent_builds is not null`) + if err != nil { + return 0, err + } + type kept struct { + name string + sent map[string]string + } + var all []kept + for rows.Next() { + var name string + var raw []byte + if err := rows.Scan(&name, &raw); err != nil { + rows.Close() + return 0, err + } + sent := map[string]string{} + if err := json.Unmarshal(raw, &sent); err != nil { + rows.Close() + return 0, fmt.Errorf("what %s was sent, in the store, cannot be read: %w", name, err) + } + all = append(all, kept{name, sent}) + } + rows.Close() + if err := rows.Err(); err != nil { + return 0, err + } + now := time.Now().UTC() + for _, k := range all { + if err := i.copySentBuildsOf(ctx, c, k.name, k.sent, now); err != nil { + return 0, fmt.Errorf("what %s was sent cannot be copied onto its assignments: %w", k.name, err) + } + if _, err := i.store.Pool().Exec(ctx, `update node set sent_builds = null where name = $1`, k.name); err != nil { + return 0, err + } + } + return len(all), nil +} + +// copySentBuildsOf is CopySentBuilds for one node, over a chain: for each module the node was sent, a standing key that +// names no commit gets it (and the build, when one is known; no rollout), and where no key stands a standing pulled-in key is +// made. A key with a commit, a hand send's unknown and a wait for a build are left as they are. +func (i *Inventory) copySentBuildsOf(ctx context.Context, c *chain.Chain, node string, sent map[string]string, at time.Time) error { + for _, module := range slices.Sorted(maps.Keys(sent)) { + commit := sent[module] + if commit == "" { + continue + } + key := assignmentKey(node, module) + build, err := i.chainBuildOf(ctx, c, module, commit) + if err != nil { + return err + } + for try := 0; ; try++ { + if try == casTries { + return fmt.Errorf("the assignment %s kept moving under the write", key) + } + cur, found, err := c.Assignments.Get(ctx, key) + if err != nil { + return err + } + if found && (cur.Value.State != "" || cur.Value.Commit != "" || !cur.Value.SentUnknown.IsZero()) { + break + } + next := cur.Value + if !found || !cur.Value.Stands() { + next = chain.Assignment{Node: node, Module: module, PulledIn: true, Since: at} + } + next.Commit, next.Build = commit, build + if next.Since.IsZero() { + next.Since = at + } + if !found { + _, err = c.Assignments.Create(ctx, key, next) + if errors.Is(err, chain.ErrExists) { + continue + } + } else { + _, err = c.Assignments.Update(ctx, key, next, cur.Revision) + if errors.Is(err, chain.ErrMoved) { + continue + } + } + if err != nil { + return err + } + break + } + } + return nil +} diff --git a/internal/inventory/chain_sent_copy_test.go b/internal/inventory/chain_sent_copy_test.go new file mode 100644 index 00000000..49bf9474 --- /dev/null +++ b/internal/inventory/chain_sent_copy_test.go @@ -0,0 +1,90 @@ +package inventory + +import ( + "testing" + "time" + + "github.com/novox/mesh-controller/internal/chain" +) + +// **What delivery 1 recorded in node.sent_builds is copied onto the keys of the assignments, once** (issue 1109, review +// round 1): a module pulled in by an assignment had no commit on its key, so it would fall out of SentBuilds. The copy +// creates a standing pulled-in key where none stands, fills a standing key that names no commit (commit, and the build +// when one is known, no rollout), and overwrites nothing: a key with a commit, a hand send's unknown, a wait for a build. +func TestWhatTheColumnRecordedIsCopiedOntoTheKeysAndOverwritesNothing(t *testing.T) { + inv, c := anInventoryOnTheChain(t) + ctx := t.Context() + at := time.Date(2026, 10, 18, 4, 0, 0, 0, time.UTC) + if _, err := c.Builds.Create(ctx, "b-wg", chain.Build{ID: "b-wg", Module: "mesh-wireguard", Commit: "c-wg"}); err != nil { + t.Fatal(err) + } + put := func(module string, a chain.Assignment) { + a.Node, a.Module = "laptop", module + if _, err := c.Assignments.Create(ctx, assignmentKey("laptop", module), a); err != nil { + t.Fatal(err) + } + } + put("notes", chain.Assignment{Assigned: at}) // assigned, no commit yet + put("web", chain.Assignment{Assigned: at, Commit: "c-web-new", Build: "b1"}) // already has one + put("hand", chain.Assignment{Assigned: at, SentUnknown: at}) // a hand send since + put("waits", chain.Assignment{Assigned: at, State: chain.StatePending}) // a wait for a build + put("ended", chain.Assignment{Assigned: at, Ended: at}) // unassigned + sent := map[string]string{"notes": "c-notes", "web": "c-web-old", "mesh-wireguard": "c-wg", "hand": "c-hand", + "waits": "c-waits", "ended": "c-ended"} + + for pass := 0; pass < 2; pass++ { // and the second pass changes nothing + if err := inv.copySentBuildsOf(ctx, c, "laptop", sent, at); err != nil { + t.Fatal(err) + } + get := func(module string) chain.Assignment { + t.Helper() + k, found, err := c.Assignments.Get(ctx, assignmentKey("laptop", module)) + if err != nil || !found { + t.Fatalf("%s: %v %v", module, found, err) + } + return k.Value + } + if a := get("notes"); a.Commit != "c-notes" || a.Rollout != "" || !a.Stands() || a.PulledIn { + t.Errorf("notes: %+v", a) + } + if a := get("web"); a.Commit != "c-web-new" || a.Build != "b1" { + t.Errorf("a key with a commit was overwritten: %+v", a) + } + if a := get("hand"); a.Commit != "" || a.SentUnknown.IsZero() { + t.Errorf("a hand send's unknown was overwritten: %+v", a) + } + if a := get("waits"); a.Commit != "" || a.State != chain.StatePending { + t.Errorf("a wait was overwritten: %+v", a) + } + if a := get("mesh-wireguard"); a.Commit != "c-wg" || a.Build != "b-wg" || !a.PulledIn || !a.Stands() || a.Rollout != "" { + t.Errorf("the pulled-in module has no standing key of its own: %+v", a) + } + if a := get("ended"); a.Commit != "c-ended" || !a.Stands() || !a.PulledIn { + t.Errorf("an ended key is not made to stand again as pulled in: %+v", a) + } + } +} + +// The copy reads the column of the store, writes the keys, and empties the column it has copied: SentBuilds then answers the +// pulled-in module, and a second copy finds nothing to do. +func TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + if _, err := inv.AddNode(ctx, "laptop"); err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `update node set sent_builds = '{"mesh-wireguard": "c-wg"}'::jsonb, sent_at = now() where name = 'laptop'`) + if n, err := inv.CopySentBuilds(ctx); err != nil || n != 1 { + t.Fatalf("the copy did %d nodes (%v)", n, err) + } + got, known, err := inv.SentBuilds(ctx, "laptop") + if err != nil || !known || got["mesh-wireguard"] != "c-wg" { + t.Fatalf("SentBuilds answers %v, known %v (%v)", got, known, err) + } + if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" { + t.Fatalf("the column was not emptied: %v", rows) + } + if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 { + t.Fatalf("a second copy did %d nodes (%v)", n, err) + } +} diff --git a/internal/inventory/chain_source_test.go b/internal/inventory/chain_source_test.go index 99f4c76c..d5f5cac2 100644 --- a/internal/inventory/chain_source_test.go +++ b/internal/inventory/chain_source_test.go @@ -21,10 +21,10 @@ var sqlStatement = regexp.MustCompile(`(?is)^\s*(select|insert|update|delete|alt // **No code names a table of the chain** (acceptance 12, issue 1096), found by reading the sources: a string literal of any // non-test file of internal/ or cmd/ that names one after from, into, update, join, delete from, references or table — -// build_gate and port_assignment are other words — or the column sent_builds. This is the static half of the proof; the +// build_gate and port_assignment are other words —. This is the static half of the proof; the // integration test reads the catalog. func TestNoCodeNamesATableOfTheChain(t *testing.T) { - names := regexp.MustCompile(`(?i)\b(from|into|update|join|delete\s+from|references|table)\s+(` + strings.Join(chainTables, "|") + `)\b|\bsent_builds\b`) + names := regexp.MustCompile(`(?i)\b(from|into|update|join|delete\s+from|references|table)\s+(` + strings.Join(chainTables, "|") + `)\b`) var found []string for _, root := range []string{"../../internal", "../../cmd"} { err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { @@ -71,11 +71,10 @@ func TestNoCodeNamesATableOfTheChain(t *testing.T) { // The scan sees what it says it sees: planted, a string naming a table is found, and a comment or a word beside is not. func TestTheSourceScanFindsATablePlanted(t *testing.T) { - names := regexp.MustCompile(`(?i)\b(from|into|update|join|delete\s+from|references|table)\s+(` + strings.Join(chainTables, "|") + `)\b|\bsent_builds\b`) + names := regexp.MustCompile(`(?i)\b(from|into|update|join|delete\s+from|references|table)\s+(` + strings.Join(chainTables, "|") + `)\b`) for text, want := range map[string]bool{ "select id from build where id = $1": true, "insert into assignment (node, module)": true, - "update node set sent_builds = $1": true, "select 1 from build_gate": false, "delete from port_assignment": false, "select a from build_request a join build b": true, @@ -116,11 +115,14 @@ func TestTheLastMigrationDropsTheTablesOfTheChain(t *testing.T) { t.Errorf("migration %d does not drop %s", last.Number, table) } } - for _, want := range []string{"drop function raise_assignment_generation()", "alter table node drop column sent_builds"} { + for _, want := range []string{"drop function raise_assignment_generation()"} { if !strings.Contains(sql, want) { t.Errorf("migration %d lacks %q", last.Number, want) } } + if strings.Contains(sql, "sent_builds") { + t.Errorf("migration %d touches node.sent_builds, which stays as the source of the copy (issue 1112 drops it)", last.Number) + } if strings.Contains(sql, "build_gate") || strings.Contains(sql, "assignment_generation;") { t.Errorf("migration %d touches a table that stays", last.Number) } diff --git a/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql b/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql index 61c129f8..411c11b3 100644 --- a/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql +++ b/internal/inventory/migrations/0105-the-chain-leaves-the-store.sql @@ -47,6 +47,6 @@ drop table pending_assignment; -- Its triggers go with it; the function they called goes here. The counter table stays: the controller raises it. drop table assignment; drop function raise_assignment_generation(); -alter table node drop column sent_builds; --- No reader of it remains. build_gate stays (issue 1097). +-- node.sent_builds STAYS, only as the source of the one-time copy onto the assignments' keys (issue 1109); dropping it is +-- issue 1112. build_gate stays (issue 1097). drop table chain_moved; diff --git a/testdata/store-tests b/testdata/store-tests index 71222538..03d58c9b 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -156,3 +156,7 @@ ./internal/inventory TestSentBuildsAnswersFromTheAssignmentsKeysOfTheNode ./internal/inventory TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown ./internal/inventory TestAFailedWriteOfTheLinksFailsTheRecordOfTheSend +./internal/inventory TestMigration0105RefusesAStoreWhoseRowsWereNotMovedAndDropsTheTablesOfAMovedOne +./internal/inventory TestMigration0105PassesAStoreThatNeverHeldARow +./cmd/mesh-controller TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments +./internal/inventory TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied From 4eeba8fb203255297c47b33b1315d8722c67476c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 05:40:50 +0200 Subject: [PATCH 08/11] The copy of sent_builds never revives an ended key; a send empties its node's column (issue 1109) A found ended or pending key is skipped and a pulled-in key is created only where no key exists. RecordSentWith and RecordSentUnder set sent_builds = null in their own update of the node row, so a machine delivery 2 has sent is never copied afterwards. These two updates and the copy's own null are the only writes to the column. --- internal/inventory/chain_links.go | 10 ++++----- internal/inventory/chain_sent_copy_test.go | 25 ++++++++++++++++++++-- internal/inventory/nodes.go | 2 +- internal/inventory/sends.go | 2 +- testdata/store-tests | 1 + 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/internal/inventory/chain_links.go b/internal/inventory/chain_links.go index 56ec03b4..37f8f3b7 100644 --- a/internal/inventory/chain_links.go +++ b/internal/inventory/chain_links.go @@ -309,8 +309,8 @@ func (i *Inventory) CopySentBuilds(ctx context.Context) (int, error) { } // copySentBuildsOf is CopySentBuilds for one node, over a chain: for each module the node was sent, a standing key that -// names no commit gets it (and the build, when one is known; no rollout), and where no key stands a standing pulled-in key is -// made. A key with a commit, a hand send's unknown and a wait for a build are left as they are. +// names no commit gets it (and the build, when one is known; no rollout), and where no key exists a standing pulled-in key is +// made. A key with a commit, an ended key, a hand send's unknown and a wait for a build are left as they are. func (i *Inventory) copySentBuildsOf(ctx context.Context, c *chain.Chain, node string, sent map[string]string, at time.Time) error { for _, module := range slices.Sorted(maps.Keys(sent)) { commit := sent[module] @@ -330,11 +330,11 @@ func (i *Inventory) copySentBuildsOf(ctx context.Context, c *chain.Chain, node s if err != nil { return err } - if found && (cur.Value.State != "" || cur.Value.Commit != "" || !cur.Value.SentUnknown.IsZero()) { - break + if found && (cur.Value.State != "" || !cur.Value.Ended.IsZero() || cur.Value.Commit != "" || !cur.Value.SentUnknown.IsZero()) { + break // a wait, an ended key (never revived), one that names a commit, or a hand send's unknown } next := cur.Value - if !found || !cur.Value.Stands() { + if !found { next = chain.Assignment{Node: node, Module: module, PulledIn: true, Since: at} } next.Commit, next.Build = commit, build diff --git a/internal/inventory/chain_sent_copy_test.go b/internal/inventory/chain_sent_copy_test.go index 49bf9474..8e797a3c 100644 --- a/internal/inventory/chain_sent_copy_test.go +++ b/internal/inventory/chain_sent_copy_test.go @@ -59,8 +59,8 @@ func TestWhatTheColumnRecordedIsCopiedOntoTheKeysAndOverwritesNothing(t *testing if a := get("mesh-wireguard"); a.Commit != "c-wg" || a.Build != "b-wg" || !a.PulledIn || !a.Stands() || a.Rollout != "" { t.Errorf("the pulled-in module has no standing key of its own: %+v", a) } - if a := get("ended"); a.Commit != "c-ended" || !a.Stands() || !a.PulledIn { - t.Errorf("an ended key is not made to stand again as pulled in: %+v", a) + if a := get("ended"); a.Commit != "" || a.Stands() { + t.Errorf("an ended key was revived or given a commit: %+v", a) } } } @@ -88,3 +88,24 @@ func TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied(t *testing.T) { t.Fatalf("a second copy did %d nodes (%v)", n, err) } } + +// A machine delivery 2 has sent has its column emptied by the send itself, so a copy after it leaves a key the send cleared +// without a commit. +func TestASendEmptiesTheColumnSoALaterCopyLeavesAClearedKeyBare(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + record, err := inv.AddNode(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `update node set sent_builds = '{"notes": "c-old"}'::jsonb where name = 'laptop'`) + if err := inv.RecordSentUnder(ctx, record.ID, "d1", map[string]string{}, 1); err != nil { + t.Fatal(err) + } + if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" { + t.Fatalf("a send left the column: %v", rows) + } + if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 { + t.Fatalf("the copy did %d nodes (%v)", n, err) + } +} diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index bf98a1a7..26f57483 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -1263,7 +1263,7 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu sentEpoch = &e } if _, err := i.store.Pool().Exec(ctx, - `update node set sent = $2, sent_at = now(), sent_epoch = $3 where id = $1`, node, digest, sentEpoch); err != nil { + `update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3 where id = $1`, node, digest, sentEpoch); err != nil { return err } // What it was sent of each module is the assignments' (SentBuilds); a write of it that fails fails this record. diff --git a/internal/inventory/sends.go b/internal/inventory/sends.go index 4b28e33e..db8b7095 100644 --- a/internal/inventory/sends.go +++ b/internal/inventory/sends.go @@ -155,7 +155,7 @@ func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, bui } defer func() { _ = tx.Rollback(ctx) }() if _, err := tx.Exec(ctx, - `update node set sent = $2, sent_at = now(), sent_epoch = $3, sent_generation = $4, + `update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3, sent_generation = $4, sent_provisioning = $5::jsonb where id = $1`, node, digest, sentEpoch, nullIfZero(generation), nullIfEmpty(s.Provisioning)); err != nil { return err diff --git a/testdata/store-tests b/testdata/store-tests index 03d58c9b..3d2db5be 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -160,3 +160,4 @@ ./internal/inventory TestMigration0105PassesAStoreThatNeverHeldARow ./cmd/mesh-controller TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments ./internal/inventory TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied +./internal/inventory TestASendEmptiesTheColumnSoALaterCopyLeavesAClearedKeyBare From 7f074d9c94c6583468991f337fac4ac67a72493e Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 13:41:25 +0200 Subject: [PATCH 09/11] A layout run again sets no pointer back and skips records already moved (issue 1121) The live start recorded its layout in chain_moved, so the marker is missing and the layout runs once more over the old keys. It wrote newest, worked, held, commit and source over the pointers of every build made since. Pointers are now made only where missing, and a record already under its new key is not read again. --- internal/inventory/chain_builds.go | 24 +++++++++++++- .../inventory/chain_builds_layout_test.go | 33 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/internal/inventory/chain_builds.go b/internal/inventory/chain_builds.go index 459ec109..c6db6b75 100644 --- a/internal/inventory/chain_builds.go +++ b/internal/inventory/chain_builds.go @@ -1206,6 +1206,28 @@ func (i *Inventory) layoutBuilds(ctx context.Context, c *chain.Chain, say func(s if r.Records%1000 == 0 { say("laying the builds out: %d of %d records, %d keys written so far", r.Records, len(ids), r.Written) } + // A record already under its new key is moved: a run again (the start before this one laid it out) neither reads its + // manifest nor writes it, and **no pointer is set back**: they are only made where missing. + own := chain.BuildKey(rec.Module, rec.ID) + if rec.State == chain.BuildAsked { + own = chain.RequestKey(rec.ID) + } + if _, there, err := c.Builds.GetRaw(ctx, own); err != nil { + return r, err + } else if there { + r.Present++ + if rec.State != chain.BuildAsked { + if wrote, err := i.pointTo(ctx, c, chain.IDKey(rec.ID), idPointer(rec.Module), false); err != nil { + return r, err + } else if wrote { + r.Written++ + } + } + if rec.State != chain.BuildAsked && rec.Module != "" { + byModule[rec.Module] = append(byModule[rec.Module], rec) + } + continue + } // A manifest already written in this run is not written again. digest := "" if body := manifestBytes([]byte(rec.Manifest)); body != nil && c.Kept { @@ -1265,7 +1287,7 @@ func (i *Inventory) layoutBuilds(ctx context.Context, c *chain.Chain, say func(s return r, err } for _, p := range points { - wrote, err := i.pointTo(ctx, c, p.key, p.value, true) + wrote, err := i.pointTo(ctx, c, p.key, p.value, false) if err != nil { return r, err } diff --git a/internal/inventory/chain_builds_layout_test.go b/internal/inventory/chain_builds_layout_test.go index b1442c0e..26e30f3a 100644 --- a/internal/inventory/chain_builds_layout_test.go +++ b/internal/inventory/chain_builds_layout_test.go @@ -426,3 +426,36 @@ func TestTheChainIsRefusedUntilTheBuildsAreLaidOutAndTheLayoutIsDoneOnce(t *test t.Fatalf("the second layout changed the bucket: %v", err) } } + +// A layout run again — the live start recorded the first in chain_moved, not in the marker — never sets a pointer back to an +// older build, and does not read a manifest of a record already under its new key. +func TestALayoutRunAgainLeavesThePointersOfTheBuildsMadeSince(t *testing.T) { + counting := &readCounting{Backend: chain.NewMemory()} + inv, c := layoutChain(counting) + ctx := context.Background() + for n := 1; n <= 2; n++ { + old := legacyBuild(n, "alpha", fmt.Sprintf("c%d", n)) + if _, err := c.Builds.Create(ctx, old.ID, old); err != nil { + t.Fatal(err) + } + } + if _, err := inv.layoutBuilds(ctx, c, nil); err != nil { + t.Fatal(err) + } + if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", 3, "c3", "s3")); err != nil { + t.Fatal(err) + } + counting.gets.Store(0) + r, err := inv.layoutBuilds(ctx, c, nil) + if err != nil || r.Written != 0 { + t.Fatalf("the second layout %+v: %v", r, err) + } + for _, key := range []string{"alpha.newest", "alpha.worked", "alpha.held"} { + if e, _, _ := counting.Get(ctx, key); string(e.Value) != "build-3" { + t.Errorf("%s = %q after the second layout; want build-3", key, e.Value) + } + } + if e, _, _ := counting.Get(ctx, "alpha.commit.c3"); pointedID(e.Value) != "build-3" { + t.Errorf("commit c3 = %s", e.Value) + } +} From cbc6bac8c070b5e6752b4432b9cea36696fb3312 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 13:51:37 +0200 Subject: [PATCH 10/11] Test chains start laid out; two tests follow delivery 2's meaning (issues 1121, 1108, 1109) The fixtures' fresh chain writes the layout marker, as the controller's start leaves a mesh, so the gate stays for the live start. A pending wait that ended unanswered is carried by the next one, so both are known. A hand send forgets what each assignment runs, so the test writes its build and commit after the send. --- internal/inventory/chain_pending_test.go | 5 +++-- internal/inventory/chain_store_test.go | 13 +++++++------ internal/inventory/fortest.go | 8 ++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/internal/inventory/chain_pending_test.go b/internal/inventory/chain_pending_test.go index 88ee416e..e9841507 100644 --- a/internal/inventory/chain_pending_test.go +++ b/internal/inventory/chain_pending_test.go @@ -53,8 +53,9 @@ func TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket(t *testing.T) { if err != nil || next.ID == p.ID { t.Fatalf("a wait after one ended: %+v %v", next, err) } - if known, _ := inv.PendingKnown(ctx, []string{p.ID, next.ID}); known[p.ID] || !known[next.ID] { - t.Fatalf("known %v: the key holds the newer one now", known) + // The earlier wait ended unmade and nobody answered its condition, so the new wait carries it (issue 1108): both are on record. + if known, _ := inv.PendingKnown(ctx, []string{p.ID, next.ID}); !known[p.ID] || !known[next.ID] { + t.Fatalf("known %v: the key holds the newer one and carries the unanswered earlier one", known) } if history, _ := inv.chain.Assignments.History(ctx, "anchor.sensors"); len(history) != 2 { t.Fatalf("the earlier wait is in the key's history: %d values", len(history)) diff --git a/internal/inventory/chain_store_test.go b/internal/inventory/chain_store_test.go index 02b3353e..11d8d68c 100644 --- a/internal/inventory/chain_store_test.go +++ b/internal/inventory/chain_store_test.go @@ -289,12 +289,6 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi t.Fatal(err) } } - // A build and a commit put in the value by another write. - cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") - cur.Value.Build, cur.Value.Commit = "build-1", "c1" - if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil { - t.Fatal(err) - } record, _ := inv.NodeByName(ctx, node) digest := "sha256:" + strings.Repeat("d", 64) plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest, @@ -304,6 +298,13 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil { t.Fatal(err) } + // A build and a commit put in the value by another write, after the send: a send that carries no builds (a hand send, as this + // one) forgets what each assignment of its machine runs (issue 1109), so they are written once it is recorded. + cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") + cur.Value.Build, cur.Value.Commit = "build-1", "c1" + if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil { + t.Fatal(err) + } inv.SetProvisioningPlans(inv.SentProvisioningPlans) written, removed, err := inv.KeepAssignmentPlans(ctx) if err != nil || written != 2 || removed != 0 { diff --git a/internal/inventory/fortest.go b/internal/inventory/fortest.go index 790401c4..4cecc83c 100644 --- a/internal/inventory/fortest.go +++ b/internal/inventory/fortest.go @@ -67,6 +67,10 @@ func ForTest(t *testing.T) *Inventory { } // The chain is on the bus in a mesh; in a test it is in memory (ADR 0408). c := chain.InMemory(nil) + // A fresh test chain has nothing to lay out: it is laid out already, as the controller's start leaves a mesh (issue 1121). + if _, err := c.Builds.CreateRaw(t.Context(), chain.LayoutMarkerKey, []byte(`{}`)); err != nil { + t.Fatal(err) + } inv.UseChain(c) testChains.Store(os.Getenv(store.Variable(Name)), c) t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) }) @@ -129,6 +133,10 @@ func ForTestBefore(t *testing.T, number int) (*Inventory, func()) { } // The chain is in memory, for what a test of the store before a migration reads of it. c := chain.InMemory(nil) + // A fresh test chain has nothing to lay out: it is laid out already, as the controller's start leaves a mesh (issue 1121). + if _, err := c.Builds.CreateRaw(t.Context(), chain.LayoutMarkerKey, []byte(`{}`)); err != nil { + t.Fatal(err) + } inv.UseChain(c) testChains.Store(os.Getenv(store.Variable(Name)), c) t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) }) From 68db18bdfc1bad87771106adc3b4870cb6662927 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 14:21:35 +0200 Subject: [PATCH 11/11] Make the second merge check pass: the gate's mesh has a chain of its own; the tests follow delivery 2's meaning (issues 1121, 1108, 1109) The merge gate raises the snapshot's mesh in a throwaway store and now gives it a chain in memory, since the chain lives only on the bus. A module with no commit is carried by no send (a send is read from the commits on the assignments' keys), so the grants step flags only a module with a commit as newly assigned. The tests that sent a module with no build record one, the merge test waits for the merges' consumer before publishing and for the last look of a window before the next test, the pending test follows a chain that deletes nothing, and a wait's settling is a value of its key. --- .../awaiting_push_integration_test.go | 15 +++++++++++++-- cmd/mesh-controller/awaiting_push_test.go | 16 +++++++++++++++- cmd/mesh-controller/data_integration_test.go | 12 +++++++++--- cmd/mesh-controller/merge_gate.go | 3 +++ cmd/mesh-controller/merge_on_the_bus_test.go | 8 ++++++++ cmd/mesh-controller/merge_window_bus.go | 6 ++++++ cmd/mesh-controller/pending_integration_test.go | 17 ++++++++++------- cmd/mesh-controller/release.go | 4 +++- internal/inventory/chain.go | 8 ++++++++ internal/inventory/chain_pending_test.go | 3 ++- 10 files changed, 77 insertions(+), 15 deletions(-) diff --git a/cmd/mesh-controller/awaiting_push_integration_test.go b/cmd/mesh-controller/awaiting_push_integration_test.go index ee881d98..d3714080 100644 --- a/cmd/mesh-controller/awaiting_push_integration_test.go +++ b/cmd/mesh-controller/awaiting_push_integration_test.go @@ -10,6 +10,7 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/overlay" ) @@ -146,8 +147,18 @@ func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing. func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) { open := aMesh(t) ctx := t.Context() - register(t, open, aKeeper()) - pushedBy(t, open, "laptop") + // What a machine was sent is read from the commits its assignments name (issue 1109): a module is carried by a send only + // when it was built from a commit, and a hand send on a machine with nothing assigned leaves no key to mark unknown. + if err := open.inventory.RegisterModule(ctx, aKeeper(), inventory.Source{BuiltFrom: "c-keeper"}); err != nil { + t.Fatal(err) + } + record, err := open.inventory.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSent(ctx, record.ID, "carried", map[string]string{overlay.Name: "c-overlay"}); err != nil { + t.Fatal(err) + } if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { t.Fatal(err) } diff --git a/cmd/mesh-controller/awaiting_push_test.go b/cmd/mesh-controller/awaiting_push_test.go index 0ec32245..58efd557 100644 --- a/cmd/mesh-controller/awaiting_push_test.go +++ b/cmd/mesh-controller/awaiting_push_test.go @@ -46,7 +46,21 @@ func pushedBy(t *testing.T, open *stores, node string) string { t.Fatal(err) } digest := digestOf(body) - if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil { + // A push records the build of each module it carries (plan.go, Allocating); a module is carried by a send only when it was + // built from a commit, for what a machine was sent is read from the commits its assignments name (issue 1109). + builds := declared.Builds + if builds == nil { + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + t.Fatal(err) + } + names := make([]string, 0, len(plan.Modules)) + for _, m := range plan.Modules { + names = append(names, m.Module) + } + builds = carriedBuilds(names, nil, current, nil) + } + if err := open.inventory.RecordSent(ctx, record.ID, digest, builds); err != nil { t.Fatal(err) } return digest diff --git a/cmd/mesh-controller/data_integration_test.go b/cmd/mesh-controller/data_integration_test.go index c1142df8..4d8a9853 100644 --- a/cmd/mesh-controller/data_integration_test.go +++ b/cmd/mesh-controller/data_integration_test.go @@ -56,13 +56,19 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent( if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { t.Fatal(err) } - register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) + // Built from a commit, so that a send carries them: a holder is asked only of what the machine was sent (issue 1109). + built := inventory.Source{BuiltFrom: "c1"} + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "keeper", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}, built); err != nil { + t.Fatal(err) + } house, err := catalogue.ParseManifest([]byte(houseManifest)) if err != nil { t.Fatal(err) } - register(t, open, house) + if err := inv.RegisterModule(ctx, house, built); err != nil { + t.Fatal(err) + } if _, err := assign(ctx, open, "laptop", "house"); err == nil { t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up") } diff --git a/cmd/mesh-controller/merge_gate.go b/cmd/mesh-controller/merge_gate.go index ec61888b..854f39ef 100644 --- a/cmd/mesh-controller/merge_gate.go +++ b/cmd/mesh-controller/merge_gate.go @@ -1120,6 +1120,9 @@ func throwawayStores(ctx context.Context, admin string) (*stores, func(), error) drop() return nil, nil, err } + // The chain lives on the bus, and the gate has none: this mesh is raised for a moment, with a chain of its own in memory + // that no other reads (issue 1121). + open.inventory.UseThrowawayChain() for _, m := range provided { if err := open.inventory.Provide(ctx, m); err != nil { open.Close() diff --git a/cmd/mesh-controller/merge_on_the_bus_test.go b/cmd/mesh-controller/merge_on_the_bus_test.go index df3825e2..3cfc5780 100644 --- a/cmd/mesh-controller/merge_on_the_bus_test.go +++ b/cmd/mesh-controller/merge_on_the_bus_test.go @@ -50,9 +50,17 @@ func theMergePathOn(t *testing.T, open *stores) func(link.SourceMoved) { t.Cleanup(func() { stop() <-done + lookingAtBatches.Wait() setWindowClock(nil) windowLook.Store(nil) }) + // The forge's merges are taken from a consumer made when the server starts serving: a merge published before it exists + // is not delivered to it. With the chain read from the bus the set-up above is quick enough to lose that race, so the + // publisher waits for the consumer. + becomes(t, 10*time.Second, "the merges' consumer being made", func() bool { + _, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName) + return err == nil + }) return func(m link.SourceMoved) { t.Helper() body, err := json.Marshal(m) diff --git a/cmd/mesh-controller/merge_window_bus.go b/cmd/mesh-controller/merge_window_bus.go index 5b4cb0e3..bd721841 100644 --- a/cmd/mesh-controller/merge_window_bus.go +++ b/cmd/mesh-controller/merge_window_bus.go @@ -145,9 +145,15 @@ func (w *busWindow) Forget(ctx context.Context, batch string) { } } +// lookingAtBatches counts the looks under way, so a test that ends the window's clock can wait for the last of them before the +// next test replaces what they read. +var lookingAtBatches sync.WaitGroup + // lookAtBatches is the look an expired window asks for: the plans held, as a merge heard holds them, and the batches // whose window closed cut (cutBatchesHeld). Waiting for the plans rather than giving up: an expiry is heard once. func lookAtBatches(ctx context.Context, open *stores) { + lookingAtBatches.Add(1) + defer lookingAtBatches.Done() release, err := open.inventory.HoldPlans(ctx, true) if err != nil { fmt.Printf("batches: a window closed and the plans could not be held for the look: %v\n", err) diff --git a/cmd/mesh-controller/pending_integration_test.go b/cmd/mesh-controller/pending_integration_test.go index 5bbf1287..ba6a82fe 100644 --- a/cmd/mesh-controller/pending_integration_test.go +++ b/cmd/mesh-controller/pending_integration_test.go @@ -542,23 +542,26 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { t.Fatalf("a module registered meanwhile: %+v", p) } - // Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is - // still open (second review, bug B). + // Ended waits are not deleted, for the chain removes no key (ADR 0408 section 4): after KeptFor they are no longer listed, open + // ones never are left out, nor one whose condition is still open (second review, bug B). if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil { t.Fatal(err) } - said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - rows, err := inv.Pending(ctx, time.Unix(0, 0)) + later := time.Now().Add(inventory.KeptFor + time.Hour) + settlePending(ctx, open, later) + rows, err := inv.Pending(ctx, later.Add(-inventory.KeptFor)) if err != nil { t.Fatal(err) } for _, r := range rows { if !r.Open() && (r.Raised == nil || r.Cleared != nil) { - t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said) + t.Fatalf("an ended pending assignment with no open condition is listed past %s: %+v", inventory.KeptFor, r) } } - if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") { - t.Fatalf("the applied row was not deleted: %v", said) + for _, r := range rows { + if r.Module == "heard" { + t.Fatalf("an applied wait is listed past %s: %+v", inventory.KeptFor, r) + } } } diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index f7939794..bd7604c7 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -1589,7 +1589,9 @@ func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalog var other []string listID := "" for _, m := range plan.Modules { - if _, was := sent[m.Module]; !was { + // A module no build is recorded for is carried with no commit and names none on its key (issue 1109), so it is "sent" + // only by the comparison of the summaries below; a module with a commit that was never sent is a new assignment. + if _, was := sent[m.Module]; !was && declared.Builds[m.Module] != "" { other = append(other, m.Module+" newly assigned") } if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) { diff --git a/internal/inventory/chain.go b/internal/inventory/chain.go index a6e39ab3..1a0fbd7e 100644 --- a/internal/inventory/chain.go +++ b/internal/inventory/chain.go @@ -71,6 +71,14 @@ func (i *Inventory) UseChain(c *chain.Chain) { } } +// UseThrowawayChain gives the inventory a chain in memory that is its own, laid out already: for a mesh raised for a moment +// beside the real one, as the merge gate raises the facts snapshot's mesh in a throwaway store (issue 1121). Nothing of it +// reaches the bus. +func (i *Inventory) UseThrowawayChain() { + i.UseChain(chain.InMemory(nil)) + i.markLaidOut() +} + // attached is the chain this inventory has, nil when none. func (i *Inventory) attached() *chain.Chain { i.state.mu.Lock() diff --git a/internal/inventory/chain_pending_test.go b/internal/inventory/chain_pending_test.go index e9841507..3d48ba19 100644 --- a/internal/inventory/chain_pending_test.go +++ b/internal/inventory/chain_pending_test.go @@ -57,7 +57,8 @@ func TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket(t *testing.T) { if known, _ := inv.PendingKnown(ctx, []string{p.ID, next.ID}); !known[p.ID] || !known[next.ID] { t.Fatalf("known %v: the key holds the newer one and carries the unanswered earlier one", known) } - if history, _ := inv.chain.Assignments.History(ctx, "anchor.sensors"); len(history) != 2 { + // The first wait, its settling and the new wait: three values, the earlier wait still in the history. + if history, _ := inv.chain.Assignments.History(ctx, "anchor.sensors"); len(history) != 3 { t.Fatalf("the earlier wait is in the key's history: %d values", len(history)) } }