novox/mesh-controller
13
0
Fork 0
Code Issues Pull Requests 4 Releases Wiki Activity

Delete the dead table paths, the move, the store's assignments mirror and its tick (issue 1110)

Browse Source 
Every method of builds, build requests, rollouts, merges and assignments reads and writes the chain alone: the else branches that read the tables, the movers and MoveChain with their tests, chain_moved reads, ChainReads and the flags that switched a kind, the store's assignments mirror (KeepAssignmentPlans needs none of it: it uses only the AssignmentMirrored type, mirrors.go, and AssignmentValues, provisioning.go) and its tick, the move code of the controller's start, and ForTest's chain_moved row. The listed store tests of the move go with their functions; the tests of the readers, requests, assignments and plans that lived beside them stay in chain_store_test.go. Tests that replayed old data migrations over the build, assignment and pin tables are removed with the tables. The static test that no plan is written but through SavePlansMoving now reads the plan's chain path.
This commit is contained in:
jschoubben
2026-10-18 05:30:19 +02:00
parent fdab58ae4f
commit 9859923664
36 changed files with 641 additions and 3331 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/chain.go
+4 -96
View File
@@ -7,15 +7,12 @@ import (
"time"
"github.com/novox/mesh-controller/internal/chain"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
// bus's key-value store, written by the controller that holds the lease. The tables of the controller's store are
// copied into them once, by the serving controller at its start, and a kind is read from its bucket only once the move
// has recorded it (inventory.ChainReads).
// bus's key-value store, written by the controller that holds the lease, and the controller's store keeps none of them.
// chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes.
func chainHolds(ctx context.Context) error {
@@ -33,97 +30,7 @@ func openTheChain(ctx context.Context) (*chain.Chain, error) {
return chain.OnTheBus(ctx, js.Conn(), chainHolds)
}
// kindChainMoveRefused is the condition of a move that stopped without switching a kind.
const kindChainMoveRefused = "chain-move-refused"
// chainMoveRetry is how long a move that failed for want of the bus waits before it tries again.
const chainMoveRetry = 30 * time.Second
// movingTheChain copies the tables into the buckets and records it, at the serving controller's start: after the buckets
// exist and the lease is held, before anything reads a kind that has not moved. **The first try is made here, before the
// controller serves**, so what it serves is read from one place. A move that fails for want of the bus is tried again
// every half minute while the controller serves from the tables; one that is refused — a count that differs, a value
// over the bus's largest — is raised as an urgent condition, and the tables stay what is read: nothing is half-switched.
func movingTheChain(ctx context.Context, inv *inventory.Inventory) {
say := func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }
// The first try is made here, before the controller serves; the tries after it, while it serves.
if movedOrStopped(ctx, inv, say) {
return
}
go func() {
for {
select {
case <-ctx.Done():
return
case <-time.After(chainMoveRetry):
}
if movedOrStopped(ctx, inv, say) {
return
}
}
}()
}
// movedOrStopped makes one try at the move and says whether there is nothing more to try: moved, refused, or the controller
// ending. False for a failure of the kind that is tried again (the bus away, a write not granted yet, the lease not held).
func movedOrStopped(ctx context.Context, inv *inventory.Inventory, say func(string, ...any)) bool {
_, err := inv.MoveChain(ctx, say)
var refusal *inventory.Refusal
switch {
case err == nil:
go clearChainMove(ctx)
go askAgainSoon(ctx, inv)
return true
case errors.As(err, &refusal):
say("%v", err)
go raiseChainMove(ctx, refusal)
return true
case ctx.Err() != nil:
return true
}
say("the move to the bus did not finish, and the tables are read until it does: %v", err)
return false
}
// chainMoveObservation is the urgent condition for a refused move.
func chainMoveObservation(r *inventory.Refusal) conditions.Observation {
return conditions.Observation{Scope: conditions.ScopeBus, ID: "chain." + r.Kind, Token: kindChainMoveRefused,
Kind: kindChainMoveRefused, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
Summary: "the controller's " + r.Kind + " were not moved to the bus: " + r.Why, Source: "chain move"}
}
// whenKept runs f once the serving controller keeps conditions: the move runs before they are kept.
func whenKept(ctx context.Context, f func(k *conditions.Keeper)) {
for conditionsFrom == nil {
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
f(conditionsFrom)
}
func raiseChainMove(ctx context.Context, r *inventory.Refusal) {
whenKept(ctx, func(k *conditions.Keeper) {
if _, err := k.Observe(ctx, chainMoveObservation(r)); err != nil {
fmt.Printf("chain: the condition for the refused move could not be raised: %v\n", err)
}
})
}
func clearChainMove(ctx context.Context) {
whenKept(ctx, func(k *conditions.Keeper) {
for _, kind := range []string{inventory.KindBuilds, inventory.KindRollouts, inventory.KindAssignments} {
key := chainMoveObservation(&inventory.Refusal{Kind: kind}).Key()
if _, err := k.Clear(ctx, key, "the "+kind+" are on the bus"); err != nil {
fmt.Printf("chain: the condition for the move of the %s could not be cleared: %v\n", kind, err)
}
}
})
}
// attachTheChain gives the inventory of the serving controller the chain on its connection, then moves what has not moved.
// attachTheChain gives the inventory of the serving controller the chain on its connection.
func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error {
bus, ok := server.Bus().(link.OverNATS)
if !ok {
@@ -134,7 +41,8 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
return err
}
inv.UseChain(c)
movingTheChain(ctx, inv)
// And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving.
go askAgainSoon(ctx, inv)
return nil
}
cmd/mesh-controller/plan_moved_every_writer_test.go
+25 -14
View File
@@ -12,9 +12,9 @@ import (
)
// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there
// after the commit; so no statement that writes the plan table may stand anywhere else, and writePlan, the one that
// does, may be called from nowhere but SavePlansMoving. Checked by reading the sources, no store needed.
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there after the
// write; so no code but chainSavePlans (which SavePlansMoving calls, and no other function) writes a plan to the rollouts
// bucket by anything but a change of the merges it holds. Checked by reading the sources, no store needed.
func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`)
var found []string
@@ -31,11 +31,13 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
return err
}
text := string(raw)
if write.MatchString(text) && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
if write.MatchString(text) {
found = append(found, path+" writes the plan table")
}
if strings.Contains(text, "writePlan(") && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
found = append(found, path+" calls writePlan")
// A plan record is written by the bucket's raw create and update, which only the plans' own file calls.
if filepath.ToSlash(path) != "../../internal/inventory/chain_rollouts.go" && filepath.ToSlash(path) != "../../internal/chain/chain.go" &&
strings.Contains(text, "chain.RolloutsOf") {
found = append(found, path+" reaches the rollouts bucket")
}
return nil
})
@@ -51,17 +53,26 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
t.Fatal(err)
}
text := string(raw)
if n := len(write.FindAllString(text, -1)); n != 1 {
t.Errorf("plans.go has %d statements that write the plan table; want the one in writePlan", n)
}
calls := regexp.MustCompile(`writePlan\(ctx, tx`).FindAllStringIndex(text, -1)
save := strings.Index(text, "func (i *Inventory) SavePlansMoving")
next := strings.Index(text[save:], "\nfunc ")
if len(calls) != 1 || calls[0][0] < save || calls[0][0] > save+next {
t.Errorf("writePlan is called %d times, and not only from SavePlansMoving", len(calls))
if next < 0 {
next = len(text) - save
}
if !strings.Contains(text[save:save+next], "PlanSaved(i, *p)") {
t.Error("SavePlansMoving does not say each plan it wrote")
if !strings.Contains(text[save:save+next], "chainSavePlans(") {
t.Error("SavePlansMoving does not write through chainSavePlans")
}
chainRaw, err := os.ReadFile("../../internal/inventory/chain_rollouts.go")
if err != nil {
t.Fatal(err)
}
chainText := string(chainRaw)
at := strings.Index(chainText, "func (i *Inventory) chainSavePlans")
end := strings.Index(chainText[at:], "\nfunc ")
if !strings.Contains(chainText[at:at+end], "PlanSaved(i, *p)") {
t.Error("chainSavePlans does not say each plan it wrote")
}
if n := strings.Count(chainText, "createRaw(ctx, c, p.ID"); n != 1 {
t.Errorf("a plan is made from %d places of chain_rollouts.go, want the one in chainSavePlans", n)
}
}
cmd/mesh-controller/record_mirrors.go
+8 -35
View File
@@ -44,7 +44,6 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
return e
}
sources := map[string]func(context.Context) (map[string][]byte, error){
recordmirror.Assignments.Name: inv.AssignmentsMirrorToWrite,
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
recordmirror.Predefined.Name: inv.PredefinedMirror,
}
@@ -55,15 +54,11 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
events := &assignmentEvents{}
for {
if holds() {
// The assignments, once they are keys of the bus, are written by the verbs that change them (novox/hq ADR
// 0408): this tick keeps the provisioner's plan in their values and says what changed; the mirror of the store
// writes them only until the move.
onTheBus := keepingAssignments(ctx, inv, announce, epoch, events, failing)
// The assignments are keys of the bus, written by the verbs that change them (novox/hq ADR 0408): this tick
// keeps the provisioner's plan in their values and says what changed. They are no mirror of the store.
keepingAssignments(ctx, inv, announce, epoch, events, failing)
for _, m := range recordmirror.Mirrors() {
if m.Name == recordmirror.Assignments.Name && onTheBus {
continue
}
why := mirrorOnce(ctx, inv, api, m, writers, sources[m.Name], announce, epoch, holds)
why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
if why != failing[m.Name] {
if why != "" {
fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
@@ -83,7 +78,7 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
}
// mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could.
func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64,
holds func() bool) string {
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
@@ -95,9 +90,6 @@ func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.Jet
return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err)
}
w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
if m.Name == recordmirror.Assignments.Name {
w.Holds = assignmentsHolds(inv, holds)
}
writers[m.Name] = w
}
want, err := source(reading)
@@ -128,21 +120,10 @@ type assignmentEvents struct {
// the values it belongs to, and one `assignment-changed` event for each key that is new, changed or ended since the last
// look — whoever wrote it, a verb in this process or a command in another. False when they are not on the bus yet.
func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce recordmirror.Announcer, epoch func() uint64,
events *assignmentEvents, failing map[string]string) bool {
events *assignmentEvents, failing map[string]string) {
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
const name = "assignments"
if _, on, err := inv.KeptAssignments(reading); err != nil || !on {
if err != nil {
// Not known to be off the bus: the store's mirror is not written over what the bus may hold.
if failing[name] == "" {
failing[name] = err.Error()
fmt.Printf("the assignments cannot be read from the bus: %v\n", err)
}
return true
}
return false
}
if _, _, err := inv.KeepAssignmentPlans(reading); err != nil {
if failing[name] != err.Error() {
failing[name] = err.Error()
@@ -152,9 +133,9 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
fmt.Printf("the provisioner's plans are kept in the assignments again\n")
failing[name] = ""
}
now, _, err := inv.KeptAssignments(reading)
now, err := inv.KeptAssignments(reading)
if err != nil {
return true
return
}
if events.known != nil {
keys := make([]string, 0, len(now)+len(events.known))
@@ -194,12 +175,4 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
}
}
events.known = now
return true
}
// assignmentsHolds is the assignments mirror's leave to write: the lease's, and the assignments neither being switched to
// the bus nor on it. Asked by the writer before every write and purge, so a pass that read its source before the switch
// writes nothing after it (ADR 0408 §4).
func assignmentsHolds(inv *inventory.Inventory, holds func() bool) func() bool {
return func() bool { return holds() && inv.AssignmentsMirrorMayWrite() }
}
cmd/mesh-controller/record_mirrors_switch_test.go
-29
View File
@@ -1,29 +0,0 @@
package main
import (
"errors"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/recordmirror"
)
// **A writer of the store's mirror of the assignments, made before the assignments are switched to the bus, writes and
// purges nothing after** (ADR 0408 §4, issue 1093): its leave to write is asked before every write, and false once they are
// switching or moved.
func TestAMirrorWriterMadeBeforeTheSwitchWritesAndPurgesNothingAfter(t *testing.T) {
inv := inventory.NewForTest()
w := &recordmirror.Writer{Mirror: recordmirror.Assignments, Holds: assignmentsHolds(inv, func() bool { return true })}
if !w.Holds() {
t.Fatal("the writer may not write before the switch")
}
inv.BeginSwitchForTest(inventory.KindAssignments)
if w.Holds() {
t.Fatal("the writer may write while the assignments are switching")
}
// Refused before it reads the bucket (it has none here) or lays or ends a key.
r, err := w.Sync(t.Context(), map[string][]byte{"anchor.notes": []byte(`{}`)})
if !errors.Is(err, recordmirror.ErrNotHeld) || r.Written != 0 || r.Removed != 0 {
t.Fatalf("the writer wrote after the switch: %+v %v", r, err)
}
}
cmd/mesh-controller/sent_required_by_integration_test.go
+1 -4
View File
@@ -85,10 +85,7 @@ func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T)
t.Fatal(err)
}
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
mirror, err := inv.AssignmentsMirror(ctx)
if err != nil {
t.Fatal(err)
}
mirror := inv.PlanValuesForTest(t)
var value inventory.AssignmentMirrored
if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil {
t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err)
internal/inventory/batches.go
+22 -85
View File
@@ -3,11 +3,8 @@ package inventory
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
"github.com/jackc/pgx/v5"
)
// The merges a batch or walk answers (novox/hq ADR 0276): every merge the controller heard into a branch a
@@ -33,138 +30,78 @@ type BatchedMerge struct {
// AddMerge keeps a merge for a plan; false when the merge was kept already, for whatever plan.
func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
return false, err
} else if on {
return i.chainAddMerge(ctx, c, m)
}
tag, err := i.store.Pool().Exec(ctx,
`insert into batched_merge (repository, branch, commit_hash, merged_at, heard_at, event, plan_id, alone)
values (lower($1), $2, $3, $4, $5, $6, nullif($7, ''), $8) on conflict do nothing`,
m.Repository, m.Branch, m.Commit, mergedAt(m.Merged), m.Heard, []byte(m.Event), m.Plan, m.Alone)
c, err := i.theChain(ctx)
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
return i.chainAddMerge(ctx, c, m)
}
// MergeOf is one kept merge; false when it was never heard.
func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (BatchedMerge, bool, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
return BatchedMerge{}, false, err
} else if on {
return i.chainMergeOf(ctx, c, repository, commit)
}
ms, err := i.merges(ctx, `where repository = lower($1) and commit_hash = $2`, repository, commit)
if err != nil || len(ms) == 0 {
c, err := i.theChain(ctx)
if err != nil {
return BatchedMerge{}, false, err
}
return ms[0], true, nil
return i.chainMergeOf(ctx, c, repository, commit)
}
// MergesOf is every merge a plan answers, oldest merge first.
func (i *Inventory) MergesOf(ctx context.Context, plan string) ([]BatchedMerge, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
return i.chainMergesOf(ctx, c, plan)
}
return i.merges(ctx, `where plan_id = $1 order by merged_at nulls last, heard_at`, plan)
return i.chainMergesOf(ctx, c, plan)
}
// LaterMergesOf is every merge of a repository's branch made after a moment and answered by a plan, the
// newest first.
func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string, after time.Time) ([]BatchedMerge, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
return i.chainLaterMergesOf(ctx, c, repository, branch, after)
}
return i.merges(ctx, `where repository = lower($1) and branch = $2 and merged_at > $3 and plan_id is not null
order by merged_at desc`, repository, branch, after)
return i.chainLaterMergesOf(ctx, c, repository, branch, after)
}
// KeptOnBranch says the controller keeps a merge of a repository's branch, and when it heard the branch's first: a merge of a branch it keeps merges of, made since then, is the record's to answer, whatever a
// module's last look says (novox/hq ADR 0276).
func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string) (bool, time.Time, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
return false, time.Time{}, err
} else if on {
return i.chainKeptOnBranch(ctx, c, repository, branch)
}
var first *time.Time
var kept bool
err := i.store.Pool().QueryRow(ctx,
`select min(heard_at), count(*) > 0 from batched_merge where repository = lower($1) and branch = $2`,
repository, branch).Scan(&first, &kept)
if err != nil || first == nil {
c, err := i.theChain(ctx)
if err != nil {
return false, time.Time{}, err
}
return kept, first.UTC(), nil
return i.chainKeptOnBranch(ctx, c, repository, branch)
}
// AloneMerges is every merge waiting to be walked on its own commit, the newest first (ADR 0276 decision 3).
func (i *Inventory) AloneMerges(ctx context.Context) ([]BatchedMerge, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
return i.chainAloneMerges(ctx, c)
}
return i.merges(ctx, `where plan_id is null and alone order by merged_at desc nulls last, heard_at desc`)
return i.chainAloneMerges(ctx, c)
}
// AnswerMerge moves a merge to the plan that answers it now; an empty plan with alone leaves it waiting to
// be walked on its own commit.
func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan string, alone bool) error {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return err
} else if on {
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}})
}
_, err := i.store.Pool().Exec(ctx,
`update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`,
repository, commit, plan, alone)
return err
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}})
}
// Batches is every batch not yet cut: assembling or queued, oldest first. One at a time is the rule; more
// are read so a fault is seen.
func (i *Inventory) Batches(ctx context.Context) ([]Plan, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
return nil, err
} else if on {
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated)
}
return i.plans(ctx, `where state in ('assembling', 'queued') order by created`)
}
func (i *Inventory) merges(ctx context.Context, tail string, args ...any) ([]BatchedMerge, error) {
rows, err := i.store.Pool().Query(ctx,
`select repository, branch, commit_hash, merged_at, heard_at, event, coalesce(plan_id, ''), alone
from batched_merge `+tail, args...)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
var out []BatchedMerge
for rows.Next() {
var m BatchedMerge
var merged *time.Time
var event []byte
if err := rows.Scan(&m.Repository, &m.Branch, &m.Commit, &merged, &m.Heard, &event, &m.Plan, &m.Alone); err != nil {
return nil, err
}
if merged != nil {
m.Merged = merged.UTC()
}
m.Heard = m.Heard.UTC()
m.Event = event
out = append(out, m)
}
if errors.Is(rows.Err(), pgx.ErrNoRows) {
return nil, nil
}
return out, rows.Err()
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated)
}
// SameRepository says two spellings of a repository are one.
internal/inventory/builds.go
+12 -283
View File
@@ -2,14 +2,9 @@ package inventory
import (
"context"
"encoding/json"
"errors"
"sort"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/novox/mesh-controller/internal/chain"
)
@@ -134,51 +129,11 @@ func (b Build) Worked() bool { return b.Failed == "" }
// whoever asked and once on the exchange when nobody was. Recording both would show one build as
// two, and which of the two is real is not a question anybody could answer afterwards.
func (i *Inventory) RecordBuild(ctx context.Context, b Build) error {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return err
} else if on {
return i.chainRecordBuild(ctx, c, b)
}
made, err := json.Marshal(b.Made)
c, err := i.theChain(ctx)
if err != nil {
return err
}
against, err := json.Marshal(b.Against)
if err != nil {
return err
}
read, err := json.Marshal(b.Read)
if err != nil {
return err
}
var sources any
if len(b.Sources) > 0 {
raw, err := json.Marshal(b.Sources)
if err != nil {
return err
}
sources = raw
}
var module *string
if b.Module != "" {
module = &b.Module
}
var asked *time.Time
if !b.Asked.IsZero() {
asked = &b.Asked
}
_, err = i.store.Pool().Exec(ctx,
`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made,
source_path, manifest, built_against, built_contexts, asked, source_fingerprint,
build_sources)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
on conflict (id) do nothing`,
b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made,
b.Path, manifestOrNil(b.Manifest), against, read, asked, b.SourceFingerprint, sources)
if err != nil {
return err
}
return i.RecordMirrored(ctx, b.ID, "", b.Mirrored)
return i.chainRecordBuild(ctx, c, b)
}
// RecordMirrored keeps that the mesh copied these bases into the artifact store (novox/hq ADR 0257):
@@ -221,79 +176,22 @@ func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Bui
if limit <= 0 {
limit = 20
}
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainBuildList(ctx, c, module, limit)
}
if limit <= 0 {
limit = 20
}
query := `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at
from build order by at desc limit $1`
args := []any{limit}
if module != "" {
query = `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at
from build where module = $2 order by at desc limit $1`
args = append(args, module)
}
rows, err := i.store.Pool().Query(ctx, query, args...)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Build
for rows.Next() {
var b Build
var made []byte
var asked *time.Time
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
&b.On, &b.Failed, &made, &asked, &b.At); err != nil {
return nil, err
}
if asked != nil {
b.Asked = *asked
}
if err := json.Unmarshal(made, &b.Made); err != nil {
return nil, err
}
out = append(out, b)
}
return out, rows.Err()
return i.chainBuildList(ctx, c, module, limit)
}
// BuildByID is one build's record, by the id its request carried — what a plan matches its outcome
// by when the outcome names no module (novox/hq ADR 0219), and what `rebuild` and `replay` read the
// repository, path, ref and commit from. False when no outcome with that id was recorded.
func (i *Inventory) BuildByID(ctx context.Context, id string) (Build, bool, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return Build{}, false, err
} else if on {
return i.chainBuildByID(ctx, c, id)
}
var b Build
var made []byte
var asked *time.Time
err := i.store.Pool().QueryRow(ctx,
`select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made,
coalesce(source_path,''), asked, at
from build where id = $1`, id).Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
&b.On, &b.Failed, &made, &b.Path, &asked, &b.At)
if errors.Is(err, pgx.ErrNoRows) {
return Build{}, false, nil
}
c, err := i.theChain(ctx)
if err != nil {
return Build{}, false, err
}
if asked != nil {
b.Asked = *asked
}
if err := json.Unmarshal(made, &b.Made); err != nil {
return Build{}, false, err
}
return b, true, nil
return i.chainBuildByID(ctx, c, id)
}
// Held is every artifact this mesh has built, keyed "<module>/<artifact>".
@@ -375,40 +273,11 @@ func registeredOnly(held, registered map[string]string) map[string]string {
// edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a
// module standing on nothing looks like: an edge the mesh has not derived is not an edge.
func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainBuiltAgainst(ctx, c)
}
rows, err := i.store.Pool().Query(ctx,
`select distinct on (module) module, built_against
from build
where module is not null and module <> '' and failed = ''
order by module, `+newestRequestFirst)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
against := map[string][]string{}
for rows.Next() {
var module string
var raw []byte
if err := rows.Scan(&module, &raw); err != nil {
return nil, err
}
if len(raw) == 0 {
continue
}
var refs []string
if err := json.Unmarshal(raw, &refs); err != nil {
continue
}
if len(refs) > 0 {
against[module] = refs
}
}
return against, rows.Err()
return i.chainBuiltAgainst(ctx, c)
}
// ReadRepositories is what each module's newest successful build read source from besides its own
@@ -423,95 +292,11 @@ func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, erro
// none — off the trunk, from a builder that predates it, or of a source not known — leaves its module read
// as before: every file of a context, and its own directory or root.
func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepository, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainReadRepositories(ctx, c)
}
// The newest build of each module whatever its outcome: a failed one newer than the newest that
// worked leaves that one's build source stale — the merge it was asked for may have changed the closure
// (novox/hq ADR 0267), so its module is read whole until a build works again.
newest := map[string]struct {
at time.Time
failed bool
}{}
tried, err := i.store.Pool().Query(ctx,
`select distinct on (module) module, coalesce(asked, at), failed <> ''
from build
where module is not null and module <> ''
order by module, `+newestRequestFirst)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
for tried.Next() {
var module string
var at time.Time
var failed bool
if err := tried.Scan(&module, &at, &failed); err != nil {
tried.Close()
return nil, err
}
newest[module] = struct {
at time.Time
failed bool
}{at, failed}
}
tried.Close()
if err := tried.Err(); err != nil {
return nil, err
}
rows, err := i.store.Pool().Query(ctx,
`select distinct on (module) module, built_contexts, build_sources, coalesce(asked, at),
(select source from module where module.name = build.module)
from build
where module is not null and module <> '' and failed = ''
order by module, `+newestRequestFirst)
if err != nil {
return nil, err
}
defer rows.Close()
read := map[string][]ReadRepository{}
for rows.Next() {
var module string
var raw, rawSources []byte
var built time.Time
var own *string
if err := rows.Scan(&module, &raw, &rawSources, &built, &own); err != nil {
return nil, err
}
var of []ReadRepository
if len(raw) > 0 {
if err := json.Unmarshal(raw, &of); err != nil {
of = nil
}
}
var sources []BuildSource
if len(rawSources) > 0 {
if err := json.Unmarshal(rawSources, &sources); err != nil {
sources = nil
}
}
whole := ""
if n, known := newest[module]; known && n.failed && n.at.After(built) && len(sources) > 0 {
sources, whole = nil, "a newer build of it failed"
}
of = WithBuildSources(of, sources)
if own != nil {
of = OwnRepositoryAsOwn(of, *own)
}
if whole != "" {
of = append(of, ReadRepository{Own: true, Whole: whole})
}
if len(of) > 0 {
for k := range of {
of[k].Built, of[k].Looked = built, newest[module].at
}
read[module] = of
}
}
return read, rows.Err()
return i.chainReadRepositories(ctx, c)
}
// WithBuildSources lays a build's build sources over the repositories it read: a context's paths on its
@@ -588,18 +373,6 @@ func namesRepository(registered, named string) bool {
return reg == name || strings.HasSuffix(reg, "/"+name)
}
// manifestOrNil keeps the difference between "declared nothing" and "predates this being kept".
//
// A build recorded before the mesh kept manifests has no manifest, and that is not the same as one
// whose manifest was empty. A replay can then say which it is holding instead of inventing an
// empty declaration for a module that certainly had one.
func manifestOrNil(raw []byte) any {
if len(raw) == 0 {
return nil
}
return raw
}
// Announceable is every build worth telling a catalogue about, oldest first.
//
// **Oldest first, because a graph is built in the order things happened.** Registering a module
@@ -614,53 +387,9 @@ func manifestOrNil(raw []byte) any {
// One row per module and commit: a module built twice at the same commit is one fact, and the
// row asked last is the one whose artifacts are current (novox/hq 04-ISSUES/219).
func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainAnnounceable(ctx, c)
}
rows, err := i.store.Pool().Query(ctx,
`select distinct on (module, commit_hash)
id, repository, ref, module, commit_hash, built_on, failed, made,
source_path, manifest, built_against, at
from build
where failed = '' and module is not null and module <> '' and commit_hash <> ''
order by module, commit_hash, `+newestRequestFirst)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Build
for rows.Next() {
var b Build
var made []byte
var manifest, against []byte
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
&b.On, &b.Failed, &made, &b.Path, &manifest, &against, &b.At); err != nil {
return nil, err
}
if err := json.Unmarshal(made, &b.Made); err != nil {
return nil, err
}
// Null rather than empty is a build recorded before the mesh kept these, and saying so is
// the point of keeping them nullable: the replay carries nothing rather than an empty
// declaration for a module that certainly had one.
if len(manifest) > 0 {
b.Manifest = manifest
}
if len(against) > 0 {
if err := json.Unmarshal(against, &b.Against); err != nil {
return nil, err
}
}
out = append(out, b)
}
if err := rows.Err(); err != nil {
return nil, err
}
// Sorted here rather than in the query, because `distinct on` fixes the ordering it needs and
// the order that matters to a catalogue is a different one.
sort.Slice(out, func(a, b int) bool { return out[a].At.Before(out[b].At) })
return out, nil
return i.chainAnnounceable(ctx, c)
}
internal/inventory/catalogue.go
+22 -80
View File
@@ -258,31 +258,17 @@ func (i *Inventory) SourceOf(ctx context.Context, module string) (Source, error)
// machines are running last week's version" is the question somebody actually has, and it is the
// one novox/hq ADR 0010 names as the thing that must not be lost.
func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) {
query := `select m.name, coalesce(n.name, '')
from module m
left join assignment a on a.module = m.name
left join node n on n.id = a.node
where m.source is not null
and m.source_head is not null
and coalesce(m.built_from, '') is distinct from m.source_head
order by m.name, n.name`
_, onTheBus, err := i.ChainReads(ctx, KindAssignments)
assigned, err := i.assignments(ctx)
if err != nil {
return nil, err
}
var assigned []assignRow
if onTheBus {
query = `select m.name, ''
rows, err := i.store.Pool().Query(ctx,
`select m.name
from module m
where m.source is not null
and m.source_head is not null
and coalesce(m.built_from, '') is distinct from m.source_head
order by m.name`
if assigned, err = i.assignments(ctx); err != nil {
return nil, err
}
}
rows, err := i.store.Pool().Query(ctx, query)
order by m.name`)
if err != nil {
return nil, err
}
@@ -290,16 +276,11 @@ func (i *Inventory) Behind(ctx context.Context) (map[string][]string, error) {
out := map[string][]string{}
for rows.Next() {
var module, node string
if err := rows.Scan(&module, &node); err != nil {
var module string
if err := rows.Scan(&module); err != nil {
return nil, err
}
if _, seen := out[module]; !seen {
out[module] = nil
}
if node != "" {
out[module] = append(out[module], node)
}
out[module] = nil
}
if err := rows.Err(); err != nil {
return nil, err
@@ -654,21 +635,11 @@ func (i *Inventory) Assign(ctx context.Context, nodeName, module string) (bool,
if err := i.runsSomewhere(ctx, module); err != nil {
return false, err
}
if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
return false, err
} else if on {
return i.chainAssign(ctx, c, node, module)
}
tag, err := i.store.Pool().Exec(ctx,
`insert into assignment (node, module) values ($1, $2) on conflict do nothing`,
node.ID, module)
if err != nil && strings.Contains(err.Error(), "assignment_module_fkey") {
return false, fmt.Errorf("%w: %s", ErrNoSuchModule, module)
}
c, err := i.theChain(ctx)
if err != nil {
return false, err
}
return tag.RowsAffected() > 0, nil
return i.chainAssign(ctx, c, node, module)
}
// Unassign takes a module off a node.
@@ -677,21 +648,12 @@ func (i *Inventory) Unassign(ctx context.Context, nodeName, module string) error
if err != nil {
return err
}
if c, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return err
}
if err := i.chainUnassign(ctx, c, node, module); err != nil {
return err
} else if on {
if err := i.chainUnassign(ctx, c, node, module); err != nil {
return err
}
} else {
tag, err := i.store.Pool().Exec(ctx,
`delete from assignment where node = $1 and module = $2`, node.ID, module)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return fmt.Errorf("%s is not assigned to %s", module, nodeName)
}
}
// The seat it held falls back to derivation rather than pointing at nothing: what the store's foreign key did.
if _, err := i.store.Pool().Exec(ctx, `delete from seat_holding where node = $1 and module = $2`, node.ID, module); err != nil {
@@ -1362,39 +1324,21 @@ type Entry struct {
// catalogue nobody lists, and the questions here — what is this, where did it come from, who is
// running it — are asked together every time.
func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
query := `select m.name, m.manifest,
coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''),
coalesce(m.built_from, ''), coalesce(m.source_head, ''),
coalesce(m.source_seen, to_timestamp(0)),
coalesce(array_agg(n.name order by n.name) filter (where n.name is not null), '{}')
from module m
left join assignment a on a.module = m.name
left join node n on n.id = a.node
group by m.name, m.manifest, m.source, m.source_path, m.source_seat, m.ref, m.built_from,
m.source_head, m.source_seen
order by m.name`
// On the bus the assignments are read from the bucket, and the machines of a module put beside it here.
_, onTheBus, err := i.ChainReads(ctx, KindAssignments)
// The machines of a module are read from the assignments bucket and put beside it here.
assigned, err := i.assignments(ctx)
if err != nil {
return nil, err
}
var assignedBy map[string][]string
if onTheBus {
query = `select m.name, m.manifest,
assignedBy := map[string][]string{}
for _, a := range assigned {
assignedBy[a.Module] = append(assignedBy[a.Module], a.Node)
}
const query = `select m.name, m.manifest,
coalesce(m.source, ''), m.source_path, m.source_seat, coalesce(m.ref, ''),
coalesce(m.built_from, ''), coalesce(m.source_head, ''),
coalesce(m.source_seen, to_timestamp(0)), '{}'::text[]
from module m
order by m.name`
rows, err := i.assignments(ctx)
if err != nil {
return nil, err
}
assignedBy = map[string][]string{}
for _, a := range rows {
assignedBy[a.Module] = append(assignedBy[a.Module], a.Node)
}
}
rows, err := i.store.Pool().Query(ctx, query)
if err != nil {
return nil, err
@@ -1414,9 +1358,7 @@ func (i *Inventory) Catalogued(ctx context.Context) ([]Entry, error) {
if source.Seen.Unix() == 0 {
source.Seen = time.Time{}
}
if onTheBus {
on = assignedBy[name]
}
on = assignedBy[name]
var m catalogue.Manifest
if err := json.Unmarshal(raw, &m); err != nil {
return nil, err
internal/inventory/chain.go
+22 -110
View File
@@ -5,55 +5,30 @@ import (
"errors"
"fmt"
"sync"
"sync/atomic"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/novox/mesh-controller/internal/chain"
)
// **The chain moves to the bus kind by kind** (novox/hq ADR 0408 §8, issue 981). The tables of builds, rollouts and
// assignments are copied once into the chain's buckets by the serving controller at its start (MoveChain), which then
// records each kind it moved in chain_moved (migration 0104). **Until a kind is recorded there nothing reads its bucket;
// once it is, nothing reads its tables**: ChainReads answers which, and every method of a kind asks it first.
// The kinds of the chain, as chain_moved names them.
const (
KindBuilds = "builds"
KindRollouts = "rollouts"
)
// **The chain lives on the bus** (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
// bus's key-value store, the controller's store keeps none of them, and every method of a kind reads and writes the chain this
// inventory is given (theChain).
// testChains are the chains in memory a test made for its store, by the store's address: every inventory opened on that
// store is given the same chain, as every process of a mesh is on the same bus. Empty outside a test.
var testChains sync.Map
// ErrNoChain is a kind moved to the bus read by an inventory that has no chain attached: a process that did not
// open the bus. Refused loudly, never answered from the tables the move left behind.
// ErrNoChain is a read or write of the chain by an inventory that has none attached: a process that did not open the bus.
// Refused loudly.
var ErrNoChain = errors.New("this kind of record lives on the bus, and this process has not opened the bus")
// movedKinds is which kinds this process has read as moved: once moved, always moved, so it is read from the store
// until it is true and not after.
type movedKinds struct {
// chainState is what an inventory keeps of the chain it is given.
type chainState struct {
mu sync.Mutex
kind map[string]bool
open ChainOpener
// view is the rollouts bucket kept in memory by one watch (issue 1092); pending are the merges added to a plan that
// is not saved yet.
view *rolloutView
pending map[string]pendingMerge
// switching is set when the assignments begin to be switched to the bus and kept once they are: the store's mirror of
// them writes nothing from then on (AssignmentsMirrorMayWrite). Cleared only if the switch fails.
switching atomic.Bool
}
// AssignmentsMirrorMayWrite says the store's mirror of the assignments may still write the bucket: the assignments are
// neither being switched to the bus nor on it. Asked before every write of the mirror's writer, not once a pass.
func (i *Inventory) AssignmentsMirrorMayWrite() bool {
i.moved.mu.Lock()
moved := i.moved.kind[KindAssignments]
i.moved.mu.Unlock()
return !moved && !i.moved.switching.Load()
}
// ChainOpener opens the chain on the bus when a process first needs it: a command that reads and never moved anything has
@@ -62,99 +37,36 @@ type ChainOpener func(ctx context.Context) (*chain.Chain, error)
// UseChainOpener gives the inventory the means to open the chain when it needs it and none is attached.
func (i *Inventory) UseChainOpener(open ChainOpener) {
i.moved.mu.Lock()
i.moved.open = open
i.moved.mu.Unlock()
i.state.mu.Lock()
i.state.open = open
i.state.mu.Unlock()
}
// UseChain attaches the chain the moved kinds are read and written through. The serving controller attaches the bus's,
// holding the lease for its writes; a test attaches one in memory.
func (i *Inventory) UseChain(c *chain.Chain) {
i.moved.mu.Lock()
defer i.moved.mu.Unlock()
i.state.mu.Lock()
defer i.state.mu.Unlock()
i.chain = c
if i.moved.view != nil {
i.moved.view.stop()
i.moved.view = nil
if i.state.view != nil {
i.state.view.stop()
i.state.view = nil
}
}
// attached is the chain this inventory has, nil when none.
func (i *Inventory) attached() *chain.Chain {
i.moved.mu.Lock()
defer i.moved.mu.Unlock()
i.state.mu.Lock()
defer i.state.mu.Unlock()
return i.chain
}
// ChainReads says whether a kind of the chain is read from its bucket: chain_moved names it. It answers the chain to read
// through, and ErrNoChain when the kind is moved and none is attached.
func (i *Inventory) ChainReads(ctx context.Context, kind string) (*chain.Chain, bool, error) {
i.moved.mu.Lock()
known := i.moved.kind[kind]
i.moved.mu.Unlock()
if !known {
var moved bool
err := i.store.Pool().QueryRow(ctx, `select counts ? $1 from chain_moved`, kind).Scan(&moved)
var pgErr *pgconn.PgError
if errors.Is(err, pgx.ErrNoRows) || (errors.As(err, &pgErr) && pgErr.Code == "42P01") {
// No row, or a store older than the migration that makes the table: nothing moved.
return nil, false, nil
}
if err != nil {
return nil, false, fmt.Errorf("whether %s moved to the bus cannot be read: %w", kind, err)
}
if !moved {
return nil, false, nil
}
i.moved.mu.Lock()
if i.moved.kind == nil {
i.moved.kind = map[string]bool{}
}
i.moved.kind[kind] = true
i.moved.mu.Unlock()
}
i.moved.mu.Lock()
defer i.moved.mu.Unlock()
if i.chain == nil && i.moved.open != nil {
c, err := i.moved.open(ctx)
if err != nil {
return nil, false, fmt.Errorf("%w: %s: %v", ErrNoChain, kind, err)
}
i.chain = c
}
if i.chain == nil {
return nil, false, fmt.Errorf("%w: %s", ErrNoChain, kind)
}
return i.chain, true, nil
}
// ChainMoved is, per kind moved, how many records the move wrote; empty before any is.
func (i *Inventory) ChainMoved(ctx context.Context) (map[string]int, error) {
var counts map[string]int
err := i.store.Pool().QueryRow(ctx, `select counts from chain_moved`).Scan(&counts)
if errors.Is(err, pgx.ErrNoRows) {
return map[string]int{}, nil
}
return counts, err
}
// NewForTest is an inventory with no store, for a test of what needs none.
func NewForTest() *Inventory { return &Inventory{moved: &movedKinds{}} }
// BeginSwitchForTest sets the flag a kind's switch sets, as a switch under way does.
func (i *Inventory) BeginSwitchForTest(kind string) {
if kind == KindAssignments {
i.moved.switching.Store(true)
}
}
// theChain is the chain this inventory writes and reads, opened if a command has not yet: for what has no table of its own
// to be read from while it is moved (a pending assignment is a value of the assignments bucket from the start).
// theChain is the chain this inventory writes and reads, opened if a command has not yet.
func (i *Inventory) theChain(ctx context.Context) (*chain.Chain, error) {
i.moved.mu.Lock()
defer i.moved.mu.Unlock()
if i.chain == nil && i.moved.open != nil {
c, err := i.moved.open(ctx)
i.state.mu.Lock()
defer i.state.mu.Unlock()
if i.chain == nil && i.state.open != nil {
c, err := i.state.open(ctx)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrNoChain, err)
}
internal/inventory/chain_assignments.go
+13 -32
View File
@@ -21,7 +21,6 @@ import (
// here (chain.Assignment.Stands). No `pending` state is written yet (issue 1094).
// KindAssignments is the kind of the chain that is each module assigned to a node, with the build it runs there.
const KindAssignments = "assignments"
func assignmentKey(node, module string) string { return node + "." + module }
@@ -31,32 +30,15 @@ type assignRow struct {
Assigned time.Time
}
// assignments is every assignment, by node then module: from the bucket when the kind is moved, else from the store.
// assignments is every assignment, by node then module, read from the assignments bucket.
// A key whose value names no time assigned is a module only pulled in by another (issue 765), and one that has ended or
// waits in a state is gone: no assignment. One on a machine the store does not know is none either, as the store took a
// removed machine's.
func (i *Inventory) assignments(ctx context.Context) ([]assignRow, error) {
c, on, err := i.ChainReads(ctx, KindAssignments)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
if !on {
rows, err := i.store.Pool().Query(ctx,
`select n.name, a.module, a.assigned from assignment a join node n on n.id = a.node order by n.name, a.module`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []assignRow
for rows.Next() {
var a assignRow
if err := rows.Scan(&a.Node, &a.Module, &a.Assigned); err != nil {
return nil, err
}
out = append(out, a)
}
return out, rows.Err()
}
all, err := c.Assignments.All(ctx)
if err != nil {
return nil, fmt.Errorf("the assignments cannot be read from the bus: %w", err)
@@ -255,8 +237,8 @@ func (i *Inventory) chainAwaitingSince(ctx context.Context, name string) (time.T
// and rollout another write put in the same value, and never ending an assignment. A plan the provisioner's rules refuse is
// refused by name and nothing is written (AssignmentValues).
func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed int, err error) {
c, on, err := i.ChainReads(ctx, KindAssignments)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return 0, 0, err
}
var plans map[string]Provisioning
@@ -342,17 +324,16 @@ type KeptValue struct {
Revision uint64
}
// KeptAssignments is every key of the assignments bucket that holds a value that stands, and false when the assignments are
// not on the bus.
// What the serving controller compares from one look to the next to say a key changed (`assignment-changed`).
func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, bool, error) {
c, on, err := i.ChainReads(ctx, KindAssignments)
if err != nil || !on {
return nil, false, err
// KeptAssignments is every key of the assignments bucket that holds a value that stands. What the serving controller
// compares from one look to the next to say a key changed (`assignment-changed`).
func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue, error) {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
all, err := c.Assignments.All(ctx)
if err != nil {
return nil, true, err
return nil, err
}
out := make(map[string]KeptValue, len(all))
for key, k := range all {
@@ -361,11 +342,11 @@ func (i *Inventory) KeptAssignments(ctx context.Context) (map[string]KeptValue,
}
body, err := json.Marshal(k.Value)
if err != nil {
return nil, true, err
return nil, err
}
out[key] = KeptValue{Value: body, Revision: k.Revision}
}
return out, true, nil
return out, nil
}
// What a pass that keeps the plans does with one key.
internal/inventory/chain_assignments_key_test.go
-12
View File
@@ -1,9 +1,7 @@
package inventory
import (
"context"
"encoding/json"
"errors"
"testing"
"time"
@@ -72,13 +70,3 @@ func TestAPassThatKeepsThePlansNeverRevivesAnEndedAssignment(t *testing.T) {
}
}
}
// **The store's mirror of the assignments is refused to its writer once the assignments are moved** (ADR 0408 §4, issue
// 1093): asked at the moment of the write, so a tick that decided before the switch lays nothing after it.
func TestTheStoresMirrorOfTheAssignmentsIsRefusedToItsWriterOnceTheyAreMoved(t *testing.T) {
inv := &Inventory{moved: &movedKinds{kind: map[string]bool{KindAssignments: true}}}
inv.UseChain(chain.InMemory(nil))
if _, err := inv.AssignmentsMirrorToWrite(context.Background()); !errors.Is(err, ErrAssignmentsMoved) {
t.Fatalf("the writer was given the store's rows: %v", err)
}
}
internal/inventory/chain_links.go
+4 -4
View File
@@ -38,8 +38,8 @@ func WithRollout(ctx context.Context, id string, modules map[string]RolloutModul
// AssignmentRuns is what each module assigned to a machine runs there, as its assignment names it: the build and the
// commit. Empty when the assignments are not on the bus, and for a module not sent yet.
func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string]chain.Ref, error) {
c, on, err := i.ChainReads(ctx, KindAssignments)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
all, err := c.Assignments.All(ctx)
@@ -58,8 +58,8 @@ func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string
// AssignmentsOn are the assignments of a machine as the chain names them, by module: the standing ones, with their links.
func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Assignment, error) {
c, on, err := i.ChainReads(ctx, KindAssignments)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
all, err := c.Assignments.All(ctx)
internal/inventory/chain_move.go
-663
View File
@@ -1,663 +0,0 @@
package inventory
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/chain"
)
// **The move** (novox/hq ADR 0408 §8, issue 981 note 7 Q2): the serving controller writes the rows of a kind into its
// bucket at its start, holding the lease and after the buckets exist, and records the kind in chain_moved. Idempotent: a
// key that exists is already moved and is never overwritten, so a bucket value written since is not laid over by the row
// it came from. It counts the rows against the keys, logs the largest value, and **refuses** (a Refusal, which the
// controller raises as an urgent condition) on a count that differs or a value over the bus's largest, with nothing
// switched: a kind is recorded moved only once all of it is in the bucket.
// Refusal is a move that stopped without switching a kind, and why. It is permanent until the cause is put right; any
// other error of a move (the bus away, a write not granted) is tried again.
type Refusal struct {
Kind, Why string
}
func (r *Refusal) Error() string {
return fmt.Sprintf("the move of %s to the bus is refused: %s", r.Kind, r.Why)
}
// MoveReport is what a move of one kind did.
type MoveReport struct {
Kind string
// Rows is how many records the tables held, Written how many keys this move made, and Present how many already
// held a value (moved by an earlier start).
Rows, Written, Present int
// TableRows is how many rows of the tables those records came from: a request and its outcome are two rows and one
// record.
TableRows int
// Largest is the biggest value, in bytes.
Largest int
// Already is a kind an earlier start moved: nothing was done.
Already bool
}
// mover is one kind of the chain: its rows, read from its tables, as the keys and values they become.
type mover struct {
kind string
// rows reads the tables: each record's key and its value, whole, and how many rows of the tables they came from.
rows func(ctx context.Context) ([]moveRow, int, error)
// create writes one record under its key; chain.ErrExists when the key holds a value.
create func(ctx context.Context, c *chain.Chain, row moveRow) error
// keys are the keys the bucket holds.
keys func(ctx context.Context, c *chain.Chain) ([]string, error)
// count is how many rows the tables hold now, counted as rows reports them.
count func(ctx context.Context, tx pgx.Tx) (int, error)
// lock keeps the tables from taking a row while the switch is recorded.
lock string
// settle runs under that lock, with the rows read again: it makes the bucket hold what the rows say now. A row the old
// controller changed after the first pass is in the bucket before the kind is recorded as moved, for before the switch
// the table is the only truth.
settle func(ctx context.Context, c *chain.Chain, rows []moveRow) error
}
type moveRow struct {
key string
value any
size int
}
func (i *Inventory) movers() []mover {
return []mover{{
kind: KindBuilds,
rows: func(ctx context.Context) ([]moveRow, int, error) {
builds, err := i.allBuildRows(ctx)
if err != nil {
return nil, 0, err
}
requests, err := i.allRequestRows(ctx)
if err != nil {
return nil, 0, err
}
// A build request is the build's record in state asked, and the request stays beside the outcome that
// answered it: one record under one id (issue 1091).
byID := make(map[string]*chain.Build, len(builds))
var order []string
for _, b := range builds {
v := toChainBuild(b)
byID[b.ID] = &v
order = append(order, b.ID)
}
for _, r := range requests {
if v, answered := byID[r.ID]; answered {
v.Request = toRequest(r)
continue
}
v := askedRecord(r)
if r.NotAsked != "" {
// Never handed over: asked again once, after the move (askAgain).
v.Request.Reask = chain.ReaskPending
}
byID[r.ID] = &v
order = append(order, r.ID)
}
out := make([]moveRow, 0, len(order))
for _, id := range order {
body, err := json.Marshal(byID[id])
if err != nil {
return nil, 0, err
}
out = append(out, moveRow{key: chainBuildKey(id), value: *byID[id], size: len(body)})
}
return out, len(builds) + len(requests), nil
},
create: func(ctx context.Context, c *chain.Chain, row moveRow) error {
_, err := c.Builds.Create(ctx, row.key, row.value.(chain.Build))
return err
},
keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Builds.Keys(ctx) },
count: func(ctx context.Context, tx pgx.Tx) (int, error) {
var n int
err := tx.QueryRow(ctx, `select (select count(*) from build) + (select count(*) from build_request)`).Scan(&n)
return n, err
},
lock: `lock table build, build_request in share mode`,
settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error {
return settleBucket(ctx, c.Builds, rows)
},
}, {
kind: KindRollouts,
rows: func(ctx context.Context) ([]moveRow, int, error) {
plans, err := i.plans(ctx, `order by created`)
if err != nil {
return nil, 0, err
}
merges, err := i.merges(ctx, ``)
if err != nil {
return nil, 0, err
}
tablePlans := len(plans)
if plans, err = holdMerges(plans, merges); err != nil {
return nil, 0, err
}
out := make([]moveRow, 0, len(plans))
for _, p := range plans {
body, err := json.Marshal(p)
if err != nil {
return nil, 0, err
}
out = append(out, moveRow{key: p.ID, value: p, size: len(body)})
}
return out, tablePlans + len(merges), nil
},
create: func(ctx context.Context, c *chain.Chain, row moveRow) error {
_, err := chain.RolloutsOf[Plan](c).Create(ctx, row.key, row.value.(Plan))
return err
},
keys: func(ctx context.Context, c *chain.Chain) ([]string, error) {
return chain.RolloutsOf[Plan](c).Keys(ctx)
},
count: func(ctx context.Context, tx pgx.Tx) (int, error) {
var n int
err := tx.QueryRow(ctx, `select (select count(*) from release_plan) + (select count(*) from batched_merge)`).Scan(&n)
return n, err
},
lock: `lock table release_plan, batched_merge in share mode`,
settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error {
return settleBucket(ctx, chain.RolloutsOf[Plan](c), rows)
},
}, {
kind: KindAssignments,
rows: func(ctx context.Context) ([]moveRow, int, error) {
assigned, err := i.assignedWithSends(ctx)
if err != nil {
return nil, 0, err
}
out := make([]moveRow, 0, len(assigned))
for _, a := range assigned {
body, err := json.Marshal(a)
if err != nil {
return nil, 0, err
}
out = append(out, moveRow{key: assignmentKey(a.Node, a.Module), value: a, size: len(body)})
}
return out, len(out), nil
},
create: createAssignment,
keys: func(ctx context.Context, c *chain.Chain) ([]string, error) { return c.Assignments.Keys(ctx) },
count: func(ctx context.Context, tx pgx.Tx) (int, error) {
var n int
err := tx.QueryRow(ctx, `select count(*) from assignment`).Scan(&n)
return n, err
},
lock: `lock table assignment in share mode`,
settle: func(ctx context.Context, c *chain.Chain, rows []moveRow) error {
have := make(map[string]bool, len(rows))
for _, r := range rows {
have[r.key] = true
if err := settleAssignment(ctx, c, r); err != nil {
return err
}
}
// An assignment the store no longer holds ended since the first pass: the bucket says so.
all, err := c.Assignments.All(ctx)
if err != nil {
return err
}
for key, k := range all {
if have[key] || k.Value.Assigned.IsZero() || !k.Value.Stands() {
continue
}
k.Value.Ended = time.Now().UTC()
if _, err := c.Assignments.Update(ctx, key, k.Value, k.Revision); err != nil && !errors.Is(err, chain.ErrMoved) {
return err
}
}
return nil
},
}}
}
// assignedWithSends is every assignment of the store as the chain keeps it: with the commit its machine was last sent for
// the module (node.sent_builds), the build that is of it and what the build published, and since when (when the machine was
// last sent). An assignment of a module the machine was never sent has none of those.
func (i *Inventory) assignedWithSends(ctx context.Context) ([]chain.Assignment, error) {
rows, err := i.store.Pool().Query(ctx,
`select n.name, a.module, a.assigned, n.sent_at, n.sent_builds
from assignment a join node n on n.id = a.node order by n.name, a.module`)
if err != nil {
return nil, err
}
var out []chain.Assignment
for rows.Next() {
var a chain.Assignment
var sentAt *time.Time
var raw []byte
if err := rows.Scan(&a.Node, &a.Module, &a.Assigned, &sentAt, &raw); err != nil {
rows.Close()
return nil, err
}
a.Assigned = a.Assigned.UTC()
var sent map[string]string
if len(raw) > 0 {
_ = json.Unmarshal(raw, &sent)
}
if commit := sent[a.Module]; commit != "" && sentAt != nil {
a.Commit, a.Since = commit, sentAt.UTC()
}
out = append(out, a)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
// The build of each commit, read once: the newest successful build of the module at it.
type at struct{ module, commit string }
var all []Build
if _, moved, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if moved {
if all, err = chainBuilds(ctx, i.attached()); err != nil {
return nil, err
}
} else if all, err = i.allBuildRows(ctx); err != nil {
return nil, err
}
newest := map[at]Build{}
for _, b := range all {
k := at{b.Module, b.Commit}
if b.Failed != "" || b.Module == "" {
continue
}
if had, has := newest[k]; !has || b.At.After(had.At) || (b.At.Equal(had.At) && b.ID > had.ID) {
newest[k] = b
}
}
for k := range out {
if out[k].Commit == "" {
continue
}
b, found := newest[at{out[k].Module, out[k].Commit}]
if !found {
continue
}
out[k].Build = b.ID
if published := publishedOf(b.Made); published != nil {
out[k].Artifact, out[k].Package = published.Artifact, published.Package
}
}
return out, nil
}
// holdMerges puts each batched merge into the record of the plan that answers it. A merge no plan answers — the plan_id
// of a merge a failed walk carried is null, or names a plan the store never kept — is held, flagged waiting, by the
// newest plan that lists it among its merges, else the record kept for the merges no plan answers: the record that held it
// last.
func holdMerges(plans []Plan, merges []BatchedMerge) ([]Plan, error) {
waitingAt := -1
at := make(map[string]int, len(plans))
for k, p := range plans {
at[p.ID] = k
}
newest := func(match func(Plan) bool) int {
best := -1
for k, p := range plans {
if match(p) && (best < 0 || p.Created.After(plans[best].Created)) {
best = k
}
}
return best
}
for _, m := range merges {
held := heldOf(m, m.Heard)
holder, answered := at[m.Plan]
if m.Plan == "" || !answered {
held.Waiting = true
holder = newest(func(p Plan) bool {
if p.Delivery == nil {
return false
}
for _, listed := range p.Delivery.Merges {
if strings.EqualFold(listed.Repository, m.Repository) && listed.Commit == m.Commit {
return true
}
}
return false
})
if holder < 0 {
// Held by the record kept for the merges no plan answers, made with the others.
if waitingAt < 0 {
plans = append(plans, Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1,
Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"})
waitingAt = len(plans) - 1
}
holder = waitingAt
}
}
plans[holder].Held = append(plans[holder].Held, held)
}
return plans, nil
}
// MoveChain moves every kind not moved yet, in order, and says what it did. It needs the chain attached and the lease
// held (the chain's writes ask). It stops at the first kind that fails.
func (i *Inventory) MoveChain(ctx context.Context, say func(format string, args ...any)) ([]MoveReport, error) {
if say == nil {
say = func(string, ...any) {}
}
if i.attached() == nil {
return nil, ErrNoChain
}
var out []MoveReport
for _, m := range i.movers() {
r, err := i.moveKind(ctx, m, say)
if err != nil {
return out, err
}
out = append(out, r)
}
return out, nil
}
// movePasses is how often a move reads the tables again when they took a row while it wrote: the controller before this
// one may still have been writing as the lease changed hands.
const movePasses = 5
func (i *Inventory) moveKind(ctx context.Context, m mover, say func(string, ...any)) (MoveReport, error) {
report := MoveReport{Kind: m.kind}
moved, err := i.ChainMoved(ctx)
if err != nil {
return report, err
}
if _, done := moved[m.kind]; done {
report.Already = true
return report, nil
}
for pass := 1; pass <= movePasses; pass++ {
rows, tableRows, err := m.rows(ctx)
if err != nil {
return report, fmt.Errorf("the %s cannot be read from the tables: %w", m.kind, err)
}
report.Rows, report.TableRows, report.Largest = len(rows), tableRows, 0
// Every value measured before any is written: a value over the bus's largest stops the move with nothing
// written (ADR 0408 §6), never cut.
for _, r := range rows {
report.Largest = max(report.Largest, r.size)
if r.size > broker.ChainValueBytes {
return report, &Refusal{m.kind, fmt.Sprintf("the value of %s is %d bytes and the bus holds at most %d; "+
"nothing is cut, so the record must be made smaller first", r.key, r.size, broker.ChainValueBytes)}
}
}
report.Written, report.Present = 0, 0
for _, r := range rows {
switch err := m.create(ctx, i.attached(), r); {
case err == nil:
report.Written++
case errors.Is(err, chain.ErrExists):
report.Present++
case errors.Is(err, chain.ErrBadKey):
return report, &Refusal{m.kind, fmt.Sprintf("%s cannot be a key of the bus: %v", r.key, err)}
default:
return report, fmt.Errorf("%s %s is not written to the bus: %w", m.kind, r.key, err)
}
}
// Rows against keys: every row's key is in the bucket.
keys, err := m.keys(ctx, i.attached())
if err != nil {
return report, fmt.Errorf("the keys of the %s cannot be read back: %w", m.kind, err)
}
have := make(map[string]bool, len(keys))
for _, k := range keys {
have[k] = true
}
for _, r := range rows {
if !have[r.key] {
return report, &Refusal{m.kind, fmt.Sprintf("the tables hold %d records and the bucket does not hold %s", len(rows), r.key)}
}
}
settled, err := i.switchKind(ctx, m, report)
if err != nil {
return report, err
}
if settled {
say("moved the %s to the bus: %d records, %d written now and %d already there, the largest %d bytes",
m.kind, report.Rows, report.Written, report.Present, report.Largest)
return report, nil
}
say("the %s took a row while they were moved, so they are read again (pass %d of %d)", m.kind, pass, movePasses)
}
return report, &Refusal{m.kind, fmt.Sprintf("the tables kept taking rows through %d passes; another controller is still writing them", movePasses)}
}
// switchKind records the kind as moved, in one transaction that holds the tables against writes and finds their count
// as the move read it: false when they took a row since.
func (i *Inventory) switchKind(ctx context.Context, m mover, r MoveReport) (bool, error) {
committed := false
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return false, err
}
defer func() { _ = tx.Rollback(ctx) }()
if _, err := tx.Exec(ctx, m.lock); err != nil {
return false, err
}
n, err := m.count(ctx, tx)
if err != nil {
return false, err
}
if n != r.TableRows {
return false, nil
}
// The store's mirror of the assignments writes nothing from here: it would lay values without the links over the keys
// the settle writes and end the ones it ended.
if m.kind == KindAssignments {
i.moved.switching.Store(true)
defer func() {
if !committed {
i.moved.switching.Store(false)
}
}()
}
// The rows read again under the lock: nothing writes them now, so what the bucket holds is made what they say.
rows, tableRows, err := m.rows(ctx)
if err != nil {
return false, err
}
if tableRows != r.TableRows {
return false, nil
}
if err := m.settle(ctx, i.attached(), rows); err != nil {
return false, fmt.Errorf("the %s changed since they were first written, and the bucket cannot be brought to them: %w", m.kind, err)
}
counts, _ := json.Marshal(map[string]int{m.kind: r.Rows})
largest, _ := json.Marshal(map[string]int{m.kind: r.Largest})
if _, err := tx.Exec(ctx,
`insert into chain_moved (counts, largest) values ($1, $2)
on conflict (only_one) do update
set counts = chain_moved.counts || excluded.counts, largest = chain_moved.largest || excluded.largest,
moved_at = now()`, counts, largest); err != nil {
return false, err
}
if err := tx.Commit(ctx); err != nil {
return false, err
}
committed = true
i.moved.mu.Lock()
if i.moved.kind == nil {
i.moved.kind = map[string]bool{}
}
i.moved.kind[m.kind] = true
i.moved.mu.Unlock()
return true, nil
}
// allBuildRows is every build of the table, whole.
func (i *Inventory) allBuildRows(ctx context.Context) ([]Build, error) {
rows, err := i.store.Pool().Query(ctx,
`select id, repository, ref, coalesce(module, ''), commit_hash, built_on, failed, made, source_path, manifest,
built_against, built_contexts, build_sources, source_fingerprint, asked, at
from build order by at, id`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Build
for rows.Next() {
var b Build
var made, manifest, against, read, sources []byte
var asked *time.Time
if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit, &b.On, &b.Failed, &made, &b.Path,
&manifest, &against, &read, &sources, &b.SourceFingerprint, &asked, &b.At); err != nil {
return nil, err
}
if asked != nil {
b.Asked = *asked
}
b.Manifest = manifest
for _, part := range []struct {
raw []byte
to any
who string
}{{made, &b.Made, "made"}, {against, &b.Against, "built_against"}, {read, &b.Read, "built_contexts"},
{sources, &b.Sources, "build_sources"}} {
if len(part.raw) == 0 {
continue
}
if err := json.Unmarshal(part.raw, part.to); err != nil {
return nil, fmt.Errorf("build %s: %s is not readable: %w", b.ID, part.who, err)
}
}
out = append(out, b)
}
return out, rows.Err()
}
// allRequestRows is every build request of the table.
func (i *Inventory) allRequestRows(ctx context.Context) ([]BuildRequest, error) {
rows, err := i.store.Pool().Query(ctx,
`select id, repository, seat, source_path, ref, commit_hash, asked_for, coalesce(not_asked, ''),
coalesce(outcome_unknown, ''), asked_at, at_terminal
from build_request order by asked_at, id`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []BuildRequest
for rows.Next() {
var a BuildRequest
if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked,
&a.OutcomeUnknown, &a.At, &a.AtTerminal); err != nil {
return nil, err
}
out = append(out, a)
}
return out, rows.Err()
}
// createAssignment writes an assignment's row under its key; chain.ErrExists when the key holds a value that stays.
func createAssignment(ctx context.Context, c *chain.Chain, row moveRow) error {
want := row.value.(chain.Assignment)
// The mirror of the store kept a value under this key until now, with the provisioner's plan beside it: the
// plan stays, the record's links are laid in. A value that names its build already is a later word, and stays.
for try := 0; try < casTries; try++ {
cur, found, err := c.Assignments.Get(ctx, row.key)
if err != nil {
return err
}
if !found {
if _, err := c.Assignments.Create(ctx, row.key, want); !errors.Is(err, chain.ErrExists) {
return err
}
continue
}
if cur.Value.Build != "" || cur.Value.Commit != "" || !cur.Value.Since.IsZero() || (want.Build == "" && want.Commit == "") {
return chain.ErrExists
}
merged := want
merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning
if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) {
continue
} else {
return err
}
}
return fmt.Errorf("the assignment %s kept moving under the write", row.key)
}
// settleBucket makes a bucket of whole values hold each row's value, by compare-and-set: a key that is absent is made, one
// whose bytes differ from the row's is written over, and one that is the same is left.
func settleBucket[T any](ctx context.Context, b chain.Bucket[T], rows []moveRow) error {
for _, r := range rows {
want, err := json.Marshal(r.value)
if err != nil {
return err
}
settled := false
for try := 0; try < casTries && !settled; try++ {
e, found, err := b.GetRaw(ctx, r.key)
if err != nil {
return err
}
switch {
case !found:
_, err = b.CreateRaw(ctx, r.key, want)
if errors.Is(err, chain.ErrExists) {
continue
}
case bytes.Equal(e.Value, want):
settled = true
continue
default:
_, err = b.UpdateRaw(ctx, r.key, want, e.Revision)
if errors.Is(err, chain.ErrMoved) {
continue
}
}
if err != nil {
return err
}
settled = true
}
if !settled {
return fmt.Errorf("%s kept moving under the write", r.key)
}
}
return nil
}
// settleAssignment makes the value under an assignment's key what its row says, keeping the plan the mirror put beside it:
// before the switch the store is the only truth of the links, so a commit it changed since the first pass is written over.
func settleAssignment(ctx context.Context, c *chain.Chain, row moveRow) error {
want := row.value.(chain.Assignment)
for try := 0; try < casTries; try++ {
cur, found, err := c.Assignments.Get(ctx, row.key)
if err != nil {
return err
}
if !found {
if _, err := c.Assignments.Create(ctx, row.key, want); errors.Is(err, chain.ErrExists) {
continue
} else {
return err
}
}
merged := want
merged.RequiredBy, merged.Provisioning = cur.Value.RequiredBy, cur.Value.Provisioning
a, _ := json.Marshal(merged)
b, _ := json.Marshal(cur.Value)
if bytes.Equal(a, b) {
return nil
}
if _, err := c.Assignments.Update(ctx, row.key, merged, cur.Revision); errors.Is(err, chain.ErrMoved) {
continue
} else {
return err
}
}
return fmt.Errorf("the assignment %s kept moving under the write", row.key)
}
internal/inventory/chain_move_test.go
-949
View File
@@ -1,949 +0,0 @@
//go:build integration
package inventory
import (
"bytes"
"context"
"encoding/json"
"errors"
"reflect"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/chain"
)
// A build row as the store held it before the move: written by SQL, the only hand-written state these tests have, for
// it stands for what the mesh held before this release.
func aBuildRow(t *testing.T, inv *Inventory, id, module, commit, failed string) {
t.Helper()
inv.ExecForTest(t,
`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, source_path, manifest,
built_against, built_contexts, asked, source_fingerprint, build_sources)
values ($1::text, 'novox/' || $2::text, 'main', $2::text, $3::text, 'ace', $4::text,
case when $4::text = ''
then ('[{"name":"app","kind":"image","reference":"artifact-store://' || $2::text || '@sha256:' || $1::text || '"}]')::jsonb
else '[]'::jsonb end,
'modules/' || $2::text, ('{"module":"' || $2::text || '"}')::jsonb,
'["artifact-store://base@sha256:aa"]'::jsonb, '[{"repository":"novox/other","ref":"main"}]'::jsonb,
now() - interval '1 hour', 'fp-' || $1::text,
('[{"paths":["modules/' || $2::text || '/module.json"]}]')::jsonb)`,
id, module, commit, failed)
}
// inTheTables is an inventory before the move: its builds and assignments are rows of the store, which a test of a data
// migration of the store (that reads those rows) needs.
func inTheTables(t *testing.T) *Inventory {
t.Helper()
inv, finish := ForTestBefore(t, 105)
finish()
return inv
}
// reportOf is the report of one kind among those a move made, which are one for each kind in the order they are moved.
func reportOf(t *testing.T, reports []MoveReport, kind string) MoveReport {
t.Helper()
for _, r := range reports {
if r.Kind == kind {
return r
}
}
t.Fatalf("no report of %s among %+v", kind, reports)
return MoveReport{}
}
// allAlready says every kind was moved by an earlier start: the second run writes nothing, and reports one for each kind.
func allAlready(t *testing.T, reports []MoveReport) {
t.Helper()
if len(reports) != 3 {
t.Fatalf("the move reported %d kinds, want 3: %+v", len(reports), reports)
}
for _, r := range reports {
if !r.Already || r.Written != 0 {
t.Fatalf("the second move of %s %+v", r.Kind, r)
}
}
}
func rowsInTheTable(t *testing.T, inv *Inventory, table string) string {
t.Helper()
rows := inv.QueryForTest(t, `select count(*) from `+table)
return rows[0][0]
}
// **The move copies every build row into the builds bucket, and a second run writes nothing** (ADR 0408 §8, issue
// 1090): each row under its id, whole; the kind recorded in chain_moved; and from then on a build is read from the
// bucket and written to it, the table taking no more.
func TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
aBuildRow(t, inv, "build-2", "notes", "c2", "")
aBuildRow(t, inv, "build-3", "wiki", "", "compile error")
// Before the move the tables are read, and the bucket is empty.
if before, err := inv.Builds(ctx, "", 10); err != nil || len(before) != 3 {
t.Fatalf("before the move the tables hold %d builds: %v", len(before), err)
}
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
t.Fatalf("moved before the move: %v", moved)
}
reports, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatal(err)
}
if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.Written != 3 || b.Largest == 0 {
t.Fatalf("the move of the builds reported %+v", reports)
}
moved, err := inv.ChainMoved(ctx)
if err != nil || moved[KindBuilds] != 3 {
t.Fatalf("chain_moved says %v: %v", moved, err)
}
kept, found, err := inv.chain.Builds.Get(ctx, "build-2")
if err != nil || !found {
t.Fatalf("build-2 is not in the bucket: %v %v", found, err)
}
if v := kept.Value; v.Module != "notes" || v.Commit != "c2" || v.Repository != "novox/notes" || v.Path != "modules/notes" ||
v.SourceFingerprint != "fp-build-2" || len(v.Made) != 1 || len(v.Against) != 1 || len(v.Read) != 1 || len(v.Sources) != 1 ||
v.Published == nil || v.Published.Artifact != "sha256:build-2" || v.Asked.IsZero() || v.At.IsZero() || len(v.Manifest) == 0 {
t.Fatalf("build-2 was moved as %+v", v)
}
// From now on the bucket is what is read and written.
if err := inv.RecordBuild(ctx, aBuild("build-4", "notes", "")); err != nil {
t.Fatal(err)
}
if n := rowsInTheTable(t, inv, "build"); n != "3" {
t.Fatalf("the table took a build after the move: %s rows", n)
}
after, err := inv.Builds(ctx, "notes", 10)
if err != nil || len(after) != 3 {
t.Fatalf("notes has %d builds on the bus: %v", len(after), err)
}
// A second run finds the kind moved and writes nothing: every key still has the one value it was made with.
again, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatal(err)
}
allAlready(t, again)
for _, id := range []string{"build-1", "build-2", "build-3", "build-4"} {
if history, err := inv.chain.Builds.History(ctx, chainBuildKey(id)); err != nil || len(history) != 1 {
t.Fatalf("%s has %d values: %v", id, len(history), err)
}
}
}
// failingToCreate is a bucket that fails to make one key while told to: the bus going away half way through a move.
type failingToCreate struct {
chain.Backend
key string
fail *atomic.Bool
}
func (f failingToCreate) Create(ctx context.Context, key string, value []byte) (uint64, error) {
if key == f.key && f.fail.Load() {
return 0, errors.New("the bus went away")
}
return f.Backend.Create(ctx, key, value)
}
// **A controller restarted half way through the move finishes it: the bucket equals the table, with no key made twice and
// no refusal** (ADR 0408 §8). Before a kind is recorded as moved nothing but the move writes its keys, and every write of
// the kind goes to its table (each method of a kind asks ChainReads first and writes the store until it says moved), so the
// table is the only truth: a key the first start made is the row it was, and the second start settles it to the row it is.
func TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
aBuildRow(t, inv, "build-2", "notes", "c2", "")
aBuildRow(t, inv, "build-3", "wiki", "c3", "")
var away atomic.Bool
away.Store(true)
builds := chain.NewMemory()
inv.UseChain(chain.New(failingToCreate{builds, "build-3", &away}, chain.NewMemory(), chain.NewMemory(),
func(context.Context) error { return nil }))
// The first start made two of the three keys and the bus went away.
_, err := inv.MoveChain(ctx, nil)
var refusal *Refusal
if err == nil || errors.As(err, &refusal) {
t.Fatalf("the first start answered %v: it should fail to be tried again, not be refused", err)
}
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
t.Fatalf("a move that did not finish recorded %v", moved)
}
if keys, _ := builds.Keys(ctx); len(keys) != 2 {
t.Fatalf("the first start made %v", keys)
}
// The controller is started again, the bus is back, and the move finishes.
away.Store(false)
reports, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatalf("the second start was refused or failed: %v", err)
}
if b := reportOf(t, reports, KindBuilds); b.Written != 1 || b.Present != 2 || b.Rows != 3 {
t.Fatalf("the move reported %+v", b)
}
if moved, _ := inv.ChainMoved(ctx); moved[KindBuilds] != 3 {
t.Fatalf("chain_moved says %v", moved)
}
// The bucket is the table: three keys, each made once and not again, each the row's own.
for _, id := range []string{"build-1", "build-2", "build-3"} {
history, err := builds.History(ctx, id)
if err != nil || len(history) != 1 {
t.Fatalf("%s has %d values: %v", id, len(history), err)
}
}
rows, err := inv.allBuildRows(ctx)
if err != nil || len(rows) != 3 {
t.Fatalf("the table holds %d builds: %v", len(rows), err)
}
for _, row := range rows {
got, found, err := inv.chain.Builds.Get(ctx, row.ID)
want, _ := json.Marshal(toChainBuild(row))
have, _ := json.Marshal(got.Value)
if err != nil || !found || string(want) != string(have) {
t.Fatalf("%s in the bucket is %s, the row says %s (%v)", row.ID, have, want, err)
}
}
}
// **A value over the bus's largest message stops the move with nothing written and nothing switched** (ADR 0408 §6):
// a Refusal, no key in the bucket, no kind recorded, and the tables still what is read.
func TestTheMoveRefusesAValueOverTheLargestMessage(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
inv.ExecForTest(t, `insert into build (id, repository, module, commit_hash, manifest)
values ('build-big', 'novox/wiki', 'wiki', 'c9', ('{"x":"' || repeat('a', 1100000) || '"}')::jsonb)`)
_, err := inv.MoveChain(ctx, nil)
var refusal *Refusal
if !errors.As(err, &refusal) || refusal.Kind != KindBuilds {
t.Fatalf("the move answered %v", err)
}
if keys, _ := inv.chain.Builds.Keys(ctx); len(keys) != 0 {
t.Fatalf("the bucket holds %v after a refused move", keys)
}
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
t.Fatalf("a refused move recorded %v", moved)
}
if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 2 {
t.Fatalf("the tables are no longer read: %d builds, %v", len(builds), err)
}
}
// losing is a bucket that says it wrote a key and kept nothing: the move's count of keys against rows must see it.
type losing struct {
chain.Backend
lose string
}
func (l losing) Create(ctx context.Context, key string, value []byte) (uint64, error) {
if key == l.lose {
return 1, nil
}
return l.Backend.Create(ctx, key, value)
}
// **The move refuses on a count that differs** (ADR 0408 §8): a row whose key is not in the bucket afterwards stops it,
// nothing is recorded, and the tables are still read.
func TestTheMoveRefusesOnACountThatDiffers(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
aBuildRow(t, inv, "build-2", "notes", "c2", "")
inv.UseChain(chain.New(losing{chain.NewMemory(), "build-2"}, chain.NewMemory(), chain.NewMemory(),
func(context.Context) error { return nil }))
_, err := inv.MoveChain(ctx, nil)
var refusal *Refusal
if !errors.As(err, &refusal) {
t.Fatalf("the move answered %v", err)
}
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
t.Fatalf("a refused move recorded %v", moved)
}
}
// **Without the lease the move writes nothing**, and fails the way the bus being away does — tried again, not refused:
// the tables are still read.
func TestTheMoveWithoutTheLeaseWritesNothing(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
mem := chain.NewMemory()
inv.UseChain(chain.New(mem, chain.NewMemory(), chain.NewMemory(),
func(context.Context) error { return errors.New("another controller holds the lease") }))
_, err := inv.MoveChain(ctx, nil)
var refusal *Refusal
if err == nil || errors.As(err, &refusal) || !errors.Is(err, chain.ErrNotHeld) {
t.Fatalf("the move answered %v", err)
}
if mem.Writes() != 0 {
t.Fatalf("the bucket took %d writes without the lease", mem.Writes())
}
if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
t.Fatalf("recorded %v", moved)
}
}
// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its
// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none.
func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
first := aBuild("build-1", "notes", "")
first.Commit = "c1"
first.Against = []string{"artifact-store://base@sha256:aa"}
first.ReuseKey = "rk-1"
second := aBuild("build-2", "notes", "")
second.Commit = "c2"
failed := aBuild("build-3", "notes", "compile error")
other := aBuild("build-4", "wiki", "")
for _, b := range []Build{first, second, failed, other, first} {
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
if n := rowsInTheTable(t, inv, "build"); n != "0" {
t.Fatalf("the table took %s builds", n)
}
kept, found, err := inv.chain.Builds.Get(ctx, "build-1")
if err != nil || !found || kept.Value.ReuseKey != "rk-1" {
t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err)
}
if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 1 {
t.Fatalf("a build recorded twice has %d values", len(history))
}
list, err := inv.Builds(ctx, "notes", 10)
if err != nil || len(list) != 3 {
t.Fatalf("notes has %d builds: %v", len(list), err)
}
one, found, err := inv.BuildByID(ctx, "build-2")
if err != nil || !found || one.Commit != "c2" || one.Module != "notes" {
t.Fatalf("build-2 reads as %+v %v %v", one, found, err)
}
if _, found, _ := inv.BuildByID(ctx, "build-nope"); found {
t.Fatal("a build nobody recorded was found")
}
if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" {
t.Fatalf("the build of notes at c1 is %q: %v", id, err)
}
held, err := inv.Held(ctx)
if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" {
t.Fatalf("held %v: %v", held, err)
}
against, err := inv.BuiltAgainst(ctx)
if err != nil || len(against["notes"]) != 0 {
// The newest successful build of notes, build-2, stood on nothing.
t.Fatalf("built against %v: %v", against, err)
}
announced, err := inv.Announceable(ctx)
if err != nil || len(announced) != 3 {
t.Fatalf("announceable %d: %v", len(announced), err)
}
prints, err := inv.Fingerprints(ctx)
if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" {
t.Fatalf("fingerprints %v: %v", prints, err)
}
}
// A build request row as the store held it before the move.
func aRequestRow(t *testing.T, inv *Inventory, id, notAsked string) {
t.Helper()
inv.ExecForTest(t,
`insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at)
values ($1::text, 'novox/notes', '', 'modules/notes', 'main', 'c1', 'merge', nullif($2::text, ''), now() - interval '1 hour')`,
id, notAsked)
}
// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue
// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none
// after the outcome is heard. The table of requests takes none.
func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true}
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
t.Fatalf("a request kept twice: %v", err)
}
if n := rowsInTheTable(t, inv, "build_request"); n != "0" {
t.Fatalf("the table took %s requests", n)
}
if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 {
t.Fatalf("a request nobody answered is a build: %+v %v", builds, err)
}
if _, found, _ := inv.BuildByID(ctx, "build-1"); found {
t.Fatal("a request nobody answered has an outcome")
}
got, found, err := inv.BuildRequestByID(ctx, "build-1")
if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() {
t.Fatalf("the request reads back as %+v %v %v", got, found, err)
}
if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil {
t.Fatal(err)
}
asked, err := inv.RequestsNamed(ctx, "notes")
if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" {
t.Fatalf("the request is %+v: %v", asked, err)
}
if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) {
t.Fatalf("requested names %v: %v", names, err)
}
// The outcome comes: the same key, now a build, with the request beside it.
built := aBuild("build-1", "notes", "")
if err := inv.RecordBuild(ctx, built); err != nil {
t.Fatal(err)
}
if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" {
t.Fatalf("the outcome reads as %+v %v %v", one, found, err)
}
answered, err := inv.RequestsNamed(ctx, "notes")
if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" {
t.Fatalf("the request answered is %+v: %v", answered, err)
}
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" {
t.Fatalf("the request was lost with the outcome: %+v", again)
}
// Heard: the marks of an unanswered request change nothing.
if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil {
t.Fatal(err)
}
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" {
t.Fatalf("an answered request was marked not asked: %+v", again)
}
if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 3 {
// asked, marked unknown, answered.
t.Fatalf("the record has %d values", len(history))
}
// An outcome heard before its request is kept, and the request lands beside it.
if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil {
t.Fatal(err)
}
if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" {
t.Fatalf("the request kept after its outcome reads %+v", late)
}
// A request older than the time they are kept is no longer listed.
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old",
At: time.Now().Add(-2 * KeptFor)}); err != nil {
t.Fatal(err)
}
if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") {
t.Fatalf("a request past its time is listed: %v", names)
}
}
// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a
// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it
// is asked and so asked at most once.
func TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
aRequestRow(t, inv, "build-1", "")
aRequestRow(t, inv, "build-2", "")
aRequestRow(t, inv, "not-asked-c1-modules-notes", "no seat holder")
if requests, err := inv.RequestsNamed(ctx, "notes"); err != nil || len(requests) != 3 {
t.Fatalf("before the move the table holds %d requests: %v", len(requests), err)
}
reports, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatal(err)
}
if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.TableRows != 4 {
t.Fatalf("the move reported %+v", b)
}
one, _, _ := inv.chain.Builds.Get(ctx, "build-1")
if one.Value.State != "" || one.Value.Request == nil || one.Value.Request.For != "merge" || one.Value.Module != "notes" {
t.Fatalf("an answered request moved as %+v", one.Value)
}
two, _, _ := inv.chain.Builds.Get(ctx, "build-2")
if two.Value.State != chain.BuildAsked || two.Value.Request.Reask != "" {
t.Fatalf("an unanswered request moved as %+v", two.Value)
}
never, _, _ := inv.chain.Builds.Get(ctx, "not-asked-c1-modules-notes")
if never.Value.State != chain.BuildAsked || never.Value.Request.NotAsked != "no seat holder" || never.Value.Request.Reask != chain.ReaskPending {
t.Fatalf("a request never handed over moved as %+v", never.Value)
}
requests, err := inv.RequestsNamed(ctx, "notes")
if err != nil || len(requests) != 3 {
t.Fatalf("after the move the bucket holds %d requests: %v", len(requests), err)
}
toAsk, err := inv.RequestsToAskAgain(ctx)
if err != nil || len(toAsk) != 1 || toAsk[0].ID != "not-asked-c1-modules-notes" {
t.Fatalf("to ask again: %+v %v", toAsk, err)
}
first, err := inv.ClaimToAskAgain(ctx, toAsk[0].ID)
second, err2 := inv.ClaimToAskAgain(ctx, toAsk[0].ID)
if err != nil || err2 != nil || !first || second {
t.Fatalf("claimed %v then %v: %v %v", first, second, err, err2)
}
if left, _ := inv.RequestsToAskAgain(ctx); len(left) != 0 {
t.Fatalf("a claimed request is still to be asked: %+v", left)
}
if err := inv.AskedAgainAs(ctx, toAsk[0].ID, "build-9"); err != nil {
t.Fatal(err)
}
if done, _, _ := inv.BuildRequestByID(ctx, toAsk[0].ID); done.NotAsked != "no seat holder" {
t.Fatalf("the request lost its words: %+v", done)
}
}
// **The move carries every plan with the merges it answers, and the plans read the same from the bucket as from the
// table** (ADR 0408 §8, ADR 0276, issue 1092): a batch with a merge, a walk that was built and rolled, a merge waiting
// for a walk of its own, and a merge whose plan the store never kept — each in the record of the plan that held it last.
func TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
batch := aBatch("plan-1", time.Now().Add(-3*time.Hour).UTC())
walk := aBatch("plan-2", time.Now().Add(-2*time.Hour).UTC())
walk.State, walk.Tier, walk.Tiers = PlanRolling, 1, [][]string{{"a"}, {"b"}}
walk.Modules = map[string]*PlanModule{"a": {State: "built"}, "b": {}}
walk.Delivery = &PlanDelivery{Awaits: "mesh-delivery", Merges: []PlanMerge{{Repository: "novox/mesh-catalog", Commit: "m-waiting"}}}
for _, p := range []*Plan{batch, walk} {
if err := inv.SavePlan(ctx, p); err != nil {
t.Fatal(err)
}
}
walk.Note = "second save"
if err := inv.SavePlan(ctx, walk); err != nil {
t.Fatal(err)
}
for _, m := range []BatchedMerge{aMerge("m-batch", "plan-1"), aMerge("m-walk", "plan-2"), aMerge("m-dangling", "plan-never-kept")} {
if added, err := inv.AddMerge(ctx, m); err != nil || !added {
t.Fatalf("added %v: %v", added, err)
}
}
waiting := aMerge("m-waiting", "plan-2")
if _, err := inv.AddMerge(ctx, waiting); err != nil {
t.Fatal(err)
}
if err := inv.AnswerMerge(ctx, waiting.Repository, waiting.Commit, "", true); err != nil {
t.Fatal(err)
}
wasPlans, err := inv.RecentPlans(ctx, 10)
if err != nil || len(wasPlans) != 2 {
t.Fatalf("the table holds %d plans: %v", len(wasPlans), err)
}
wasMerge, _, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk")
reports, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatal(err)
}
var moved MoveReport
for _, r := range reports {
if r.Kind == KindRollouts {
moved = r
}
}
// Two plans, and the record of the merge whose plan the store never kept.
if moved.Rows != 3 || moved.TableRows != 6 || moved.Written != 3 {
t.Fatalf("the move of the rollouts reported %+v", moved)
}
nowPlans, err := inv.RecentPlans(ctx, 10)
if err != nil || len(nowPlans) != 2 {
t.Fatalf("the bucket holds %d plans: %v", len(nowPlans), err)
}
for k := range wasPlans {
was, now := wasPlans[k], nowPlans[k]
if was.ID != now.ID || was.State != now.State || was.Tier != now.Tier || was.Revision != now.Revision || was.Note != now.Note ||
!was.Created.Equal(now.Created) || !reflect.DeepEqual(was.Tiers, now.Tiers) || len(was.Modules) != len(now.Modules) {
t.Errorf("plan %s was %+v and is %+v", was.ID, was, now)
}
}
if open, _ := inv.OpenPlans(ctx); len(open) != 1 || open[0].ID != "plan-2" {
t.Fatalf("open plans %+v", open)
}
if batches, _ := inv.Batches(ctx); len(batches) != 1 || batches[0].ID != "plan-1" {
t.Fatalf("batches %+v", batches)
}
if got, _ := inv.MergesOf(ctx, "plan-1"); len(got) != 1 || got[0].Commit != "m-batch" {
t.Fatalf("the batch answers %+v", got)
}
nowMerge, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk")
if !known || nowMerge.Plan != "plan-2" || nowMerge.Branch != wasMerge.Branch || !nowMerge.Merged.Equal(wasMerge.Merged) ||
!nowMerge.Heard.Equal(wasMerge.Heard) || !sameEvent(nowMerge.Event, wasMerge.Event) {
t.Fatalf("a merge was %+v and is %+v", wasMerge, nowMerge)
}
alone, _ := inv.AloneMerges(ctx)
if len(alone) != 1 || alone[0].Commit != "m-waiting" || alone[0].Plan != "" {
t.Fatalf("waiting: %+v", alone)
}
if dangling, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-dangling"); !known || dangling.Plan != "" || dangling.Alone {
t.Fatalf("a merge whose plan was never kept reads as %+v %v", dangling, known)
}
if n := rowsInTheTable(t, inv, "release_plan"); n != "2" {
t.Fatalf("the table is %s plans", n)
}
// From now on a plan is saved to the bucket and the table takes none.
later := aBatch("plan-3", time.Now().UTC())
if err := inv.SavePlan(ctx, later); err != nil {
t.Fatal(err)
}
if n := rowsInTheTable(t, inv, "release_plan"); n != "2" {
t.Fatalf("the table took a plan after the move: %s", n)
}
again, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatal(err)
}
allAlready(t, again)
}
// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a
// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module
// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the
// assignment goes with it.
func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
inv, node := aNodeWithModules(t, "postgres", "web")
ctx := t.Context()
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
start := generationNow(t, inv)
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added {
t.Fatalf("assigned %v: %v", added, err)
}
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added {
t.Fatalf("assigned again %v: %v", added, err)
}
if n := rowsInTheTable(t, inv, "assignment"); n != "0" {
t.Fatalf("the table took %s assignments", n)
}
kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() {
t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err)
}
if generationNow(t, inv) <= start {
t.Fatal("an assignment left the generation where it was")
}
if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) {
t.Fatalf("a module the mesh does not know: %v", err)
}
if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) {
t.Fatalf("assigned on %s: %v %v", node, on, err)
}
if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) {
t.Fatalf("running: %v %v", running, err)
}
entries, err := inv.Catalogued(ctx)
if err != nil {
t.Fatal(err)
}
for _, e := range entries {
switch e.Manifest.Module {
case "postgres":
if !reflect.DeepEqual(e.On, []string{node}) {
t.Errorf("postgres is on %v", e.On)
}
case "web":
if len(e.On) != 0 {
t.Errorf("web is on %v", e.On)
}
}
}
if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) {
t.Fatalf("a module still assigned was forgotten: %v", err)
}
// A seat is held by an assignment, or by nothing.
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil {
t.Fatal("a seat was handed to a module that is not assigned")
}
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil {
t.Fatalf("a seat handed to an assignment: %v", err)
}
before := generationNow(t, inv)
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
t.Fatal(err)
}
if generationNow(t, inv) <= before {
t.Fatal("an unassignment left the generation where it was")
}
// No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history.
gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() {
t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err)
}
if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() ||
history[1].Value.Stands() {
t.Fatalf("the earlier values of the key: %+v", history)
}
if on, _ := inv.Assigned(ctx, node); len(on) != 0 {
t.Fatalf("an ended assignment is still assigned: %v", on)
}
if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 {
t.Fatalf("an ended assignment is running: %v", running)
}
if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 {
t.Fatalf("the seat the assignment held is held by %+v: %v", held, err)
}
if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" {
t.Fatalf("an unassignment of nothing: %v", err)
}
if err := inv.ForgetModule(ctx, "postgres"); err != nil {
t.Fatalf("a module nothing is assigned could not be forgotten: %v", err)
}
// Made again after it ended: a fresh value over the ended one, which stays in history.
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
t.Fatal(added, err)
}
if added, err := inv.Assign(ctx, node, "web"); err != nil || added {
t.Fatal(added, err)
}
if err := inv.Unassign(ctx, node, "web"); err != nil {
t.Fatal(err)
}
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
t.Fatalf("assigned again after it ended: %v %v", added, err)
}
again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web")
if !again.Value.Stands() || !again.Value.Ended.IsZero() {
t.Fatalf("the assignment made again is %+v", again.Value)
}
if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() {
t.Fatalf("the values of the key: %+v", history)
}
}
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
// ends an assignment.
func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) {
inv, node := aNodeWithModules(t, "dunst", "web")
ctx := t.Context()
for _, m := range []string{"dunst", "web"} {
if _, err := inv.Assign(ctx, node, m); err != nil {
t.Fatal(err)
}
}
// A build and a commit put in the value by another write.
cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
cur.Value.Build, cur.Value.Commit = "build-1", "c1"
if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil {
t.Fatal(err)
}
record, _ := inv.NodeByName(ctx, node)
digest := "sha256:" + strings.Repeat("d", 64)
plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest,
Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc",
Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}}
raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`)
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil {
t.Fatal(err)
}
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
written, removed, err := inv.KeepAssignmentPlans(ctx)
if err != nil || written != 2 || removed != 0 {
t.Fatalf("kept: %d written, %d ended: %v", written, removed, err)
}
dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() {
t.Fatalf("dunst's value is %+v", dunst.Value)
}
if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil {
t.Fatalf("web has a plan: %+v", web.Value)
}
notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify")
if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil {
t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found)
}
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
t.Fatalf("a pulled-in module is assigned: %v", on)
}
if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 {
t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err)
}
// Sent again without the pulled-in module: its value ends, the assignments stay.
only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`)
if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil {
t.Fatal(err)
}
if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 {
t.Fatalf("ended %d: %v", removed, err)
}
if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() {
t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value)
}
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
t.Fatalf("assigned: %v", on)
}
}
// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value**
// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build
// already is left as it is.
func TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
for _, name := range []string{"anchor", "laptop"} {
if _, err := inv.AddNode(ctx, name); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"notes", "wiki"} {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
t.Fatal(err)
}
}
for _, a := range [][2]string{{"anchor", "notes"}, {"anchor", "wiki"}, {"laptop", "notes"}} {
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
t.Fatal(err)
}
}
aBuildRow(t, inv, "build-notes", "notes", "c-notes", "")
anchor, _ := inv.NodeByName(ctx, "anchor")
if err := inv.RecordSent(ctx, anchor.ID, "sent-1", map[string]string{"notes": "c-notes"}); err != nil {
t.Fatal(err)
}
// The mirror of the store put a value under one key, with a plan.
if _, err := inv.chain.Assignments.Create(ctx, "anchor.wiki", chain.Assignment{Node: "anchor", Module: "wiki",
Assigned: time.Now().UTC(), Provisioning: json.RawMessage(`{"plan":1}`)}); err != nil {
t.Fatal(err)
}
reports, err := inv.MoveChain(ctx, nil)
if err != nil {
t.Fatal(err)
}
var moved MoveReport
for _, r := range reports {
if r.Kind == KindAssignments {
moved = r
}
}
if moved.Rows != 3 || moved.Written+moved.Present != 3 {
t.Fatalf("the move of the assignments reported %+v", moved)
}
notes, _, _ := inv.chain.Assignments.Get(ctx, "anchor.notes")
if notes.Value.Commit != "c-notes" || notes.Value.Build != "build-notes" || notes.Value.Since.IsZero() || notes.Value.Assigned.IsZero() {
t.Fatalf("anchor.notes moved as %+v", notes.Value)
}
if notes.Value.Artifact != "sha256:build-notes" {
t.Fatalf("what the build published: %+v", notes.Value)
}
laptop, _, _ := inv.chain.Assignments.Get(ctx, "laptop.notes")
if laptop.Value.Commit != "" || laptop.Value.Build != "" {
t.Fatalf("a machine never sent has a build: %+v", laptop.Value)
}
wiki, _, _ := inv.chain.Assignments.Get(ctx, "anchor.wiki")
if string(wiki.Value.Provisioning) != `{"plan":1}` {
t.Fatalf("the plan the mirror put in the value was lost: %+v", wiki.Value)
}
// From now on the bucket is what is written.
if _, err := inv.Assign(ctx, "laptop", "wiki"); err != nil {
t.Fatal(err)
}
if n := rowsInTheTable(t, inv, "assignment"); n != "3" {
t.Fatalf("the table took an assignment after the move: %s", n)
}
if on, _ := inv.Assigned(ctx, "laptop"); !reflect.DeepEqual(on, []string{"notes", "wiki"}) {
t.Fatalf("assigned on laptop: %v", on)
}
}
// hooked runs a function the first time a key is made in a bucket: the old controller writing the store between the move's
// first pass and its switch.
type hooked struct {
chain.Backend
once *sync.Once
then func()
}
func (h hooked) Create(ctx context.Context, key string, value []byte) (uint64, error) {
rev, err := h.Backend.Create(ctx, key, value)
h.once.Do(h.then)
return rev, err
}
// **A row the old controller changes after the move's first pass is in the bucket when the kind is recorded as moved**
// (ADR 0408 §8, issue 981 note 7 Q2): before the switch the store is the only truth, so under the lock the move reads the
// rows again and writes over every key that differs — a plan's state and a build's failure — and ends an assignment the
// store dropped. The count alone would have kept the older value and refused the next save of the plan.
func TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved(t *testing.T) {
inv, finish := ForTestBefore(t, 105)
finish()
ctx := context.Background()
aBuildRow(t, inv, "build-1", "notes", "c1", "")
for _, n := range []string{"anchor"} {
if _, err := inv.AddNode(ctx, n); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"notes", "wiki"} {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"notes", "wiki"} {
if _, err := inv.Assign(ctx, "anchor", m); err != nil {
t.Fatal(err)
}
}
p := aBatch("plan-1", time.Now().Add(-time.Hour).UTC())
p.State, p.Tiers = PlanBuilding, [][]string{{"notes"}}
if err := inv.SavePlan(ctx, p); err != nil {
t.Fatal(err)
}
// Between a kind's first pass and its switch the old controller saves the plan again, fails the build and unassigns wiki:
// each is done when the first key of its kind is made, so the first pass has read the rows as they were.
after := func(sql string) hooked {
return hooked{chain.NewMemory(), &sync.Once{}, func() { inv.ExecForTest(t, sql) }}
}
inv.UseChain(chain.New(
after(`update build set failed = 'it broke later' where id = 'build-1'`),
after(`update release_plan set state = 'rolling', revision = revision + 1, note = 'the old controller was here' where id = 'plan-1'`),
after(`delete from assignment where module = 'wiki'`),
func(context.Context) error { return nil }))
if _, err := inv.MoveChain(ctx, nil); err != nil {
t.Fatal(err)
}
now, err := inv.PlanByID(ctx, "plan-1")
if err != nil {
t.Fatal(err)
}
if now.State != PlanRolling || now.Note != "the old controller was here" || now.Revision != 2 {
t.Fatalf("the bucket holds a plan from before the change: %s %q revision %d", now.State, now.Note, now.Revision)
}
if b, found, _ := inv.BuildByID(ctx, "build-1"); !found || b.Failed != "it broke later" {
t.Fatalf("the bucket holds the build as it was: %+v", b)
}
if on, _ := inv.Assigned(ctx, "anchor"); !slices.Equal(on, []string{"notes"}) {
t.Fatalf("anchor is assigned %v, though the store dropped wiki", on)
}
// The plan can be saved at the revision the store had.
now.Note = "saved after the move"
if err := inv.SavePlan(ctx, &now); err != nil {
t.Fatalf("the next save of the plan is refused: %v", err)
}
}
// sameEvent says two encodings of the forge's announcement are one: the store keeps it as jsonb, which spells it with its own
// white space, and whoever reads it (the cut, mesh-delivery) decodes it, so the white space is no part of what it says.
func sameEvent(a, b json.RawMessage) bool {
var x, y bytes.Buffer
return json.Compact(&x, a) == nil && json.Compact(&y, b) == nil && x.String() == y.String()
}
internal/inventory/chain_moved_test.go
-45
View File
@@ -1,45 +0,0 @@
package inventory
import (
"strings"
"testing"
)
// **The move of the chain has a switch, made by the last migration and written by no migration** (ADR 0408 §8, issue
// 1089): migration 0104 makes the one-row table chain_moved, and does not drop a table of the chain, which a later
// delivery does once the move is read back.
func TestMigration0104MakesTheSwitchOfTheChainsMoveAndNothingElse(t *testing.T) {
migrations, err := Migrations()
if err != nil {
t.Fatal(err)
}
var found bool
for _, m := range migrations {
if m.Number != 104 {
continue
}
found = true
sql := strings.ToLower(m.SQL)
if !strings.Contains(sql, "create table chain_moved") || !strings.Contains(sql, "check (only_one)") {
t.Errorf("migration 0104 does not make the one-row chain_moved:\n%s", m.SQL)
}
for _, line := range strings.Split(sql, "\n") {
if strings.HasPrefix(strings.TrimSpace(line), "--") {
continue
}
for _, verb := range []string{"drop table", "insert ", "delete ", "update "} {
if strings.Contains(line, verb) {
t.Errorf("migration 0104 does more than make the switch: %q", line)
}
}
// The one other change: a seat's holder is no longer a foreign key to the assignment (issue 1093); the tables of
// the chain are dropped by a later migration (issue 1096).
if strings.Contains(line, "alter ") && !strings.Contains(line, "alter table seat_holding drop constraint") {
t.Errorf("migration 0104 alters more than the seat holdings' foreign key: %q", line)
}
}
}
if !found {
t.Fatal("no migration 0104")
}
}
internal/inventory/chain_pending_test.go
-3
View File
@@ -59,9 +59,6 @@ func TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket(t *testing.T) {
if history, _ := inv.chain.Assignments.History(ctx, "anchor.sensors"); len(history) != 2 {
t.Fatalf("the earlier wait is in the key's history: %d values", len(history))
}
if n := rowsInTheTable(t, inv, "pending_assignment"); n != "0" {
t.Fatalf("the table took %s pending assignments", n)
}
}
// **A pending value is skipped by what reads assignments** (issue 1094): the machine is assigned nothing by it, the module
internal/inventory/chain_requests.go
+6 -6
View File
@@ -123,8 +123,8 @@ func (i *Inventory) chainRequests(ctx context.Context, c *chain.Chain) ([]Reques
// RequestsToAskAgain are the requests the move found never handed over (issue 981 note 7 Q5), not yet asked again: each is
// asked once, and never twice.
func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, error) {
c, on, err := i.ChainReads(ctx, KindBuilds)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
all, err := c.Builds.All(ctx)
@@ -144,8 +144,8 @@ func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, err
// ClaimToAskAgain takes a request to ask again, by compare-and-set: true for the one process that took it. **Claimed
// before it is asked and settled after**, so a crash between loses the ask rather than making it twice.
func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error) {
c, on, err := i.ChainReads(ctx, KindBuilds)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return false, err
}
cur, found, err := c.Builds.Get(ctx, chainBuildKey(id))
@@ -164,8 +164,8 @@ func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error
// AskedAgainAs records the build id a claimed request was asked as.
func (i *Inventory) AskedAgainAs(ctx context.Context, id, asked string) error {
c, on, err := i.ChainReads(ctx, KindBuilds)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return err
}
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.Reask = asked })
internal/inventory/chain_rollouts.go
+27 -27
View File
@@ -139,17 +139,17 @@ func (v *rolloutView) list() []viewEntry {
// rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole.
func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) {
i.moved.mu.Lock()
v := i.moved.view
i.state.mu.Lock()
v := i.state.view
if v == nil || v.chain != c || v.dead.Load() {
if v != nil {
v.stop()
}
v = newRolloutView(c)
i.moved.view = v
i.state.view = v
v.start()
}
i.moved.mu.Unlock()
i.state.mu.Unlock()
select {
case <-v.ready:
case <-ctx.Done():
@@ -165,11 +165,11 @@ func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutVi
// stopViews ends the view's watch: the inventory is closing.
func (i *Inventory) stopViews() {
i.moved.mu.Lock()
defer i.moved.mu.Unlock()
if i.moved.view != nil {
i.moved.view.stop()
i.moved.view = nil
i.state.mu.Lock()
defer i.state.mu.Unlock()
if i.state.view != nil {
i.state.view.stop()
i.state.view = nil
}
}
@@ -492,12 +492,12 @@ func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m Batched
return false, err
}
if !found {
i.moved.mu.Lock()
if i.moved.pending == nil {
i.moved.pending = map[string]pendingMerge{}
i.state.mu.Lock()
if i.state.pending == nil {
i.state.pending = map[string]pendingMerge{}
}
i.moved.pending[key] = pendingMerge{plan: m.Plan, m: held}
i.moved.mu.Unlock()
i.state.pending[key] = pendingMerge{plan: m.Plan, m: held}
i.state.mu.Unlock()
}
return true, nil
}
@@ -712,12 +712,12 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []
held, has := copies[key]
if !has {
// A merge carried for a plan not saved yet goes where it is moved.
i.moved.mu.Lock()
if pm, pending := i.moved.pending[key]; pending && m.Plan != "" {
i.state.mu.Lock()
if pm, pending := i.state.pending[key]; pending && m.Plan != "" {
pm.plan, pm.m.Alone = m.Plan, m.Alone
i.moved.pending[key] = pm
i.state.pending[key] = pm
}
i.moved.mu.Unlock()
i.state.mu.Unlock()
continue
}
w := winner(held)
@@ -803,14 +803,14 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []
if e := edits[p.ID]; e != nil {
held = e.apply(held)
}
i.moved.mu.Lock()
for key, pm := range i.moved.pending {
i.state.mu.Lock()
for key, pm := range i.state.pending {
if pm.plan == p.ID {
held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held)
_ = key
}
}
i.moved.mu.Unlock()
i.state.mu.Unlock()
record.Held = held
entered, duration := tierLeftOf(old, *p, time.Now())
record.TierEntered, record.Updated = entered, time.Now().UTC()
@@ -859,13 +859,13 @@ func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []
}
}
i.moved.mu.Lock()
for key, pm := range i.moved.pending {
i.state.mu.Lock()
for key, pm := range i.state.pending {
if _, saved := done[pm.plan]; saved {
delete(i.moved.pending, key)
delete(i.state.pending, key)
}
}
i.moved.mu.Unlock()
i.state.mu.Unlock()
for _, p := range plans {
w := done[p.ID]
p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held
@@ -908,8 +908,8 @@ func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uin
// RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired
// at the next cut (ADR 0408, issue 1092). It says how many it repaired.
func (i *Inventory) RepairMerges(ctx context.Context) (int, error) {
c, on, err := i.ChainReads(ctx, KindRollouts)
if err != nil || !on {
c, err := i.theChain(ctx)
if err != nil {
return 0, err
}
v, err := i.rolloutView(ctx, c)
internal/inventory/chain_rollouts_test.go
+6 -9
View File
@@ -15,7 +15,7 @@ import (
func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) {
t.Helper()
c := chain.InMemory(nil)
inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
inv.UseChain(c)
t.Cleanup(inv.stopViews)
return inv, c
@@ -117,10 +117,10 @@ func TestAMergeAddedToAPlanNotSavedYetIsWrittenByItsSave(t *testing.T) {
}
return errors.New("the lease is not held")
})
other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded}
other := &Inventory{state: &chainState{}, recorded: inv.recorded}
other.UseChain(away)
t.Cleanup(other.stopViews)
other.moved.pending = inv.moved.pending
other.state.pending = inv.state.pending
p := aBatch("plan-new", plansAt)
if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); !errors.Is(err, chain.ErrNotHeld) {
t.Fatalf("the save answered %v", err)
@@ -158,7 +158,7 @@ func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *test
var fail atomic.Bool
rollouts := failingOn{Backend: chain.NewMemory(), key: "plan-a", fail: &fail}
c = chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil })
inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
inv.UseChain(c)
t.Cleanup(inv.stopViews)
ctx := context.Background()
@@ -190,9 +190,6 @@ func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *test
if n := len(mergeHolders(v.list())["novox/mesh-catalog|m1"]); n != 2 {
t.Fatalf("the merge is held by %d records, want the two of a stopped move", n)
}
inv.moved.mu.Lock()
inv.moved.kind = map[string]bool{KindRollouts: true}
inv.moved.mu.Unlock()
repaired := 0
for k, copies := range mergeHolders(v.list()) {
if len(copies) < 2 {
@@ -310,7 +307,7 @@ func (c *countingBackend) Keys(ctx context.Context) ([]string, error) {
func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) {
counting := &countingBackend{Backend: chain.NewMemory()}
c := chain.New(chain.NewMemory(), counting, chain.NewMemory(), func(context.Context) error { return nil })
inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
inv.UseChain(c)
t.Cleanup(inv.stopViews)
ctx := context.Background()
@@ -337,7 +334,7 @@ func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) {
t.Fatalf("the bucket was read %d times and its keys %d times for the surveys", counting.gets.Load()-before, counting.keys.Load())
}
// Another process writes; the view follows.
other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded}
other := &Inventory{state: &chainState{}, recorded: inv.recorded}
other.UseChain(c)
t.Cleanup(other.stopViews)
late := aBatch("plan-4", plansAt.Add(time.Hour))
internal/inventory/chain_store_test.go
+346
View File
@@ -0,0 +1,346 @@
//go:build integration
package inventory
import (
"context"
"errors"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
)
// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its
// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none.
func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
first := aBuild("build-1", "notes", "")
first.Commit = "c1"
first.Against = []string{"artifact-store://base@sha256:aa"}
first.ReuseKey = "rk-1"
second := aBuild("build-2", "notes", "")
second.Commit = "c2"
failed := aBuild("build-3", "notes", "compile error")
other := aBuild("build-4", "wiki", "")
for _, b := range []Build{first, second, failed, other, first} {
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
kept, found, err := inv.chain.Builds.Get(ctx, "build-1")
if err != nil || !found || kept.Value.ReuseKey != "rk-1" {
t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err)
}
if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 1 {
t.Fatalf("a build recorded twice has %d values", len(history))
}
list, err := inv.Builds(ctx, "notes", 10)
if err != nil || len(list) != 3 {
t.Fatalf("notes has %d builds: %v", len(list), err)
}
one, found, err := inv.BuildByID(ctx, "build-2")
if err != nil || !found || one.Commit != "c2" || one.Module != "notes" {
t.Fatalf("build-2 reads as %+v %v %v", one, found, err)
}
if _, found, _ := inv.BuildByID(ctx, "build-nope"); found {
t.Fatal("a build nobody recorded was found")
}
if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" {
t.Fatalf("the build of notes at c1 is %q: %v", id, err)
}
held, err := inv.Held(ctx)
if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" {
t.Fatalf("held %v: %v", held, err)
}
against, err := inv.BuiltAgainst(ctx)
if err != nil || len(against["notes"]) != 0 {
// The newest successful build of notes, build-2, stood on nothing.
t.Fatalf("built against %v: %v", against, err)
}
announced, err := inv.Announceable(ctx)
if err != nil || len(announced) != 3 {
t.Fatalf("announceable %d: %v", len(announced), err)
}
prints, err := inv.Fingerprints(ctx)
if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" {
t.Fatalf("fingerprints %v: %v", prints, err)
}
}
// A build request row as the store held it before the move.
// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue
// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none
// after the outcome is heard. The table of requests takes none.
func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true}
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuildRequest(ctx, ask); err != nil {
t.Fatalf("a request kept twice: %v", err)
}
if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 {
t.Fatalf("a request nobody answered is a build: %+v %v", builds, err)
}
if _, found, _ := inv.BuildByID(ctx, "build-1"); found {
t.Fatal("a request nobody answered has an outcome")
}
got, found, err := inv.BuildRequestByID(ctx, "build-1")
if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() {
t.Fatalf("the request reads back as %+v %v %v", got, found, err)
}
if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil {
t.Fatal(err)
}
asked, err := inv.RequestsNamed(ctx, "notes")
if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" {
t.Fatalf("the request is %+v: %v", asked, err)
}
if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) {
t.Fatalf("requested names %v: %v", names, err)
}
// The outcome comes: the same key, now a build, with the request beside it.
built := aBuild("build-1", "notes", "")
if err := inv.RecordBuild(ctx, built); err != nil {
t.Fatal(err)
}
if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" {
t.Fatalf("the outcome reads as %+v %v %v", one, found, err)
}
answered, err := inv.RequestsNamed(ctx, "notes")
if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" {
t.Fatalf("the request answered is %+v: %v", answered, err)
}
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" {
t.Fatalf("the request was lost with the outcome: %+v", again)
}
// Heard: the marks of an unanswered request change nothing.
if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil {
t.Fatal(err)
}
if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" {
t.Fatalf("an answered request was marked not asked: %+v", again)
}
if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 3 {
// asked, marked unknown, answered.
t.Fatalf("the record has %d values", len(history))
}
// An outcome heard before its request is kept, and the request lands beside it.
if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil {
t.Fatal(err)
}
if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" {
t.Fatalf("the request kept after its outcome reads %+v", late)
}
// A request older than the time they are kept is no longer listed.
if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old",
At: time.Now().Add(-2 * KeptFor)}); err != nil {
t.Fatal(err)
}
if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") {
t.Fatalf("a request past its time is listed: %v", names)
}
}
// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a
// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it
// is asked and so asked at most once.
// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a
// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module
// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the
// assignment goes with it.
func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
inv, node := aNodeWithModules(t, "postgres", "web")
ctx := t.Context()
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
start := generationNow(t, inv)
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added {
t.Fatalf("assigned %v: %v", added, err)
}
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added {
t.Fatalf("assigned again %v: %v", added, err)
}
kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() {
t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err)
}
if generationNow(t, inv) <= start {
t.Fatal("an assignment left the generation where it was")
}
if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) {
t.Fatalf("a module the mesh does not know: %v", err)
}
if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) {
t.Fatalf("assigned on %s: %v %v", node, on, err)
}
if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) {
t.Fatalf("running: %v %v", running, err)
}
entries, err := inv.Catalogued(ctx)
if err != nil {
t.Fatal(err)
}
for _, e := range entries {
switch e.Manifest.Module {
case "postgres":
if !reflect.DeepEqual(e.On, []string{node}) {
t.Errorf("postgres is on %v", e.On)
}
case "web":
if len(e.On) != 0 {
t.Errorf("web is on %v", e.On)
}
}
}
if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) {
t.Fatalf("a module still assigned was forgotten: %v", err)
}
// A seat is held by an assignment, or by nothing.
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil {
t.Fatal("a seat was handed to a module that is not assigned")
}
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil {
t.Fatalf("a seat handed to an assignment: %v", err)
}
before := generationNow(t, inv)
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
t.Fatal(err)
}
if generationNow(t, inv) <= before {
t.Fatal("an unassignment left the generation where it was")
}
// No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history.
gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() {
t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err)
}
if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() ||
history[1].Value.Stands() {
t.Fatalf("the earlier values of the key: %+v", history)
}
if on, _ := inv.Assigned(ctx, node); len(on) != 0 {
t.Fatalf("an ended assignment is still assigned: %v", on)
}
if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 {
t.Fatalf("an ended assignment is running: %v", running)
}
if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 {
t.Fatalf("the seat the assignment held is held by %+v: %v", held, err)
}
if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" {
t.Fatalf("an unassignment of nothing: %v", err)
}
if err := inv.ForgetModule(ctx, "postgres"); err != nil {
t.Fatalf("a module nothing is assigned could not be forgotten: %v", err)
}
// Made again after it ended: a fresh value over the ended one, which stays in history.
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
t.Fatal(added, err)
}
if added, err := inv.Assign(ctx, node, "web"); err != nil || added {
t.Fatal(added, err)
}
if err := inv.Unassign(ctx, node, "web"); err != nil {
t.Fatal(err)
}
if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
t.Fatalf("assigned again after it ended: %v %v", added, err)
}
again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web")
if !again.Value.Stands() || !again.Value.Ended.IsZero() {
t.Fatalf("the assignment made again is %+v", again.Value)
}
if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() {
t.Fatalf("the values of the key: %+v", history)
}
}
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
// ends an assignment.
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
// ends an assignment.
func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) {
inv, node := aNodeWithModules(t, "dunst", "web")
ctx := t.Context()
for _, m := range []string{"dunst", "web"} {
if _, err := inv.Assign(ctx, node, m); err != nil {
t.Fatal(err)
}
}
// A build and a commit put in the value by another write.
cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
cur.Value.Build, cur.Value.Commit = "build-1", "c1"
if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil {
t.Fatal(err)
}
record, _ := inv.NodeByName(ctx, node)
digest := "sha256:" + strings.Repeat("d", 64)
plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest,
Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc",
Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}}
raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`)
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil {
t.Fatal(err)
}
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
written, removed, err := inv.KeepAssignmentPlans(ctx)
if err != nil || written != 2 || removed != 0 {
t.Fatalf("kept: %d written, %d ended: %v", written, removed, err)
}
dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() {
t.Fatalf("dunst's value is %+v", dunst.Value)
}
if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil {
t.Fatalf("web has a plan: %+v", web.Value)
}
notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify")
if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil {
t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found)
}
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
t.Fatalf("a pulled-in module is assigned: %v", on)
}
if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 {
t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err)
}
// Sent again without the pulled-in module: its value ends, the assignments stay.
only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`)
if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil {
t.Fatal(err)
}
if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 {
t.Fatalf("ended %d: %v", removed, err)
}
if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() {
t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value)
}
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
t.Fatalf("assigned: %v", on)
}
}
// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value**
// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build
// already is left as it is.
internal/inventory/collection.go
+19 -70
View File
@@ -2,7 +2,6 @@ package inventory
import (
"context"
"encoding/json"
"sort"
"strings"
@@ -327,84 +326,34 @@ type recentBuild struct {
// recentBuilds is, of each module the mesh still holds, its KeptBuilds newest successful builds' artifacts and bases.
func (i *Inventory) recentBuilds(ctx context.Context) ([]recentBuild, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
builds, err := i.chainRecentMade(ctx, c)
if err != nil {
return nil, err
}
out := make([]recentBuild, 0, len(builds))
for _, b := range builds {
out = append(out, recentBuild{made: b.Made, against: b.Against})
}
return out, nil
}
rows, err := i.store.Pool().Query(ctx,
`select made, built_against from (
select b.made, b.built_against,
row_number() over (partition by b.module order by b.at desc, b.id desc) as back
from build b
join module m on m.name = b.module
where b.failed = '' and b.module is not null and b.module <> ''
) ranked where back <= $1`, KeptBuilds)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
var out []recentBuild
for rows.Next() {
var raw, against []byte
if err := rows.Scan(&raw, &against); err != nil {
return nil, err
}
var r recentBuild
if err := json.Unmarshal(raw, &r.made); err != nil {
r.made = nil
}
if err := json.Unmarshal(against, &r.against); err != nil {
r.against = nil
}
out = append(out, r)
builds, err := i.chainRecentMade(ctx, c)
if err != nil {
return nil, err
}
return out, rows.Err()
out := make([]recentBuild, 0, len(builds))
for _, b := range builds {
out = append(out, recentBuild{made: b.Made, against: b.Against})
}
return out, nil
}
// everyReferenceMade is every artifact reference any successful build recorded, oldest build first.
func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) {
var made [][]Artifact
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
builds, err := i.chainEveryReferenceMade(ctx, c)
if err != nil {
return nil, err
}
for _, b := range builds {
made = append(made, b.Made)
}
} else {
rows, err := i.store.Pool().Query(ctx,
`select made from build where failed = '' and module is not null and module <> ''
order by at asc, id asc`)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var raw []byte
if err := rows.Scan(&raw); err != nil {
return nil, err
}
var a []Artifact
if err := json.Unmarshal(raw, &a); err != nil {
continue
}
made = append(made, a)
}
if err := rows.Err(); err != nil {
return nil, err
}
}
builds, err := i.chainEveryReferenceMade(ctx, c)
if err != nil {
return nil, err
}
var made [][]Artifact
for _, b := range builds {
made = append(made, b.Made)
}
seen := map[string]bool{}
var out []string
internal/inventory/durations.go
-42
View File
@@ -2,11 +2,7 @@ package inventory
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
)
// The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066.
@@ -148,41 +144,3 @@ func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) {
}
return tag.RowsAffected(), nil
}
// planTierLeft is the measurement a plan's save makes when it leaves a tier: the tier moved on, or
// the plan ended. Read in the save's own transaction, so two saves cannot both measure one tier.
func planTierLeft(ctx context.Context, tx pgx.Tx, p Plan, now time.Time) (entered time.Time, err error) {
var oldTier int
var oldState string
var since time.Time
err = tx.QueryRow(ctx,
`select tier, state, coalesce(tier_entered, created) from release_plan where id = $1 for update`,
p.ID).Scan(&oldTier, &oldState, &since)
if errors.Is(err, pgx.ErrNoRows) {
return now, nil // a new plan enters its first tier now
}
if err != nil {
return time.Time{}, err
}
if oldState == PlanAssembling || oldState == PlanQueued {
return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276)
}
wasOpen := oldState == PlanBuilding || oldState == PlanRolling
if !wasOpen || (oldTier == p.Tier && p.Open()) {
return since, nil // still in the tier, or already ended
}
var modules []string
if oldTier >= 0 && oldTier < len(p.Tiers) {
modules = p.Tiers[oldTier]
}
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, oldTier, len(p.Tiers), modules, p.State)
if p.Open() {
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, oldTier, len(p.Tiers), modules, p.Tier)
}
_, err = tx.Exec(ctx,
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
values ($1, $2, '', $3, $4, $5, $6) on conflict do nothing`,
DurationPlanTier, p.Repository, fmt.Sprintf("%s/tier-%d", p.ID, oldTier), since,
now.Sub(since).Milliseconds(), detail)
return now, err
}
internal/inventory/fortest.go
+20 -7
View File
@@ -65,16 +65,11 @@ func ForTest(t *testing.T) *Inventory {
if _, err := inv.store.Migrate(t.Context(), migrations); err != nil {
t.Fatal(err)
}
// The chain is on the bus in a mesh; in a test it is in memory, with every kind moved, as a mesh raised after this
// release has them (ADR 0408, issue 1089). The tests of the move itself use ForTestBefore.
// The chain is on the bus in a mesh; in a test it is in memory (ADR 0408).
c := chain.InMemory(nil)
inv.UseChain(c)
testChains.Store(os.Getenv(store.Variable(Name)), c)
t.Cleanup(func() { testChains.Delete(os.Getenv(store.Variable(Name))) })
if _, err := inv.store.Pool().Exec(t.Context(),
`insert into chain_moved (counts, largest) values ($1, $1)`, `{"builds": 0, "rollouts": 0, "assignments": 0}`); err != nil {
t.Fatal(err)
}
return inv
}
@@ -132,7 +127,7 @@ func ForTestBefore(t *testing.T, number int) (*Inventory, func()) {
if err := inv.Ready(t.Context(), 20*time.Second); err != nil {
t.Fatal(err)
}
// The chain is in memory and nothing is moved: the tests of a move seed the tables and move them.
// The chain is in memory, for what a test of the store before a migration reads of it.
c := chain.InMemory(nil)
inv.UseChain(c)
testChains.Store(os.Getenv(store.Variable(Name)), c)
@@ -183,3 +178,21 @@ func (i *Inventory) QueryForTest(t *testing.T, sql string, args ...any) [][]stri
}
return out
}
// PlanValuesForTest runs the pass that keeps the provisioner's plans in the values of the assignments (KeepAssignmentPlans)
// and answers the values that stand, by key: what a reader of the bucket reads.
func (i *Inventory) PlanValuesForTest(t *testing.T) map[string][]byte {
t.Helper()
if _, _, err := i.KeepAssignmentPlans(t.Context()); err != nil {
t.Fatal(err)
}
kept, err := i.KeptAssignments(t.Context())
if err != nil {
t.Fatal(err)
}
out := make(map[string][]byte, len(kept))
for key, v := range kept {
out[key] = v.Value
}
return out
}
internal/inventory/gate.go
+12 -86
View File
@@ -2,8 +2,6 @@ package inventory
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -241,65 +239,20 @@ func (i *Inventory) RestoreModule(ctx context.Context, b Build) error {
// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told
// from one that does (the bus's planned step, ADR 0236).
func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainBuildFingerprints(ctx, c, module)
}
rows, err := i.store.Pool().Query(ctx,
`select commit_hash, made, coalesce(manifest::text, '') from build
where module = $1 and failed = '' and commit_hash <> '' order by at desc limit 50`, module)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]string{}
for rows.Next() {
var commit, manifest string
var made []byte
if err := rows.Scan(&commit, &made, &manifest); err != nil {
return nil, err
}
if _, seen := out[commit]; seen {
continue
}
sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...))
out[commit] = hex.EncodeToString(sum[:])
}
return out, rows.Err()
return i.chainBuildFingerprints(ctx, c, module)
}
// Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint.
func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainFingerprints(ctx, c)
}
rows, err := i.store.Pool().Query(ctx,
`select module, commit_hash, made, coalesce(manifest::text, '') from build
where module is not null and failed = '' and commit_hash <> '' order by at desc`)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]map[string]string{}
for rows.Next() {
var module, commit, manifest string
var made []byte
if err := rows.Scan(&module, &commit, &made, &manifest); err != nil {
return nil, err
}
if out[module] == nil {
out[module] = map[string]string{}
}
if _, seen := out[module][commit]; seen {
continue
}
sum := sha256.Sum256(append(append(made, 0), []byte(manifest)...))
out[module][commit] = hex.EncodeToString(sum[:])
}
return out, rows.Err()
return i.chainFingerprints(ctx, c)
}
// PassedCommits is, per module, the commits a build of which passed its gate on some machine.
@@ -326,19 +279,11 @@ func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bo
// BuildOf is the id of the newest successful build of a module from a commit; empty when none is
// recorded (a manifest handed over by hand).
func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return "", err
} else if on {
return i.chainBuildOf(ctx, c, module, commit)
}
var id string
err := i.store.Pool().QueryRow(ctx,
`select id from build where module = $1 and commit_hash = $2 and failed = '' order by at desc limit 1`,
module, commit).Scan(&id)
if errors.Is(err, pgx.ErrNoRows) {
return "", nil
}
return id, err
return i.chainBuildOf(ctx, c, module, commit)
}
// ManifestAt is the manifest a module was registered with at a commit (or a commit it abbreviates): the
@@ -372,38 +317,19 @@ func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (cata
// manifestOf is the manifest a build was recorded with; nil when it kept none.
func (i *Inventory) manifestOf(ctx context.Context, id string) ([]byte, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainManifestOf(ctx, c, id)
}
var manifest []byte
if err := i.store.Pool().QueryRow(ctx, `select manifest from build where id = $1`, id).Scan(&manifest); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
return manifest, nil
return i.chainManifestOf(ctx, c, id)
}
// newestManifestAt is the newest successful build of a module from a commit, or one it abbreviates, that kept a manifest:
// its id and manifest; false when there is none.
func (i *Inventory) newestManifestAt(ctx context.Context, module, commit string) (string, []byte, bool, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return "", nil, false, err
} else if on {
return i.chainManifestBuild(ctx, c, module, commit)
}
var id string
var raw []byte
err := i.store.Pool().QueryRow(ctx,
`select id, manifest from build
where module = $1 and failed = '' and manifest is not null and manifest::text <> 'null'
and (commit_hash = $2 or starts_with(commit_hash, $2))
order by at desc limit 1`, module, commit).Scan(&id, &raw)
if errors.Is(err, pgx.ErrNoRows) {
return "", nil, false, nil
}
c, err := i.theChain(ctx)
if err != nil {
return "", nil, false, err
}
return id, raw, true, nil
return i.chainManifestBuild(ctx, c, module, commit)
}
internal/inventory/given.go
+6 -15
View File
@@ -93,28 +93,19 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s
if declared == "" {
return nil, nil
}
// Only a module assigned there: from the bucket when the assignments are on the bus, else the store's join.
_, onTheBus, err := i.ChainReads(ctx, KindAssignments)
// Only a module assigned there, from the assignments bucket.
assignedHere := map[string]bool{}
modules, err := i.assignedModules(ctx, node)
if err != nil {
return nil, err
}
assignedHere := map[string]bool{}
join := `join assignment a on a.node = s.node and a.module = s.module`
if onTheBus {
join = ``
modules, err := i.assignedModules(ctx, node)
if err != nil {
return nil, err
}
for _, m := range modules {
assignedHere[m] = true
}
for _, m := range modules {
assignedHere[m] = true
}
rows, err := i.store.Pool().Query(ctx,
`select s.node, s.module, s.name, s.replace_after_start
from module_secret s
join node n on n.id = s.node
`+join+`
where n.name = $1 and n.sent = $2 and n.sent_at > s.replace_after_start
and s.origin = 'accepted' and s.replace_after_start is not null
and (not n.adopted or exists (select 1 from taken t where t.node = s.node and t.module = s.module))
@@ -134,7 +125,7 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s
rows.Close()
return nil, err
}
if onTheBus && !assignedHere[d.module] {
if !assignedHere[d.module] {
continue
}
dues = append(dues, d)
internal/inventory/mirrored_test.go
-35
View File
@@ -5,7 +5,6 @@ package inventory
import (
"context"
"fmt"
"os"
"slices"
"strings"
"testing"
@@ -148,40 +147,6 @@ func TestAPersonRecordsACopyAndTheFirstRecordStands(t *testing.T) {
}
}
// The migration records the copies the build records already name: only a module's own on-<argument>
// repository, and only once, by the earliest build naming it.
func TestTheCopiesTheRecordsAlreadyNameAreRecorded(t *testing.T) {
inv := inTheTables(t)
ctx := context.Background()
copyRef := mirrorRef("route-proxy/on-go_base", 1)
notACopy := ref("go-base", "build", 2)
builtOn(t, inv, "b1", "route-proxy", 1, nil, copyRef, notACopy)
builtOn(t, inv, "b2", "route-proxy", 2, nil, copyRef)
// The rows a mesh had before the table: written by builds that did not say what they copied.
if _, err := inv.store.Pool().Exec(ctx, `truncate artifact_mirrored`); err != nil {
t.Fatal(err)
}
sql, err := os.ReadFile("migrations/0081-a-mirrored-base-is-recorded.sql")
if err != nil {
t.Fatal(err)
}
_, backfill, _ := strings.Cut(string(sql), "insert into artifact_mirrored")
if _, err := inv.store.Pool().Exec(ctx, "insert into artifact_mirrored"+backfill); err != nil {
t.Fatal(err)
}
var reference, build string
var n int
if err := inv.store.Pool().QueryRow(ctx, `select count(*) from artifact_mirrored`).Scan(&n); err != nil || n != 1 {
t.Fatalf("%d copies recorded (%v); want the one", n, err)
}
if err := inv.store.Pool().QueryRow(ctx, `select reference, build from artifact_mirrored`).Scan(&reference, &build); err != nil {
t.Fatal(err)
}
if reference != copyRef || build != "b1" {
t.Fatalf("recorded %s by %s", reference, build)
}
}
// A copy let go of and copied again by a later build is held again: its record is back, and the sweep
// decides about it as before.
func TestACopyLetGoOfAndCopiedAgainIsHeldAgain(t *testing.T) {
internal/inventory/mirrors.go
+2 -34
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"time"
)
@@ -70,39 +69,8 @@ type DeclaredMirrored struct {
var DeclaredMirroredFields = []string{"kind", "default", "why", "type", "description", "enum", "minimum", "maximum",
"pattern", "trusted", "predefined", "required", "runtime"}
// ErrAssignmentsMoved is a write of the store's mirror of the assignments, refused because the assignments are keys of the bus.
var ErrAssignmentsMoved = errors.New("the assignments are keys of the bus, and the store's rows are no longer what they say")
// AssignmentsMirrorToWrite is AssignmentsMirror for the mirror's writer, which lays what it returns over the bucket and
// ends the keys it does not return: **refused once the assignments are moved** (ADR 0408 §4), asked at the moment of the
// write, so a tick that decided before the switch cannot lay mirror-shaped values over the links after it.
func (i *Inventory) AssignmentsMirrorToWrite(ctx context.Context) (map[string][]byte, error) {
if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
return nil, err
} else if on {
return nil, ErrAssignmentsMoved
}
return i.AssignmentsMirror(ctx)
}
// AssignmentsMirror is every assignment's mirrored value, keyed `<node>.<module>`, with the provisioner's plan of
// each assignment of a module handed to it, and a value of its own for each module handed to it on a node that
// nothing assigned there, only pulled in by another (issue 765).
func (i *Inventory) AssignmentsMirror(ctx context.Context) (map[string][]byte, error) {
var plans map[string]Provisioning
if i.provisioningPlans != nil {
var err error
if plans, err = i.provisioningPlans(ctx); err != nil {
return nil, fmt.Errorf("composing the provisioner's plans: %w", err)
}
}
assignments, err := i.assignmentsMirrored(ctx)
if err != nil {
return nil, err
}
return AssignmentValues(assignments, plans)
}
// assignmentsMirrored is every standing assignment as the provisioner's plans are matched to it (SentProvisioningPlans,
// KeepAssignmentPlans).
func (i *Inventory) assignmentsMirrored(ctx context.Context) ([]AssignmentMirrored, error) {
rows, err := i.assignments(ctx)
if err != nil {
internal/inventory/nodes.go
+4 -21
View File
@@ -34,11 +34,9 @@ type Inventory struct {
// configValues reads a module's config values from mesh-config once the move of config has run (ReadsConfigFrom,
// issue 664); nil reads none.
configValues ConfigValues
// chain is the controller's chain on the bus (novox/hq ADR 0408, issue 1089), once attached (UseChain); moved is which
// kinds of it have been moved from the tables, as chain_moved says (ChainReads). A kind not moved is read and written
// in the tables as before.
// chain is the controller's chain on the bus (novox/hq ADR 0408), once attached (UseChain).
chain *chain.Chain
moved *movedKinds
state *chainState
// recorded, when set, takes the durations a save of the chain measures in place of the store: a test with no store.
recorded func(context.Context, Duration) error
}
@@ -49,7 +47,7 @@ func Open(ctx context.Context) (*Inventory, error) {
if err != nil {
return nil, err
}
inv := &Inventory{store: s, moved: &movedKinds{}}
inv := &Inventory{store: s, state: &chainState{}}
// A test's store has the chain a test made for it in memory, shared by every inventory opened on that store.
if c, ok := testChains.Load(os.Getenv(store.Variable(Name))); ok {
inv.chain = c.(*chain.Chain)
@@ -1278,22 +1276,7 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu
// read from: every change a push carries happened after the last push, and an assignment is the one
// that says when.
func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time, error) {
if _, on, err := i.ChainReads(ctx, KindAssignments); err != nil {
return time.Time{}, err
} else if on {
return i.chainAwaitingSince(ctx, name)
}
var since time.Time
err := i.store.Pool().QueryRow(ctx,
`select coalesce(
(select min(a.assigned) from assignment a
where a.node = n.id and (n.sent_at is null or a.assigned > n.sent_at)),
n.sent_at, n.created)
from node n where n.name = $1`, name).Scan(&since)
if errors.Is(err, pgx.ErrNoRows) {
return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
}
return since, err
return i.chainAwaitingSince(ctx, name)
}
// SentBuilds is the build of each module a machine was last sent, by its name: module to the commit
internal/inventory/pending.go
+14 -81
View File
@@ -9,8 +9,6 @@ import (
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/novox/mesh-controller/internal/chain"
)
@@ -71,85 +69,40 @@ type RequestOutcome struct {
// RecordBuildRequest keeps one build request, once it is asked. Idempotent on the id. Requests older than
// KeptFor are deleted in the same act.
func (i *Inventory) RecordBuildRequest(ctx context.Context, a BuildRequest) error {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return err
} else if on {
return i.chainRecordBuildRequest(ctx, c, a)
}
at := a.At
if at.IsZero() {
at = time.Now().UTC()
}
var notAsked *string
if a.NotAsked != "" {
notAsked = &a.NotAsked
}
if _, err := i.store.Pool().Exec(ctx,
`insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at,
at_terminal)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
on conflict (id) do nothing`,
a.ID, a.Repository, a.Seat, strings.Trim(a.Path, "/"), a.Ref, a.Commit, a.For, notAsked, at,
a.AtTerminal); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return err
}
_, err := i.store.Pool().Exec(ctx, `delete from build_request where asked_at < $1`, time.Now().Add(-KeptFor))
return err
return i.chainRecordBuildRequest(ctx, c, a)
}
// BuildRequestByID is the build request kept under this id, and whether one was kept.
func (i *Inventory) BuildRequestByID(ctx context.Context, id string) (BuildRequest, bool, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return BuildRequest{}, false, err
} else if on {
return i.chainBuildRequestByID(ctx, c, id)
}
var a BuildRequest
err := i.store.Pool().QueryRow(ctx,
`select id, repository, seat, source_path, ref, commit_hash, asked_for, at_terminal, asked_at
from build_request where id = $1`, id).Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit,
&a.For, &a.AtTerminal, &a.At)
if errors.Is(err, pgx.ErrNoRows) {
return BuildRequest{}, false, nil
}
return a, err == nil, err
return i.chainBuildRequestByID(ctx, c, id)
}
// MarkNotAsked says a kept build request was never handed over: the words are kept, unless its outcome was
// heard first.
func (i *Inventory) MarkNotAsked(ctx context.Context, id, why string) error {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return err
} else if on {
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.NotAsked = why })
}
if heard, err := i.outcomeHeard(ctx, id); err != nil || heard {
c, err := i.theChain(ctx)
if err != nil {
return err
}
_, err := i.store.Pool().Exec(ctx, `update build_request set not_asked = $2 where id = $1`, id, why)
return err
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.NotAsked = why })
}
// MarkOutcomeUnknown says the asker of a build it handed over stopped waiting for it: asked, outcome unknown.
// The build may still run; its outcome, when heard, is the last word.
func (i *Inventory) MarkOutcomeUnknown(ctx context.Context, id, why string) error {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return err
} else if on {
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.OutcomeUnknown = why })
}
if heard, err := i.outcomeHeard(ctx, id); err != nil || heard {
c, err := i.theChain(ctx)
if err != nil {
return err
}
_, err := i.store.Pool().Exec(ctx, `update build_request set outcome_unknown = $2 where id = $1`, id, why)
return err
}
// outcomeHeard says a build of this id was recorded in the table: its outcome was taken in.
func (i *Inventory) outcomeHeard(ctx context.Context, id string) (bool, error) {
var heard bool
err := i.store.Pool().QueryRow(ctx, `select exists (select 1 from build where id = $1)`, id).Scan(&heard)
return heard, err
return i.chainMarkRequest(ctx, c, id, func(r *chain.Request) { r.OutcomeUnknown = why })
}
// RequestsNamed is every build request kept whose directory names the module, or whose build said it built
@@ -187,31 +140,11 @@ func (i *Inventory) RequestedNames(ctx context.Context) ([]string, error) {
}
func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, err
} else if on {
return i.chainRequests(ctx, c)
}
rows, err := i.store.Pool().Query(ctx,
`select a.id, a.repository, a.seat, a.source_path, a.ref, a.commit_hash, a.asked_for,
coalesce(a.not_asked, ''), coalesce(a.outcome_unknown, ''), a.asked_at,
b.id is not null, coalesce(b.failed, ''), coalesce(b.module, ''), coalesce(b.at, a.asked_at)
from build_request a left join build b on b.id = a.id
order by a.asked_at desc, a.id desc`)
c, err := i.theChain(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
var out []RequestOutcome
for rows.Next() {
var a RequestOutcome
if err := rows.Scan(&a.ID, &a.Repository, &a.Seat, &a.Path, &a.Ref, &a.Commit, &a.For, &a.NotAsked,
&a.OutcomeUnknown, &a.At, &a.Heard, &a.Failed, &a.Module, &a.HeardAt); err != nil {
return nil, err
}
out = append(out, a)
}
return out, rows.Err()
return i.chainRequests(ctx, c)
}
// The states of a pending assignment.
internal/inventory/pin_completed_test.go
-88
View File
@@ -1,88 +0,0 @@
//go:build integration
package inventory
import (
"context"
"os"
"testing"
)
// A pin made before it named the module (migration 0051, novox/hq #258): completed where the node it
// names answers once, left for a person where it answers twice.
func legacyPin(t *testing.T, inv *Inventory, node, provision, provider string) {
t.Helper()
_, err := inv.store.Pool().Exec(context.Background(),
`insert into provision_pin (node, name, provider)
select u.id, $2, p.id from node u, node p where u.name = $1 and p.name = $3`,
node, provision, provider)
if err != nil {
t.Fatal(err)
}
}
func completeEarlierPins(t *testing.T, inv *Inventory) {
t.Helper()
sql, err := os.ReadFile("migrations/0051-a-pin-made-before-is-completed.sql")
if err != nil {
t.Fatal(err)
}
if _, err := inv.store.Pool().Exec(context.Background(), string(sql)); err != nil {
t.Fatal(err)
}
}
func TestAPinMadeBeforeIsCompletedWhenTheNodeAnswersOnce(t *testing.T) {
inv := inTheTables(t)
ctx := context.Background()
for _, n := range []string{"user", "provider"} {
if _, err := inv.AddNode(ctx, n); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"postgres", "redis"} {
if err := inv.RegisterModule(ctx, manifest(m, []string{m + "-database"}, nil), Source{}); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "provider", m); err != nil {
t.Fatal(err)
}
}
legacyPin(t, inv, "user", "postgres-database", "provider")
completeEarlierPins(t, inv)
pins, err := inv.PinsFor(ctx, "user")
if err != nil {
t.Fatal(err)
}
if got := pins["postgres-database"]; got.Node != "provider" || got.Module != "postgres" {
t.Fatalf("the record was not completed with the one module that answers: %+v", got)
}
}
func TestAPinMadeBeforeIsLeftOpenWhenTheNodeAnswersTwice(t *testing.T) {
inv := inTheTables(t)
ctx := context.Background()
for _, n := range []string{"user", "provider"} {
if _, err := inv.AddNode(ctx, n); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"public-acme", "step-ca"} {
if err := inv.RegisterModule(ctx, manifest(m, []string{"acme-ca"}, nil), Source{}); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "provider", m); err != nil {
t.Fatal(err)
}
}
legacyPin(t, inv, "user", "acme-ca", "provider")
completeEarlierPins(t, inv)
pins, err := inv.PinsFor(ctx, "user")
if err != nil {
t.Fatal(err)
}
if got := pins["acme-ca"]; got.Node != "provider" || got.Module != "" {
t.Fatalf("a node that answers twice was guessed for: %+v", got)
}
}
internal/inventory/plans.go
+21 -243
View File
@@ -2,15 +2,12 @@ package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/novox/mesh-controller/internal/chain"
)
@@ -680,140 +677,12 @@ type MergeMove struct {
// leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by
// somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once
// the transaction is committed, in order.
func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) (err error) {
if c, on, cerr := i.ChainReads(ctx, KindRollouts); cerr != nil {
return cerr
} else if on {
return i.chainSavePlans(ctx, c, plans, moves)
}
epoch, err := i.actingEpoch(ctx)
if err != nil {
if len(plans) > 0 {
return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err)
}
return err
}
// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
// measure one twice.
tx, err := i.store.Pool().Begin(ctx)
func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error {
c, err := i.theChain(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
type written struct {
revision int64
entered time.Time
}
done := make([]written, len(plans))
// **The save that starts or ends a walk is marked** (novox/hq ADR 0308 decision 1), in the same write, and
// unmarked should the write not be made, so the next save marks it again.
marked := time.Now()
undo := make([]func(), 0, len(plans))
defer func() {
if err != nil {
for _, u := range undo {
u()
}
}
}()
for _, p := range plans {
undo = append(undo, p.markWalk(marked))
}
for n, p := range plans {
revision, entered, err := writePlan(ctx, tx, p, epoch)
if err != nil {
return err
}
done[n] = written{revision, entered}
}
for _, m := range moves {
if _, err := tx.Exec(ctx,
`update batched_merge set plan_id = nullif($3, ''), alone = $4 where repository = lower($1) and commit_hash = $2`,
m.Repository, m.Commit, m.Plan, m.Alone); err != nil {
return err
}
}
if err := tx.Commit(ctx); err != nil {
return err
}
for n, p := range plans {
p.Revision, p.TierEntered = done[n].revision, done[n].entered
if epoch != nil {
p.Epoch = uint64(*epoch)
} else {
p.Epoch = 0
}
if PlanSaved != nil {
PlanSaved(i, *p)
}
}
return nil
}
// writePlan writes one plan within a transaction, by compare-and-set on its revision: its new revision, and when
// it entered its tier.
func writePlan(ctx context.Context, tx pgx.Tx, p *Plan, epoch *int64) (int64, time.Time, error) {
tiers, err := json.Marshal(p.Tiers)
if err != nil {
return 0, time.Time{}, err
}
modules, err := json.Marshal(p.Modules)
if err != nil {
return 0, time.Time{}, err
}
var release, delivery, commits, times, backups []byte
if p.Backups != nil {
if backups, err = json.Marshal(p.Backups); err != nil {
return 0, time.Time{}, err
}
}
if p.Times != nil {
if times, err = json.Marshal(p.Times); err != nil {
return 0, time.Time{}, err
}
}
if len(p.Commits) > 0 {
if commits, err = json.Marshal(p.Commits); err != nil {
return 0, time.Time{}, err
}
}
if p.Release != nil {
if release, err = json.Marshal(p.Release); err != nil {
return 0, time.Time{}, err
}
}
if p.Delivery != nil {
if delivery, err = json.Marshal(p.Delivery); err != nil {
return 0, time.Time{}, err
}
}
entered, err := planTierLeft(ctx, tx, *p, time.Now())
if err != nil {
return 0, time.Time{}, err
}
var revision int64
err = tx.QueryRow(ctx,
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note,
branch, tier_entered, revision, epoch, release, delivery, merged_at, commits, times, backups)
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14, $15, $16, $17, $18, $19)
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch,
release = excluded.release, delivery = excluded.delivery, repository = excluded.repository,
commit_hash = excluded.commit_hash, merged_at = excluded.merged_at, commits = excluded.commits,
created = excluded.created, times = excluded.times, backups = excluded.backups
where release_plan.revision = $12
returning revision`,
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered,
p.Revision, epoch, release, delivery, mergedAt(p.Merged), commits, times, backups).Scan(&revision)
if errors.Is(err, pgx.ErrNoRows) {
// The row is there and at another revision — moved since this was read, or there already
// when this one is new: either way not this writer's to overwrite. (A plan saved before plans
// had revisions is at zero, and its first save here is from a read at zero.)
return 0, time.Time{}, fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
}
return revision, entered, err
return i.chainSavePlans(ctx, c, plans, moves)
}
// short is a commit as a person reads it.
@@ -824,145 +693,54 @@ func short(commit string) string {
return commit
}
// mergedAt is a plan's merge time as the store keeps it: null when not known.
func mergedAt(t time.Time) *time.Time {
if t.IsZero() {
return nil
}
return &t
}
// OpenPlans is every plan still being worked, oldest first.
func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
}
return i.plans(ctx, `where state in ('building', 'rolling') order by created`)
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
}
// PlansSince is every plan made after a moment, and every plan still being worked, oldest first.
func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated)
}
return i.plans(ctx, `where created > $1 or state in ('building', 'rolling') order by created`, since)
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated)
}
// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
c, err := i.theChain(ctx)
if err != nil {
return nil, err
} else if on {
plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest)
if err == nil && limit >= 0 && len(plans) > limit {
plans = plans[:limit]
}
return plans, err
}
return i.plans(ctx, `order by created desc limit $1`, limit)
plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest)
if err == nil && limit >= 0 && len(plans) > limit {
plans = plans[:limit]
}
return plans, err
}
// NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh
// planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it
// recorded when its merge was made.
func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
return Plan{}, false, err
} else if on {
return i.chainNewestMergeOf(ctx, c, repository, branch)
}
plans, err := i.plans(ctx, `where lower(repository) = lower($1) and branch = $2 and merged_at is not null
and release is null order by merged_at desc, created desc limit 1`, repository, branch)
if err != nil || len(plans) == 0 {
c, err := i.theChain(ctx)
if err != nil {
return Plan{}, false, err
}
return plans[0], true, nil
return i.chainNewestMergeOf(ctx, c, repository, branch)
}
// PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a
// caller of the plans verb.
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
if c, on, err := i.ChainReads(ctx, KindRollouts); err != nil {
return Plan{}, err
} else if on {
return i.chainPlanByID(ctx, c, id)
}
plans, err := i.plans(ctx, `where id = $1`, id)
c, err := i.theChain(ctx)
if err != nil {
return Plan{}, err
}
if len(plans) == 0 {
return Plan{}, fmt.Errorf("no plan %s", id)
}
return plans[0], nil
}
func (i *Inventory) plans(ctx context.Context, tail string, args ...any) ([]Plan, error) {
rows, err := i.store.Pool().Query(ctx,
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery, merged_at, commits, times, backups
from release_plan `+tail, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Plan
for rows.Next() {
var p Plan
var tiers, modules, release, delivery, commits, times, backups []byte
var epoch int64
var merged *time.Time
if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch, &release,
&delivery, &merged, &commits, &times, &backups); err != nil {
return nil, err
}
if len(backups) > 0 {
if err := json.Unmarshal(backups, &p.Backups); err != nil {
return nil, err
}
}
if len(times) > 0 {
if err := json.Unmarshal(times, &p.Times); err != nil {
return nil, err
}
}
if len(commits) > 0 {
if err := json.Unmarshal(commits, &p.Commits); err != nil {
return nil, err
}
}
if merged != nil {
p.Merged = merged.UTC()
}
if len(release) > 0 {
if err := json.Unmarshal(release, &p.Release); err != nil {
return nil, err
}
}
if len(delivery) > 0 {
if err := json.Unmarshal(delivery, &p.Delivery); err != nil {
return nil, err
}
}
p.Epoch = uint64(epoch)
if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
return nil, err
}
if err := json.Unmarshal(modules, &p.Modules); err != nil {
return nil, err
}
if p.Modules == nil {
p.Modules = map[string]*PlanModule{}
}
out = append(out, p)
}
if errors.Is(rows.Err(), pgx.ErrNoRows) {
return nil, nil
}
return out, rows.Err()
return i.chainPlanByID(ctx, c, id)
}
internal/inventory/sent_plans_test.go
+3 -6
View File
@@ -48,10 +48,7 @@ func TestASendKeepsThePlansItHandedAndTheMirrorCarriesThem(t *testing.T) {
t.Fatalf("the plans mirrored are %+v (%v)", plans, err)
}
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
mirror, err := inv.AssignmentsMirror(ctx)
if err != nil {
t.Fatal(err)
}
mirror := inv.PlanValuesForTest(t)
var dunst, web AssignmentMirrored
if err := json.Unmarshal(mirror[node+".dunst"], &dunst); err != nil {
t.Fatal(err)
@@ -81,8 +78,8 @@ func TestASendKeepsThePlansItHandedAndTheMirrorCarriesThem(t *testing.T) {
if sent, _, err := inv.SentTo(ctx, node); err != nil || sent.Provisioned != nil {
t.Fatalf("a send handing nothing says it handed %v (%v)", sent.Provisioned, err)
}
if mirror, err := inv.AssignmentsMirror(ctx); err != nil || strings.Contains(string(mirror[node+".dunst"]), "provisioning") {
t.Fatalf("dunst's assignment still carries a plan: %s (%v)", mirror[node+".dunst"], err)
if mirror := inv.PlanValuesForTest(t); strings.Contains(string(mirror[node+".dunst"]), "provisioning") {
t.Fatalf("dunst's assignment still carries a plan: %s", mirror[node+".dunst"])
} else if _, has := mirror[node+".notify"]; has {
t.Fatalf("a module the machine no longer runs is mirrored: %s", mirror[node+".notify"])
}
internal/inventory/source.go
+2 -30
View File
@@ -32,39 +32,11 @@ type sourceRow struct {
// sourceRows is every successful build of each module (of one module, when named), newest request
// first.
func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][]sourceRow, []string, error) {
if c, on, err := i.ChainReads(ctx, KindBuilds); err != nil {
return nil, nil, err
} else if on {
return i.chainSourceRows(ctx, c, module)
}
query := `select b.module, b.id, b.commit_hash, b.source_fingerprint, b.made, b.manifest,
coalesce(g.verdict = 'failed', false)
from build b left join build_gate g on g.build = b.id
where b.module is not null and b.module <> '' and b.failed = ''`
args := []any{}
if module != "" {
query += ` and b.module = $1`
args = append(args, module)
}
rows, err := i.store.Pool().Query(ctx, query+` order by b.module, coalesce(b.asked, b.at) desc, b.at desc`, args...)
c, err := i.theChain(ctx)
if err != nil {
return nil, nil, err
}
defer rows.Close()
out := map[string][]sourceRow{}
var order []string
for rows.Next() {
var m string
var r sourceRow
if err := rows.Scan(&m, &r.id, &r.commit, &r.fingerprint, &r.made, &r.manifest, &r.failedGate); err != nil {
return nil, nil, err
}
if _, seen := out[m]; !seen {
order = append(order, m)
}
out[m] = append(out[m], r)
}
return out, order, rows.Err()
return i.chainSourceRows(ctx, c, module)
}
// standing is, of a module's builds newest first, the index of the build that stands for the build at
internal/recordmirror/mirror.go
+1 -1
View File
@@ -60,7 +60,7 @@ var (
)
// Mirrors is every mirror this package writes, in the order they are written.
func Mirrors() []Mirror { return []Mirror{Assignments, DeclaredConfig, Predefined} }
func Mirrors() []Mirror { return []Mirror{DeclaredConfig, Predefined} }
// The changes an event says.
const (
internal/recordmirror/mirror_store_test.go
+19 -77
View File
@@ -12,30 +12,19 @@ import (
"github.com/novox/mesh-controller/internal/inventory"
)
// **Design 57's F3, end to end** (novox/hq issue 652): from the controller's store to a real bus. An
// assignment changes exactly one key of the assignments mirror and raises exactly one `assignment-changed`;
// a registration changes exactly one key of the declared-config mirror and raises exactly one
// `declared-config-changed`; and the same records written again change nothing and say nothing — a module
// registered again unchanged included, although the store's registration time moved.
// **Design 57's F3, end to end** (novox/hq issue 652): from the controller's store to a real bus. A registration
// changes exactly one key of the declared-config mirror and raises exactly one `declared-config-changed`; and the same
// records written again change nothing and say nothing — a module registered again unchanged included, although the
// store's registration time moved. (The assignments are keys of the bus, no mirror of the store: ADR 0408.)
func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *testing.T) {
inv := inventory.ForTest(t)
ctx := t.Context()
js, api := aMirrorBus(t)
assignments := aWriter(t, js, api, Assignments)
declared := aWriter(t, js, api, DeclaredConfig)
heardAssignments := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+Assignments.Event)
heardDeclared := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+DeclaredConfig.Event)
pass := func() (Result, Result) {
pass := func() Result {
t.Helper()
a, err := inv.AssignmentsMirror(ctx)
if err != nil {
t.Fatal(err)
}
ra, err := assignments.Sync(ctx, a)
if err != nil {
t.Fatal(err)
}
d, err := inv.DeclaredConfigMirror(ctx)
if err != nil {
t.Fatal(err)
@@ -44,7 +33,7 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes
if err != nil {
t.Fatal(err)
}
return ra, rd
return rd
}
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
@@ -56,72 +45,37 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes
t.Fatal(err)
}
pass()
assignmentWrites, declaredWrites := writes(t, api, Assignments.Bucket), writes(t, api, DeclaredConfig.Bucket)
assignmentsSaid, declaredSaid := len(settle(heardAssignments, 0)), len(settle(heardDeclared, 1))
declaredWrites := writes(t, api, DeclaredConfig.Bucket)
declaredSaid := len(settle(heardDeclared, 1))
// **A registration**: a module new to the catalogue.
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "notes", Version: "1"}, inventory.Source{}); err != nil {
t.Fatal(err)
}
ra, rd := pass()
if ra != (Result{}) || rd != (Result{Written: 1}) {
t.Errorf("a registration wrote assignments %+v and declared config %+v", ra, rd)
if rd := pass(); rd != (Result{Written: 1}) {
t.Errorf("a registration wrote declared config %+v", rd)
}
if got := writes(t, api, DeclaredConfig.Bucket) - declaredWrites; got != 1 {
t.Errorf("a registration changed %d keys of the declared config", got)
}
if got := writes(t, api, Assignments.Bucket) - assignmentWrites; got != 0 {
t.Errorf("a registration changed %d keys of the assignments", got)
}
said := settle(heardDeclared, declaredSaid+1)[declaredSaid:]
if len(said) != 1 || said[0].Key != "notes" || said[0].Change != Added {
t.Errorf("a registration raised %+v", said)
}
declaredWrites, declaredSaid = writes(t, api, DeclaredConfig.Bucket), declaredSaid+len(said)
// **An assignment.**
if _, err := inv.Assign(ctx, "anchor", "ui"); err != nil {
t.Fatal(err)
}
ra, rd = pass()
if ra != (Result{Written: 1}) || rd != (Result{}) {
t.Errorf("an assignment wrote assignments %+v and declared config %+v", ra, rd)
}
if got := writes(t, api, Assignments.Bucket) - assignmentWrites; got != 1 {
t.Errorf("an assignment changed %d keys of the assignments", got)
}
said = settle(heardAssignments, assignmentsSaid+1)[assignmentsSaid:]
if len(said) != 1 || said[0].Key != "anchor.ui" || said[0].Change != Added {
t.Errorf("an assignment raised %+v", said)
}
assignmentWrites, assignmentsSaid = writes(t, api, Assignments.Bucket), assignmentsSaid+len(said)
e, err := assignments.KV.Get(ctx, "anchor.ui")
if err != nil {
t.Fatal(err)
}
var value inventory.AssignmentMirrored
if err := json.Unmarshal(e.Value(), &value); err != nil || value.Node != "anchor" || value.Module != "ui" ||
value.Assigned.IsZero() {
t.Errorf("the assignment is mirrored as %s (%v)", e.Value(), err)
}
// **The same records written again change nothing**: an assignment made again, and a module registered
// again with the manifest it had.
if _, err := inv.Assign(ctx, "anchor", "ui"); err != nil {
t.Fatal(err)
}
// **The same records written again change nothing**: a module registered again with the manifest it had.
if err := inv.RegisterModule(ctx, ui, inventory.Source{}); err != nil {
t.Fatal(err)
}
if ra, rd := pass(); ra != (Result{}) || rd != (Result{}) {
t.Errorf("unchanged records wrote assignments %+v and declared config %+v", ra, rd)
if rd := pass(); rd != (Result{}) {
t.Errorf("unchanged records wrote declared config %+v", rd)
}
if writes(t, api, Assignments.Bucket) != assignmentWrites || writes(t, api, DeclaredConfig.Bucket) != declaredWrites {
if writes(t, api, DeclaredConfig.Bucket) != declaredWrites {
t.Error("unchanged records changed a key")
}
if a, d := settle(heardAssignments, assignmentsSaid+1), settle(heardDeclared, declaredSaid+1); len(a) != assignmentsSaid ||
len(d) != declaredSaid {
t.Errorf("unchanged records raised %+v and %+v", a[assignmentsSaid:], d[declaredSaid:])
if d := settle(heardDeclared, declaredSaid+1); len(d) != declaredSaid {
t.Errorf("unchanged records raised %+v", d[declaredSaid:])
}
// **A declaration changed by a registration**: one key, one event, naming the field that moved.
@@ -131,8 +85,8 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes
if err := inv.RegisterModule(ctx, ui, inventory.Source{}); err != nil {
t.Fatal(err)
}
if ra, rd := pass(); ra != (Result{}) || rd != (Result{Written: 1}) {
t.Errorf("a changed declaration wrote assignments %+v and declared config %+v", ra, rd)
if rd := pass(); rd != (Result{Written: 1}) {
t.Errorf("a changed declaration wrote declared config %+v", rd)
}
said = settle(heardDeclared, declaredSaid+1)[declaredSaid:]
if len(said) != 1 || said[0].Key != "ui" || said[0].Change != Changed || strings.Join(said[0].Fields, ",") != "settings" {
@@ -146,25 +100,13 @@ func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *tes
t.Errorf("the declaration is mirrored as %s, want %s", got.Value(), want)
}
// **An unassignment** removes its key, said once.
if err := inv.Unassign(ctx, "anchor", "ui"); err != nil {
t.Fatal(err)
}
if ra, _ := pass(); ra != (Result{Removed: 1}) {
t.Errorf("an unassignment did %+v", ra)
}
said = settle(heardAssignments, assignmentsSaid+1)[assignmentsSaid:]
if len(said) != 1 || said[0].Key != "anchor.ui" || said[0].Change != Removed {
t.Errorf("an unassignment raised %+v", said)
}
// **A module no longer registered** has its key removed, said once as `removed` (ADR 0356 §4).
if err := inv.ForgetModule(ctx, "notes"); err != nil {
t.Fatal(err)
}
declaredWrites = writes(t, api, DeclaredConfig.Bucket)
declaredSaid = len(heardDeclared.all())
if _, rd := pass(); rd != (Result{Removed: 1}) {
if rd := pass(); rd != (Result{Removed: 1}) {
t.Errorf("a module forgotten did %+v", rd)
}
if got := writes(t, api, DeclaredConfig.Bucket) - declaredWrites; got != 1 {
testdata/store-tests
Vendored
-9
View File
@@ -139,22 +139,13 @@
./cmd/mesh-controller TestADeliveryGroupMovingAModuleMachinesRunPassesTheGate
./cmd/mesh-controller TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt
./cmd/mesh-controller TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheBus
./internal/inventory TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing
./internal/inventory TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing
./internal/inventory TestTheMoveRefusesAValueOverTheLargestMessage
./internal/inventory TestTheMoveRefusesOnACountThatDiffers
./internal/inventory TestTheMoveWithoutTheLeaseWritesNothing
./internal/inventory TestBuildsOnTheBusAreReadBackByEveryReader
./internal/inventory TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace
./internal/inventory TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain
./internal/inventory TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame
./cmd/mesh-controller TestAMergeWhosePlanCannotBeSavedIsHeldAndRecordedWhenHandedOverAgain
./internal/inventory TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt
./internal/inventory TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves
./internal/inventory TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent
./cmd/mesh-controller TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments
./cmd/mesh-controller TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment
./internal/inventory TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved
./internal/inventory TestAPendingAssignmentIsAKeyOfTheAssignmentsBucket
./internal/inventory TestARecordedPendingIsSkippedByAssignedAndTheJoins
./internal/inventory TestTwoClaimsOfOnePendingMakeOneAndTheOtherSeesFalse