From fc55785956b37dc2acecb4eb0f02e20570bd9f4b Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 02:39:11 +0200 Subject: [PATCH] Add the chain package and its buckets: builds, rollouts and assignments with history 64 (issue 1089) internal/chain holds typed Build, Rollout and Assignment and a Backend with no delete or purge, in memory and on the bus. The builds and rollouts buckets are asserted with history 64, no age and values up to just under a mebibyte; the assignments bucket gets the same, keeping its marker age because a stream cannot be made to disallow message ages. Migration 0104 makes the one-row chain_moved switch. The controller keeps no consolidated schema, so 0104 is the whole schema change. Nothing reads the buckets yet. --- internal/broker/controller_buckets.go | 83 ++++- internal/broker/controller_buckets_test.go | 53 ++- internal/broker/nats.go | 10 + internal/broker/testdata/composed.conf | 2 +- internal/broker/writers.go | 8 + internal/broker/writers_test.go | 3 + internal/catalogue/state.go | 2 +- internal/chain/backend.go | 230 +++++++++++++ internal/chain/bus.go | 132 ++++++++ internal/chain/chain.go | 143 ++++++++ internal/chain/chain_test.go | 312 ++++++++++++++++++ internal/chain/records.go | 147 +++++++++ internal/inventory/chain_moved_test.go | 40 +++ .../0104-the-chain-moves-to-the-bus.sql | 16 + internal/recordmirror/readers_test.go | 3 + 15 files changed, 1175 insertions(+), 9 deletions(-) create mode 100644 internal/chain/backend.go create mode 100644 internal/chain/bus.go create mode 100644 internal/chain/chain.go create mode 100644 internal/chain/chain_test.go create mode 100644 internal/chain/records.go create mode 100644 internal/inventory/chain_moved_test.go create mode 100644 internal/inventory/migrations/0104-the-chain-moves-to-the-bus.sql diff --git a/internal/broker/controller_buckets.go b/internal/broker/controller_buckets.go index 088f1edf..529eb2f3 100644 --- a/internal/broker/controller_buckets.go +++ b/internal/broker/controller_buckets.go @@ -92,9 +92,41 @@ const ( MirrorPredefined = "predefined" ) -// ControllerMirrors are the local names of the controller's record mirrors, which a module may read. +// The chain's buckets (novox/hq ADR 0408, issue 1089): builds, rollouts and assignments are one chain kept on the bus +// alone, one bucket each, one key per record, written by the lease holder by compare-and-set. The assignments bucket is +// AssignmentsBucket above, which keeps its name and its readers (ADR 0408 §3). Nothing is removed from them (§4), they keep +// ChainHistory earlier revisions of every key (§5), and a value is never cut (§6). +var ( + BuildsBucket = BucketName(ControllerSeat, ChainBuilds) + RolloutsBucket = BucketName(ControllerSeat, ChainRollouts) +) + +// The chain buckets' local names, as a reader names them in `reads` after `mesh-controller.`. +const ( + ChainBuilds = "builds" + ChainRollouts = "rollouts" +) + +// The chain buckets' bounds. ChainHistory is the bus's largest history (ADR 0408 §5). A value is at most ChainValueBytes, +// just under the server's largest message of a mebibyte, which also carries the subject and the headers (ADR 0408 §6): +// the controller writes none larger and the move refuses to. The buckets' size is the arithmetic of the survey of +// issue 981: 8585 builds of 1 to 40 KB, 1049 rollouts of about 2 KB a revision and 64 revisions, so 512 MiB for the two +// that grow, and the assignments, 285 of under a kilobyte, 64 MiB. +const ( + ChainHistory = 64 + ChainValueBytes = 1<<20 - 8<<10 + ChainBuildsBytes = 512 << 20 + ChainRolloutsBytes = 512 << 20 + ChainAssignmentBytes = 64 << 20 +) + +// ChainBuckets are the three buckets of the chain: builds, rollouts and assignments. +func ChainBuckets() []string { return []string{BuildsBucket, RolloutsBucket, AssignmentsBucket} } + +// ControllerMirrors are the local names of the controller's buckets a module may read: the record mirrors and the +// chain's buckets. func ControllerMirrors() []string { - return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined} + return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts} } // The mirrors' caps (design 54 §5): one value per key, no age, 64 KiB a value, 64 MiB a bucket. @@ -151,14 +183,14 @@ func IsControllerBucket(bucket string) bool { bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket || bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket || bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket || - bucket == DeliveryMergesBucket + bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket } // ControllerBuckets are the controller's own buckets, in the order they are asserted. func ControllerBuckets() []string { return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket, AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket, - DeliveryMergesBucket} + DeliveryMergesBucket, BuildsBucket, RolloutsBucket} } // ControllerBucketsAsserter is what raising the controller's buckets needs of a connection. @@ -315,11 +347,36 @@ func (j *JetStream) EnsureControllerBuckets() error { }); err != nil { return fmt.Errorf("asserting bucket %s: %w", DeliveryMergesBucket, err) } + // **The chain: no age, no removal, history 64, no cut** (ADR 0408 §4 to §6). A key is never removed, so no purge + // marker needs an age and a key that expired would be a record the bus forgot. + for _, c := range []struct { + bucket, why string + bytes int64 + }{ + {BuildsBucket, "every build, one key each by its id, with its repository, commit, reuse key and what it " + + "published (novox/hq ADR 0408, design 47): written by the controller alone, never removed, read through `reads`", + ChainBuildsBytes}, + {RolloutsBucket, "every rollout, one key each by its id: the commit it fixed per repository, its tiers, the " + + "builds it carried and the assignments it changed (novox/hq ADR 0408, design 47): written by the " + + "controller alone, never removed, read through `reads`", ChainRolloutsBytes}, + } { + if _, err := js.CreateOrUpdateKeyValue(ctx, chainBucket(c.bucket, c.why, c.bytes, 0)); err != nil { + return fmt.Errorf("asserting bucket %s: %w", c.bucket, err) + } + } + // **The assignments keep the marker age they had as a mirror.** A stream that allows messages an age cannot be + // made to disallow them (the server refuses: "message TTL status can not be disabled"), so the bucket that was + // the assignments mirror is asserted with the age it was made with. Nothing is purged from it, so no marker is ever + // written and the age ends nothing. + if _, err := js.CreateOrUpdateKeyValue(ctx, chainBucket(AssignmentsBucket, + "which module is assigned to which node, keyed ., with the build, commit and rollout it runs "+ + "there (novox/hq ADR 0408, design 47): written by the controller alone, never removed, read through `reads`", + ChainAssignmentBytes, MirrorMarkerTTL)); err != nil { + return fmt.Errorf("asserting bucket %s: %w", AssignmentsBucket, err) + } // **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose // marker lives a day; a key that expired would be a record the bus forgot while the store still holds it. for _, m := range []struct{ bucket, why string }{ - {AssignmentsBucket, "which module is assigned to which node, keyed . (novox/hq ADR 0289, " + - "design 54): written by the controller alone from its store when an assignment changes, read through `reads`"}, {DeclaredConfigBucket, "each registered module's config declarations as its manifest holds them, keyed by " + "the module (novox/hq ADR 0348, design 57 F3): written by the controller alone from its store when a " + "registration changes them, read through `reads`"}, @@ -342,6 +399,20 @@ func (j *JetStream) EnsureControllerBuckets() error { return nil } +// chainBucket is the options of a bucket of the chain: sixty-four revisions of each key, no age, a value up to +// ChainValueBytes, on disk. +func chainBucket(bucket, why string, bytes int64, markers time.Duration) jetstream.KeyValueConfig { + return jetstream.KeyValueConfig{ + Bucket: bucket, + Description: why, + History: ChainHistory, + MaxValueSize: ChainValueBytes, + MaxBytes: bytes, + Storage: jetstream.FileStorage, + LimitMarkerTTL: markers, + } +} + // EnsureLeaseBucket creates the lease's bucket if absent and brings its options to match (to-be 45 // §6). **Before the lease is taken, by any candidate**: it is the lease's own precondition, and asserting // a bucket that exists changes nothing. File storage, so its revisions — the epochs — outlive a restart of diff --git a/internal/broker/controller_buckets_test.go b/internal/broker/controller_buckets_test.go index 7b9d9f9b..1db7465c 100644 --- a/internal/broker/controller_buckets_test.go +++ b/internal/broker/controller_buckets_test.go @@ -4,6 +4,8 @@ import ( "slices" "strings" "testing" + + "github.com/nats-io/nats.go/jetstream" ) // **The controller may write every bucket it writes** (novox/hq to-be 45 §1, issue 269). Writing a @@ -118,7 +120,7 @@ func TestTheMirrorsListAndTheWritersTableAgree(t *testing.T) { module := Principal{Kind: KindModule, Module: ControllerSeat, Node: "one"} rowed := map[string]bool{} for _, row := range WritersTable { - if !strings.HasSuffix(row.State, " mirror") { + if !strings.HasSuffix(row.State, " mirror") && !strings.HasSuffix(row.State, " of the chain") { continue } for _, subject := range row.Subjects { @@ -254,3 +256,52 @@ func TestTheControllerMayAskMeshConfigInItsOwnName(t *testing.T) { // **Delivering this change loses no merge** (review of issue 973): a merge published under the old filter of the // `controller` consumer and never acknowledged by it is delivered by the merges consumer once the assertion has // made it, because it starts where `controller` has acknowledged up to, not from now. + +// **The chain's buckets keep sixty-four revisions and no age** (ADR 0408 §4 to §6, issue 1089): the bus's largest +// history, no age that could forget a key, a value up to just under the server's largest message, on disk. +func TestTheChainBucketsKeepSixtyFourRevisionsAndNoAge(t *testing.T) { + if want := []string{"mesh-controller_builds", "mesh-controller_rollouts", "mesh-controller_assignments"}; !slices.Equal(ChainBuckets(), want) { + t.Fatalf("the chain's buckets are %v, not %v", ChainBuckets(), want) + } + for _, bucket := range ChainBuckets() { + cfg := chainBucket(bucket, "why", ChainBuildsBytes, 0) + if cfg.History != 64 || cfg.TTL != 0 || cfg.Storage != jetstream.FileStorage { + t.Errorf("%s keeps %d revisions, age %v, on %v", bucket, cfg.History, cfg.TTL, cfg.Storage) + } + if cfg.MaxValueSize <= MirrorValueBytes || cfg.MaxValueSize >= 1<<20 { + t.Errorf("%s takes a value of %d bytes, not just under a mebibyte", bucket, cfg.MaxValueSize) + } + if cfg.Replicas > 1 { + t.Errorf("%s is replicated %d times", bucket, cfg.Replicas) + } + } +} + +// **Every bucket of the chain is the controller's, listed for a reader, and has a writer row** (ADR 0408 §3). +func TestEveryChainBucketIsOnTheControllersListAndHasAWriterRow(t *testing.T) { + controller := Principal{Kind: KindController} + module := Principal{Kind: KindModule, Module: ControllerSeat, Node: "one"} + for _, bucket := range ChainBuckets() { + if !IsControllerBucket(bucket) || !slices.Contains(ControllerBuckets(), bucket) { + t.Errorf("%s is not on the controller's list of buckets", bucket) + } + local := strings.TrimPrefix(bucket, ControllerSeat+"_") + if !slices.Contains(ControllerMirrors(), local) { + t.Errorf("a module cannot name %s in `reads`", bucket) + } + rows := 0 + for _, row := range WritersTable { + for _, subject := range row.Subjects { + if subject == "$KV."+bucket+".>" { + rows++ + if row.Writes == nil || !row.Writes(controller, subject) || row.Writes(module, subject) { + t.Errorf("%q is not written by the controller alone", row.State) + } + } + } + } + if rows != 1 { + t.Errorf("%s has %d writer rows", bucket, rows) + } + } +} diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 07d30883..9408ee8a 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -403,6 +403,16 @@ var DegradableGrants = []DegradableGrant{ Reason: "refused: the controller then keeps no merge mesh-delivery announced, so the build of a moved module is " + "refused as it is without the operator's Approve — failing closed (ADR 0420 §7, issue 1025)", }, + { + Subject: "$KV." + BuildsBucket + ".>", + Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " + + "first push, and writes the chain only after that, holding the lease (ADR 0408 §2 and §8, issue 1089)", + }, + { + Subject: "$KV." + RolloutsBucket + ".>", + Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " + + "first push, and writes the chain only after that, holding the lease (ADR 0408 §2 and §8, issue 1089)", + }, { Subject: "$KV." + RolloutSilentBucket + ".>", Reason: "refused: the controller looks at the plans when a wait for a machine's report is up, by a timer of its own " + diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index d9e24d27..fc21c54a 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/broker/writers.go b/internal/broker/writers.go index bf7a5960..44d71a1a 100644 --- a/internal/broker/writers.go +++ b/internal/broker/writers.go @@ -196,6 +196,14 @@ var WritersTable = []WriterRow{ {State: "the assignments mirror", Writer: "controller (lease holder), from its store, only on a change", KeptIn: "key-value " + AssignmentsBucket, Others: "read through `reads` as mesh-controller." + MirrorAssignments, Subjects: kvOf(AssignmentsBucket), Writes: isController}, + // The chain (novox/hq ADR 0408): builds and rollouts beside the assignments above, which keep their bucket. Not + // mirrors of the store but the record, written by the lease holder by compare-and-set and removed by nobody. + {State: "the builds of the chain", Writer: "controller (lease holder), compare-and-set on the key's revision", + KeptIn: "key-value " + BuildsBucket + ", sixty-four revisions of each key, none removed", + Others: "read through `reads` as mesh-controller." + ChainBuilds, Subjects: kvOf(BuildsBucket), Writes: isController}, + {State: "the rollouts of the chain", Writer: "controller (lease holder), compare-and-set on the key's revision", + KeptIn: "key-value " + RolloutsBucket + ", sixty-four revisions of each key, none removed", + Others: "read through `reads` as mesh-controller." + ChainRollouts, Subjects: kvOf(RolloutsBucket), Writes: isController}, {State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change", KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig, Subjects: kvOf(DeclaredConfigBucket), Writes: isController}, diff --git a/internal/broker/writers_test.go b/internal/broker/writers_test.go index 6f929662..4ac55259 100644 --- a/internal/broker/writers_test.go +++ b/internal/broker/writers_test.go @@ -38,6 +38,9 @@ var designRows = []string{ "a message a consumer gave up on", // The record mirrors, rows of to-be 45 §1's table by design 54 §2 (ADR 0289 §9). "the assignments mirror", + // The chain beside the assignments, the record itself and no mirror of one (ADR 0408, design 47, issue 1089). + "the builds of the chain", + "the rollouts of the chain", "the declared-config mirror", "the predefined-keys mirror", // The provisioner client's word on what it placed (novox/hq ADR 0349 §4, design 57 P6). diff --git a/internal/catalogue/state.go b/internal/catalogue/state.go index b36d9564..1d08e229 100644 --- a/internal/catalogue/state.go +++ b/internal/catalogue/state.go @@ -130,7 +130,7 @@ func StateProblems(m Manifest) []string { // controller's account alone (the writers table); declared as a module's state, they would be asserted a // second way and granted to the module's own principal as well. Named here so a read of one is a read of // kept state. The bus's list is broker.ControllerMirrors; a test keeps the two agreeing. -var ControllerMirrors = []string{"assignments", "declared-config", "predefined"} +var ControllerMirrors = []string{"assignments", "declared-config", "predefined", "builds", "rollouts"} // StateReadsNothingDeclares is every read across a catalogue whose owner is present and declares no // such state. An absent owner says nothing — a module may be installed long before the one whose diff --git a/internal/chain/backend.go b/internal/chain/backend.go new file mode 100644 index 00000000..caf52128 --- /dev/null +++ b/internal/chain/backend.go @@ -0,0 +1,230 @@ +package chain + +import ( + "context" + "errors" + "sort" + "sync" + "time" +) + +// Entry is one key's value at a revision. Caught is set on the entry a watch sends once, after the values the bucket +// held when it began, and carries no key. +type Entry struct { + Key string + Value []byte + Revision uint64 + At time.Time + Caught bool +} + +// Backend is one bucket of the chain. **It has no delete and no purge**: no key of the chain is ever removed (ADR +// 0408 §4), so nothing that holds a Backend can remove one. +type Backend interface { + // Get is one key's value and revision; false when it holds none. + Get(ctx context.Context, key string) (Entry, bool, error) + // Create writes a key that holds nothing and fails with ErrExists when it holds something. + Create(ctx context.Context, key string, value []byte) (uint64, error) + // Update writes a key at the revision it was read at and fails with ErrMoved when it moved. + Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) + // Keys is every key. An error is an error: never an empty bucket. + Keys(ctx context.Context) ([]string, error) + // Watch sends every key's current value, then a Caught entry, then each later write, until the context ends. + Watch(ctx context.Context) (<-chan Entry, error) + // History is a key's earlier values and its current one, oldest first, at most the bucket keeps. + History(ctx context.Context, key string) ([]Entry, error) +} + +var ( + // ErrMoved is a compare-and-set that lost: somebody wrote the key since it was read. + ErrMoved = errors.New("the key was written by somebody else since it was read") + // ErrExists is a create of a key that holds a value. + ErrExists = errors.New("the key already holds a value") + // ErrNotHeld is a write refused because this controller does not hold the lease. + ErrNotHeld = errors.New("this controller does not hold the lease, so it writes nothing of the chain") + // ErrTooLarge is a value over the largest the bus holds: never cut (ADR 0408 §6), so refused. + ErrTooLarge = errors.New("the value is larger than the bus holds, and a value of the chain is never cut") +) + +// Memory is a Backend held in this process, for tests and for nothing else. It keeps what the bucket keeps: every +// earlier value, up to History of them, and a revision that rises with every write of the bucket. +type Memory struct { + // Keeps is how many values of a key it keeps; zero keeps sixty-four. + Keeps int + + mu sync.Mutex + values map[string][]Entry + revision uint64 + watchers []chan Entry + // Now is the clock the entries are dated by; nil is time.Now. + Now func() time.Time + // writes counts the creates and updates taken. + writes int + // fail is what every call answers, when set. + fail error +} + +// NewMemory is an empty bucket. +func NewMemory() *Memory { return &Memory{values: map[string][]Entry{}} } + +// SetFail makes every call answer err from now on, or none when err is nil: a bus that is away. +func (m *Memory) SetFail(err error) { + m.mu.Lock() + defer m.mu.Unlock() + m.fail = err +} + +// Writes is how many writes the bucket took. +func (m *Memory) Writes() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.writes +} + +func (m *Memory) keep() int { + if m.Keeps > 0 { + return m.Keeps + } + return 64 +} + +func (m *Memory) now() time.Time { + if m.Now != nil { + return m.Now() + } + return time.Now() +} + +func (m *Memory) Get(_ context.Context, key string) (Entry, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return Entry{}, false, m.fail + } + h := m.values[key] + if len(h) == 0 { + return Entry{}, false, nil + } + return clone(h[len(h)-1]), true, nil +} + +func (m *Memory) Create(_ context.Context, key string, value []byte) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return 0, m.fail + } + if len(m.values[key]) > 0 { + return 0, ErrExists + } + return m.write(key, value), nil +} + +func (m *Memory) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return 0, m.fail + } + h := m.values[key] + if len(h) == 0 || h[len(h)-1].Revision != revision { + return 0, ErrMoved + } + return m.write(key, value), nil +} + +// write is called with the lock held. +func (m *Memory) write(key string, value []byte) uint64 { + m.revision++ + m.writes++ + e := Entry{Key: key, Value: append([]byte(nil), value...), Revision: m.revision, At: m.now()} + h := append(m.values[key], e) + if len(h) > m.keep() { + h = h[len(h)-m.keep():] + } + m.values[key] = h + for _, w := range m.watchers { + select { + case w <- clone(e): + default: + } + } + return e.Revision +} + +func (m *Memory) Keys(_ context.Context) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return nil, m.fail + } + out := make([]string, 0, len(m.values)) + for k := range m.values { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (m *Memory) Watch(ctx context.Context) (<-chan Entry, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return nil, m.fail + } + // Room for the current values, the caught mark, and a burst of later writes. + w := make(chan Entry, len(m.values)+1+1024) + keys := make([]string, 0, len(m.values)) + for k := range m.values { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + h := m.values[k] + w <- clone(h[len(h)-1]) + } + w <- Entry{Caught: true} + m.watchers = append(m.watchers, w) + out := make(chan Entry) + go func() { + defer close(out) + for { + select { + case <-ctx.Done(): + m.mu.Lock() + for i, x := range m.watchers { + if x == w { + m.watchers = append(m.watchers[:i], m.watchers[i+1:]...) + break + } + } + m.mu.Unlock() + return + case e := <-w: + select { + case out <- e: + case <-ctx.Done(): + } + } + } + }() + return out, nil +} + +func (m *Memory) History(_ context.Context, key string) ([]Entry, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.fail != nil { + return nil, m.fail + } + out := make([]Entry, 0, len(m.values[key])) + for _, e := range m.values[key] { + out = append(out, clone(e)) + } + return out, nil +} + +func clone(e Entry) Entry { + e.Value = append([]byte(nil), e.Value...) + return e +} diff --git a/internal/chain/bus.go b/internal/chain/bus.go new file mode 100644 index 00000000..97c5b872 --- /dev/null +++ b/internal/chain/bus.go @@ -0,0 +1,132 @@ +package chain + +import ( + "context" + "errors" + "fmt" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" +) + +// OnTheBus opens the chain's three buckets on a connection. The controller asserts them at its start +// (broker.EnsureControllerBuckets), so one older than this has not, and the error says so. +func OnTheBus(ctx context.Context, conn *nats.Conn, holds func() bool) (*Chain, error) { + api, err := jetstream.New(conn) + if err != nil { + return nil, err + } + open := func(bucket string) (Backend, error) { + kv, err := api.KeyValue(ctx, bucket) + if err != nil { + return nil, fmt.Errorf("the bucket %s is not on the bus — the controller asserts it at its start, so one "+ + "older than this has not: %w", bucket, err) + } + return busBucket{kv}, nil + } + builds, err := open(broker.BuildsBucket) + if err != nil { + return nil, err + } + rollouts, err := open(broker.RolloutsBucket) + if err != nil { + return nil, err + } + assignments, err := open(broker.AssignmentsBucket) + if err != nil { + return nil, err + } + return New(builds, rollouts, assignments, holds), nil +} + +// busBucket is a Backend on a key-value bucket. It calls no delete and no purge: the type has none to call. +type busBucket struct{ kv jetstream.KeyValue } + +func entryOf(e jetstream.KeyValueEntry) Entry { + return Entry{Key: e.Key(), Value: e.Value(), Revision: e.Revision(), At: e.Created()} +} + +func (b busBucket) Get(ctx context.Context, key string) (Entry, bool, error) { + e, err := b.kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return Entry{}, false, nil + } + if err != nil { + return Entry{}, false, err + } + return entryOf(e), true, nil +} + +func (b busBucket) Create(ctx context.Context, key string, value []byte) (uint64, error) { + rev, err := b.kv.Create(ctx, key, value) + if errors.Is(err, jetstream.ErrKeyExists) { + return 0, ErrExists + } + return rev, err +} + +func (b busBucket) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { + rev, err := b.kv.Update(ctx, key, value, revision) + var apiErr *jetstream.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence { + return 0, ErrMoved + } + return rev, err +} + +func (b busBucket) Keys(ctx context.Context) ([]string, error) { + keys, err := b.kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +func (b busBucket) Watch(ctx context.Context) (<-chan Entry, error) { + w, err := b.kv.WatchAll(ctx) + if err != nil { + return nil, err + } + out := make(chan Entry) + go func() { + defer close(out) + defer func() { _ = w.Stop() }() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + return + } + v := Entry{Caught: true} + if e != nil { + v = entryOf(e) + } + select { + case out <- v: + case <-ctx.Done(): + return + } + } + } + }() + return out, nil +} + +func (b busBucket) History(ctx context.Context, key string) ([]Entry, error) { + all, err := b.kv.History(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + out := make([]Entry, 0, len(all)) + for _, e := range all { + out = append(out, entryOf(e)) + } + return out, nil +} diff --git a/internal/chain/chain.go b/internal/chain/chain.go new file mode 100644 index 00000000..d0ec79a7 --- /dev/null +++ b/internal/chain/chain.go @@ -0,0 +1,143 @@ +package chain + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/novox/mesh-controller/internal/broker" +) + +// Bucket is one kind of record of the chain, typed. Reads need nothing; **every write asks first whether this +// controller holds the lease** and writes nothing when it does not (ADR 0408 §2). +type Bucket[T any] struct { + b Backend + holds func() bool +} + +// Kept is a record as the bucket holds it: with its revision, the one a write must find. +type Kept[T any] struct { + Key string + Value T + Revision uint64 +} + +func (s Bucket[T]) guard(value []byte) error { + if s.holds == nil || !s.holds() { + return ErrNotHeld + } + if len(value) > broker.ChainValueBytes { + return fmt.Errorf("%w: %d bytes, the most is %d", ErrTooLarge, len(value), broker.ChainValueBytes) + } + return nil +} + +func decode[T any](e Entry) (Kept[T], error) { + var v T + if err := json.Unmarshal(e.Value, &v); err != nil { + return Kept[T]{}, fmt.Errorf("the value of %s cannot be read: %w", e.Key, err) + } + return Kept[T]{Key: e.Key, Value: v, Revision: e.Revision}, nil +} + +// Get is one record; false when the key holds none. +func (s Bucket[T]) Get(ctx context.Context, key string) (Kept[T], bool, error) { + e, ok, err := s.b.Get(ctx, key) + if err != nil || !ok { + return Kept[T]{}, false, err + } + e.Key = key + k, err := decode[T](e) + return k, err == nil, err +} + +// Create writes a key that holds nothing: ErrExists when it holds something. It returns the revision. +func (s Bucket[T]) Create(ctx context.Context, key string, v T) (uint64, error) { + body, err := json.Marshal(v) + if err != nil { + return 0, err + } + if err := s.guard(body); err != nil { + return 0, err + } + return s.b.Create(ctx, key, body) +} + +// Update writes a key at the revision it was read at: ErrMoved when it moved, the key unchanged. +func (s Bucket[T]) Update(ctx context.Context, key string, v T, revision uint64) (uint64, error) { + body, err := json.Marshal(v) + if err != nil { + return 0, err + } + if err := s.guard(body); err != nil { + return 0, err + } + return s.b.Update(ctx, key, body, revision) +} + +// All is every record, by key. An error is an error: never an empty bucket. +func (s Bucket[T]) All(ctx context.Context) (map[string]Kept[T], error) { + keys, err := s.b.Keys(ctx) + if err != nil { + return nil, err + } + out := make(map[string]Kept[T], len(keys)) + for _, key := range keys { + k, ok, err := s.Get(ctx, key) + if err != nil { + return nil, err + } + if ok { + out[key] = k + } + } + return out, nil +} + +// History is a key's earlier records and its current one, oldest first. +func (s Bucket[T]) History(ctx context.Context, key string) ([]Kept[T], error) { + all, err := s.b.History(ctx, key) + if err != nil { + return nil, err + } + out := make([]Kept[T], 0, len(all)) + for _, e := range all { + e.Key = key + k, err := decode[T](e) + if err != nil { + return nil, err + } + out = append(out, k) + } + return out, nil +} + +// Keys is every key of the bucket. +func (s Bucket[T]) Keys(ctx context.Context) ([]string, error) { return s.b.Keys(ctx) } + +// Watch is the bucket's raw entries: its current values, a Caught mark, then each write. +func (s Bucket[T]) Watch(ctx context.Context) (<-chan Entry, error) { return s.b.Watch(ctx) } + +// Chain is the three buckets of the chain. +type Chain struct { + Builds Bucket[Build] + Rollouts Bucket[Rollout] + Assignments Bucket[Assignment] +} + +// New is a chain over three backends, written only while holds says this controller holds the lease. +func New(builds, rollouts, assignments Backend, holds func() bool) *Chain { + return &Chain{ + Builds: Bucket[Build]{builds, holds}, + Rollouts: Bucket[Rollout]{rollouts, holds}, + Assignments: Bucket[Assignment]{assignments, holds}, + } +} + +// InMemory is a chain held in this process, for tests, written whenever holds says so; nil always holds. +func InMemory(holds func() bool) *Chain { + if holds == nil { + holds = func() bool { return true } + } + return New(NewMemory(), NewMemory(), NewMemory(), holds) +} diff --git a/internal/chain/chain_test.go b/internal/chain/chain_test.go new file mode 100644 index 00000000..bc95b001 --- /dev/null +++ b/internal/chain/chain_test.go @@ -0,0 +1,312 @@ +package chain + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" +) + +func at(h int) time.Time { return time.Date(2026, 10, 18, h, 0, 0, 0, time.UTC) } + +// **A build is kept whole and read back with its reuse key and what it published** (ADR 0408 §1): its module, +// repository, commit, reuse key, and an artifact by digest or a package release by name and version. +func TestABuildIsKeptWholeAndReadBackWithItsReuseKeyAndWhatItPublished(t *testing.T) { + ctx := context.Background() + c := InMemory(nil) + artifact := Build{ID: "b1", Repository: "mesh-catalog", Module: "notes", Commit: "c1", ReuseKey: "rk-1", + Published: &Published{Artifact: "sha256:aa"}, Asked: at(1), At: at(2), + Made: []Artifact{{Name: "notes", Kind: "image", Reference: "sha256:aa"}}} + pkg := Build{ID: "b2", Repository: "mesh-sdk", Module: "sdk", Commit: "c2", ReuseKey: "rk-2", + Published: &Published{Package: &Package{Name: "mesh-sdk", Version: "1.4.0"}}} + for _, b := range []Build{artifact, pkg} { + if _, err := c.Builds.Create(ctx, b.ID, b); err != nil { + t.Fatal(err) + } + } + got, ok, err := c.Builds.Get(ctx, "b1") + if err != nil || !ok { + t.Fatalf("read b1: %v %v", ok, err) + } + if got.Value.Module != "notes" || got.Value.Repository != "mesh-catalog" || got.Value.Commit != "c1" || + got.Value.ReuseKey != "rk-1" || got.Value.Published.Artifact != "sha256:aa" || len(got.Value.Made) != 1 { + t.Fatalf("the build came back as %+v", got.Value) + } + got, _, _ = c.Builds.Get(ctx, "b2") + if p := got.Value.Published; p == nil || p.Package == nil || p.Package.Name != "mesh-sdk" || p.Package.Version != "1.4.0" { + t.Fatalf("the package came back as %+v", p) + } + if _, ok, _ := c.Builds.Get(ctx, "nope"); ok { + t.Fatal("a key nobody wrote holds a build") + } +} + +// **A write against a revision that moved is refused and nothing is laid over it** (ADR 0408 §2). +func TestAWriteAgainstAMovedRevisionIsRefusedAndNothingIsLaidOver(t *testing.T) { + ctx := context.Background() + c := InMemory(nil) + first, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}) + if err != nil { + t.Fatal(err) + } + if _, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes"}); !errors.Is(err, ErrExists) { + t.Fatalf("a create of a key that holds a value answered %v", err) + } + second, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c2"}, first) + if err != nil { + t.Fatal(err) + } + // A writer that read the first revision writes now. + if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "stale"}, first); !errors.Is(err, ErrMoved) { + t.Fatalf("a write at a moved revision answered %v", err) + } + got, _, _ := c.Assignments.Get(ctx, "anchor.notes") + if got.Value.Commit != "c2" || got.Revision != second { + t.Fatalf("the key that moved was laid over: %+v at %d", got.Value, got.Revision) + } +} + +// **No key of the chain is ever removed** (ADR 0408 §4): the Backend has no delete and no purge, so nothing that holds +// one can remove a key; and an assignment of a module taken off its node is kept, with its earlier values. +func TestNoKeyOfTheChainIsEverRemoved(t *testing.T) { + var backend Backend = NewMemory() + if _, ok := backend.(interface { + Delete(context.Context, string, uint64) error + }); ok { + t.Fatal("the backend can delete") + } + if _, ok := backend.(interface { + Purge(context.Context, string) error + }); ok { + t.Fatal("the backend can purge") + } + if _, ok := any(InMemory(nil).Builds).(interface { + Delete(context.Context, string) error + }); ok { + t.Fatal("a bucket of the chain can delete") + } + ctx := context.Background() + c := InMemory(nil) + rev, _ := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}) + if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}, rev); err != nil { + t.Fatal(err) + } + keys, err := c.Assignments.Keys(ctx) + if err != nil || len(keys) != 1 || keys[0] != "anchor.notes" { + t.Fatalf("keys %v %v", keys, err) + } + history, err := c.Assignments.History(ctx, "anchor.notes") + if err != nil || len(history) != 2 { + t.Fatalf("the key keeps %d values: %v", len(history), err) + } +} + +// **Each key keeps up to sixty-four earlier values** (ADR 0408 §5), oldest first. +func TestAKeyKeepsItsEarlierValuesUpToSixtyFour(t *testing.T) { + ctx := context.Background() + c := InMemory(nil) + rev, _ := c.Rollouts.Create(ctx, "r1", Rollout{ID: "r1", State: "n0"}) + for i := 1; i <= 70; i++ { + var err error + if rev, err = c.Rollouts.Update(ctx, "r1", Rollout{ID: "r1", State: "n" + string(rune('A'+i%26))}, rev); err != nil { + t.Fatal(err) + } + } + history, err := c.Rollouts.History(ctx, "r1") + if err != nil { + t.Fatal(err) + } + if len(history) != broker.ChainHistory { + t.Fatalf("the key keeps %d values and the bus keeps %d", len(history), broker.ChainHistory) + } + for i := 1; i < len(history); i++ { + if history[i].Revision <= history[i-1].Revision { + t.Fatal("the history is not oldest first") + } + } +} + +// **A value over the largest message is refused, not cut, and the key is unchanged** (ADR 0408 §6). +func TestAValueOverTheLargestMessageIsRefusedNotCut(t *testing.T) { + ctx := context.Background() + c := InMemory(nil) + small := Build{ID: "b1", Repository: "r", Commit: "c"} + rev, err := c.Builds.Create(ctx, "b1", small) + if err != nil { + t.Fatal(err) + } + huge := Build{ID: "b1", Repository: "r", Commit: "c", Manifest: json.RawMessage(`"` + strings.Repeat("x", 1<<20) + `"`)} + if _, err := c.Builds.Update(ctx, "b1", huge, rev); !errors.Is(err, ErrTooLarge) { + t.Fatalf("a value of a mebibyte answered %v", err) + } + if _, err := c.Builds.Create(ctx, "b2", huge); !errors.Is(err, ErrTooLarge) { + t.Fatalf("a created value of a mebibyte answered %v", err) + } + got, _, _ := c.Builds.Get(ctx, "b1") + if got.Revision != rev || len(got.Value.Manifest) != 0 { + t.Fatalf("the key was changed: %+v", got) + } + if _, ok, _ := c.Builds.Get(ctx, "b2"); ok { + t.Fatal("a refused create left a key") + } + // The largest that fits is written whole. + fits := Build{ID: "b3", Manifest: json.RawMessage(`"` + strings.Repeat("x", broker.ChainValueBytes-1024) + `"`)} + if _, err := c.Builds.Create(ctx, "b3", fits); err != nil { + t.Fatalf("a value under the largest was refused: %v", err) + } + got, _, _ = c.Builds.Get(ctx, "b3") + if len(got.Value.Manifest) != len(fits.Manifest) { + t.Fatal("a value under the largest was cut") + } +} + +// **A rollout names every assignment it changed, with the build before and after** (ADR 0408 §1). +func TestARolloutNamesEveryAssignmentItChangedWithBuildBeforeAndAfter(t *testing.T) { + ctx := context.Background() + c := InMemory(nil) + r := Rollout{ID: "r1", State: "rolling", Commits: []Commit{{Repository: "mesh-catalog", Commit: "c2"}}, + Tiers: [][]string{{"notes"}}, Links: Links{Builds: []string{"b2"}, Changed: []Change{ + {Node: "anchor", Module: "notes", Before: Ref{Build: "b1", Commit: "c1"}, After: Ref{Build: "b2", Commit: "c2"}}, + {Node: "g14", Module: "notes", After: Ref{Build: "b2", Commit: "c2"}}}}} + if _, err := c.Rollouts.Create(ctx, "r1", r); err != nil { + t.Fatal(err) + } + got, _, _ := c.Rollouts.Get(ctx, "r1") + if len(got.Value.Changed) != 2 || got.Value.Changed[0].Before.Build != "b1" || got.Value.Changed[0].After.Build != "b2" || + got.Value.Changed[1].Before.Build != "" || got.Value.Changed[1].After.Commit != "c2" { + t.Fatalf("the rollout came back as %+v", got.Value) + } + // And in the contract's words: `changed`, with `before` and `after` naming `build` and `commit`. + raw, _ := json.Marshal(got.Value) + for _, want := range []string{`"changed":[`, `"before":{"build":"b1","commit":"c1"}`, `"after":{"build":"b2","commit":"c2"}`, + `"builds":["b2"]`, `"commits":[`, `"tiers":[["notes"]]`} { + if !strings.Contains(string(raw), want) { + t.Errorf("the rollout's value lacks %s: %s", want, raw) + } + } +} + +// **An assignment names its build, artifact or package, commit and rollout**, and keeps the fields its readers read +// under the names they read them by (ADR 0408 §3). +func TestAnAssignmentNamesBuildArtifactCommitAndRollout(t *testing.T) { + a := Assignment{Node: "anchor", Module: "notes", Assigned: at(3), RequiredBy: "wiki", + Provisioning: json.RawMessage(`{"plan":1}`), Build: "b2", Artifact: "sha256:bb", Commit: "c2", Rollout: "r1", Since: at(4)} + raw, _ := json.Marshal(a) + for _, want := range []string{`"node":"anchor"`, `"module":"notes"`, `"assigned":"2026-10-18T03:00:00Z"`, + `"required-by":"wiki"`, `"provisioning":{"plan":1}`, `"build":"b2"`, `"artifact":"sha256:bb"`, `"commit":"c2"`, + `"rollout":"r1"`, `"since":"2026-10-18T04:00:00Z"`} { + if !strings.Contains(string(raw), want) { + t.Errorf("the assignment's value lacks %s: %s", want, raw) + } + } + if a.Key() != "anchor.notes" { + t.Errorf("key %s", a.Key()) + } + // An assignment of today carries none of the new fields, and says none: a reader of today reads the bytes it did. + raw, _ = json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at(3)}) + if string(raw) != `{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z"}` { + t.Errorf("an assignment without a chain is %s", raw) + } + // A package in place of an artifact. + raw, _ = json.Marshal(Assignment{Node: "a", Module: "m", Package: &Package{Name: "n", Version: "1"}}) + if !strings.Contains(string(raw), `"package":{"name":"n","version":"1"}`) { + t.Errorf("the package is %s", raw) + } +} + +// **The writer without the lease writes nothing** (ADR 0408 §2): it reads, and every create and update is refused with +// the bucket untouched. +func TestTheWriterWithoutTheLeaseWritesNothing(t *testing.T) { + ctx := context.Background() + held := true + builds, rollouts, assignments := NewMemory(), NewMemory(), NewMemory() + c := New(builds, rollouts, assignments, func() bool { return held }) + rev, err := c.Builds.Create(ctx, "b1", Build{ID: "b1"}) + if err != nil { + t.Fatal(err) + } + held = false + if _, err := c.Builds.Create(ctx, "b2", Build{ID: "b2"}); !errors.Is(err, ErrNotHeld) { + t.Fatalf("a create without the lease answered %v", err) + } + if _, err := c.Builds.Update(ctx, "b1", Build{ID: "b1", Failed: "x"}, rev); !errors.Is(err, ErrNotHeld) { + t.Fatalf("an update without the lease answered %v", err) + } + if _, err := c.Rollouts.Create(ctx, "r", Rollout{ID: "r"}); !errors.Is(err, ErrNotHeld) { + t.Fatalf("a rollout written without the lease: %v", err) + } + if _, err := c.Assignments.Create(ctx, "a.b", Assignment{}); !errors.Is(err, ErrNotHeld) { + t.Fatalf("an assignment written without the lease: %v", err) + } + if builds.Writes() != 1 || rollouts.Writes() != 0 || assignments.Writes() != 0 { + t.Fatalf("writes %d %d %d", builds.Writes(), rollouts.Writes(), assignments.Writes()) + } + // Reads still answer. + if _, ok, err := c.Builds.Get(ctx, "b1"); err != nil || !ok { + t.Fatalf("a read without the lease: %v %v", ok, err) + } + if all, err := c.Builds.All(ctx); err != nil || len(all) != 1 { + t.Fatalf("all: %v %v", all, err) + } + // And a controller with no lease function at all holds nothing. + if _, err := New(NewMemory(), NewMemory(), NewMemory(), nil).Builds.Create(ctx, "b", Build{}); !errors.Is(err, ErrNotHeld) { + t.Fatalf("a chain with no lease wrote: %v", err) + } +} + +// **A bucket that is away is an error, never an empty bucket** (ADR 0227 rule 4). +func TestABucketThatIsAwayIsAnErrorNotAnEmptyOne(t *testing.T) { + ctx := context.Background() + m := NewMemory() + c := New(m, NewMemory(), NewMemory(), func() bool { return true }) + if _, err := c.Builds.Create(ctx, "b1", Build{ID: "b1"}); err != nil { + t.Fatal(err) + } + away := errors.New("the bus is away") + m.SetFail(away) + if all, err := c.Builds.All(ctx); !errors.Is(err, away) || all != nil { + t.Fatalf("all answered %v %v", all, err) + } + if _, _, err := c.Builds.Get(ctx, "b1"); !errors.Is(err, away) { + t.Fatalf("get answered %v", err) + } +} + +// A watch hands over every key's value, says it has, and then each write. +func TestAWatchHandsOverTheValuesThenEachWrite(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + c := InMemory(nil) + if _, err := c.Builds.Create(ctx, "b1", Build{ID: "b1"}); err != nil { + t.Fatal(err) + } + w, err := c.Builds.Watch(ctx) + if err != nil { + t.Fatal(err) + } + next := func() Entry { + select { + case e := <-w: + return e + case <-ctx.Done(): + t.Fatal("the watch said nothing") + return Entry{} + } + } + if e := next(); e.Key != "b1" || e.Caught { + t.Fatalf("first %+v", e) + } + if e := next(); !e.Caught { + t.Fatalf("second %+v", e) + } + if _, err := c.Builds.Create(ctx, "b2", Build{ID: "b2"}); err != nil { + t.Fatal(err) + } + if e := next(); e.Key != "b2" { + t.Fatalf("third %+v", e) + } +} diff --git a/internal/chain/records.go b/internal/chain/records.go new file mode 100644 index 00000000..01b54f9c --- /dev/null +++ b/internal/chain/records.go @@ -0,0 +1,147 @@ +// Package chain keeps the controller's chain on the bus alone (novox/hq ADR 0408, design 47): module, commit, build, +// published artifact or package, rollout and assignment are one chain, a bucket each of the bus's key-value store, +// each the only source of its kind, written by the controller that holds the lease by compare-and-set. +// +// **No key is removed.** A Backend has no delete and no purge (ADR 0408 §4): an assignment of a module taken off its +// node is kept, saying so. **A value is never cut** (§6): one over the largest the bus holds is refused, and the key +// stays as it was. **Each key keeps its earlier values** (§5), read back with History. +package chain + +import ( + "encoding/json" + "time" +) + +// Ref names a build and the commit it was made from: one end of a change. +type Ref struct { + Build string `json:"build,omitempty"` + Commit string `json:"commit,omitempty"` +} + +// Package is a release a build published to the package registry. +type Package struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// Published is what a build published: an artifact by its digest, or a package release by name and version. +type Published struct { + Artifact string `json:"artifact,omitempty"` + Package *Package `json:"package,omitempty"` +} + +// Artifact is one thing a build made. +type Artifact struct { + Name string `json:"name"` + Kind string `json:"kind"` + Reference string `json:"reference"` +} + +// ReadRepository is a repository a build read source from besides the module's own. +type ReadRepository struct { + Repository string `json:"repository"` + Ref string `json:"ref,omitempty"` + Paths []string `json:"paths,omitempty"` + Own bool `json:"own,omitempty"` +} + +// Source is the build source a build read in one repository. +type Source struct { + Repository string `json:"repository,omitempty"` + Ref string `json:"ref,omitempty"` + Paths []string `json:"paths"` +} + +// The states of a build record. A record in state asked is a request that no outcome has answered (issue 1091); one +// without a state is an outcome. +const BuildAsked = "asked" + +// Build is one attempt, whichever way it went, kept under its id. It names its module, its repository and commit, its +// reuse key (novox/hq ADR 0350) and what it published (ADR 0408 §1). +type Build struct { + ID string `json:"id"` + State string `json:"state,omitempty"` + Repository string `json:"repository"` + Ref string `json:"ref,omitempty"` + Module string `json:"module,omitempty"` + Commit string `json:"commit,omitempty"` + On string `json:"on,omitempty"` + Path string `json:"path,omitempty"` + Manifest json.RawMessage `json:"manifest,omitempty"` + Against []string `json:"against,omitempty"` + Mirrored []string `json:"mirrored,omitempty"` + Read []ReadRepository `json:"read,omitempty"` + Sources []Source `json:"sources,omitempty"` + SourceFingerprint string `json:"source-fingerprint,omitempty"` + Failed string `json:"failed,omitempty"` + Made []Artifact `json:"made,omitempty"` + Asked time.Time `json:"asked,omitzero"` + At time.Time `json:"at,omitzero"` + ReuseKey string `json:"reuse-key,omitempty"` + Published *Published `json:"published,omitempty"` +} + +// Commit is one repository's commit a rollout carries. +type Commit struct { + Repository string `json:"repository"` + Branch string `json:"branch,omitempty"` + Commit string `json:"commit"` + Merged time.Time `json:"merged,omitzero"` +} + +// Change is one assignment a rollout changed: the build and commit it ran before and the ones it runs after. +type Change struct { + Node string `json:"node"` + Module string `json:"module"` + Before Ref `json:"before"` + After Ref `json:"after"` +} + +// Step is one tier sent to one machine. +type Step struct { + Tier int `json:"tier"` + Node string `json:"node"` + Sent *time.Time `json:"sent,omitempty"` + Reported *time.Time `json:"reported,omitempty"` +} + +// Links are the fields a rollout gains on the chain: the builds it carried, the machine steps it has taken and the +// assignments it changed, each with the build before and after. The controller's plan embeds them, so the plan's own +// fields and these are one value (ADR 0408 §1). +type Links struct { + Builds []string `json:"builds,omitempty"` + Steps []Step `json:"steps,omitempty"` + Changed []Change `json:"changed,omitempty"` +} + +// Rollout is the controller's record of one batch's delivery, kept under the batch's id: the commit it fixed per +// repository, its tiers, the builds it carried and the assignments it changed (ADR 0408 §1). The controller's plan +// carries more (its modules, its times, its backups) beside these, under the same key. +type Rollout struct { + ID string `json:"id"` + State string `json:"state,omitempty"` + Commits []Commit `json:"commits,omitempty"` + Tiers [][]string `json:"tiers,omitempty"` + Links +} + +// Assignment is one module on one node, kept under `.`. The first five fields are the ones its readers +// have always read, with the names they have always had (novox/hq ADR 0408 §3); the rest name the chain: the build, the +// artifact (by digest) or the package it runs, its commit, the rollout that put it there and since when. +type Assignment struct { + Node string `json:"node"` + Module string `json:"module"` + Assigned time.Time `json:"assigned,omitzero"` + RequiredBy string `json:"required-by,omitempty"` + Provisioning json.RawMessage `json:"provisioning,omitempty"` + + Build string `json:"build,omitempty"` + Artifact string `json:"artifact,omitempty"` + Package *Package `json:"package,omitempty"` + Commit string `json:"commit,omitempty"` + Rollout string `json:"rollout,omitempty"` + Since time.Time `json:"since,omitzero"` +} + +// Key is an assignment's key. +func (a Assignment) Key() string { return a.Node + "." + a.Module } diff --git a/internal/inventory/chain_moved_test.go b/internal/inventory/chain_moved_test.go new file mode 100644 index 00000000..5298aeef --- /dev/null +++ b/internal/inventory/chain_moved_test.go @@ -0,0 +1,40 @@ +package inventory + +import ( + "strings" + "testing" +) + +// **The move of the chain has a switch, made by the last migration and written by no migration** (ADR 0408 §8, issue +// 1089): migration 0104 makes the one-row table chain_moved, and does not touch a table of the chain, which a later +// delivery drops once the move is read back. +func TestMigration0104MakesTheSwitchOfTheChainsMoveAndNothingElse(t *testing.T) { + migrations, err := Migrations() + if err != nil { + t.Fatal(err) + } + var found bool + for _, m := range migrations { + if m.Number != 104 { + continue + } + found = true + sql := strings.ToLower(m.SQL) + if !strings.Contains(sql, "create table chain_moved") || !strings.Contains(sql, "check (only_one)") { + t.Errorf("migration 0104 does not make the one-row chain_moved:\n%s", m.SQL) + } + for _, line := range strings.Split(sql, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "--") { + continue + } + for _, verb := range []string{"drop ", "insert ", "delete ", "update ", "alter "} { + if strings.Contains(line, verb) { + t.Errorf("migration 0104 does more than make the switch: %q", line) + } + } + } + } + if !found { + t.Fatal("no migration 0104") + } +} diff --git a/internal/inventory/migrations/0104-the-chain-moves-to-the-bus.sql b/internal/inventory/migrations/0104-the-chain-moves-to-the-bus.sql new file mode 100644 index 00000000..92702cdb --- /dev/null +++ b/internal/inventory/migrations/0104-the-chain-moves-to-the-bus.sql @@ -0,0 +1,16 @@ +-- Builds, rollouts and assignments move to the bus (novox/hq ADR 0408 §8, design 47 milestone 5, issue 1089). +-- +-- This migration makes the switch alone. The rows of build, build_request, release_plan, batched_merge, assignment and +-- pending_assignment are written into the chain's buckets by the serving controller at its start, holding the lease, +-- because a migration runs when the store is opened, before any connection to the bus. This row is the switch: while it +-- is absent nothing reads the buckets; once the move has counted every row against the keys it wrote, it is written, and +-- nothing reads the tables. One row at most; never written by a migration, so a delivery alone changes nothing. The +-- controller keeps no consolidated schema, so this migration is the whole of the change; the tables go by a later one. +create table chain_moved ( + only_one boolean primary key default true check (only_one), + moved_at timestamptz not null default now(), + -- The counts the move read from the tables and wrote as keys, per kind, and the largest value it wrote: what a + -- reader of the move reads back. + counts jsonb not null, + largest jsonb not null +); diff --git a/internal/recordmirror/readers_test.go b/internal/recordmirror/readers_test.go index 591b57d9..864a22ea 100644 --- a/internal/recordmirror/readers_test.go +++ b/internal/recordmirror/readers_test.go @@ -20,6 +20,9 @@ import ( // (internal/catalogue/state.go), and each mirror's own writer (this package and the loop that runs it). The // audit of a mirror against its store (ADR 0289 §4, issue 419) joins this list when it is built. var mayNameAMirror = []string{ + // The chain opens its buckets, the assignments' among them, which stopped being a mirror when the chain took them + // (ADR 0408 §3, issue 1089). + "internal/chain/bus.go", "cmd/mesh-controller/record_mirrors.go", "internal/broker/controller_buckets.go", // The grant of a machine's own keys of the assignments mirror to its provisioner client (design 57 P3).