57/F3: Keep the assignments and each module's declared config on the bus (issue 652) #315
@@ -264,6 +264,9 @@ func serve(ctx context.Context) (err error) {
|
||||
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9): kept current while this
|
||||
// controller holds the lease, read by the build seat from the artifact store.
|
||||
go exportingFacts(ctx, open, bus.Conn.ConnectedServerVersion)
|
||||
// The record mirrors (novox/hq ADR 0289, design 57 F3): the assignments and each registered module's
|
||||
// declared config on the bus, written from the store only when a record changed, by the lease holder.
|
||||
go mirroringRecords(ctx, inv, bus.Conn, bus)
|
||||
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
|
||||
// to-be 45 §6).
|
||||
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/recordmirror"
|
||||
)
|
||||
|
||||
// The controller's record mirrors on the bus (novox/hq ADR 0289, design 54, design 57 F3): the assignments,
|
||||
// and each registered module's declared config, which mesh-config resolves a module's config values from.
|
||||
//
|
||||
// **Built from the store, on a tick of their own, by the lease holder alone.** A record changes through a
|
||||
// verb that runs as a command of its own, a reconcile, a merge's registration or a pending assignment made,
|
||||
// so no one call site sees every change; what every change has in common is that it is committed to the
|
||||
// store. So the serving controller reads the store every mirrorEvery and writes only what differs from what
|
||||
// the bucket holds (ADR 0289 §1): a quiet mesh writes nothing, and a change rolled back is never read. A
|
||||
// mirror is therefore at most mirrorEvery behind its store.
|
||||
|
||||
// mirrorEvery is how often the mirrors are compared with the store: two small reads of the store.
|
||||
const mirrorEvery = 10 * time.Second
|
||||
|
||||
// mirroringRecords keeps the mirrors current for as long as this controller serves: ctx ends when it stops.
|
||||
func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, announce recordmirror.Announcer) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
fmt.Printf("the record mirrors are not kept: %v\n", err)
|
||||
return
|
||||
}
|
||||
holds := func() bool {
|
||||
epoch, err := theLease.epoch(ctx)
|
||||
return err == nil && epoch > 0
|
||||
}
|
||||
epoch := func() uint64 {
|
||||
e, _ := theLease.epoch(ctx)
|
||||
return e
|
||||
}
|
||||
sources := map[string]func(context.Context) (map[string][]byte, error){
|
||||
recordmirror.Assignments.Name: inv.AssignmentsMirror,
|
||||
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
||||
}
|
||||
writers := map[string]*recordmirror.Writer{}
|
||||
failing := map[string]string{}
|
||||
tick := time.NewTicker(mirrorEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
if holds() {
|
||||
for _, m := range recordmirror.Mirrors() {
|
||||
why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
|
||||
if why != failing[m.Name] {
|
||||
if why != "" {
|
||||
fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
|
||||
} else {
|
||||
fmt.Printf("the %s mirror is kept again\n", m.Name)
|
||||
}
|
||||
failing[m.Name] = why
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could.
|
||||
func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
|
||||
source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64,
|
||||
holds func() bool) string {
|
||||
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
w := writers[m.Name]
|
||||
if w == nil {
|
||||
kv, err := api.KeyValue(reading, m.Bucket)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err)
|
||||
}
|
||||
w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
|
||||
writers[m.Name] = w
|
||||
}
|
||||
want, err := source(reading)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("the store cannot be read: %v", err)
|
||||
}
|
||||
lost := w.LostEvents
|
||||
r, err := w.Sync(reading, want)
|
||||
if r.Written+r.Removed > 0 {
|
||||
fmt.Printf("the %s mirror moved: %d written, %d removed\n", m.Name, r.Written, r.Removed)
|
||||
}
|
||||
if w.LostEvents > lost {
|
||||
fmt.Printf("the %s mirror lost %d event(s): the bucket holds the change, its notice was refused twice\n",
|
||||
m.Name, w.LostEvents-lost)
|
||||
}
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -40,6 +40,39 @@ var (
|
||||
AskedBucket = BucketName(ControllerSeat, "asked")
|
||||
)
|
||||
|
||||
// The controller's record mirrors (novox/hq ADR 0289, design 54, design 57 package F3): a bucket per kind of
|
||||
// record, one key per record, written from the store by the lease holder only when the record changed, with
|
||||
// one event on the controller's seat per write. Read by whatever names them in `reads` as
|
||||
// `mesh-controller.<mirror>` (ADR 0201 §1), written by nobody else. **No secret in either** (ADR 0201): an
|
||||
// assignment's value names its module and node; a module's declared config is what its manifest, which is
|
||||
// public, declares.
|
||||
//
|
||||
// AssignmentsBucket holds which module is assigned to which node, keyed `<node>.<module>`; and
|
||||
// DeclaredConfigBucket each registered module's config declarations as its manifest holds them, keyed by
|
||||
// the module: what mesh-config resolves a config value from (ADR 0348 clauses 2 and 4).
|
||||
var (
|
||||
AssignmentsBucket = BucketName(ControllerSeat, MirrorAssignments)
|
||||
DeclaredConfigBucket = BucketName(ControllerSeat, MirrorDeclaredConfig)
|
||||
)
|
||||
|
||||
// The mirrors' local names, as a reader names them in `reads` after `mesh-controller.`.
|
||||
const (
|
||||
MirrorAssignments = "assignments"
|
||||
MirrorDeclaredConfig = "declared-config"
|
||||
)
|
||||
|
||||
// ControllerMirrors are the local names of the controller's record mirrors, which a module may read.
|
||||
func ControllerMirrors() []string { return []string{MirrorAssignments, MirrorDeclaredConfig} }
|
||||
|
||||
// The mirrors' caps (design 54 §5): one value per key, no age, 64 KiB a value, 64 MiB a bucket.
|
||||
const (
|
||||
MirrorValueBytes = 64 << 10
|
||||
MirrorBytes = 64 << 20
|
||||
// MirrorMarkerTTL is the longest a purge's marker is kept (design 54 §2): the bucket allows a
|
||||
// per-message age for it, and the writer gives each marker a day.
|
||||
MirrorMarkerTTL = 24 * time.Hour
|
||||
)
|
||||
|
||||
// AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own.
|
||||
const AskedKeptFor = 30 * 24 * time.Hour
|
||||
|
||||
@@ -66,12 +99,14 @@ const (
|
||||
// IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares.
|
||||
func IsControllerBucket(bucket string) bool {
|
||||
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
||||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket
|
||||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
|
||||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket
|
||||
}
|
||||
|
||||
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||
func ControllerBuckets() []string {
|
||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket}
|
||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
|
||||
AssignmentsBucket, DeclaredConfigBucket}
|
||||
}
|
||||
|
||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||
@@ -168,6 +203,27 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", AskedBucket, err)
|
||||
}
|
||||
// **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose
|
||||
// marker lives a day; a key that expired would be a record the bus forgot while the store still holds it.
|
||||
for _, m := range []struct{ bucket, why string }{
|
||||
{AssignmentsBucket, "which module is assigned to which node, keyed <node>.<module> (novox/hq ADR 0289, " +
|
||||
"design 54): written by the controller alone from its store when an assignment changes, read through `reads`"},
|
||||
{DeclaredConfigBucket, "each registered module's config declarations as its manifest holds them, keyed by " +
|
||||
"the module (novox/hq ADR 0348, design 57 F3): written by the controller alone from its store when a " +
|
||||
"registration changes them, read through `reads`"},
|
||||
} {
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: m.bucket,
|
||||
Description: m.why,
|
||||
History: 1,
|
||||
MaxValueSize: MirrorValueBytes,
|
||||
MaxBytes: MirrorBytes,
|
||||
Storage: jetstream.FileStorage,
|
||||
LimitMarkerTTL: MirrorMarkerTTL,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", m.bucket, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,8 @@ func TestTheControllerMayWriteEveryBucketItWrites(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"$KV." + CallsBucket + ".>", "$KV." + HandActsBucket + ".>"}
|
||||
want := []string{"$KV." + CallsBucket + ".>", "$KV." + HandActsBucket + ".>",
|
||||
"$KV." + AssignmentsBucket + ".>", "$KV." + DeclaredConfigBucket + ".>"}
|
||||
for _, seat := range seatsTheControllerAsks {
|
||||
if hasCancelledSet(seat) {
|
||||
want = append(want, "$KV."+CancelledSetName(seat)+".>")
|
||||
@@ -37,6 +38,34 @@ func TestTheControllerMayWriteEveryBucketItWrites(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// **A module reads a record mirror through `reads`, and never writes it** (ADR 0201 §1, design 54 §5,
|
||||
// design 57 F3): granted the direct reads and watches of the bucket, and refused at composition a grant to
|
||||
// write it, whose writer is the controller.
|
||||
func TestAModuleReadsTheControllersMirrorsAndWritesNone(t *testing.T) {
|
||||
reader := Principal{Kind: KindModule, Node: "one", Module: "mesh-config", PasswordHash: "x",
|
||||
Reads: []string{"mesh-controller." + MirrorAssignments, "mesh-controller." + MirrorDeclaredConfig}}
|
||||
p, err := PermissionsFor(reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, bucket := range []string{AssignmentsBucket, DeclaredConfigBucket} {
|
||||
for _, subject := range []string{"$JS.API.STREAM.INFO.KV_" + bucket, "$JS.API.DIRECT.GET.KV_" + bucket + ".>",
|
||||
"$JS.API.CONSUMER.CREATE.KV_" + bucket + ".>"} {
|
||||
if !slices.Contains(p.Publish, subject) {
|
||||
t.Errorf("a module reading %s may not publish %s", bucket, subject)
|
||||
}
|
||||
}
|
||||
for _, subject := range p.Publish {
|
||||
if SubjectsOverlap(subject, "$KV."+bucket+".>") {
|
||||
t.Errorf("a module reading %s may write it through %s", bucket, subject)
|
||||
}
|
||||
}
|
||||
if err := CheckWriters(reader, []string{"$KV." + bucket + ".>"}); err == nil {
|
||||
t.Errorf("a module granted a write of %s would compose", bucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A machine's node tools may say they are there, as that machine and no other** (novox/hq to-be 45
|
||||
// S11), and the controller may hear the bus's advisories and ask who answers — read-only, named.
|
||||
func TestTheWatchedSignalsMayBeSaidAndHeard(t *testing.T) {
|
||||
@@ -96,3 +125,56 @@ func TestWhatTheControllerAskedIsBounded(t *testing.T) {
|
||||
t.Errorf("history %d, age %s, bytes %d", status.History(), status.TTL(), info.Config.MaxBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// **A listed mirror is no undeclared bucket** (ADR 0356 §3, ADR 0201 §16): the controller's list of its mirrors
|
||||
// stands for their declaration, so the report of state nothing declares never names one.
|
||||
func TestAListedMirrorIsNotReportedAsUndeclared(t *testing.T) {
|
||||
names := []string{"gone_state"}
|
||||
for _, m := range ControllerMirrors() {
|
||||
names = append(names, BucketName(ControllerSeat, m))
|
||||
}
|
||||
undeclared, err := RaiseBuckets(fakeBuckets{names: names}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(undeclared, []string{"gone_state"}) {
|
||||
t.Fatalf("the report of undeclared state names %v", undeclared)
|
||||
}
|
||||
}
|
||||
|
||||
// **The mirrors' list and the writers table agree** (ADR 0356 §3, to-be 45 §1): every listed mirror is a row of
|
||||
// the table whose one writer is the controller, on its bucket alone; and every row that writes a bucket of the
|
||||
// controller's named as a mirror is on the list.
|
||||
func TestTheMirrorsListAndTheWritersTableAgree(t *testing.T) {
|
||||
controller := Principal{Kind: KindController}
|
||||
module := Principal{Kind: KindModule, Module: ControllerSeat, Node: "one"}
|
||||
rowed := map[string]bool{}
|
||||
for _, row := range WritersTable {
|
||||
if !strings.HasSuffix(row.State, " mirror") {
|
||||
continue
|
||||
}
|
||||
for _, subject := range row.Subjects {
|
||||
bucket := strings.TrimSuffix(strings.TrimPrefix(subject, "$KV."), ".>")
|
||||
rowed[bucket] = true
|
||||
if row.Writes == nil || !row.Writes(controller, subject) || row.Writes(module, subject) {
|
||||
t.Errorf("%q is not written by the controller alone", row.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
listed := map[string]bool{}
|
||||
for _, m := range ControllerMirrors() {
|
||||
bucket := BucketName(ControllerSeat, m)
|
||||
listed[bucket] = true
|
||||
if !rowed[bucket] {
|
||||
t.Errorf("the mirror %s is listed and no row of the writers table writes it", bucket)
|
||||
}
|
||||
if !IsControllerBucket(bucket) || !slices.Contains(ControllerBuckets(), bucket) {
|
||||
t.Errorf("the mirror %s is listed and not asserted as the controller's own", bucket)
|
||||
}
|
||||
}
|
||||
for bucket := range rowed {
|
||||
if !listed[bucket] {
|
||||
t.Errorf("the writers table writes the mirror %s, which the list does not name", bucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,7 +172,9 @@ func TestRaisingLogsReportsWhatNothingDeclares(t *testing.T) {
|
||||
// **No path of the mesh deletes or purges a log or a bucket, or recreates one** (novox/hq ADR 0297 §2,
|
||||
// ADR 0201 §15–16): no code of this repository outside its tests calls the client's stream or bucket
|
||||
// delete, or purges a stream, but for the controller lease's own stream, which purges its own history
|
||||
// below a sequence (internal/lease). A new call is a decision, not a refactor.
|
||||
// below a sequence (internal/lease), and the controller's record mirrors, which purge the key of a record
|
||||
// the store no longer holds (internal/recordmirror; novox/hq ADR 0289, design 54 §2). A new call is a decision,
|
||||
// not a refactor.
|
||||
func TestNoPathDeletesALogOrABucket(t *testing.T) {
|
||||
removers := map[string]bool{"DeleteStream": true, "DeleteKeyValue": true, "PurgeStream": true,
|
||||
"DeleteObjectStore": true}
|
||||
@@ -197,7 +199,9 @@ func TestNoPathDeletesALogOrABucket(t *testing.T) {
|
||||
return true
|
||||
}
|
||||
name := sel.Sel.Name
|
||||
if removers[name] || (name == "Purge" && !strings.Contains(filepath.ToSlash(path), "internal/lease/")) {
|
||||
decided := strings.Contains(filepath.ToSlash(path), "internal/lease/") ||
|
||||
strings.Contains(filepath.ToSlash(path), "internal/recordmirror/")
|
||||
if removers[name] || (name == "Purge" && !decided) {
|
||||
found = append(found, path+": "+name)
|
||||
}
|
||||
return true
|
||||
|
||||
@@ -34,6 +34,8 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
|
||||
states = append(states, link.KeyChecked)
|
||||
// And a walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||
states = append(states, link.KeyPlanMoved)
|
||||
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
|
||||
states = append(states, link.KeyAssignmentChanged, link.KeyDeclaredConfigChanged)
|
||||
for _, event := range states {
|
||||
if !slices.Contains(broker.ControllerStates, event) {
|
||||
t.Errorf("the mesh states %q and its account may not publish it", event)
|
||||
@@ -42,6 +44,11 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
|
||||
if len(broker.ControllerStates) != len(states) {
|
||||
t.Errorf("the grant permits %v, which is more than the mesh states", broker.ControllerStates)
|
||||
}
|
||||
// The record mirrors a module may read are the ones the bus holds (design 54 §5, design 57 F3).
|
||||
if !slices.Equal(catalogue.ControllerMirrors, broker.ControllerMirrors()) {
|
||||
t.Errorf("the catalogue lets a module read the controller's mirrors %v and the bus holds %v",
|
||||
catalogue.ControllerMirrors, broker.ControllerMirrors())
|
||||
}
|
||||
// **And the seat says it.** A seat carries the protocol of its role (novox/hq ADR 0129), so the
|
||||
// facts the control plane states are the seat's `emits` — which is what lets anything else declare
|
||||
// that it consumes them, and what the subject-agreement check reads to know they have an owner.
|
||||
|
||||
@@ -313,7 +313,10 @@ var ControllerStates = []string{"applied", "refused", "built-before",
|
||||
// the pull request's status, and anybody else may read.
|
||||
"checked",
|
||||
// And a walk kept in a new state (novox/hq ADR 0239): what the delivery it walks reads its steps from.
|
||||
"plan-moved"}
|
||||
"plan-moved",
|
||||
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and
|
||||
// each registered module's declared config (design 57 F3).
|
||||
"assignment-changed", "declared-config-changed"}
|
||||
|
||||
// BusAdvisories are what the bus server says about the mesh's own account that the controller
|
||||
// reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
@@ -158,6 +158,15 @@ var WritersTable = []WriterRow{
|
||||
{State: "a message a consumer gave up on", Writer: "controller", KeptIn: "the bus, the stream " + DeadLettersStream,
|
||||
Others: "read, delivered again or dropped through the controller's dead-letters verb",
|
||||
Subjects: []string{deadLetterPrefix + ">", againPrefix + ">"}, Writes: isController},
|
||||
// The record mirrors (novox/hq ADR 0289 §9, design 54 §2): rows of this table, so a grant that lets
|
||||
// anything else write them is refused at composition. Built so far: the assignments and each registered
|
||||
// module's declared config (design 57 F3).
|
||||
{State: "the assignments mirror", Writer: "controller (lease holder), from its store, only on a change",
|
||||
KeptIn: "key-value " + AssignmentsBucket, Others: "read through `reads` as mesh-controller." + MirrorAssignments,
|
||||
Subjects: kvOf(AssignmentsBucket), Writes: isController},
|
||||
{State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change",
|
||||
KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig,
|
||||
Subjects: kvOf(DeclaredConfigBucket), Writes: isController},
|
||||
}
|
||||
|
||||
// CheckWriters refuses a grant that lets a principal publish on a subject the writers table gives
|
||||
|
||||
@@ -34,6 +34,9 @@ var designRows = []string{
|
||||
"the operator-channel's open messages",
|
||||
"the facts snapshot",
|
||||
"a message a consumer gave up on",
|
||||
// The record mirrors, rows of to-be 45 §1's table by design 54 §2 (ADR 0289 §9).
|
||||
"the assignments mirror",
|
||||
"the declared-config mirror",
|
||||
}
|
||||
|
||||
func TestTheWritersTableIsTheDesigns(t *testing.T) {
|
||||
|
||||
@@ -107,7 +107,9 @@ var defaultSeats = append([]Seat{
|
||||
// A pull request's merge check, judged (novox/hq to-be 45 §9).
|
||||
"checked",
|
||||
// A walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||
"plan-moved"},
|
||||
"plan-moved",
|
||||
// A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
|
||||
"assignment-changed", "declared-config-changed"},
|
||||
Serves: ControllerVerbs},
|
||||
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
|
||||
// served by whichever module holds the seat with tools of these names.
|
||||
|
||||
@@ -124,6 +124,14 @@ func StateProblems(m Manifest) []string {
|
||||
return problems
|
||||
}
|
||||
|
||||
// ControllerMirrors are the controller's record mirrors a module may read, each `mesh-controller.<name>`
|
||||
// (novox/hq ADR 0289, design 54 §5, design 57 F3). **The controller keeps them, not its manifest's `state`**:
|
||||
// they are asserted with the controller's own buckets, with options of their own, and written by the
|
||||
// controller's account alone (the writers table); declared as a module's state, they would be asserted a
|
||||
// second way and granted to the module's own principal as well. Named here so a read of one is a read of
|
||||
// kept state. The bus's list is broker.ControllerMirrors; a test keeps the two agreeing.
|
||||
var ControllerMirrors = []string{"assignments", "declared-config"}
|
||||
|
||||
// StateReadsNothingDeclares is every read across a catalogue whose owner is present and declares no
|
||||
// such state. An absent owner says nothing — a module may be installed long before the one whose
|
||||
// state it reads, as a consumer may before its emitter (design 32 §1).
|
||||
@@ -136,6 +144,11 @@ func StateReadsNothingDeclares(manifests []Manifest) []string {
|
||||
}
|
||||
declared[m.Module] = own
|
||||
}
|
||||
if own, present := declared[ControllerSeatName]; present {
|
||||
for _, name := range ControllerMirrors {
|
||||
own[name] = true
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
for _, m := range manifests {
|
||||
for _, r := range m.Reads {
|
||||
|
||||
@@ -67,6 +67,21 @@ func TestAReadNamesStateItsOwnerKeeps(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// **The controller's record mirrors are kept state** (design 54 §5, design 57 F3): a module may read
|
||||
// `mesh-controller.assignments` and `mesh-controller.declared-config` although the controller's manifest
|
||||
// declares no `state`; a read of anything else of the controller's is still named.
|
||||
func TestAReadOfTheControllersMirrorsIsAReadOfKeptState(t *testing.T) {
|
||||
controller := Manifest{Module: ControllerSeatName}
|
||||
reader := Manifest{Module: "mesh-config", Reads: []string{"mesh-controller.assignments", "mesh-controller.declared-config"}}
|
||||
if p := StateReadsNothingDeclares([]Manifest{controller, reader}); len(p) != 0 {
|
||||
t.Fatalf("a read of the controller's mirrors was refused: %v", p)
|
||||
}
|
||||
other := Manifest{Module: "other", Reads: []string{"mesh-controller.calls"}}
|
||||
if p := StateReadsNothingDeclares([]Manifest{controller, other}); len(p) != 1 {
|
||||
t.Fatalf("a read of the controller's own calls was not named: %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// **Across the whole catalogue**: every state name is local, and every read whose owner is present
|
||||
// names state that owner keeps.
|
||||
func TestEveryManifestsStateIsLocalAndEveryReadIsKept(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The values of the controller's record mirrors, built from the store (novox/hq ADR 0289, design 54 §1,
|
||||
// design 57 package F3).
|
||||
//
|
||||
// **Each value is built from a named list of fields, and nothing else** (design 54 §1): no epoch, no time
|
||||
// of writing and no time of the last look, so any lease holder builds the same bytes from the same record
|
||||
// and a pass over unchanged records writes nothing (ADR 0289 §1, §4). The encoding is fixed: the fields in
|
||||
// their struct's order, a declaration's maps sorted.
|
||||
|
||||
// AssignmentMirrored is one assignment as the assignments mirror holds it: the module, the node, and since
|
||||
// when it is assigned there. Design 54 §1 also lists whether it is pending, the builds sent and running and
|
||||
// the seats and provisions it holds there; those join with issue 419, which builds the rest of design 54.
|
||||
type AssignmentMirrored struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Assigned time.Time `json:"assigned"`
|
||||
}
|
||||
|
||||
// DeclaredConfigMirrored is one registered module's config declarations as the declared-config mirror holds
|
||||
// them: the module, and each key its manifest's `settings` declares with that declaration's listed fields.
|
||||
// An empty object when the module declares none: registered, declaring nothing.
|
||||
type DeclaredConfigMirrored struct {
|
||||
Module string `json:"module"`
|
||||
Settings map[string]DeclaredMirrored `json:"settings"`
|
||||
}
|
||||
|
||||
// DeclaredMirrored is one key's declaration as the mirror holds it: **its listed fields and no other**
|
||||
// (ADR 0289 §12). Today a declaration is kind, default and why (catalogue.SettingDeclaration); a field the
|
||||
// manifest gains reaches the bus only once it is added here, deliberately, and a test fails until it is
|
||||
// listed or refused (ADR 0348 clause 4's fields arrive with package K1). A default is never a secret: the
|
||||
// catalogue refuses a default for a key whose name says secret.
|
||||
type DeclaredMirrored struct {
|
||||
Kind string `json:"kind"`
|
||||
Default json.RawMessage `json:"default"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// DeclaredMirroredFields are the fields of a declaration the mirror carries, by their manifest names.
|
||||
var DeclaredMirroredFields = []string{"kind", "default", "why"}
|
||||
|
||||
// AssignmentsMirror is every assignment's mirrored value, keyed `<node>.<module>`.
|
||||
func (i *Inventory) AssignmentsMirror(ctx context.Context) (map[string][]byte, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select n.name, a.module, a.assigned from assignment a join node n on n.id = a.node`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string][]byte{}
|
||||
for rows.Next() {
|
||||
var a AssignmentMirrored
|
||||
if err := rows.Scan(&a.Node, &a.Module, &a.Assigned); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.Assigned = a.Assigned.UTC()
|
||||
value, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[a.Node+"."+a.Module] = value
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// DeclaredConfigMirror is every registered module's mirrored config declarations, keyed by the module.
|
||||
func (i *Inventory) DeclaredConfigMirror(ctx context.Context) (map[string][]byte, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select name, coalesce(manifest->'settings', '{}'::jsonb) from module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string][]byte{}
|
||||
for rows.Next() {
|
||||
var (
|
||||
d DeclaredConfigMirrored
|
||||
raw []byte
|
||||
)
|
||||
if err := rows.Scan(&d.Module, &raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var declared map[string]DeclaredMirrored
|
||||
if err := json.Unmarshal(raw, &declared); err != nil {
|
||||
return nil, fmt.Errorf("the declarations of %s cannot be read: %w", d.Module, err)
|
||||
}
|
||||
d.Settings = map[string]DeclaredMirrored{}
|
||||
for key, decl := range declared {
|
||||
if len(decl.Default) > 0 {
|
||||
if decl.Default, err = canonical(decl.Default); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
d.Settings[key] = decl
|
||||
}
|
||||
value, err := json.Marshal(d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[d.Module] = value
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// canonical is a JSON document encoded the one way: objects' keys sorted, numbers as written.
|
||||
func canonical(raw []byte) ([]byte, error) {
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.UseNumber()
|
||||
var v any
|
||||
if err := dec.Decode(&v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(v)
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// jsonFields is the names a struct's fields are encoded under.
|
||||
func jsonFields(t reflect.Type) []string {
|
||||
var out []string
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
name, _, _ := strings.Cut(t.Field(i).Tag.Get("json"), ",")
|
||||
if name != "" && name != "-" {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// **A declaration reaches the bus with its listed fields and no other** (ADR 0289 §12): the mirror's value
|
||||
// carries exactly DeclaredMirroredFields, and every field a manifest's declaration has is on that list. A
|
||||
// field a declaration gains (package K1 widens it, ADR 0348 clause 4) fails here until it is listed — read,
|
||||
// judged not a secret and carried — so nothing reaches the declared-config mirror unreviewed.
|
||||
func TestTheDeclaredConfigMirrorCarriesOnlyItsListedFields(t *testing.T) {
|
||||
if got := jsonFields(reflect.TypeOf(DeclaredMirrored{})); !slices.Equal(got, DeclaredMirroredFields) {
|
||||
t.Errorf("the mirrored declaration carries %v, its list is %v", got, DeclaredMirroredFields)
|
||||
}
|
||||
for _, field := range jsonFields(reflect.TypeOf(catalogue.SettingDeclaration{})) {
|
||||
if !slices.Contains(DeclaredMirroredFields, field) {
|
||||
t.Errorf("a declaration has the field %q, which the declared-config mirror does not list: list it "+
|
||||
"in DeclaredMirrored (and say why it is no secret), or say here why it stays off the bus", field)
|
||||
}
|
||||
}
|
||||
if got := jsonFields(reflect.TypeOf(DeclaredConfigMirrored{})); !slices.Equal(got, []string{"module", "settings"}) {
|
||||
t.Errorf("the mirrored module carries %v", got)
|
||||
}
|
||||
}
|
||||
@@ -81,6 +81,11 @@ const (
|
||||
// steps from. Said after every save of the serving controller; a delivery's holder also asks for the
|
||||
// walks it follows, so a save made elsewhere is found by comparison, never lost.
|
||||
KeyPlanMoved = "plan-moved"
|
||||
// KeyAssignmentChanged and KeyDeclaredConfigChanged: a key of the controller's assignments or declared-config
|
||||
// mirror was written or removed (novox/hq ADR 0289, design 54 §3, design 57 F3): which key, how and at which
|
||||
// revision of the bucket — never the value, which the bucket holds.
|
||||
KeyAssignmentChanged = "assignment-changed"
|
||||
KeyDeclaredConfigChanged = "declared-config-changed"
|
||||
)
|
||||
|
||||
// Applied is what a machine now runs, as the mesh states it.
|
||||
|
||||
@@ -0,0 +1,344 @@
|
||||
// Package recordmirror writes the controller's record mirrors (novox/hq ADR 0289, design 54): a bucket per kind
|
||||
// of record, one key per record, written from the store only when the record changed, and one event on the
|
||||
// controller's seat per write.
|
||||
//
|
||||
// **The store stays the record.** A mirror is built from what the store committed, so a change rolled back
|
||||
// is never written. Every pass reads what the bucket holds and compares it byte for byte with what the store
|
||||
// says: a pass over unchanged records writes nothing and says nothing (ADR 0289 §1), and a value somebody
|
||||
// else wrote behind the writer's back (a former lease holder) differs from the store and is put right on
|
||||
// the next pass. Each write names the revision it read (compare-and-set), so a write in between is noticed,
|
||||
// never overwritten blind.
|
||||
//
|
||||
// **Only the lease holder writes** (ADR 0289 §9, design 54 §2): a pass begins by asking whether this
|
||||
// controller holds the lease, and asks again before every write; without it, it writes nothing.
|
||||
//
|
||||
// **The bucket is the authority, the event a notice** (design 54 §3): the event is sent after the write
|
||||
// succeeded, so there is no event for a value the bucket does not hold. A refused event is tried once more
|
||||
// and then counted as lost; a watcher that must not miss a change watches the bucket.
|
||||
//
|
||||
// What this package does not do yet is design 54 §4's audit (probe D17 and its healer H6), which compares a
|
||||
// mirror with its store behind the writer's back; novox/hq issue 419 carries it.
|
||||
package recordmirror
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A Mirror is one kind of record the controller keeps on the bus: its local name, its bucket and the event
|
||||
// each write raises on the controller's seat.
|
||||
type Mirror struct {
|
||||
Name string
|
||||
Bucket string
|
||||
Event string
|
||||
}
|
||||
|
||||
// The mirrors this package writes (novox/hq design 57, package F3). The assignments mirror is design 54's,
|
||||
// keyed `<node>.<module>`; the declared config is each registered module's config declarations as its
|
||||
// manifest holds them, keyed by the module, which mesh-config resolves a module's config values from
|
||||
// (ADR 0348 clauses 2 and 4).
|
||||
var (
|
||||
Assignments = Mirror{Name: broker.MirrorAssignments, Bucket: broker.AssignmentsBucket,
|
||||
Event: link.KeyAssignmentChanged}
|
||||
DeclaredConfig = Mirror{Name: broker.MirrorDeclaredConfig, Bucket: broker.DeclaredConfigBucket,
|
||||
Event: link.KeyDeclaredConfigChanged}
|
||||
)
|
||||
|
||||
// Mirrors is every mirror this package writes, in the order they are written.
|
||||
func Mirrors() []Mirror { return []Mirror{Assignments, DeclaredConfig} }
|
||||
|
||||
// The changes an event says.
|
||||
const (
|
||||
Added = "added"
|
||||
Changed = "changed"
|
||||
Removed = "removed"
|
||||
)
|
||||
|
||||
// Event is what one write says on the controller's seat (design 54 §3): which mirror and key, what kind of
|
||||
// change, the names of the top-level fields that changed, the bucket's revision of the write, the lease's
|
||||
// epoch and when. Never the value: a watcher reads it from the bucket at that revision or later.
|
||||
type Event struct {
|
||||
Mirror string `json:"mirror"`
|
||||
Key string `json:"key"`
|
||||
Change string `json:"change"`
|
||||
Fields []string `json:"fields,omitempty"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Epoch uint64 `json:"epoch,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// Announcer says an event on a seat; link.OverNATS is one, and so is link.Bus.
|
||||
type Announcer interface {
|
||||
PublishSeatEvent(ctx context.Context, seat, event string, body []byte) error
|
||||
}
|
||||
|
||||
// MarkerTTL is how long the marker a purge leaves is kept (design 54 §2): a day, so the markers of removed
|
||||
// records do not pile up in a bucket that has no age.
|
||||
const MarkerTTL = 24 * time.Hour
|
||||
|
||||
// ErrNotHeld is a pass refused because this controller does not hold the lease: it wrote nothing.
|
||||
var ErrNotHeld = errors.New("this controller does not hold the lease, so it writes no mirror")
|
||||
|
||||
// Writer writes one mirror. Not safe for use by two goroutines at once: one controller's one loop owns it.
|
||||
type Writer struct {
|
||||
Mirror Mirror
|
||||
KV jetstream.KeyValue
|
||||
Announce Announcer
|
||||
// Epoch is the lease's epoch the events carry; nil carries none.
|
||||
Epoch func() uint64
|
||||
// Holds says this controller holds the lease. Asked before every pass and every write; a writer
|
||||
// without it writes nothing (design 54 §2). Required: nil holds nothing.
|
||||
Holds func() bool
|
||||
// Now is the clock the events are dated by; nil is time.Now.
|
||||
Now func() time.Time
|
||||
|
||||
known map[string]kept
|
||||
// LostEvents counts the events refused twice since the writer was made.
|
||||
LostEvents int
|
||||
}
|
||||
|
||||
type kept struct {
|
||||
value []byte
|
||||
revision uint64
|
||||
}
|
||||
|
||||
// Result is what one pass did: keys written, keys removed, and keys refused with why.
|
||||
type Result struct {
|
||||
Written int
|
||||
Removed int
|
||||
Refused int
|
||||
}
|
||||
|
||||
// Sync makes the bucket hold want, one key per record: writes each value that differs from what the key
|
||||
// holds, purges each key no record is left for, and raises one event per write. A key the bus cannot hold
|
||||
// is refused alone, and the others are written; the error then names it.
|
||||
func (w *Writer) Sync(ctx context.Context, want map[string][]byte) (Result, error) {
|
||||
var r Result
|
||||
if !w.holds() {
|
||||
return r, ErrNotHeld
|
||||
}
|
||||
// What the bucket holds now, every pass: a value written behind this writer's back differs from the
|
||||
// store's and is written again below.
|
||||
if err := w.fill(ctx); err != nil {
|
||||
return r, fmt.Errorf("reading the %s mirror: %w", w.Mirror.Name, err)
|
||||
}
|
||||
var problems []error
|
||||
keys := make([]string, 0, len(want))
|
||||
for k := range want {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
value := want[key]
|
||||
before, had := w.known[key]
|
||||
if had && bytes.Equal(before.value, value) {
|
||||
continue
|
||||
}
|
||||
if err := validKey(key); err != nil {
|
||||
r.Refused++
|
||||
problems = append(problems, err)
|
||||
continue
|
||||
}
|
||||
if !w.holds() {
|
||||
return r, errors.Join(append(problems, ErrNotHeld)...)
|
||||
}
|
||||
revision, wrote, err := w.put(ctx, key, value, before, had)
|
||||
if err != nil {
|
||||
problems = append(problems, fmt.Errorf("writing %s: %w", key, err))
|
||||
continue
|
||||
}
|
||||
w.known[key] = kept{value: append([]byte(nil), value...), revision: revision}
|
||||
if !wrote {
|
||||
continue
|
||||
}
|
||||
r.Written++
|
||||
change, fields := Added, []string(nil)
|
||||
if had {
|
||||
change, fields = Changed, changedFields(before.value, value)
|
||||
}
|
||||
w.say(ctx, key, change, fields, revision)
|
||||
}
|
||||
gone := make([]string, 0)
|
||||
for key := range w.known {
|
||||
if _, ok := want[key]; !ok {
|
||||
gone = append(gone, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(gone)
|
||||
for _, key := range gone {
|
||||
k := w.known[key]
|
||||
if !w.holds() {
|
||||
return r, errors.Join(append(problems, ErrNotHeld)...)
|
||||
}
|
||||
err := w.KV.Purge(ctx, key, jetstream.LastRevision(k.revision), jetstream.PurgeTTL(MarkerTTL))
|
||||
if refusedByRevision(err) {
|
||||
// Read it again: purged already, or moved by another writer.
|
||||
if e, gerr := w.KV.Get(ctx, key); errors.Is(gerr, jetstream.ErrKeyNotFound) {
|
||||
delete(w.known, key)
|
||||
continue
|
||||
} else if gerr == nil && w.holds() {
|
||||
err = w.KV.Purge(ctx, key, jetstream.LastRevision(e.Revision()), jetstream.PurgeTTL(MarkerTTL))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
problems = append(problems, fmt.Errorf("removing %s: %w", key, err))
|
||||
continue
|
||||
}
|
||||
delete(w.known, key)
|
||||
r.Removed++
|
||||
revision := uint64(0)
|
||||
if status, err := w.KV.Status(ctx); err == nil {
|
||||
// The purge's own sequence is the stream's last one only when nothing wrote in between, which no
|
||||
// other writer may; said as the bucket's revision after it.
|
||||
if s, ok := status.(interface{ StreamInfo() *jetstream.StreamInfo }); ok {
|
||||
revision = s.StreamInfo().State.LastSeq
|
||||
}
|
||||
}
|
||||
w.say(ctx, key, Removed, nil, revision)
|
||||
}
|
||||
return r, errors.Join(problems...)
|
||||
}
|
||||
|
||||
// put writes one value by compare-and-set and answers its revision, and whether this call wrote it (false
|
||||
// when the bucket already held it).
|
||||
func (w *Writer) put(ctx context.Context, key string, value []byte, before kept, had bool) (uint64, bool, error) {
|
||||
var (
|
||||
revision uint64
|
||||
err error
|
||||
)
|
||||
if had {
|
||||
revision, err = w.KV.Update(ctx, key, value, before.revision)
|
||||
} else {
|
||||
revision, err = w.KV.Create(ctx, key, value)
|
||||
}
|
||||
if !refusedByRevision(err) {
|
||||
return revision, err == nil, err
|
||||
}
|
||||
// Refused: another writer in between, or this writer's own write whose acknowledgement was lost.
|
||||
e, gerr := w.KV.Get(ctx, key)
|
||||
switch {
|
||||
case errors.Is(gerr, jetstream.ErrKeyNotFound):
|
||||
if !w.holds() {
|
||||
return 0, false, errors.New("the key moved and this controller no longer holds the lease")
|
||||
}
|
||||
revision, err = w.KV.Create(ctx, key, value)
|
||||
return revision, err == nil, err
|
||||
case gerr != nil:
|
||||
return 0, false, gerr
|
||||
case bytes.Equal(e.Value(), value):
|
||||
return e.Revision(), false, nil
|
||||
case !w.holds():
|
||||
return 0, false, errors.New("the key moved and this controller no longer holds the lease")
|
||||
}
|
||||
revision, err = w.KV.Update(ctx, key, value, e.Revision())
|
||||
return revision, err == nil, err
|
||||
}
|
||||
|
||||
// say raises one write's event, tried once more when refused and then counted as lost.
|
||||
func (w *Writer) say(ctx context.Context, key, change string, fields []string, revision uint64) {
|
||||
if w.Announce == nil {
|
||||
return
|
||||
}
|
||||
e := Event{Mirror: w.Mirror.Name, Key: key, Change: change, Fields: fields, Revision: revision, At: w.now().UTC()}
|
||||
if w.Epoch != nil {
|
||||
e.Epoch = w.Epoch()
|
||||
}
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
w.LostEvents++
|
||||
return
|
||||
}
|
||||
for try := 0; try < 2; try++ {
|
||||
if err = w.Announce.PublishSeatEvent(ctx, broker.ControllerSeat, w.Mirror.Event, body); err == nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
w.LostEvents++
|
||||
}
|
||||
|
||||
// fill reads what the bucket holds now, so a controller started again writes only what differs.
|
||||
func (w *Writer) fill(ctx context.Context) error {
|
||||
w.known = map[string]kept{}
|
||||
watch, err := w.KV.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = watch.Stop() }()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case e := <-watch.Updates():
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
w.known[e.Key()] = kept{value: append([]byte(nil), e.Value()...), revision: e.Revision()}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Writer) holds() bool { return w.Holds != nil && w.Holds() }
|
||||
|
||||
func (w *Writer) now() time.Time {
|
||||
if w.Now == nil {
|
||||
return time.Now()
|
||||
}
|
||||
return w.Now()
|
||||
}
|
||||
|
||||
// refusedByRevision is the server refusing a compare-and-set: the key is not at the revision named.
|
||||
func refusedByRevision(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return true
|
||||
}
|
||||
var apiErr *jetstream.APIError
|
||||
return errors.As(err, &apiErr) && apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence
|
||||
}
|
||||
|
||||
// keyChars is what a bucket's key may be made of.
|
||||
var keyChars = regexp.MustCompile(`^[-/_=.a-zA-Z0-9]+$`)
|
||||
|
||||
// validKey refuses a key the bus cannot hold, naming it.
|
||||
func validKey(key string) error {
|
||||
if key == "" || !keyChars.MatchString(key) || key[0] == '.' || key[len(key)-1] == '.' {
|
||||
return fmt.Errorf("%q cannot be a key of a bucket: letters, digits, '-', '_', '=', '/' and '.', not at "+
|
||||
"either end", key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// changedFields names the top-level fields whose values differ between two encoded objects, sorted; nil
|
||||
// when either is not an object.
|
||||
func changedFields(before, after []byte) []string {
|
||||
var a, b map[string]json.RawMessage
|
||||
if json.Unmarshal(before, &a) != nil || json.Unmarshal(after, &b) != nil || a == nil || b == nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for k, v := range b {
|
||||
if old, ok := a[k]; !ok || !bytes.Equal(old, v) {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
for k := range a {
|
||||
if _, ok := b[k]; !ok {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package recordmirror
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Design 57's F3, end to end** (novox/hq issue 652): from the controller's store to a real bus. An
|
||||
// assignment changes exactly one key of the assignments mirror and raises exactly one `assignment-changed`;
|
||||
// a registration changes exactly one key of the declared-config mirror and raises exactly one
|
||||
// `declared-config-changed`; and the same records written again change nothing and say nothing — a module
|
||||
// registered again unchanged included, although the store's registration time moved.
|
||||
func TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := t.Context()
|
||||
js, api := aMirrorBus(t)
|
||||
assignments := aWriter(t, js, api, Assignments)
|
||||
declared := aWriter(t, js, api, DeclaredConfig)
|
||||
heardAssignments := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+Assignments.Event)
|
||||
heardDeclared := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+DeclaredConfig.Event)
|
||||
|
||||
pass := func() (Result, Result) {
|
||||
t.Helper()
|
||||
a, err := inv.AssignmentsMirror(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ra, err := assignments.Sync(ctx, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := inv.DeclaredConfigMirror(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rd, err := declared.Sync(ctx, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ra, rd
|
||||
}
|
||||
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ui := catalogue.Manifest{Module: "ui", Version: "1", Settings: map[string]catalogue.SettingDeclaration{
|
||||
"font-size": {Kind: catalogue.KindPreference, Default: 14, Why: "readable on the desktop"}}}
|
||||
if err := inv.RegisterModule(ctx, ui, inventory.Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pass()
|
||||
assignmentWrites, declaredWrites := writes(t, api, Assignments.Bucket), writes(t, api, DeclaredConfig.Bucket)
|
||||
assignmentsSaid, declaredSaid := len(settle(heardAssignments, 0)), len(settle(heardDeclared, 1))
|
||||
|
||||
// **A registration**: a module new to the catalogue.
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "notes", Version: "1"}, inventory.Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ra, rd := pass()
|
||||
if ra != (Result{}) || rd != (Result{Written: 1}) {
|
||||
t.Errorf("a registration wrote assignments %+v and declared config %+v", ra, rd)
|
||||
}
|
||||
if got := writes(t, api, DeclaredConfig.Bucket) - declaredWrites; got != 1 {
|
||||
t.Errorf("a registration changed %d keys of the declared config", got)
|
||||
}
|
||||
if got := writes(t, api, Assignments.Bucket) - assignmentWrites; got != 0 {
|
||||
t.Errorf("a registration changed %d keys of the assignments", got)
|
||||
}
|
||||
said := settle(heardDeclared, declaredSaid+1)[declaredSaid:]
|
||||
if len(said) != 1 || said[0].Key != "notes" || said[0].Change != Added {
|
||||
t.Errorf("a registration raised %+v", said)
|
||||
}
|
||||
declaredWrites, declaredSaid = writes(t, api, DeclaredConfig.Bucket), declaredSaid+len(said)
|
||||
|
||||
// **An assignment.**
|
||||
if _, err := inv.Assign(ctx, "anchor", "ui"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ra, rd = pass()
|
||||
if ra != (Result{Written: 1}) || rd != (Result{}) {
|
||||
t.Errorf("an assignment wrote assignments %+v and declared config %+v", ra, rd)
|
||||
}
|
||||
if got := writes(t, api, Assignments.Bucket) - assignmentWrites; got != 1 {
|
||||
t.Errorf("an assignment changed %d keys of the assignments", got)
|
||||
}
|
||||
said = settle(heardAssignments, assignmentsSaid+1)[assignmentsSaid:]
|
||||
if len(said) != 1 || said[0].Key != "anchor.ui" || said[0].Change != Added {
|
||||
t.Errorf("an assignment raised %+v", said)
|
||||
}
|
||||
assignmentWrites, assignmentsSaid = writes(t, api, Assignments.Bucket), assignmentsSaid+len(said)
|
||||
e, err := assignments.KV.Get(ctx, "anchor.ui")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var value inventory.AssignmentMirrored
|
||||
if err := json.Unmarshal(e.Value(), &value); err != nil || value.Node != "anchor" || value.Module != "ui" ||
|
||||
value.Assigned.IsZero() {
|
||||
t.Errorf("the assignment is mirrored as %s (%v)", e.Value(), err)
|
||||
}
|
||||
|
||||
// **The same records written again change nothing**: an assignment made again, and a module registered
|
||||
// again with the manifest it had.
|
||||
if _, err := inv.Assign(ctx, "anchor", "ui"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, ui, inventory.Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ra, rd := pass(); ra != (Result{}) || rd != (Result{}) {
|
||||
t.Errorf("unchanged records wrote assignments %+v and declared config %+v", ra, rd)
|
||||
}
|
||||
if writes(t, api, Assignments.Bucket) != assignmentWrites || writes(t, api, DeclaredConfig.Bucket) != declaredWrites {
|
||||
t.Error("unchanged records changed a key")
|
||||
}
|
||||
if a, d := settle(heardAssignments, assignmentsSaid+1), settle(heardDeclared, declaredSaid+1); len(a) != assignmentsSaid ||
|
||||
len(d) != declaredSaid {
|
||||
t.Errorf("unchanged records raised %+v and %+v", a[assignmentsSaid:], d[declaredSaid:])
|
||||
}
|
||||
|
||||
// **A declaration changed by a registration**: one key, one event, naming the field that moved.
|
||||
ui.Version = "2"
|
||||
ui.Settings["font-size"] = catalogue.SettingDeclaration{Kind: catalogue.KindPreference, Default: 16,
|
||||
Why: "readable on the desktop"}
|
||||
if err := inv.RegisterModule(ctx, ui, inventory.Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ra, rd := pass(); ra != (Result{}) || rd != (Result{Written: 1}) {
|
||||
t.Errorf("a changed declaration wrote assignments %+v and declared config %+v", ra, rd)
|
||||
}
|
||||
said = settle(heardDeclared, declaredSaid+1)[declaredSaid:]
|
||||
if len(said) != 1 || said[0].Key != "ui" || said[0].Change != Changed || strings.Join(said[0].Fields, ",") != "settings" {
|
||||
t.Errorf("a changed declaration raised %+v", said)
|
||||
}
|
||||
got, err := declared.KV.Get(ctx, "ui")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := `{"module":"ui","settings":{"font-size":{"kind":"preference","default":16,"why":"readable on the desktop"}}}`; string(got.Value()) != want {
|
||||
t.Errorf("the declaration is mirrored as %s, want %s", got.Value(), want)
|
||||
}
|
||||
|
||||
// **An unassignment** removes its key, said once.
|
||||
if err := inv.Unassign(ctx, "anchor", "ui"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ra, _ := pass(); ra != (Result{Removed: 1}) {
|
||||
t.Errorf("an unassignment did %+v", ra)
|
||||
}
|
||||
said = settle(heardAssignments, assignmentsSaid+1)[assignmentsSaid:]
|
||||
if len(said) != 1 || said[0].Key != "anchor.ui" || said[0].Change != Removed {
|
||||
t.Errorf("an unassignment raised %+v", said)
|
||||
}
|
||||
|
||||
// **A module no longer registered** has its key removed, said once as `removed` (ADR 0356 §4).
|
||||
if err := inv.ForgetModule(ctx, "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declaredWrites = writes(t, api, DeclaredConfig.Bucket)
|
||||
declaredSaid = len(heardDeclared.all())
|
||||
if _, rd := pass(); rd != (Result{Removed: 1}) {
|
||||
t.Errorf("a module forgotten did %+v", rd)
|
||||
}
|
||||
if got := writes(t, api, DeclaredConfig.Bucket) - declaredWrites; got != 1 {
|
||||
t.Errorf("a module forgotten changed %d keys of the declared config", got)
|
||||
}
|
||||
if _, err := declared.KV.Get(ctx, "notes"); err == nil {
|
||||
t.Error("a module forgotten keeps its key")
|
||||
}
|
||||
said = settle(heardDeclared, declaredSaid+1)[declaredSaid:]
|
||||
if len(said) != 1 || said[0].Key != "notes" || said[0].Change != Removed {
|
||||
t.Errorf("a module forgotten raised %+v", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
package recordmirror
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// A record mirror against a real bus (novox/hq design 54 §2, §3; ADR 0289): a change of a record changes
|
||||
// exactly one key and raises exactly one event; a pass over unchanged records writes nothing and says
|
||||
// nothing; a record gone is a purge said as `removed`.
|
||||
|
||||
// aMirrorBus is a bus of this test alone with the mesh's streams and the controller's buckets, as the
|
||||
// controller asserts them at its start, and a counter of the writes made to one bucket.
|
||||
func aMirrorBus(t *testing.T) (*broker.JetStream, jetstream.JetStream) {
|
||||
t.Helper()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js, api
|
||||
}
|
||||
|
||||
// heard counts what is said on one subject, each body kept.
|
||||
type heard struct {
|
||||
mu sync.Mutex
|
||||
bodies []Event
|
||||
}
|
||||
|
||||
func (h *heard) all() []Event {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return append([]Event(nil), h.bodies...)
|
||||
}
|
||||
|
||||
func listen(t *testing.T, conn *nats.Conn, subject string) *heard {
|
||||
t.Helper()
|
||||
h := &heard{}
|
||||
sub, err := conn.Subscribe(subject, func(m *nats.Msg) {
|
||||
var e Event
|
||||
if err := json.Unmarshal(m.Data, &e); err != nil {
|
||||
t.Errorf("an event that is not one: %s", m.Data)
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.bodies = append(h.bodies, e)
|
||||
h.mu.Unlock()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
if err := conn.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// writes is how many messages the bucket's stream has taken: every put and every purge is one.
|
||||
func writes(t *testing.T, api jetstream.JetStream, bucket string) uint64 {
|
||||
t.Helper()
|
||||
s, err := api.Stream(t.Context(), "KV_"+bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := s.Info(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.State.LastSeq
|
||||
}
|
||||
|
||||
// settle waits until n events are heard or a second passed, and answers what was heard.
|
||||
func settle(h *heard, n int) []Event {
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) && len(h.all()) < n {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
// And a moment more, so an event too many is heard too.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
return h.all()
|
||||
}
|
||||
|
||||
func aWriter(t *testing.T, js *broker.JetStream, api jetstream.JetStream, m Mirror) *Writer {
|
||||
t.Helper()
|
||||
kv, err := api.KeyValue(t.Context(), m.Bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &Writer{Mirror: m, KV: kv, Announce: link.OverNATS{Conn: js.Conn(), JS: js.Context()},
|
||||
Epoch: func() uint64 { return 7 }, Holds: func() bool { return true }}
|
||||
}
|
||||
|
||||
func TestNatsAMirrorChangesOneKeyAndRaisesOneEventPerChange(t *testing.T) {
|
||||
for _, m := range []Mirror{Assignments, DeclaredConfig} {
|
||||
t.Run(m.Name, func(t *testing.T) {
|
||||
js, api := aMirrorBus(t)
|
||||
events := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+m.Event)
|
||||
w := aWriter(t, js, api, m)
|
||||
ctx := t.Context()
|
||||
|
||||
first := map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)}
|
||||
if _, err := w.Sync(ctx, first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := writes(t, api, m.Bucket)
|
||||
heardBefore := len(settle(events, 1))
|
||||
|
||||
// One record changes, one is added: two keys, two events, nothing else.
|
||||
second := map[string][]byte{"anchor.gitea": []byte(`{"a":2}`), "anchor.forgejo": []byte(`{"a":1}`)}
|
||||
r, err := w.Sync(ctx, second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Written != 2 || r.Removed != 0 {
|
||||
t.Errorf("a pass over one changed and one new record wrote %+v", r)
|
||||
}
|
||||
if got := writes(t, api, m.Bucket) - before; got != 2 {
|
||||
t.Errorf("the bucket took %d writes for two changed records", got)
|
||||
}
|
||||
said := settle(events, heardBefore+2)[heardBefore:]
|
||||
if len(said) != 2 {
|
||||
t.Fatalf("two changes raised %d events: %+v", len(said), said)
|
||||
}
|
||||
changes := map[string]Event{}
|
||||
for _, e := range said {
|
||||
changes[e.Key] = e
|
||||
}
|
||||
if e := changes["anchor.gitea"]; e.Change != Changed || !slices.Equal(e.Fields, []string{"a"}) ||
|
||||
e.Mirror != m.Name || e.Epoch != 7 || e.Revision == 0 {
|
||||
t.Errorf("the change of anchor.gitea was said as %+v", e)
|
||||
}
|
||||
if e := changes["anchor.forgejo"]; e.Change != Added {
|
||||
t.Errorf("the new anchor.forgejo was said as %+v", e)
|
||||
}
|
||||
|
||||
// **An unchanged pass writes nothing**, and says nothing (ADR 0289 §1).
|
||||
heardBefore = len(events.all())
|
||||
before = writes(t, api, m.Bucket)
|
||||
r, err = w.Sync(ctx, second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r != (Result{}) {
|
||||
t.Errorf("a pass over unchanged records did %+v", r)
|
||||
}
|
||||
if got := writes(t, api, m.Bucket) - before; got != 0 {
|
||||
t.Errorf("a pass over unchanged records wrote the bucket %d times", got)
|
||||
}
|
||||
if said := settle(events, heardBefore+1)[heardBefore:]; len(said) != 0 {
|
||||
t.Errorf("a pass over unchanged records raised %+v", said)
|
||||
}
|
||||
|
||||
// **A record gone is a purge**, one write and one `removed`.
|
||||
before = writes(t, api, m.Bucket)
|
||||
r, err = w.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":2}`)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Removed != 1 || r.Written != 0 {
|
||||
t.Errorf("a record gone did %+v", r)
|
||||
}
|
||||
if got := writes(t, api, m.Bucket) - before; got != 1 {
|
||||
t.Errorf("a record gone wrote the bucket %d times", got)
|
||||
}
|
||||
if _, err := w.KV.Get(ctx, "anchor.forgejo"); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
t.Errorf("the key of a record gone is still read: %v", err)
|
||||
}
|
||||
said = settle(events, heardBefore+1)[heardBefore:]
|
||||
if len(said) != 1 || said[0].Change != Removed || said[0].Key != "anchor.forgejo" {
|
||||
t.Errorf("a record gone was said as %+v", said)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// **A controller started again writes nothing for what the bucket already holds** (design 54 §2: the writer's
|
||||
// memory is filled from the bucket), and a key nobody holds a record for is removed.
|
||||
func TestNatsAWriterStartedAgainWritesOnlyWhatDiffers(t *testing.T) {
|
||||
js, api := aMirrorBus(t)
|
||||
ctx := t.Context()
|
||||
first := aWriter(t, js, api, Assignments)
|
||||
held := map[string][]byte{"anchor.gitea": []byte(`{"a":1}`), "anchor.forgejo": []byte(`{"a":1}`)}
|
||||
if _, err := first.Sync(ctx, held); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := writes(t, api, Assignments.Bucket)
|
||||
again := aWriter(t, js, api, Assignments)
|
||||
r, err := again.Sync(ctx, held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r != (Result{}) || writes(t, api, Assignments.Bucket) != before {
|
||||
t.Errorf("a writer started again over unchanged records did %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// **Another writer in between is no lost value** (design 54 §2, compare-and-set): a write refused because the
|
||||
// key moved reads it again, takes the revision when it already holds the value, and otherwise writes again.
|
||||
func TestNatsAWriteRefusedByCompareAndSetReadsAgain(t *testing.T) {
|
||||
js, api := aMirrorBus(t)
|
||||
ctx := t.Context()
|
||||
w := aWriter(t, js, api, Assignments)
|
||||
if _, err := w.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Behind its back, as a former lease holder would.
|
||||
if _, err := w.KV.Put(ctx, "anchor.gitea", []byte(`{"a":9}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := w.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":2}`)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Written != 1 {
|
||||
t.Errorf("a refused write was not made again: %+v", r)
|
||||
}
|
||||
e, err := w.KV.Get(ctx, "anchor.gitea")
|
||||
if err != nil || string(e.Value()) != `{"a":2}` {
|
||||
t.Fatalf("the bucket holds %v, %v", e, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A value written behind the writer's back is put right on the next pass** (ADR 0289 §10): a former lease
|
||||
// holder's write differs from the store, and the store's value is written again, said as one change.
|
||||
func TestNatsAForeignWriteIsCorrectedOnTheNextPass(t *testing.T) {
|
||||
js, api := aMirrorBus(t)
|
||||
ctx := t.Context()
|
||||
events := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+Assignments.Event)
|
||||
w := aWriter(t, js, api, Assignments)
|
||||
store := map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)}
|
||||
if _, err := w.Sync(ctx, store); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heardBefore := len(settle(events, 1))
|
||||
if _, err := w.KV.Put(ctx, "anchor.gitea", []byte(`{"a":9}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := w.Sync(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r != (Result{Written: 1}) {
|
||||
t.Errorf("a pass after a foreign write did %+v", r)
|
||||
}
|
||||
e, err := w.KV.Get(ctx, "anchor.gitea")
|
||||
if err != nil || string(e.Value()) != `{"a":1}` {
|
||||
t.Fatalf("the bucket still holds %v, %v", e, err)
|
||||
}
|
||||
if said := settle(events, heardBefore+1)[heardBefore:]; len(said) != 1 || said[0].Change != Changed {
|
||||
t.Errorf("the correction was said as %+v", said)
|
||||
}
|
||||
// And put right, it stays: the next pass writes nothing.
|
||||
if r, err := w.Sync(ctx, store); err != nil || r != (Result{}) {
|
||||
t.Errorf("a pass after the correction did %+v, %v", r, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A writer without the lease writes nothing** (ADR 0289 §9, design 54 §2): not a new key, not a change, not
|
||||
// a removal, and it says so.
|
||||
func TestNatsAWriterWithoutTheLeaseWritesNothing(t *testing.T) {
|
||||
js, api := aMirrorBus(t)
|
||||
ctx := t.Context()
|
||||
held := aWriter(t, js, api, Assignments)
|
||||
if _, err := held.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := writes(t, api, Assignments.Bucket)
|
||||
for _, holds := range []func() bool{nil, func() bool { return false }} {
|
||||
w := aWriter(t, js, api, Assignments)
|
||||
w.Holds = holds
|
||||
r, err := w.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":2}`), "anchor.forgejo": []byte(`{"a":1}`)})
|
||||
if !errors.Is(err, ErrNotHeld) || r != (Result{}) {
|
||||
t.Errorf("a writer without the lease did %+v, %v", r, err)
|
||||
}
|
||||
r, err = w.Sync(ctx, map[string][]byte{})
|
||||
if !errors.Is(err, ErrNotHeld) || r != (Result{}) {
|
||||
t.Errorf("a writer without the lease removed: %+v, %v", r, err)
|
||||
}
|
||||
}
|
||||
// And one that loses it mid-pass stops before its next write.
|
||||
asked := 0
|
||||
w := aWriter(t, js, api, Assignments)
|
||||
w.Holds = func() bool { asked++; return asked <= 2 }
|
||||
r, err := w.Sync(ctx, map[string][]byte{"anchor.a": []byte(`{}`), "anchor.b": []byte(`{}`), "anchor.gitea": []byte(`{"a":1}`)})
|
||||
if !errors.Is(err, ErrNotHeld) || r.Written != 1 {
|
||||
t.Errorf("a writer that lost the lease mid-pass did %+v, %v", r, err)
|
||||
}
|
||||
if got := writes(t, api, Assignments.Bucket) - before; got != 1 {
|
||||
t.Errorf("writers without the lease wrote the bucket %d times", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A removal asks for the lease too** (ADR 0289 §9): a writer that held the lease when its pass began and lost
|
||||
// it before its first removal purges nothing, and says it does not hold the lease.
|
||||
func TestNatsARemovalWithoutTheLeaseRemovesNothing(t *testing.T) {
|
||||
js, api := aMirrorBus(t)
|
||||
ctx := t.Context()
|
||||
held := aWriter(t, js, api, Assignments)
|
||||
if _, err := held.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := writes(t, api, Assignments.Bucket)
|
||||
asked := 0
|
||||
w := aWriter(t, js, api, Assignments)
|
||||
w.Holds = func() bool { asked++; return asked == 1 }
|
||||
r, err := w.Sync(ctx, map[string][]byte{})
|
||||
if !errors.Is(err, ErrNotHeld) || r != (Result{}) || asked != 2 {
|
||||
t.Errorf("a removal without the lease did %+v, %v (the lease asked %d times)", r, err, asked)
|
||||
}
|
||||
if got := writes(t, api, Assignments.Bucket) - before; got != 0 {
|
||||
t.Errorf("a removal without the lease wrote the bucket %d times", got)
|
||||
}
|
||||
if _, err := w.KV.Get(ctx, "anchor.gitea"); err != nil {
|
||||
t.Errorf("the key was removed without the lease: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A key the bus cannot hold is refused alone**, and the others are written.
|
||||
func TestAKeyTheBusCannotHoldIsRefusedAlone(t *testing.T) {
|
||||
if validKey("anchor.gitea") != nil || validKey("bad key") == nil || validKey(".lead") == nil ||
|
||||
validKey("") == nil || validKey("trail.") == nil {
|
||||
t.Error("the keys a bucket may hold are not told apart")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFieldsThatChangedAreNamed(t *testing.T) {
|
||||
got := changedFields([]byte(`{"a":1,"b":{"x":1},"c":"same"}`), []byte(`{"a":2,"b":{"x":1},"d":true,"c":"same"}`))
|
||||
if !slices.Equal(got, []string{"a", "d"}) {
|
||||
t.Errorf("the fields that changed are %v", got)
|
||||
}
|
||||
if got := changedFields(nil, []byte(`{"a":1}`)); got != nil {
|
||||
t.Errorf("an added key names fields %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package recordmirror
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// mayNameAMirror are the files of the controller that may name a record mirror's bucket outside their tests:
|
||||
// where the buckets are defined, asserted and granted (internal/broker), the list a reader names in `reads`
|
||||
// (internal/catalogue/state.go), and each mirror's own writer (this package and the loop that runs it). The
|
||||
// audit of a mirror against its store (ADR 0289 §4, issue 419) joins this list when it is built.
|
||||
var mayNameAMirror = []string{
|
||||
"cmd/mesh-controller/record_mirrors.go",
|
||||
"internal/broker/controller_buckets.go",
|
||||
"internal/broker/writers.go",
|
||||
"internal/catalogue/state.go",
|
||||
}
|
||||
|
||||
// The packages whose names stand for a mirror, by import path, and the names in each.
|
||||
const (
|
||||
brokerPath = "github.com/novox/mesh-controller/internal/broker"
|
||||
recordmirrorPath = "github.com/novox/mesh-controller/internal/recordmirror"
|
||||
)
|
||||
|
||||
var mirrorNames = map[string][]string{
|
||||
brokerPath: {"AssignmentsBucket", "DeclaredConfigBucket", "MirrorAssignments", "MirrorDeclaredConfig",
|
||||
"ControllerMirrors"},
|
||||
recordmirrorPath: {"Assignments", "DeclaredConfig", "Mirrors"},
|
||||
}
|
||||
|
||||
// **Nothing in the controller reads a mirror to decide anything** (ADR 0289 §19 as ADR 0356 §2 changes it): no
|
||||
// file of the controller's packages outside its tests names a mirror's bucket but the audit and a mirror's own
|
||||
// writer. A reader elsewhere fails here, naming the file and what it named.
|
||||
//
|
||||
// **What it catches**: a name that stands for a mirror, qualified by the package it comes from under any
|
||||
// local name the file imports it as (`broker.AssignmentsBucket`, `b.AssignmentsBucket`); the same names bare
|
||||
// in a file of package broker or package recordmirror; and a string literal holding a mirror's bucket name.
|
||||
// **What it does not catch**: a bucket name built at run time, such as `BucketName(ControllerSeat,
|
||||
// "assignments")` or literals concatenated into one, and a mirror reached through a value passed in from an
|
||||
// allowed file. Those are deliberate evasions rather than ordinary code; a reviewer judges them.
|
||||
func TestOnlyTheAuditAndAMirrorsWriterReadAMirror(t *testing.T) {
|
||||
var buckets []string
|
||||
for _, m := range broker.ControllerMirrors() {
|
||||
buckets = append(buckets, broker.BucketName(broker.ControllerSeat, m))
|
||||
}
|
||||
root := filepath.Join("..", "..")
|
||||
found := mirrorReaders(t, root, buckets)
|
||||
if len(found) > 0 {
|
||||
t.Fatalf("a package of the controller reads a record mirror, which only the audit and the mirror's "+
|
||||
"writer may (ADR 0289 §19, ADR 0356 §2):\n %s", strings.Join(found, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// **The reader test sees what it says it sees**: in a planted tree, each planted reader is named, and an allowed
|
||||
// file and a selector on a value are not.
|
||||
func TestTheReaderTestFindsPlantedReaders(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
plant := func(path, src string) {
|
||||
full := filepath.Join(root, path)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(full, []byte(src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
plant("cmd/mesh-controller/qualified.go", "package main\n\nimport \""+brokerPath+"\"\n\nvar a = broker.AssignmentsBucket\n")
|
||||
plant("internal/x/aliased.go", "package x\n\nimport b \""+brokerPath+"\"\n\nvar a = b.DeclaredConfigBucket\n")
|
||||
plant("internal/broker/zz_reader.go", "package broker\n\nfunc zzR() string { return AssignmentsBucket }\n")
|
||||
plant("internal/inventory/literal.go", "package inventory\n\nvar a = \"mesh-controller_declared-config\"\n")
|
||||
plant("internal/y/writer.go", "package y\n\nimport \""+recordmirrorPath+"\"\n\nvar a = recordmirror.Assignments\n")
|
||||
// Allowed: a mirror's definition and its writer.
|
||||
plant("internal/broker/controller_buckets.go", "package broker\n\nvar AssignmentsBucket = \"x\"\n")
|
||||
plant("internal/broker/other.go", "package broker\n\nfunc other() string { return \"nothing\" }\n")
|
||||
// A selector on a value (v.AssignmentsBucket) is no use of the mirror, but declaring a field of a mirror's
|
||||
// name in package broker is named, on the safe side; and a bare use inside a call under a selector is found.
|
||||
plant("internal/broker/field.go", "package broker\n\ntype s struct{ AssignmentsBucket string }\n\nfunc f(v s) string { return v.AssignmentsBucket }\n")
|
||||
plant("internal/broker/nested.go", "package broker\n\nfunc g() int { return len(h(DeclaredConfigBucket).x) }\n")
|
||||
found := mirrorReaders(t, root, []string{"mesh-controller_assignments", "mesh-controller_declared-config"})
|
||||
want := []string{
|
||||
"cmd/mesh-controller/qualified.go: broker.AssignmentsBucket",
|
||||
"internal/broker/field.go: AssignmentsBucket",
|
||||
"internal/broker/nested.go: DeclaredConfigBucket",
|
||||
"internal/broker/zz_reader.go: AssignmentsBucket",
|
||||
"internal/inventory/literal.go: \"mesh-controller_declared-config\"",
|
||||
"internal/x/aliased.go: b.DeclaredConfigBucket",
|
||||
"internal/y/writer.go: recordmirror.Assignments",
|
||||
}
|
||||
slices.Sort(found)
|
||||
if !slices.Equal(found, want) {
|
||||
t.Errorf("the planted readers were found as\n %s\nwant\n %s", strings.Join(found, "\n "),
|
||||
strings.Join(want, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// mirrorReaders is every place under root's cmd and internal, outside tests and the allowed files, that names a
|
||||
// mirror.
|
||||
func mirrorReaders(t *testing.T, root string, buckets []string) []string {
|
||||
t.Helper()
|
||||
var found []string
|
||||
for _, dir := range []string{"cmd", "internal"} {
|
||||
err := filepath.WalkDir(filepath.Join(root, dir), func(path string, e fs.DirEntry, err error) error {
|
||||
if err != nil || e.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if strings.HasPrefix(rel, "internal/recordmirror/") || slices.Contains(mayNameAMirror, rel) {
|
||||
return nil
|
||||
}
|
||||
f, err := parser.ParseFile(token.NewFileSet(), path, nil, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Each import's local name, mapped to the package it stands for.
|
||||
imported := map[string]string{}
|
||||
for _, imp := range f.Imports {
|
||||
p, _ := strconv.Unquote(imp.Path.Value)
|
||||
if _, ok := mirrorNames[p]; !ok {
|
||||
continue
|
||||
}
|
||||
local := filepath.Base(p)
|
||||
if imp.Name != nil {
|
||||
local = imp.Name.Name
|
||||
}
|
||||
imported[local] = p
|
||||
}
|
||||
// A file of the package that defines the names may use them bare.
|
||||
var bare []string
|
||||
switch f.Name.Name {
|
||||
case "broker":
|
||||
bare = mirrorNames[brokerPath]
|
||||
case "recordmirror":
|
||||
bare = mirrorNames[recordmirrorPath]
|
||||
}
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
switch n := n.(type) {
|
||||
case *ast.SelectorExpr:
|
||||
pkg, ok := n.X.(*ast.Ident)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if p, imp := imported[pkg.Name]; imp && slices.Contains(mirrorNames[p], n.Sel.Name) {
|
||||
found = append(found, rel+": "+pkg.Name+"."+n.Sel.Name)
|
||||
}
|
||||
// Neither a selector's own name nor its package is a bare use.
|
||||
return false
|
||||
case *ast.Ident:
|
||||
if slices.Contains(bare, n.Name) {
|
||||
found = append(found, rel+": "+n.Name)
|
||||
}
|
||||
case *ast.BasicLit:
|
||||
for _, b := range buckets {
|
||||
if n.Kind == token.STRING && strings.Contains(n.Value, b) {
|
||||
found = append(found, rel+": "+n.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
Vendored
+1
-1
@@ -8,7 +8,7 @@ private addresses. That is acceptable while mesh-controller is private; if it ev
|
||||
a second place to clean besides the catalogue itself.
|
||||
|
||||
mesh-catalog e0bc919d4945bb5f13f971f74643a0b714cf45da modules/*/module.json, modules/nats/Dockerfile
|
||||
mesh-host 7a41e120635b3f7b6cb94cca8b3fc4093e4e6110 examples/foundation-first-node.lock
|
||||
mesh-host c8920622516548978d92e734f3991a262f98a742 examples/foundation-first-node.lock
|
||||
mesh-sdk 5933eb757d443293631be2bed27a96288c2348d8 conformance/events/module-event.json
|
||||
|
||||
To move them, from this repository's root, with the two repositories checked out beside it:
|
||||
|
||||
+79
-29
@@ -28,11 +28,12 @@
|
||||
// image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The
|
||||
// registry prefix is the lab's; the digests are upstream's.
|
||||
//
|
||||
// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host
|
||||
// creates is owned by root, and the database runs as somebody else inside the container — so it
|
||||
// could not write, and the container crash-looped. A named volume lets the image set up its own
|
||||
// ownership, and outlives the container, which is what you want for the thing holding the mesh's
|
||||
// state.
|
||||
// **The store is raised where the `postgres` module keeps it** (novox/hq ADR 0338): the container
|
||||
// `postgres`, on the module's pgvector image, its data in the directory /var/lib/mesh-store owned by
|
||||
// the image's postgres user, its superuser password read from the module's own-secret file — so the
|
||||
// store step adopts it as it runs. Pinned upstream by the module's own digest, not through the lab's
|
||||
// registry prefix, because the two must be the same reference (mesh-host internal/bootstrap holds
|
||||
// them to each other against the module's manifest).
|
||||
|
||||
{
|
||||
"declaration": 1,
|
||||
@@ -83,17 +84,28 @@
|
||||
"boot": "enabled",
|
||||
"restart-on": ["base-filter"]
|
||||
},
|
||||
// The store's data is a directory owned by the image's postgres user (999), made before the
|
||||
// container starts: a directory the runtime made would be root's, and the database could not
|
||||
// write it. Measured on the module's image 2026-10-13: it initialises a new store on a directory
|
||||
// made so (novox/hq ADR 0338 §5).
|
||||
{
|
||||
"id": "store-data",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-store",
|
||||
"mode": "0700",
|
||||
"owner": "999:70"
|
||||
},
|
||||
{
|
||||
"id": "store",
|
||||
"type": "container",
|
||||
"name": "mesh-store",
|
||||
"image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a",
|
||||
"name": "postgres",
|
||||
"image": "pgvector/pgvector@sha256:cf134a767f474095eeba57e0117be8e568e011a63f33fbf252f14c9b760f8e6f",
|
||||
"env": {
|
||||
"POSTGRES_PASSWORD": "bootstrap",
|
||||
"PGDATA": "/var/lib/postgresql/data/pgdata"
|
||||
},
|
||||
"ports": ["5432:5432"],
|
||||
"volumes": ["mesh-store-data:/var/lib/postgresql/data"]
|
||||
"volumes": ["/var/lib/mesh-store:/var/lib/postgresql/data"]
|
||||
},
|
||||
// Over TCP, not the socket. While the store initialises it runs a temporary server on the
|
||||
// socket ONLY, then stops it and starts the real one — so a socket check passes, the action
|
||||
@@ -102,21 +114,21 @@
|
||||
{
|
||||
"id": "store-ready",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"in": "postgres",
|
||||
"command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
|
||||
"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
|
||||
},
|
||||
{
|
||||
"id": "inventory-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"in": "postgres",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
|
||||
},
|
||||
{
|
||||
"id": "identity-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"in": "postgres",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
|
||||
},
|
||||
@@ -126,20 +138,44 @@
|
||||
{
|
||||
"id": "licences-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"in": "postgres",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
|
||||
},
|
||||
{
|
||||
"id": "context-schemas",
|
||||
"type": "action",
|
||||
"command": ["docker", "run", "--rm", "--network", "container:mesh-store",
|
||||
"command": ["docker", "run", "--rm", "--network", "container:postgres",
|
||||
"-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||
"-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||
"-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"migrate"],
|
||||
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
||||
"verify": ["sh", "-c", "docker exec postgres psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec postgres psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec postgres psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
||||
},
|
||||
// **The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333 §1): the container
|
||||
// `mesh-broker-nats`, its configuration and user list in /var/lib/nats-module/conf, its streams in
|
||||
// /var/lib/mesh-broker-nats and its certificate in /var/lib/mesh-broker-tls — the names and paths the
|
||||
// catalogue's module declares. So adopting it as that module at the broker step keeps it: the
|
||||
// module's built image takes the place of the upstream one below, once, on the same streams, users
|
||||
// and certificate, and nothing else moves. mesh-host's internal/bootstrap holds the two to each other
|
||||
// against the module's own manifest (busLayout), at the broker step and in its tests.
|
||||
//
|
||||
// Directories on the machine, not volumes the runtime names: the module mounts these paths, and the
|
||||
// control plane, once it runs as the mesh's process, reads the certificate at its path on the machine.
|
||||
// The certificate's directory is readable by everyone because the certificate is public and the control
|
||||
// plane runs as nobody; its key is written 0600 by the program that makes it.
|
||||
{
|
||||
"id": "bus-tls-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-broker-tls",
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "bus-data-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-broker-nats",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "bus-certificate",
|
||||
@@ -151,49 +187,63 @@
|
||||
// it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
|
||||
// this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
|
||||
// to ask an authority of.
|
||||
// `--user 0:0` because the volume is root's and this image runs as nobody, which is right
|
||||
// for the long-running control plane and wrong for a one-shot writing into a fresh volume.
|
||||
"command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
|
||||
// `--user 0:0` because the directory is root's and this image runs as nobody, which is right
|
||||
// for the long-running control plane and wrong for a one-shot writing into a fresh directory.
|
||||
"command": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"broker", "certificate", "--into", "/tls"],
|
||||
"verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
|
||||
"verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"broker", "certificate", "--check", "--into", "/tls"]
|
||||
},
|
||||
// **Seeds, written only where nothing is** (create-once). The bus's configuration and user list are
|
||||
// the bundle's until the bus is adopted, and the module's from then on (ADR 0333 §4): an installer
|
||||
// run again after the adoption applies this bundle again, and an ordinary file would put the
|
||||
// bundle's one user back over every user the mesh composed, and leave the node unable to reach its
|
||||
// bus. Until the adoption the installer places the composed list over the seed (enrol.go).
|
||||
{
|
||||
"id": "bus-conf-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-bus-conf",
|
||||
"path": "/var/lib/nats-module/conf",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "bus-conf",
|
||||
"type": "file",
|
||||
"path": "/var/lib/mesh-bus-conf/nats.conf",
|
||||
"path": "/var/lib/nats-module/conf/nats.conf",
|
||||
"mode": "0644",
|
||||
"content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
|
||||
"create-once": true,
|
||||
"content": "port: 4222\nhttp: 0.0.0.0:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
|
||||
},
|
||||
{
|
||||
"id": "bus-accounts",
|
||||
"type": "file",
|
||||
"path": "/var/lib/mesh-bus-conf/accounts.conf",
|
||||
"path": "/var/lib/nats-module/conf/accounts.conf",
|
||||
"mode": "0600",
|
||||
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
|
||||
"create-once": true,
|
||||
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_assignments.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_declared-config.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.assignment-changed\", \"mesh.seat.mesh-controller.event.declared-config-changed\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
|
||||
},
|
||||
// The upstream server, until the broker step adopts it: the `nats` module's own image, built by the
|
||||
// mesh, takes its place there once, on these same directories (ADR 0333 §3), as the bus's planned
|
||||
// step (ADR 0236). The release is the one the module's image is built on (its NATS_BASE, 2.11.17),
|
||||
// pinned upstream by that same digest: the adoption changes no server release, so it can be undone
|
||||
// by putting the old image back and is said reversible. The published port is the
|
||||
// node's bus port, 5671 unless given, which every enrolment token names; the installer gives the
|
||||
// module the setting that maps its declared 4222 to it (ADR 0333 §2).
|
||||
{
|
||||
"id": "broker",
|
||||
"type": "container",
|
||||
"name": "mesh-broker",
|
||||
"image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
|
||||
"name": "mesh-broker-nats",
|
||||
"image": "nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00",
|
||||
"ports": ["5671:4222", "127.0.0.1:8222:8222"],
|
||||
"volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"],
|
||||
"volumes": ["/var/lib/mesh-broker-nats:/data", "/var/lib/nats-module/conf:/etc/nats:ro", "/var/lib/mesh-broker-tls:/tls:ro"],
|
||||
"args": ["-c", "/etc/nats/nats.conf", "-js"]
|
||||
},
|
||||
{
|
||||
"id": "broker-ready",
|
||||
"type": "action",
|
||||
"command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
||||
"verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"]
|
||||
"command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
||||
"verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671'"]
|
||||
},
|
||||
{
|
||||
"id": "control-plane",
|
||||
@@ -202,7 +252,7 @@
|
||||
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"network": "host",
|
||||
"args": ["serve"],
|
||||
"volumes": ["mesh-broker-tls:/broker-tls:ro"],
|
||||
"volumes": ["/var/lib/mesh-broker-tls:/broker-tls:ro"],
|
||||
"env": {
|
||||
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||
|
||||
Vendored
+1
@@ -49,6 +49,7 @@
|
||||
./internal/inventory TestACredentialIssuedOutsideIsNeverMade
|
||||
./internal/inventory TestOneAcceptForAProviderIsSealedToEveryConsumer
|
||||
./internal/inventory TestASharedOwnSecretIssuedOutsideIsNeverMade
|
||||
./internal/recordmirror TestNatsAnAssignmentAndARegistrationEachChangeOneKeyAndRaiseOneEvent
|
||||
./cmd/mesh-controller TestOneAcceptForAProviderIsPlacedForEveryConsumer
|
||||
./cmd/mesh-controller TestReplay164
|
||||
./cmd/mesh-controller TestAnApprovedLayerOpensTheWalkThatSendsIt
|
||||
|
||||
Reference in New Issue
Block a user