mesh-sdk/go/buscompose/controller_buckets.go

package buscompose

import (
	"context"
	"fmt"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
	"github.com/nats-io/nats.go/jetstream"
)

// The controller's own key-value buckets (novox/hq to-be 45 §1, §6, §7).
//
// **What the controller must remember across its own restart, it keeps on the bus.** A call's
// outcome lived in the memory of the process that served it (novox/hq issue 265), so a controller
// replaced while a push ran answered "no such call" for the one thing its caller had been told to
// ask about. The bus already outlives the controller and is the shape ADR 0201 gives a module's
// current state: one value per key, written by one owner, read by anybody granted it. These are the
// controller's, written by it alone — the writers table of to-be 45 §1 — and asserted on every start
// like the streams, so a bus raised from nothing has them before the first call is served.

// CallsBucket keeps every call of the mesh's own verbs and what came of it; HandActsBucket every act
// a person did by hand, with why; ConditionsBucket every condition open now (to-be 45 §2), one key
// each, and ConditionHistoryBucket every transition of one — raised, changed, silenced, cleared —
// for ninety days.
//
// **The history is a bucket of its own** because its keys expire and an open condition's must not:
// a bucket has one age for every key, and a condition open longer than the history is kept would
// otherwise vanish from the store while still true.
var (
	CallsBucket            = BucketName(ControllerSeat, "calls")
	HandActsBucket         = BucketName(ControllerSeat, "hand-acts")
	ConditionsBucket       = BucketName(ControllerSeat, "conditions")
	ConditionHistoryBucket = BucketName(ControllerSeat, "condition-history")
	// LeaseBucket holds the controller's lease (to-be 45 §6): one key, `holder`, which the instance
	// allowed to act writes by compare-and-set and renews; its revision when taken is the epoch.
	LeaseBucket = BucketName(ControllerSeat, "lease")
	// AskedBucket keeps what the controller asked the operator about its conditions (novox/hq ADR 0259):
	// each ask by its id, its options and the actions they stand for, how it ended and whether the
	// controller acted on its warrant — so a restart neither asks twice nor acts twice.
	AskedBucket = BucketName(ControllerSeat, "asked")
	// DeliveryMergesBucket holds the merges mesh-delivery announced it made, each with the ask whose Approve it merged
	// on (novox/hq ADR 0420 §7, issue 1025): per merge a key `<owner>/<repo>/<number>/<head sha>` holding the
	// announcement, kept DeliveryMergesKeptFor. What lets the build of a moved module register it from its new
	// repository: the controller reads the ask's warrant from the router and accepts only the operator's Approve of
	// those heads. Written by the controller alone, from the announcement, when it holds the lease.
	DeliveryMergesBucket = BucketName(ControllerSeat, "delivery-merges")
)

// DeliveryMergesKeptFor is how long a merge mesh-delivery announced is kept: a week, as the events are.
const DeliveryMergesKeptFor = 7 * 24 * time.Hour

// The controller's record mirrors (novox/hq ADR 0289, design 54, design 57 package F3): a bucket per kind of
// record, one key per record, written from the store by the lease holder only when the record changed, with
// one event on the controller's seat per write. Read by whatever names them in `reads` as
// `mesh-controller.<mirror>` (ADR 0201 §1), written by nobody else. **No secret in either** (ADR 0201): an
// assignment's value names its module and node; a module's declared config is what its manifest, which is
// public, declares.
//
// AssignmentsBucket holds which module is assigned to which node, keyed `<node>.<module>`; and
// DeclaredConfigBucket each registered module's config declarations as its manifest holds them, keyed by
// the module: what mesh-config resolves a config value from (ADR 0348 clauses 2 and 4); and PredefinedBucket the
// list of predefined keys the mesh published (the newest trunk build's of the catalogue, ADR 0348 §7), one key
// `list`: what mesh-config publishes as mesh-config_predefined (design 57 K3, issue 662).
var (
	AssignmentsBucket    = BucketName(ControllerSeat, MirrorAssignments)
	DeclaredConfigBucket = BucketName(ControllerSeat, MirrorDeclaredConfig)
	PredefinedBucket     = BucketName(ControllerSeat, MirrorPredefined)
	// DeclaredPackagesBucket holds, keyed by the node, each package the modules assigned to it declare, present or absent,
	// with the declaring modules: what the pacman module's verbs read to say a package is declared (ADR 0507 §5).
	DeclaredPackagesBucket = BucketName(ControllerSeat, MirrorDeclaredPackages)
)

// The mirrors' local names, as a reader names them in `reads` after `mesh-controller.`.
const (
	MirrorAssignments    = "assignments"
	MirrorDeclaredConfig = "declared-config"
	MirrorPredefined     = "predefined"
	// MirrorDeclaredPackages is the local name of the packages each node's modules declare (ADR 0507 §5, issue 1840/S3).
	MirrorDeclaredPackages = "declared-packages"
)

// The chain's buckets (novox/hq ADR 0408, issue 1089): builds, rollouts and assignments are one chain kept on the bus
// alone, one bucket each, one key per record, written by the lease holder by compare-and-set. The assignments bucket is
// AssignmentsBucket above, which keeps its name and its readers (ADR 0408 §3). Nothing is removed from them (§4), they keep
// ChainHistory earlier revisions of every key (§5), and a value is never cut (§6).
var (
	BuildsBucket   = BucketName(ControllerSeat, ChainBuilds)
	RolloutsBucket = BucketName(ControllerSeat, ChainRollouts)
)

// **The assignments bucket keeps the marker age (MirrorMarkerTTL) it was made with as a mirror.** A stream that allows
// messages an age cannot be made to disallow them (the server refuses the update: "message TTL status can not be
// disabled", nats-server stream.go), so asserting the bucket without the age would fail the controller's start on a
// mesh that has the bucket. The chain never purges or removes a key (ADR 0408 §4), so no marker is ever written and the
// age ends nothing; it is kept only so the stream's configuration stays one the server accepts.
//
// The chain buckets' local names, as a reader names them in `reads` after `mesh-controller.`.
const (
	ChainBuilds   = "builds"
	ChainRollouts = "rollouts"
)

// The chain buckets' bounds. ChainHistory is the bus's largest history (ADR 0408 §5). A value is at most ChainValueBytes,
// just under the server's largest message of a mebibyte, which also carries the subject and the headers (ADR 0408 §6):
// the controller writes none larger and the move refuses to. The buckets' size is the arithmetic of the survey of
// issue 981: 8585 builds of 1 to 40 KB, 1049 rollouts of about 2 KB a revision and 64 revisions, so 512 MiB for the two
// that grow, and the assignments, 285 of under a kilobyte, 64 MiB.
const (
	ChainHistory         = 64
	ChainValueBytes      = 1<<20 - 8<<10
	ChainBuildsBytes     = 512 << 20
	ChainRolloutsBytes   = 512 << 20
	ChainAssignmentBytes = 64 << 20
)

// GrantsBucket holds, keyed `<module>.<fingerprint hex>`, each fingerprint of a module whose bus credential the controller
// has issued (novox/hq ADR 0528 clause 2, ADR 0429 §6 rule 6, issue 1394): written by the controller alone once the bus's
// user list composed with the module's manifest at that fingerprint was sent to the bus's machine; read by the assigner,
// whose sixth rule moves a node's want to a fingerprint only once its grant exists. The name and layout are mesh-sdk's
// (delivery.ControllerGrantsBucket, ControllerGrantKey).
var GrantsBucket = BucketName(ControllerSeat, ControllerGrants)

// ControllerGrants is the grants bucket's local name, as a reader names it in `reads` after `mesh-controller.`.
const ControllerGrants = "grants"

// ChainBuckets are the three buckets of the chain: builds, rollouts and assignments.
func ChainBuckets() []string { return []string{BuildsBucket, RolloutsBucket, AssignmentsBucket} }

// MirrorsWrittenByAnother are the controller's mirrors whose writer is not the controller, each with its reason: the rule
// that a mirror is written by the controller alone (TestTheMirrorsListAndTheWritersTableAgree) has these exceptions and no
// other.
var MirrorsWrittenByAnother = map[string]string{}

// The mirrors' caps (design 54 §5): one value per key, no age, 64 KiB a value, 64 MiB a bucket.
const (
	MirrorValueBytes = 64 << 10
	MirrorBytes      = 64 << 20
	// MirrorMarkerTTL is the longest a purge's marker is kept (design 54 §2): the bucket allows a
	// per-message age for it, and the writer gives each marker a day.
	MirrorMarkerTTL = 24 * time.Hour
)

// PlanStateBuckets are the buckets of the plans' state that release 6 deletes (novox/hq ADR 0539 clause 13): the merge
// window's clock, the silent sends' waits and the building plans' hold. The controller deletes each at its first start
// and says it once (composed_start.go); none is asserted, granted or written any more.
var PlanStateBuckets = []string{BucketName(ControllerSeat, "merge-window"), BucketName(ControllerSeat, "rollout-silent"),
	BucketName(ControllerSeat, "plans-building")}

// AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own.
const AskedKeptFor = 30 * 24 * time.Hour

// LeaseTTL is how long the lease's key lives unrenewed (to-be 45 §6): fifteen seconds, renewed
// every five. The bucket's age, so the bus forgets a holder that stopped renewing.
const LeaseTTL = 15 * time.Second

// The bounds to-be 45 §6 sets for calls: the last thousand, or fourteen days, whichever is fewer.
// A call is two keys — its record, and its answer apart so a listing does not read every answer —
// so the stream holds twice as many messages as it keeps calls.
const (
	KeptCallsDurably = 1000
	CallsKeptFor     = 14 * 24 * time.Hour
	// CallAnswerBytes is the most of one answer kept: a whole declaration is far smaller, and an
	// answer larger is cut and says so.
	CallAnswerBytes = 64 << 10
	// HandActsKeptFor is as long as a condition's history (to-be 45 §2): an act by hand is read
	// back beside what it addressed.
	HandActsKeptFor = 90 * 24 * time.Hour
	// ConditionHistoryKeptFor is how long a condition's transitions are kept (to-be 45 §2).
	ConditionHistoryKeptFor = 90 * 24 * time.Hour
)

// IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares.
func IsControllerBucket(bucket string) bool {
	return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
		bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
		bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
		bucket == DeclaredPackagesBucket ||
		bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket ||
		bucket == delivery.HaveBucket || bucket == StatesAnnouncedBucket ||
		bucket == GrantsBucket || bucket == delivery.ControllerComposedBucket
}

// ControllerBuckets are the controller's own buckets, in the order they are asserted.
func ControllerBuckets() []string {
	return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
		AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, DeclaredPackagesBucket,
		DeliveryMergesBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket, StatesAnnouncedBucket, GrantsBucket,
		delivery.ControllerComposedBucket}
}

// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
type ControllerBucketsAsserter interface {
	EnsureControllerBuckets() error
}

// chainBucket is the options of a bucket of the chain: sixty-four revisions of each key, no age, a value up to
// ChainValueBytes, on disk.
func chainBucket(bucket, why string, bytes int64, markers time.Duration) jetstream.KeyValueConfig {
	return jetstream.KeyValueConfig{
		Bucket:         bucket,
		Description:    why,
		History:        ChainHistory,
		MaxValueSize:   ChainValueBytes,
		MaxBytes:       bytes,
		Storage:        jetstream.FileStorage,
		LimitMarkerTTL: markers,
	}
}

// EnsureLeaseBucket creates the lease's bucket if absent and brings its options to match (to-be 45
// §6). **Before the lease is taken, by any candidate**: it is the lease's own precondition, and asserting
// a bucket that exists changes nothing. File storage, so its revisions — the epochs — outlive a restart of
// the bus; one raised again from nothing is moved past the highest epoch issued when the lease is taken.
func EnsureLeaseBucket(ctx context.Context, js jetstream.JetStream) error {
	if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
		Bucket: LeaseBucket,
		Description: "the controller's lease (novox/hq to-be 45 §6): the key `holder`, written by compare-and-set " +
			"by the one controller instance that may act and renewed every five seconds; its revision when taken " +
			"is the epoch every declaration and plan write carries",
		History:      1,
		TTL:          LeaseTTL,
		MaxValueSize: 4 << 10,
		MaxBytes:     1 << 20,
		Storage:      jetstream.FileStorage,
	}); err != nil {
		return fmt.Errorf("asserting bucket %s: %w", LeaseBucket, err)
	}
	return nil
}

// HaveBucketConfig is the options of the have bucket (novox/hq ADR 0429 §1): delivery.HaveHistory revisions of each
// key, no age, 64 MiB on disk like the assignments. A module's have is JSON of about 500 bytes, but a node's own key
// carries its whole facts (its profile among them), several kilobytes on a desktop: a value is as large as the chain's
// buckets take, since a value is never cut (novox/hq ADR 0408 §6, issue 1162).
func HaveBucketConfig() jetstream.KeyValueConfig {
	return jetstream.KeyValueConfig{
		Bucket: delivery.HaveBucket,
		Description: "what each machine has of each module, one key `<node>.<module>` (and `<node>.node` for the machine's " +
			"own facts) (novox/hq ADR 0429, design 47): written by each machine's node-engine under its own node's keys " +
			"alone, never by the controller; read by the delivery view and the watchman",
		History:      delivery.HaveHistory,
		MaxValueSize: ChainValueBytes,
		MaxBytes:     64 << 20,
		Storage:      jetstream.FileStorage,
	}
}