issue 974: a rollout's wait for a machine is a key on the bus that its report deletes and its expiry names silent
This commit is contained in:
@@ -43,6 +43,12 @@ var (
|
||||
// `<batch>.most`, made once when it opens and living the window's most. The first to expire closes the
|
||||
// batch, and the controller's watcher hears the delete marker. Written by the lease holder alone.
|
||||
MergeWindowBucket = BucketName(ControllerSeat, "merge-window")
|
||||
// RolloutSilentBucket holds the wait of every machine a rollout has sent a tier to (novox/hq ADR 0406 §5, issue
|
||||
// 974): per send a key `<plan>.<tier>.<node>` that lives ApplySilentAfter, made when the controller sends that
|
||||
// machine and deleted by the machine's report on that send. Only a missing report lets it expire, and the
|
||||
// controller's watcher hearing the expiry ends the wait and names the machine silent. Written by the lease
|
||||
// holder alone.
|
||||
RolloutSilentBucket = BucketName(ControllerSeat, "rollout-silent")
|
||||
)
|
||||
|
||||
// The controller's record mirrors (novox/hq ADR 0289, design 54, design 57 package F3): a bucket per kind of
|
||||
@@ -88,6 +94,9 @@ const (
|
||||
// that is up to hear it, short enough to leave nothing behind.
|
||||
const MergeWindowMarkerTTL = time.Minute
|
||||
|
||||
// RolloutSilentMarkerTTL is how long the marker an expired silent key leaves is kept, for the same reason.
|
||||
const RolloutSilentMarkerTTL = time.Minute
|
||||
|
||||
// AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own.
|
||||
const AskedKeptFor = 30 * 24 * time.Hour
|
||||
|
||||
@@ -116,13 +125,13 @@ func IsControllerBucket(bucket string) bool {
|
||||
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
||||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
|
||||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
|
||||
bucket == MergeWindowBucket
|
||||
bucket == MergeWindowBucket || bucket == RolloutSilentBucket
|
||||
}
|
||||
|
||||
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||
func ControllerBuckets() []string {
|
||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket}
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket}
|
||||
}
|
||||
|
||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||
@@ -234,6 +243,21 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", MergeWindowBucket, err)
|
||||
}
|
||||
// **The rollout's silent keys: keys that expire** (ADR 0406 §5, issue 974). A key lives as long as the machine
|
||||
// may stay silent; the machine's report deletes it, and a marker of expiry is what the watcher hears.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: RolloutSilentBucket,
|
||||
Description: "the wait of every machine a rollout sent a tier to (novox/hq ADR 0406): `<plan>.<tier>.<node>` " +
|
||||
"lives the time a machine may stay silent and is deleted by its report; one that expires names the " +
|
||||
"machine silent; written by the controller alone",
|
||||
History: 1,
|
||||
MaxValueSize: 4 << 10,
|
||||
MaxBytes: 8 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
LimitMarkerTTL: RolloutSilentMarkerTTL,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", RolloutSilentBucket, err)
|
||||
}
|
||||
// **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose
|
||||
// marker lives a day; a key that expired would be a record the bus forgot while the store still holds it.
|
||||
for _, m := range []struct{ bucket, why string }{
|
||||
|
||||
@@ -322,6 +322,73 @@ func TestTheMergeWindowsKeysExpireAndAWatcherHearsIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// **A rollout's wait for a machine is a bucket whose keys expire** (novox/hq ADR 0406 §5, issue 974): a key made with a
|
||||
// time to live goes when it is up, and a watcher hears it go as a purge; a key a report deleted is a delete. The
|
||||
// controller may write it, and it is no undeclared state.
|
||||
func TestTheRolloutsSilentKeysExpireAsAPurgeAndAReportsDeleteIsNot(t *testing.T) {
|
||||
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(controller.Publish, "$KV."+RolloutSilentBucket+".>") {
|
||||
t.Error("the controller may not write the rollout's silent keys' bucket")
|
||||
}
|
||||
if !IsControllerBucket(RolloutSilentBucket) || !slices.Contains(ControllerBuckets(), RolloutSilentBucket) {
|
||||
t.Error("the rollout's silent keys' bucket is not asserted as the controller's own")
|
||||
}
|
||||
js, err := Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, RolloutSilentBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer w.Stop()
|
||||
if _, err := kv.Create(ctx, "plan-1.0.ace", []byte("x"), jetstream.KeyTTL(time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := kv.Delete(ctx, "plan-1.0.ace"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Create(ctx, "plan-1.0.g14", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case e := <-w.Updates():
|
||||
if e == nil || e.Operation() == jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
if e.Key() == "plan-1.0.ace" && e.Operation() != jetstream.KeyValueDelete {
|
||||
t.Fatalf("a report's delete was heard as %v", e.Operation())
|
||||
}
|
||||
if e.Key() == "plan-1.0.g14" {
|
||||
if e.Operation() != jetstream.KeyValuePurge {
|
||||
t.Fatalf("an expiry was heard as %v, not a purge", e.Operation())
|
||||
}
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("no watcher heard the key expire")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The merges have a consumer of their own, which a reset of the events consumer leaves alone** (issue 973, novox/hq
|
||||
// ADR 0407 decision 3): healer H4 re-makes the events consumer from now, dropping what it holds pending; a merge
|
||||
// waiting in the merges consumer is not that consumer's, and nothing would catch it up.
|
||||
|
||||
@@ -378,6 +378,11 @@ var DegradableGrants = []DegradableGrant{
|
||||
Reason: "refused: the controller closes a batch's window by a timer of its own for that batch, as the " +
|
||||
"window setting says (ADR 0407 decision 1, issue 973)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + RolloutSilentBucket + ".>",
|
||||
Reason: "refused: the controller advances a rollout on the machines' reports alone, and a machine never heard " +
|
||||
"from is found when the plan is next looked at (ADR 0406 §5, issue 974)",
|
||||
},
|
||||
}
|
||||
|
||||
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
@@ -117,11 +117,13 @@ var WritersTable = []WriterRow{
|
||||
Subjects: []string{"mesh.control.*.report", "mesh.control.*.health", "mesh.control.*.cli"}, Writes: ownMachine},
|
||||
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
|
||||
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
|
||||
// And the clock of an open batch's merge window, a plan before it is one (novox/hq ADR 0407 decision 1): keys that
|
||||
// expire in a bucket of their own, written by the lease holder.
|
||||
// And the clock of an open batch's merge window, a plan before it is one (novox/hq ADR 0407 decision 1), and the
|
||||
// wait of a rollout for each machine it sent (ADR 0406 §5, issue 974): keys that expire in buckets of their own,
|
||||
// written by the lease holder.
|
||||
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
|
||||
KeptIn: "the controller's store; a batch's merge window in key-value " + MergeWindowBucket, Others: "read through plans",
|
||||
Subjects: kvOf(MergeWindowBucket), Writes: isController},
|
||||
KeptIn: "the controller's store; a batch's merge window in key-value " + MergeWindowBucket +
|
||||
", a rollout's wait for a machine's report in key-value " + RolloutSilentBucket, Others: "read through plans",
|
||||
Subjects: append(kvOf(MergeWindowBucket), kvOf(RolloutSilentBucket)...), Writes: isController},
|
||||
{State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " +
|
||||
ConditionHistoryBucket + "); how often each was observed, when last and its newest evidence in the " +
|
||||
"controller's store, so a look that changes nothing writes no bucket", Others: "raise or clear only through observations the controller reads",
|
||||
|
||||
Reference in New Issue
Block a user