issue 1025: the operator's Approve of a reviewed group registers a moved module from its new repository (ADR 0420 §7)
This commit is contained in:
@@ -56,8 +56,17 @@ var (
|
||||
// waiting request for a head and trunk commit a key names drops it. **Written by the build seat's holders, several by
|
||||
// design** — each the key of the check it runs.
|
||||
CheckRunningBucket = BucketName(ControllerSeat, "checks-running")
|
||||
// DeliveryMergesBucket holds the merges mesh-delivery announced it made, each with the ask whose Approve it merged
|
||||
// on (novox/hq ADR 0420 §7, issue 1025): per merge a key `<owner>/<repo>/<number>/<head sha>` holding the
|
||||
// announcement, kept DeliveryMergesKeptFor. What lets the build of a moved module register it from its new
|
||||
// repository: the controller reads the ask's warrant from the router and accepts only the operator's Approve of
|
||||
// those heads. Written by the controller alone, from the announcement, when it holds the lease.
|
||||
DeliveryMergesBucket = BucketName(ControllerSeat, "delivery-merges")
|
||||
)
|
||||
|
||||
// DeliveryMergesKeptFor is how long a merge mesh-delivery announced is kept: a week, as the events are.
|
||||
const DeliveryMergesKeptFor = 7 * 24 * time.Hour
|
||||
|
||||
// The controller's record mirrors (novox/hq ADR 0289, design 54, design 57 package F3): a bucket per kind of
|
||||
// record, one key per record, written from the store by the lease holder only when the record changed, with
|
||||
// one event on the controller's seat per write. Read by whatever names them in `reads` as
|
||||
@@ -141,13 +150,15 @@ func IsControllerBucket(bucket string) bool {
|
||||
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
|
||||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
|
||||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
|
||||
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket
|
||||
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket ||
|
||||
bucket == DeliveryMergesBucket
|
||||
}
|
||||
|
||||
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||
func ControllerBuckets() []string {
|
||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket}
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket,
|
||||
DeliveryMergesBucket}
|
||||
}
|
||||
|
||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||
@@ -290,6 +301,20 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", CheckRunningBucket, err)
|
||||
}
|
||||
// **The merges mesh-delivery announced: keys that age** (ADR 0420 §7, issue 1025). One message a key; a week.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: DeliveryMergesBucket,
|
||||
Description: "the merges mesh-delivery announced it made, each with the ask whose Approve it merged on (novox/hq " +
|
||||
"ADR 0420): `<owner>/<repo>/<number>/<head sha>` holds the announcement, kept a week; what lets the build " +
|
||||
"of a moved module register it from its new repository; written by the controller alone",
|
||||
History: 1,
|
||||
TTL: DeliveryMergesKeptFor,
|
||||
MaxValueSize: 16 << 10,
|
||||
MaxBytes: 16 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", DeliveryMergesBucket, err)
|
||||
}
|
||||
// **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose
|
||||
// marker lives a day; a key that expired would be a record the bus forgot while the store still holds it.
|
||||
for _, m := range []struct{ bucket, why string }{
|
||||
|
||||
@@ -397,6 +397,11 @@ var DegradableGrants = []DegradableGrant{
|
||||
Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +
|
||||
"be said is printed and goes on (ADR 0406 §5, issue 974)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + DeliveryMergesBucket + ".>",
|
||||
Reason: "refused: the controller then keeps no merge mesh-delivery announced, so the build of a moved module is " +
|
||||
"refused as it is without the operator's Approve — failing closed (ADR 0420 §7, issue 1025)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + RolloutSilentBucket + ".>",
|
||||
Reason: "refused: the controller looks at the plans when a wait for a machine's report is up, by a timer of its own " +
|
||||
|
||||
@@ -242,7 +242,7 @@ func eventsTheControllerFollows() []string {
|
||||
out = append(out, subject)
|
||||
}
|
||||
}
|
||||
return out
|
||||
return append(out, ControllerHears...)
|
||||
}
|
||||
|
||||
// DeliverSubjectFor is where a push consumer's messages land.
|
||||
@@ -428,6 +428,19 @@ var ControllerFollows = []string{
|
||||
DecidedSubject,
|
||||
}
|
||||
|
||||
// ControllerHears are events the controller's events consumer takes beside the ones it follows, and that need no
|
||||
// subscribe grant of their own. **A push consumer delivers onto its deliver subject** (`_DELIVER.controller.>`, already
|
||||
// granted), so the subject a consumer filters on is never one the controller subscribes: listing this one in
|
||||
// ControllerFollows would add a subscribe grant the installer's carried user list, which lives in mesh-host, would
|
||||
// have to carry too (TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose), for nothing the consumer uses.
|
||||
//
|
||||
// A merge mesh-delivery made, announced (novox/hq ADR 0420 §7, issue 1025): with the ask whose Approve it merged on and
|
||||
// the heads that Approve covered, kept a week to read the warrant a moved module's registration needs.
|
||||
var ControllerHears = []string{DeliveryMergedSubject}
|
||||
|
||||
// DeliveryMergedSubject is where mesh-delivery says it merged a pull request.
|
||||
var DeliveryMergedSubject = moduleEventSubject("mesh-delivery", "merged")
|
||||
|
||||
// AsksSeat is the seat an ask is made on and its warrant heard from (novox/hq ADR 0259): the router's.
|
||||
const AsksSeat = "operator-channel"
|
||||
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
@@ -130,8 +130,11 @@ var WritersTable = []WriterRow{
|
||||
// written by the lease holder.
|
||||
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
|
||||
KeptIn: "the controller's store; a batch's merge window in key-value " + MergeWindowBucket +
|
||||
", a rollout's wait for a machine's report in key-value " + RolloutSilentBucket, Others: "read through plans",
|
||||
Subjects: append(kvOf(MergeWindowBucket), kvOf(RolloutSilentBucket)...), Writes: isController},
|
||||
", a rollout's wait for a machine's report in key-value " + RolloutSilentBucket +
|
||||
", and the merges mesh-delivery announced, for a moved module's registration (ADR 0420 §7), in key-value " +
|
||||
DeliveryMergesBucket, Others: "read through plans",
|
||||
Subjects: append(append(kvOf(MergeWindowBucket), kvOf(RolloutSilentBucket)...), kvOf(DeliveryMergesBucket)...),
|
||||
Writes: isController},
|
||||
{State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " +
|
||||
ConditionHistoryBucket + "); how often each was observed, when last and its newest evidence in the " +
|
||||
"controller's store, so a look that changes nothing writes no bucket", Others: "raise or clear only through observations the controller reads",
|
||||
|
||||
@@ -51,6 +51,8 @@ var Contracts = map[string]Contract{
|
||||
"at the router, and the controller acts on it once, recording that it did under the ask's id — so a word " +
|
||||
"heard again, or late, does nothing more (novox/hq ADR 0259)",
|
||||
Tests: []string{"TestAWarrantIsActedOnOnce"}},
|
||||
KindDeliveryMerged: {Unordered: "mesh-delivery's word that it merged a pull request, by the pull request and its head: " +
|
||||
"kept under that key once, so a word heard again or late rewrites the same value (novox/hq ADR 0420 §7)"},
|
||||
KindCatchUp: {Unordered: "a catalogue asking what it missed: answered from the record, whenever asked"},
|
||||
KindProvisioner: {Unordered: "a provider's newest word about a consumer, said again every fifteen minutes " +
|
||||
"while it holds (ADR 0224): the condition keeps the last observed, and S8 says when the words stop. " +
|
||||
|
||||
@@ -18,6 +18,7 @@ func TestEveryConsumedKindHasAContract(t *testing.T) {
|
||||
subjects := []string{EnrolSubject, BuiltSubject, ReportSubject("anchor"), AliveSubject("anchor"),
|
||||
ToolsAliveSubject("anchor"), HealthSubject("anchor"), "mesh.mod.postgres.event.provisioner.failing"}
|
||||
subjects = append(subjects, broker.ControllerFollows...)
|
||||
subjects = append(subjects, broker.ControllerHears...)
|
||||
kinds := map[string]bool{}
|
||||
for _, s := range subjects {
|
||||
kind, known := kindOfSubject(s)
|
||||
|
||||
@@ -47,6 +47,9 @@ const (
|
||||
// KindDecided is the router's warrant for an ask the controller made, or that ask's end without one
|
||||
// (novox/hq ADR 0259): said to the controller alone, under its own name.
|
||||
KindDecided = "decided"
|
||||
// KindDeliveryMerged is mesh-delivery saying it merged a pull request, with the ask whose Approve it merged on
|
||||
// (novox/hq ADR 0420 §7, issue 1025).
|
||||
KindDeliveryMerged = "delivery-merged"
|
||||
)
|
||||
|
||||
// Control is one thing a node or a module said, as the controller must act on it.
|
||||
|
||||
@@ -62,7 +62,8 @@ func Nats(js *broker.JetStream) Inbound {
|
||||
// whatever was asked for — and not at all when nothing was.
|
||||
func (n *natsInbound) Also(kind string) error {
|
||||
switch kind {
|
||||
case KindModuleMoved, KindCatchUp, KindSourceMoved, KindProvisioner, KindPullUpdated, KindDecided:
|
||||
case KindModuleMoved, KindCatchUp, KindSourceMoved, KindProvisioner, KindPullUpdated, KindDecided,
|
||||
KindDeliveryMerged:
|
||||
n.follows[kind] = true
|
||||
return nil
|
||||
default:
|
||||
@@ -301,6 +302,8 @@ func kindOfSubject(subject string) (string, bool) {
|
||||
return KindPullUpdated, true
|
||||
case broker.DecidedSubject:
|
||||
return KindDecided, true
|
||||
case broker.DeliveryMergedSubject:
|
||||
return KindDeliveryMerged, true
|
||||
case BuildOutcome(), BuildOutcomeOf(TheBuildMachineBefore):
|
||||
// A build's outcome is the role's event now, so it arrives on the events stream rather than
|
||||
// the control branch — and is acted on by the same handler, because what the controller does
|
||||
|
||||
@@ -98,6 +98,8 @@ type Server struct {
|
||||
healths Healths
|
||||
// decider acts on the operator's warrants for what the controller asked (novox/hq ADR 0259).
|
||||
decider Decider
|
||||
// deliveryMerges keeps mesh-delivery's merges (KeepsDeliveryMerges).
|
||||
deliveryMerges DeliveryMergeKeeper
|
||||
|
||||
log *log.Logger
|
||||
// giveUp is how long one message is held for the store; zero means GiveUpAfter.
|
||||
@@ -155,6 +157,21 @@ func (s *Server) Decides(d Decider) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeliveryMergeKeeper is what the controller does with mesh-delivery's word that it merged a pull request (novox/hq
|
||||
// ADR 0420 §7): keep it, to be read when a moved module's build is taken in.
|
||||
type DeliveryMergeKeeper interface {
|
||||
KeepDeliveryMerge(ctx context.Context, body []byte) error
|
||||
}
|
||||
|
||||
// KeepsDeliveryMerges says what to do about mesh-delivery's merges, and asks for them to be delivered.
|
||||
func (s *Server) KeepsDeliveryMerges(k DeliveryMergeKeeper) error {
|
||||
if err := s.inbound.Also(KindDeliveryMerged); err != nil {
|
||||
return err
|
||||
}
|
||||
s.deliveryMerges = k
|
||||
return nil
|
||||
}
|
||||
|
||||
// Answers says what to do about a catalogue's catch-up request, and asks for them to be delivered.
|
||||
func (s *Server) Answers(r Replayer) error {
|
||||
if err := s.inbound.Also(KindCatchUp); err != nil {
|
||||
@@ -229,6 +246,8 @@ func (s *Server) act(ctx context.Context, m Control) {
|
||||
s.pullUpdated(ctx, m)
|
||||
case KindDecided:
|
||||
s.decided(ctx, m)
|
||||
case KindDeliveryMerged:
|
||||
s.deliveryMerged(ctx, m)
|
||||
default:
|
||||
// Dropped: a message nothing understands will not be understood on the next attempt
|
||||
// either, and asking for it again would spin.
|
||||
@@ -728,6 +747,24 @@ func (s *Server) decided(ctx context.Context, m Control) {
|
||||
_ = m.Took()
|
||||
}
|
||||
|
||||
// deliveryMerged hands mesh-delivery's word on a merge to its keeper; a failure to keep it is held for the store
|
||||
// like any word that must not be lost, and a word that cannot be read is said and let go.
|
||||
func (s *Server) deliveryMerged(ctx context.Context, m Control) {
|
||||
if s.deliveryMerges == nil {
|
||||
_ = m.Took()
|
||||
return
|
||||
}
|
||||
err := s.deliveryMerges.KeepDeliveryMerge(ctx, m.Body())
|
||||
switch s.decide(ctx, m, "mesh-delivery's word on a merge", "", "", err) {
|
||||
case Hold:
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.log.Printf("mesh-delivery's word on a merge could not be kept: %v", err)
|
||||
}
|
||||
_ = m.Took()
|
||||
}
|
||||
|
||||
// saysWhatItDid states what a machine now runs, or what it would not take, as a fact on the bus
|
||||
// (novox/hq ADR 0134).
|
||||
//
|
||||
|
||||
Reference in New Issue
Block a user