From fd522f432200d6b72d5c9a7ecc09f84e64973e7b Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 13:18:27 +0200 Subject: [PATCH 01/12] issue 974: a rollout's wait for a machine is a key on the bus that its report deletes and its expiry names silent --- cmd/mesh-controller/rollout_silent_bus.go | 121 ++++++++++++++++++ .../rollout_silent_bus_test.go | 120 +++++++++++++++++ internal/broker/controller_buckets.go | 28 +++- internal/broker/controller_buckets_test.go | 67 ++++++++++ internal/broker/nats.go | 5 + internal/broker/testdata/composed.conf | 2 +- internal/broker/writers.go | 10 +- 7 files changed, 346 insertions(+), 7 deletions(-) create mode 100644 cmd/mesh-controller/rollout_silent_bus.go create mode 100644 cmd/mesh-controller/rollout_silent_bus_test.go diff --git a/cmd/mesh-controller/rollout_silent_bus.go b/cmd/mesh-controller/rollout_silent_bus.go new file mode 100644 index 00000000..ca21ce66 --- /dev/null +++ b/cmd/mesh-controller/rollout_silent_bus.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" +) + +// A rollout's wait for a machine is a key on the bus (novox/hq ADR 0406 §5, issue 974): each send to a machine puts +// a key `..` that lives as long as the machine may stay silent (ApplySilentAfter); the machine's +// report on that send deletes it, so only a missing report lets it expire; and the expiry, heard by a watcher, +// ends the wait for that machine, which the plan then names silent. No timer looks every so often whether a +// machine has reported. + +// silentWait is one machine's wait on one send of a plan's tier. +type silentWait struct { + Plan string + Tier int + Node string +} + +// silentKey is a wait's key. A plan id may hold a dot, so it is read from the right (parseSilentKey). +func silentKey(w silentWait) string { return w.Plan + "." + strconv.Itoa(w.Tier) + "." + w.Node } + +// parseSilentKey reads a key back: the node is the last token, the tier the one before, the plan what is left. +func parseSilentKey(key string) (silentWait, bool) { + last := strings.LastIndexByte(key, '.') + if last < 1 || last == len(key)-1 { + return silentWait{}, false + } + rest, node := key[:last], key[last+1:] + mid := strings.LastIndexByte(rest, '.') + if mid < 1 || mid == len(rest)-1 { + return silentWait{}, false + } + tier, err := strconv.Atoi(rest[mid+1:]) + if err != nil || tier < 0 { + return silentWait{}, false + } + return silentWait{Plan: rest[:mid], Tier: tier, Node: node}, true +} + +// silentClock is the bus's side of the wait for a machine's report. +type silentClock interface { + // Arm puts a wait's key, to live ttl: made again if the machine is sent again. + Arm(ctx context.Context, w silentWait, ttl time.Duration) error + // Reported takes a wait's key away: the machine reported on the send, and nothing is to wake the controller. + Reported(ctx context.Context, w silentWait) error +} + +// busSilent is the clock on the controller's bucket. +type busSilent struct { + kv jetstream.KeyValue +} + +// newBusSilent opens the bucket and starts the watcher, which names each wait whose key expired until the context +// ends. Only expiry wakes it: the marker of an expiry is a purge, and a key a report deleted, or one Arm made again, +// leaves a delete (the vendored client reads the server's MaxAge marker as a purge, kv.go:971). +func newBusSilent(ctx context.Context, api jetstream.JetStream, silent func(silentWait)) (*busSilent, error) { + kv, err := api.KeyValue(ctx, broker.RolloutSilentBucket) + if err != nil { + return nil, fmt.Errorf("the rollout's silent keys' bucket: %w", err) + } + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + return nil, fmt.Errorf("watching the rollout's silent keys: %w", err) + } + go func() { + defer w.Stop() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil || e.Operation() != jetstream.KeyValuePurge { + continue // a put, or a delete: a report's, or a send made again + } + if wait, ok := parseSilentKey(e.Key()); ok && silent != nil { + silent(wait) + } + } + } + }() + return &busSilent{kv: kv}, nil +} + +func (s *busSilent) Arm(ctx context.Context, w silentWait, ttl time.Duration) error { + key := silentKey(w) + // Made again, not changed: a key's time to live is set when it is made. + if err := s.kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + _, err := s.kv.Create(ctx, key, []byte(time.Now().UTC().Format(time.RFC3339)), jetstream.KeyTTL(leastSecond(ttl))) + return err +} + +func (s *busSilent) Reported(ctx context.Context, w silentWait) error { + if err := s.kv.Delete(ctx, silentKey(w)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} + +// leastSecond rounds a time to live up to a whole second, at least one: the server rounds a key's time down to +// whole seconds, so a fraction would close the key before the time it stands for. +func leastSecond(d time.Duration) time.Duration { + if d < time.Second { + return time.Second + } + return (d + time.Second - 1) / time.Second * time.Second +} diff --git a/cmd/mesh-controller/rollout_silent_bus_test.go b/cmd/mesh-controller/rollout_silent_bus_test.go new file mode 100644 index 00000000..b9cc46f0 --- /dev/null +++ b/cmd/mesh-controller/rollout_silent_bus_test.go @@ -0,0 +1,120 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +// silentOnABus is the rollout's silent keys on a real bus, and the waits its watcher heard expire. +func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + api, err := jetstream.New(conn) + if err != nil { + t.Fatal(err) + } + heard := make(chan silentWait, 16) + ctx, stop := context.WithCancel(context.Background()) + t.Cleanup(stop) + s, err := newBusSilent(ctx, api, func(w silentWait) { heard <- w }) + if err != nil { + t.Fatal(err) + } + return s, heard +} + +func hearsSilent(t *testing.T, heard <-chan silentWait, within time.Duration) (silentWait, bool) { + t.Helper() + select { + case w := <-heard: + return w, true + case <-time.After(within): + return silentWait{}, false + } +} + +// A machine that never reports on its send is named silent when its key expires (issue 974, novox/hq ADR 0406 §5): +// the watcher names the plan, the tier and the machine, which a plan id or a machine name with a dot does not blur. +func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { + s, heard := silentOnABus(t) + want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"} + if err := s.Arm(t.Context(), want, time.Second); err != nil { + t.Fatal(err) + } + if got, ok := hearsSilent(t, heard, 10*time.Second); !ok || got != want { + t.Fatalf("heard %+v (%v); want %+v", got, ok, want) + } +} + +// A machine that reports in time is never named silent: the report deletes the key, and a delete is not an expiry. +func TestAReportDeletesTheKeyAndNoOneIsNamedSilent(t *testing.T) { + s, heard := silentOnABus(t) + ctx := t.Context() + w := silentWait{Plan: "plan-1", Tier: 0, Node: "g14"} + if err := s.Arm(ctx, w, time.Second); err != nil { + t.Fatal(err) + } + if err := s.Reported(ctx, w); err != nil { + t.Fatal(err) + } + if got, ok := hearsSilent(t, heard, 2500*time.Millisecond); ok { + t.Fatalf("%+v was named silent after it reported", got) + } + // Reporting twice, or on a send that put no key, is no error. + if err := s.Reported(ctx, w); err != nil { + t.Fatalf("a second report: %v", err) + } +} + +// A send made again to a machine puts its key again with the full time, and only the last send's key can expire. +func TestASendMadeAgainPutsTheKeyAgain(t *testing.T) { + s, heard := silentOnABus(t) + ctx := t.Context() + w := silentWait{Plan: "plan-3", Tier: 1, Node: "novox"} + if err := s.Arm(ctx, w, 2*time.Second); err != nil { + t.Fatal(err) + } + time.Sleep(1200 * time.Millisecond) + if err := s.Arm(ctx, w, 2*time.Second); err != nil { + t.Fatal(err) + } + if got, ok := hearsSilent(t, heard, 1500*time.Millisecond); ok { + t.Fatalf("%+v expired at the first send's time, not the second's", got) + } + if got, ok := hearsSilent(t, heard, 5*time.Second); !ok || got != w { + t.Fatalf("heard %+v (%v); want %+v after the second send's time", got, ok, w) + } +} + +func TestASilentKeyIsParsedFromTheRightOfTheNode(t *testing.T) { + for _, w := range []silentWait{{"plan-1", 0, "ace"}, {"a.b", 12, "g14"}} { + got, ok := parseSilentKey(silentKey(w)) + if !ok || got != w { + t.Errorf("%+v became %+v (%v)", w, got, ok) + } + } + for _, bad := range []string{"", "plan", "plan.x.ace", "plan..ace", ".1.ace"} { + if _, ok := parseSilentKey(bad); ok { + t.Errorf("%q was read as a wait", bad) + } + } +} diff --git a/internal/broker/controller_buckets.go b/internal/broker/controller_buckets.go index 1c534d0d..b47d9111 100644 --- a/internal/broker/controller_buckets.go +++ b/internal/broker/controller_buckets.go @@ -43,6 +43,12 @@ var ( // `.most`, made once when it opens and living the window's most. The first to expire closes the // batch, and the controller's watcher hears the delete marker. Written by the lease holder alone. MergeWindowBucket = BucketName(ControllerSeat, "merge-window") + // RolloutSilentBucket holds the wait of every machine a rollout has sent a tier to (novox/hq ADR 0406 §5, issue + // 974): per send a key `..` that lives ApplySilentAfter, made when the controller sends that + // machine and deleted by the machine's report on that send. Only a missing report lets it expire, and the + // controller's watcher hearing the expiry ends the wait and names the machine silent. Written by the lease + // holder alone. + RolloutSilentBucket = BucketName(ControllerSeat, "rollout-silent") ) // The controller's record mirrors (novox/hq ADR 0289, design 54, design 57 package F3): a bucket per kind of @@ -88,6 +94,9 @@ const ( // that is up to hear it, short enough to leave nothing behind. const MergeWindowMarkerTTL = time.Minute +// RolloutSilentMarkerTTL is how long the marker an expired silent key leaves is kept, for the same reason. +const RolloutSilentMarkerTTL = time.Minute + // AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own. const AskedKeptFor = 30 * 24 * time.Hour @@ -116,13 +125,13 @@ func IsControllerBucket(bucket string) bool { return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket || bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket || bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket || - bucket == MergeWindowBucket + bucket == MergeWindowBucket || bucket == RolloutSilentBucket } // ControllerBuckets are the controller's own buckets, in the order they are asserted. func ControllerBuckets() []string { return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket, - AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket} + AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket} } // ControllerBucketsAsserter is what raising the controller's buckets needs of a connection. @@ -234,6 +243,21 @@ func (j *JetStream) EnsureControllerBuckets() error { }); err != nil { return fmt.Errorf("asserting bucket %s: %w", MergeWindowBucket, err) } + // **The rollout's silent keys: keys that expire** (ADR 0406 §5, issue 974). A key lives as long as the machine + // may stay silent; the machine's report deletes it, and a marker of expiry is what the watcher hears. + if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: RolloutSilentBucket, + Description: "the wait of every machine a rollout sent a tier to (novox/hq ADR 0406): `..` " + + "lives the time a machine may stay silent and is deleted by its report; one that expires names the " + + "machine silent; written by the controller alone", + History: 1, + MaxValueSize: 4 << 10, + MaxBytes: 8 << 20, + Storage: jetstream.FileStorage, + LimitMarkerTTL: RolloutSilentMarkerTTL, + }); err != nil { + return fmt.Errorf("asserting bucket %s: %w", RolloutSilentBucket, err) + } // **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose // marker lives a day; a key that expired would be a record the bus forgot while the store still holds it. for _, m := range []struct{ bucket, why string }{ diff --git a/internal/broker/controller_buckets_test.go b/internal/broker/controller_buckets_test.go index 65dcd948..e34e0d3e 100644 --- a/internal/broker/controller_buckets_test.go +++ b/internal/broker/controller_buckets_test.go @@ -322,6 +322,73 @@ func TestTheMergeWindowsKeysExpireAndAWatcherHearsIt(t *testing.T) { } } +// **A rollout's wait for a machine is a bucket whose keys expire** (novox/hq ADR 0406 §5, issue 974): a key made with a +// time to live goes when it is up, and a watcher hears it go as a purge; a key a report deleted is a delete. The +// controller may write it, and it is no undeclared state. +func TestTheRolloutsSilentKeysExpireAsAPurgeAndAReportsDeleteIsNot(t *testing.T) { + controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(controller.Publish, "$KV."+RolloutSilentBucket+".>") { + t.Error("the controller may not write the rollout's silent keys' bucket") + } + if !IsControllerBucket(RolloutSilentBucket) || !slices.Contains(ControllerBuckets(), RolloutSilentBucket) { + t.Error("the rollout's silent keys' bucket is not asserted as the controller's own") + } + js, err := Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer js.Close() + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + kv, err := api.KeyValue(ctx, RolloutSilentBucket) + if err != nil { + t.Fatal(err) + } + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + t.Fatal(err) + } + defer w.Stop() + if _, err := kv.Create(ctx, "plan-1.0.ace", []byte("x"), jetstream.KeyTTL(time.Hour)); err != nil { + t.Fatal(err) + } + if err := kv.Delete(ctx, "plan-1.0.ace"); err != nil { + t.Fatal(err) + } + if _, err := kv.Create(ctx, "plan-1.0.g14", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil { + t.Fatal(err) + } + for { + select { + case e := <-w.Updates(): + if e == nil || e.Operation() == jetstream.KeyValuePut { + continue + } + if e.Key() == "plan-1.0.ace" && e.Operation() != jetstream.KeyValueDelete { + t.Fatalf("a report's delete was heard as %v", e.Operation()) + } + if e.Key() == "plan-1.0.g14" { + if e.Operation() != jetstream.KeyValuePurge { + t.Fatalf("an expiry was heard as %v, not a purge", e.Operation()) + } + return + } + case <-ctx.Done(): + t.Fatal("no watcher heard the key expire") + } + } +} + // **The merges have a consumer of their own, which a reset of the events consumer leaves alone** (issue 973, novox/hq // ADR 0407 decision 3): healer H4 re-makes the events consumer from now, dropping what it holds pending; a merge // waiting in the merges consumer is not that consumer's, and nothing would catch it up. diff --git a/internal/broker/nats.go b/internal/broker/nats.go index e12966bc..396a353c 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -378,6 +378,11 @@ var DegradableGrants = []DegradableGrant{ Reason: "refused: the controller closes a batch's window by a timer of its own for that batch, as the " + "window setting says (ADR 0407 decision 1, issue 973)", }, + { + Subject: "$KV." + RolloutSilentBucket + ".>", + Reason: "refused: the controller advances a rollout on the machines' reports alone, and a machine never heard " + + "from is found when the plan is next looked at (ADR 0406 §5, issue 974)", + }, } // perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index eb556826..6178d798 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/broker/writers.go b/internal/broker/writers.go index 7e50b9ce..1b03593f 100644 --- a/internal/broker/writers.go +++ b/internal/broker/writers.go @@ -117,11 +117,13 @@ var WritersTable = []WriterRow{ Subjects: []string{"mesh.control.*.report", "mesh.control.*.health", "mesh.control.*.cli"}, Writes: ownMachine}, {State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket, Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController}, - // And the clock of an open batch's merge window, a plan before it is one (novox/hq ADR 0407 decision 1): keys that - // expire in a bucket of their own, written by the lease holder. + // And the clock of an open batch's merge window, a plan before it is one (novox/hq ADR 0407 decision 1), and the + // wait of a rollout for each machine it sent (ADR 0406 §5, issue 974): keys that expire in buckets of their own, + // written by the lease holder. {State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision", - KeptIn: "the controller's store; a batch's merge window in key-value " + MergeWindowBucket, Others: "read through plans", - Subjects: kvOf(MergeWindowBucket), Writes: isController}, + KeptIn: "the controller's store; a batch's merge window in key-value " + MergeWindowBucket + + ", a rollout's wait for a machine's report in key-value " + RolloutSilentBucket, Others: "read through plans", + Subjects: append(kvOf(MergeWindowBucket), kvOf(RolloutSilentBucket)...), Writes: isController}, {State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " + ConditionHistoryBucket + "); how often each was observed, when last and its newest evidence in the " + "controller's store, so a look that changes nothing writes no bucket", Others: "raise or clear only through observations the controller reads", From 8af77c2c75c6524e7f8715a455a7074dc7e4b0ff Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 13:28:34 +0200 Subject: [PATCH 02/12] issue 974: a rollout advances on events (a look asked by each plan saved, report and expired silent key), the 30 s ticker goes, and each tier send is said as sent --- cmd/mesh-controller/plan_looks.go | 568 ++++++++++++++++++ cmd/mesh-controller/plan_looks_test.go | 259 ++++++++ cmd/mesh-controller/push.go | 15 +- cmd/mesh-controller/release_plan.go | 82 +-- cmd/mesh-controller/rollout_silent_bus.go | 37 +- .../rollout_silent_bus_test.go | 8 +- cmd/mesh-controller/status_summary.go | 5 + internal/broker/nats.go | 9 +- internal/broker/states_agreement_test.go | 2 + internal/broker/streams.go | 2 + internal/broker/testdata/composed.conf | 2 +- internal/catalogue/seats.go | 2 + internal/link/events.go | 4 + 13 files changed, 931 insertions(+), 64 deletions(-) create mode 100644 cmd/mesh-controller/plan_looks.go create mode 100644 cmd/mesh-controller/plan_looks_test.go diff --git a/cmd/mesh-controller/plan_looks.go b/cmd/mesh-controller/plan_looks.go new file mode 100644 index 00000000..10ad6832 --- /dev/null +++ b/cmd/mesh-controller/plan_looks.go @@ -0,0 +1,568 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A rollout advances on events, not on a timer (novox/hq ADR 0406 §5, issue 974). One look at the open plans runs on +// each thing that can let a plan move: a plan saved (a build's `built`, a tier's send kept, a command's move), a +// machine's report on a send, a silent key's expiry, and the controller taking the lease. The looks are asked for +// with askToLook and run one at a time by a worker, so an ask made while a look runs is not lost and many asks are +// one look. Only two things are still started by time, both named and both one-shot: a failed advance step is +// looked at again (retryLook), and the end of a walk whose end is not known is recorded when it can be +// (armWalkEnd). + +// looker runs the look at the plans that events ask for. +type looker struct { + signal chan struct{} // one place: asks made together are one look + look func(ctx context.Context) (failed bool) + // gap is the least time between two looks, so a plan that kept changing could not make the worker spin. + gap time.Duration + // first runs before the first look: what a controller that took the lease sets up. + first func(ctx context.Context) + + mu sync.Mutex + retry *time.Timer + backoff time.Duration + base time.Duration + most time.Duration +} + +const ( + retryLookFirst = 5 * time.Second + retryLookMost = time.Minute + lookGap = 200 * time.Millisecond +) + +func newLooker(look func(ctx context.Context) bool) *looker { + return &looker{signal: make(chan struct{}, 1), look: look, gap: lookGap, base: retryLookFirst, most: retryLookMost} +} + +// ask has a look run, soon; never blocks. +func (l *looker) ask() { + select { + case l.signal <- struct{}{}: + default: + } +} + +// run takes the asks one at a time until the context ends. +func (l *looker) run(ctx context.Context) { + if l.first != nil { + l.first(ctx) + } + for { + select { + case <-ctx.Done(): + l.mu.Lock() + if l.retry != nil { + l.retry.Stop() + } + l.mu.Unlock() + return + case <-l.signal: + } + l.after(l.look(ctx)) + if l.gap > 0 { + select { + case <-ctx.Done(): + case <-time.After(l.gap): + } + } + } +} + +// after takes the outcome of a look: a failed one is looked at again later, a good one forgets the failures. +func (l *looker) after(failed bool) { + l.mu.Lock() + defer l.mu.Unlock() + if !failed { + l.backoff = 0 + if l.retry != nil { + l.retry.Stop() + l.retry = nil + } + return + } + if l.retry != nil { + return // one is armed already + } + switch { + case l.backoff == 0: + l.backoff = l.base + case l.backoff*2 > l.most: + l.backoff = l.most + default: + l.backoff *= 2 + } + // retries: an advance step that failed (store, bus or send). One at a time, five seconds doubling to a minute, forgotten by + // the first look that fails nothing; the step is otherwise started by an event, and a failure is none. + l.retry = time.AfterFunc(l.backoff, func() { + l.mu.Lock() + l.retry = nil + l.mu.Unlock() + l.ask() + }) +} + +// theLooker is the serving controller's; nil in a command, whose looks are its own. +var ( + lookerMu sync.RWMutex + theLooker *looker +) + +// askToLook has the serving controller look at the open plans soon: a plan was saved, a machine reported on a send, +// a silent key expired. Nothing in a process that is not serving. +func askToLook() { + lookerMu.RLock() + l := theLooker + lookerMu.RUnlock() + if l != nil { + l.ask() + } +} + +// startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a +// plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker. +func startPlanLooks(ctx context.Context, open *stores) { + l := newLooker(func(ctx context.Context) bool { return lookAtPlans(ctx, open) }) + l.first = func(ctx context.Context) { rearmWalkEnds(ctx, open) } + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + go func() { + <-ctx.Done() + lookerMu.Lock() + if theLooker == l { + theLooker = nil + } + lookerMu.Unlock() + }() + go l.run(ctx) + l.ask() +} + +// planLookPasses is the most times one look goes over the plans again because the first pass changed which are open. +const planLookPasses = 6 + +// lookAtPlans moves every open plan as far as the facts allow, as the ticker did each pass: what waits for a gate is +// released, the plans advanced, the wait for each machine's report kept as a key on the bus, and the phases of the +// walks that ended recorded. It says whether a step failed. Waits for the plans rather than giving up: an ask is +// answered once. +func lookAtPlans(ctx context.Context, open *stores) (failed bool) { + release, err := open.inventory.HoldPlans(ctx, true) + if err != nil { + fmt.Printf("plans: cannot hold them: %v\n", err) + return true + } + func() { + defer release() + // What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353). + if _, err := releaseBacklog(ctx, open, ""); err != nil { + fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err) + failed = true + } + // Again while the plans that are open changed: a plan that ended lets a deferred one before it start, and a + // batch cut makes one. + for pass := 0; pass < planLookPasses; pass++ { + before := openPlansNow(ctx, open.inventory) + if advanceHeld(ctx, open) { + failed = true + } + if openPlansNow(ctx, open.inventory) == before { + break + } + } + if err := syncSilentKeys(ctx, open); err != nil { + fmt.Printf("plans: the wait for the machines' reports could not be kept on the bus: %v\n", err) + } + }() + keepWalkPhases(ctx, open) + return failed +} + +// advancePlans is one pass over the open plans taking the plans without waiting for them, for a caller that wants a +// single step rather than the look (the tests of the steps). +func advancePlans(ctx context.Context, open *stores) { + release, err := open.inventory.HoldPlans(ctx, false) + if err != nil { + if !errors.Is(err, inventory.ErrPlansBusy) { + fmt.Printf("plans: cannot hold them: %v\n", err) + } + return + } + defer release() + if _, err := releaseBacklog(ctx, open, ""); err != nil { + fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err) + } + advanceHeld(ctx, open) +} + +// openPlansNow is which plans are open and in what state, for telling a pass that changed them from one that did not. +func openPlansNow(ctx context.Context, inv *inventory.Inventory) string { + plans, err := inv.OpenPlans(ctx) + if err != nil { + return "unreadable " + time.Now().String() + } + ids := make([]string, 0, len(plans)) + for _, p := range plans { + ids = append(ids, p.ID+":"+string(p.State)) + } + sort.Strings(ids) + return strings.Join(ids, ",") +} + +// --- the wait for a machine's report, as keys on the bus --- + +var ( + silentMu sync.RWMutex + silentKeyed silentClock // nil in a process that is not the serving controller +) + +// setSilentClock makes clock the rollout's; nil takes it away. +func setSilentClock(c silentClock) { + silentMu.Lock() + defer silentMu.Unlock() + silentKeyed = c +} + +func silentClockOf() silentClock { + silentMu.RLock() + defer silentMu.RUnlock() + return silentKeyed +} + +// silentWaitsOf is every machine an open plan waits on for a report, with when it was sent: the machines a walk sent +// that have not reported on the send and are not silent yet (walkReports), the machine a gate of a first machine +// judges, and the machines a release's gates wait on. Pure. +func silentWaitsOf(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[silentWait]time.Time { + waits := map[silentWait]time.Time{} + put := func(w silentWait, at time.Time) { + if at.IsZero() { + return + } + if was, ok := waits[w]; !ok || at.Before(was) { + waits[w] = at + } + } + if !p.Open() { + return waits + } + if r := p.Release; r != nil { + gates := []*inventory.PlanGate{r.Gate} + for _, g := range r.Gates { + gates = append(gates, g) + } + for _, g := range gates { + if g == nil || g.Verdict != "" || g.Silent || g.Pending || g.Since == nil { + continue + } + for _, node := range g.Machines { + put(silentWait{Plan: p.ID, Tier: 0, Node: node}, *g.Since) + } + } + return waits + } + if p.Waiting() { + return waits + } + tierOf := func(module string) int { + for i, tier := range p.Tiers { + if slices.Contains(tier, module) { + return i + } + } + return 0 + } + waiting, _, _ := walkReports(p, reports, now, prov) + for module, s := range p.Modules { + if s == nil { + continue + } + for _, node := range waiting { + sent, ok := s.Rest[node] + if !ok { + continue + } + at := sent.At + if at.IsZero() && s.SentAt != nil { + at = *s.SentAt + } + put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, at) + } + if g := s.Gate; g != nil && g.Verdict == "" && !g.Silent && g.Since != nil && s.FirstAt != nil { + for _, node := range g.Machines { + put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, *g.Since) + } + } + } + return waits +} + +// silentTTL is how long a wait's key lives: what is left of ApplySilentAfter from the send, and a second over, so the +// look it wakes finds the machine silent and not a fraction early. +func silentTTL(sent, now time.Time) time.Duration { + left := sent.Add(inventory.ApplySilentAfter).Sub(now) + time.Second + if left < time.Second { + return time.Second + } + return left +} + +// silentDiff is what the keys on the bus need to be as the waits are: the waits to put a key for (none yet, or the +// key is of an earlier send) and the keys to delete (the wait is over). Pure. +func silentDiff(want, held map[silentWait]time.Time) (arm, drop []silentWait) { + for w, sent := range want { + if was, ok := held[w]; !ok || !was.Equal(sent) { + arm = append(arm, w) + } + } + for w := range held { + if _, ok := want[w]; !ok { + drop = append(drop, w) + } + } + return arm, drop +} + +// syncSilentKeys keeps the keys on the bus as the open plans' waits are: a wait without a key gets one that lives out +// its time, a send made again gets its key again, and a key whose wait is over — the machine reported on the send, the +// gate has its verdict, the plan ended — is deleted. A report deletes the key by way of the look it asks for (the +// look alone knows the report was on that send). Only the lease holder looks, so only it writes. +func syncSilentKeys(ctx context.Context, open *stores) error { + c := silentClockOf() + inv := open.inventory + plans, err := inv.OpenPlans(ctx) + if err != nil { + return err + } + reports, err := inv.LastReports(ctx) + if err != nil { + return err + } + now := time.Now().UTC() + want := map[silentWait]time.Time{} + for _, p := range plans { + for w, at := range silentWaitsOf(p, reports, now, provLookup(ctx)) { + want[w] = at + } + } + if c == nil { + // No bucket to put them in: each wait is looked at by a timer of its own, once. + for w, sent := range want { + silentByTimer(w, sent, silentTTL(sent, now)) + } + return nil + } + held, err := c.Held(ctx) + if err != nil { + return err + } + arm, drop := silentDiff(want, held) + var first error + for _, w := range arm { + if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil { + silentByTimer(w, want[w], silentTTL(want[w], now)) + if first == nil { + first = err + } + } + } + for _, w := range drop { + if err := c.Reported(ctx, w); err != nil && first == nil { + first = err + } + } + return first +} + +var silentTimers = struct { + mu sync.Mutex + timers map[silentWait]time.Time +}{timers: map[silentWait]time.Time{}} + +// silentByTimer looks at the plans when a wait's time is up, for a process whose bus will not take the key +// (DegradableGrants says why that can be): never silently never. Once for each send. +func silentByTimer(w silentWait, sent time.Time, ttl time.Duration) { + silentTimers.mu.Lock() + defer silentTimers.mu.Unlock() + if at, ok := silentTimers.timers[w]; ok && at.Equal(sent) { + return + } + silentTimers.timers[w] = sent + // retries: a rollout's wait for a machine, where the bus would not take its key. One per send, once. + time.AfterFunc(ttl, func() { + silentTimers.mu.Lock() + if silentTimers.timers[w].Equal(sent) { + delete(silentTimers.timers, w) + } + silentTimers.mu.Unlock() + askToLook() + }) +} + +// startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the +// controller is the lease holder here, the only one that writes it. The watcher asks for a look when a key expires. +func startRolloutSilent(ctx context.Context, js *broker.JetStream) error { + api, err := jetstream.New(js.Conn()) + if err != nil { + return err + } + clock, err := newBusSilent(ctx, api, func(silentWait) { askToLook() }) + if err != nil { + return err + } + setSilentClock(clock) + go func() { + <-ctx.Done() + setSilentClock(nil) + }() + return nil +} + +// --- the end of a walk whose end is not known --- + +var walkEnds = struct { + mu sync.Mutex + timers map[string]*time.Timer +}{timers: map[string]*time.Timer{}} + +// armWalkEnd has the walk recorded once its end can be known: a walk that ended with a machine silent is recorded +// without its total ApplySilentAfter and a minute after its last save (recordWalkPhases), which no event says. Armed at +// each save of the done walk, so it counts from the last. +func armWalkEnd(id string, savedAt time.Time) { + d := time.Until(savedAt.Add(inventory.ApplySilentAfter + time.Minute + time.Second)) + if d <= 0 { + return + } + walkEnds.mu.Lock() + defer walkEnds.mu.Unlock() + if t := walkEnds.timers[id]; t != nil { + t.Stop() + } + // retries: recording the end of a walk whose end is not known. One per walk, once, from its last save; the first + // look after the lease was taken arms it again for the walks still unrecorded. + walkEnds.timers[id] = time.AfterFunc(d, func() { + walkEnds.mu.Lock() + delete(walkEnds.timers, id) + walkEnds.mu.Unlock() + askToLook() + }) +} + +// rearmWalkEnds is armWalkEnd for the walks that ended lately, for a controller that has just taken the lease. +func rearmWalkEnds(ctx context.Context, open *stores) { + recent, err := open.inventory.RecentPlans(ctx, 30) + if err != nil { + fmt.Printf("plans: the walks ended lately could not be read to record their phases: %v\n", err) + return + } + for _, p := range recent { + if p.State == inventory.PlanDone && p.Release == nil && time.Since(p.Updated) <= 2*time.Hour { + armWalkEnd(p.ID, p.Updated) + } + } +} + +// planKept is what the serving controller does when a plan was saved: a look is asked for, so a send, a built tier or +// a command's move is followed by the plan's next step; and a walk that ended is armed to be recorded. +func planKept(p inventory.Plan) { + if p.State == inventory.PlanDone && p.Release == nil { + armWalkEnd(p.ID, time.Now()) + } + askToLook() +} + +// --- `sent`: a tier sent to a machine, said --- + +// sendMade is one machine a plan's tier was sent to. +type sendMade struct { + Tier int + Node string + Modules []string + At time.Time +} + +// sendsMade collects the sends one step of a plan made, to be said once the plan is saved. +type sendsMade struct { + mu sync.Mutex + list []sendMade +} + +type sendsMadeKey struct{} + +func withSendsMade(ctx context.Context, s *sendsMade) context.Context { + return context.WithValue(ctx, sendsMadeKey{}, s) +} + +// noteSends records that modules of the plan's tier were sent to the nodes. +func noteSends(ctx context.Context, tier int, nodes, modules []string, at time.Time) { + s, _ := ctx.Value(sendsMadeKey{}).(*sendsMade) + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + for _, n := range nodes { + s.list = append(s.list, sendMade{Tier: tier, Node: n, Modules: slices.Clone(modules), At: at}) + } +} + +// planSentBody is the body of the seat event `sent` (novox/hq ADR 0406 §5, issue 974): the plan, the tier, the machine +// and the modules it was sent, and when. +type planSentBody struct { + Event string `json:"event"` + Plan string `json:"plan"` + Tier int `json:"tier"` + Node string `json:"node"` + Modules []string `json:"modules"` + At time.Time `json:"at"` +} + +// sentBodies are the events a step's sends are said as. +func sentBodies(plan string, sends *sendsMade) []planSentBody { + sends.mu.Lock() + defer sends.mu.Unlock() + out := make([]planSentBody, 0, len(sends.list)) + for _, s := range sends.list { + out = append(out, planSentBody{Event: link.KeySent, Plan: plan, Tier: s.Tier, Node: s.Node, + Modules: s.Modules, At: s.At}) + } + return out +} + +// saySends says each send a step made as `sent`, once the plan that records it is saved. What an outsider hears; the +// controller's own next step does not wait for it. +func saySends(ctx context.Context, p inventory.Plan, sends *sendsMade) { + bus := checkEvents + if bus == nil { + return + } + for _, body := range sentBodies(p.ID, sends) { + raw, err := json.Marshal(body) + if err != nil { + continue + } + go func() { + stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer stop() + if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySent, raw); err != nil { + fmt.Printf("%s: tier %d was sent to %s, and could not be said as sent: %v\n", body.Plan, body.Tier, body.Node, err) + } + }() + } +} diff --git a/cmd/mesh-controller/plan_looks_test.go b/cmd/mesh-controller/plan_looks_test.go new file mode 100644 index 00000000..fb3dc659 --- /dev/null +++ b/cmd/mesh-controller/plan_looks_test.go @@ -0,0 +1,259 @@ +package main + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/testbus" +) + +func rolloutOf(sentAgo time.Duration, reported bool) (inventory.Plan, []inventory.Reported) { + sent := time.Now().Add(-sentAgo).UTC() + p := inventory.Plan{ID: "plan-9", State: inventory.PlanRolling, Tiers: [][]string{{"a", "b"}}, Tier: 0, + Modules: map[string]*inventory.PlanModule{ + "a": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}}, + "b": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}}, + }} + r := inventory.Reported{Node: "ace", Sent: &sent} + if reported { + at := sent.Add(time.Minute) + r.At, r.Current, r.Declared, r.Outcome = &at, true, "d1", inventory.OutcomeApplied + } + return p, []inventory.Reported{r} +} + +// A machine a plan sent and heard nothing from is a wait with a key, from when it was sent (issue 974, novox/hq ADR +// 0406 §5); one that reported on the send, or that is silent already, is none. +func TestAPlanWaitsOnTheMachinesItSentAndHeardNothingFrom(t *testing.T) { + now := time.Now().UTC() + p, reports := rolloutOf(5*time.Minute, false) + waits := silentWaitsOf(p, reports, now, nil) + if len(waits) != 1 { + t.Fatalf("waits = %v; want the one machine", waits) + } + for w, at := range waits { + if w.Plan != "plan-9" || w.Node != "ace" || w.Tier != 0 || now.Sub(at) < 4*time.Minute || now.Sub(at) > 6*time.Minute { + t.Errorf("wait %+v sent at %v is wrong", w, at) + } + } + p, reports = rolloutOf(5*time.Minute, true) + if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 { + t.Errorf("a machine that reported on its send is waited on: %v", waits) + } + p, reports = rolloutOf(inventory.ApplySilentAfter+time.Minute, false) + if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 { + t.Errorf("a machine silent past its time is waited on again: %v", waits) + } + p, reports = rolloutOf(time.Minute, false) + p.State = inventory.PlanDone + if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 { + t.Errorf("a plan that ended is waiting: %v", waits) + } +} + +// A release's open gate waits on its machines from when it began judging, and its verdict ends the wait. +func TestAReleasesOpenGateWaitsOnItsMachines(t *testing.T) { + since := time.Now().Add(-time.Minute).UTC() + g := &inventory.PlanGate{Machines: []string{"g14", "novox"}, Since: &since} + p := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: g}} + waits := silentWaitsOf(p, nil, time.Now(), nil) + if len(waits) != 2 { + t.Fatalf("waits = %v; want g14 and novox", waits) + } + g.Verdict = inventory.GatePassed + if waits := silentWaitsOf(p, nil, time.Now(), nil); len(waits) != 0 { + t.Errorf("a gate with its verdict still waits: %v", waits) + } +} + +// The keys follow the waits: a wait with no key, or a key of an earlier send, is armed; a key with no wait is deleted. +func TestTheKeysFollowTheWaits(t *testing.T) { + t1 := time.Now().UTC() + t2 := t1.Add(time.Minute) + a, b, c := silentWait{"p", 0, "a"}, silentWait{"p", 0, "b"}, silentWait{"p", 0, "c"} + arm, drop := silentDiff(map[silentWait]time.Time{a: t1, b: t2, c: t1}, map[silentWait]time.Time{a: t1, b: t1, silentWait{"p", 1, "a"}: t1}) + if len(arm) != 2 || !(arm[0] == b || arm[1] == b) || !(arm[0] == c || arm[1] == c) { + t.Errorf("armed %v; want b (sent again) and c (no key)", arm) + } + if len(drop) != 1 || drop[0] != (silentWait{"p", 1, "a"}) { + t.Errorf("dropped %v; want the key with no wait", drop) + } + if ttl := silentTTL(t1, t1.Add(inventory.ApplySilentAfter+time.Hour)); ttl != time.Second { + t.Errorf("a wait long over has a key of %v; want the least, one second", ttl) + } +} + +func lookerCounting(failures *atomic.Int32, looks *atomic.Int32, block chan struct{}) *looker { + l := newLooker(func(ctx context.Context) bool { + looks.Add(1) + if block != nil { + <-block + } + return failures.Add(-1) >= 0 + }) + l.gap, l.base, l.most = 0, 20*time.Millisecond, 80*time.Millisecond + return l +} + +func eventually(t *testing.T, what string, ok func() bool) { + t.Helper() + for end := time.Now().Add(3 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) { + if ok() { + return + } + } + t.Fatalf("never: %s", what) +} + +// Asks made while a look runs are not lost and are one look after it, not one each. +func TestAsksMadeWhileALookRunsAreOneLookAfterIt(t *testing.T) { + var failures, looks atomic.Int32 + block := make(chan struct{}) + l := lookerCounting(&failures, &looks, block) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + l.ask() + eventually(t, "the first look starts", func() bool { return looks.Load() == 1 }) + for i := 0; i < 20; i++ { + l.ask() + } + block <- struct{}{} // the first look ends + eventually(t, "the asks made meanwhile are looked at", func() bool { return looks.Load() == 2 }) + block <- struct{}{} + time.Sleep(60 * time.Millisecond) + if n := looks.Load(); n != 2 { + t.Errorf("%d looks for 21 asks; want 2", n) + } +} + +// A look that failed a step is looked at again, one at a time with a longer wait each time, and a look that fails +// nothing ends the retries (novox/hq ADR 0406 §5; the one named retry). +func TestAFailedLookIsLookedAtAgainAndAGoodOneEndsTheRetries(t *testing.T) { + var failures, looks atomic.Int32 + failures.Store(3) // three looks fail, then one is good + l := lookerCounting(&failures, &looks, nil) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + l.ask() + eventually(t, "four looks", func() bool { return looks.Load() == 4 }) + time.Sleep(200 * time.Millisecond) + if n := looks.Load(); n != 4 { + t.Errorf("%d looks; the good one should have ended the retries at 4", n) + } + l.mu.Lock() + armed, backoff := l.retry != nil, l.backoff + l.mu.Unlock() + if armed || backoff != 0 { + t.Errorf("a retry is armed (%v) or the wait kept (%v) after a good look", armed, backoff) + } +} + +func TestTheRetryWaitDoublesToItsMost(t *testing.T) { + l := newLooker(func(context.Context) bool { return true }) + l.base, l.most = 5*time.Second, time.Minute + var seen []time.Duration + for i := 0; i < 6; i++ { + l.after(true) + seen = append(seen, l.backoff) + l.retry.Stop() + l.retry = nil + } + want := []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, time.Minute, time.Minute} + for i := range want { + if seen[i] != want[i] { + t.Fatalf("waits %v; want %v", seen, want) + } + } +} + +// A plan saved asks for a look, and a walk saved done is armed to be recorded once its end can be known. +func TestAPlanSavedAsksForALookAndADoneWalkIsArmedToBeRecorded(t *testing.T) { + var failures, looks atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + + planKept(inventory.Plan{ID: "walk-1", State: inventory.PlanDone}) + t.Cleanup(func() { + walkEnds.mu.Lock() + defer walkEnds.mu.Unlock() + if t := walkEnds.timers["walk-1"]; t != nil { + t.Stop() + } + delete(walkEnds.timers, "walk-1") + }) + eventually(t, "a look", func() bool { return looks.Load() == 1 }) + walkEnds.mu.Lock() + _, armed := walkEnds.timers["walk-1"] + walkEnds.mu.Unlock() + if !armed { + t.Error("a walk saved done is not armed to be recorded") + } + askToLook() // no panic, no looker, once it is taken away +} + +// The tier-sends a step made are said once each, per machine, with the modules it was sent. +func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) { + sends := &sendsMade{} + ctx := withSendsMade(context.Background(), sends) + at := time.Now().UTC() + noteSends(ctx, 2, []string{"ace"}, []string{"a", "b"}, at) + noteSends(ctx, 2, []string{"g14"}, []string{"a"}, at) + noteSends(context.Background(), 2, []string{"nobody"}, []string{"a"}, at) // outside a step: nothing kept + got := sentBodies("plan-9", sends) + if len(got) != 2 || got[0].Node != "ace" || len(got[0].Modules) != 2 || got[0].Tier != 2 || got[0].Plan != "plan-9" || + got[0].Event != "sent" || !got[0].At.Equal(at) || got[1].Node != "g14" { + t.Errorf("sent bodies = %+v", got) + } +} + +// A silent key that expires asks for a look; a key a report deleted does not (issue 974, novox/hq ADR 0406 §5). +func TestAnExpiredSilentKeyAsksForALookAndAReportsDeleteDoesNot(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + var failures, looks atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock(); setSilentClock(nil) }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + if err := startRolloutSilent(ctx, js); err != nil { + t.Fatal(err) + } + c := silentClockOf() + w := silentWait{Plan: "plan-5", Tier: 0, Node: "ace"} + if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { + t.Fatal(err) + } + if err := c.Reported(ctx, w); err != nil { + t.Fatal(err) + } + time.Sleep(1800 * time.Millisecond) + if n := looks.Load(); n != 0 { + t.Fatalf("a key a report deleted asked for %d looks", n) + } + if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { + t.Fatal(err) + } + eventually(t, "the expiry asks for a look", func() bool { return looks.Load() == 1 }) +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index d1953be8..af6d40e1 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -180,9 +180,15 @@ func serve(ctx context.Context) (err error) { // And build results nobody was waiting for. A build triggered any other way than `build` // would otherwise be reported into the void, which is the same as not reporting it. server.Records(builds{inv, open}) - // Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for - // machines to report moves when they have, and a plan left by a replaced controller resumes. - go planTicker(ctx, open) + // Open plans move on events (novox/hq ADR 0406 §5, issue 974): a tier is asked on the `built` and the send of the + // tier before it, a walk ends on its machines' reports, a silent machine is a key that expired, and a plan left + // by a replaced controller resumes from the first look, which this asks for. No timer looks every so often. + // The wait for each machine's report is a key on the bus that expires; where the bus will not take it a timer of its + // own for that wait looks instead. + if err := startRolloutSilent(ctx, server.JetStream()); err != nil { + fmt.Printf("plans: the wait for the machines' reports could not be put on the bus (%v); each is looked at by a timer of its own\n", err) + } + startPlanLooks(ctx, open) // And the merge window (novox/hq ADR 0276, ADR 0407): a batch is cut into its walk when the window closes, // which no merge or outcome says. The window is two keys on the bus that expire and a watcher that hears it; // nothing polls (issue 973). @@ -254,6 +260,9 @@ func serve(ctx context.Context) (err error) { // And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239). inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { sayPlanMoved(ctx, bus, open.inventory, p) + // And the plan's next step is looked at (ADR 0406 §5): a build, a send kept or a command's move is followed + // by the look that asks the next tier, in place of the timer that did. + planKept(p) // A walk ended has taken what it reuses: its merges' check builds go (novox/hq ADR 0350 §5). if p.WalkSaid == inventory.WalkEnded { go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p) diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 20f15956..64e0fa8d 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -24,9 +24,9 @@ import ( // // The handler that hears the merge computes the plan from the catalogue's one dependency relation, // writes it to the store, asks the first tier and returns — the receive loop is never held by a -// build. Every outcome taken in advances the plan it belongs to; a ticker advances what outcomes -// alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the -// plan where it left it. +// build. Every outcome taken in advances the plan it belongs to; a look at the plans, asked for by each event that can +// move one (plan_looks.go, ADR 0406 §5), advances what outcomes alone cannot (a tier waiting for machines to report); +// a controller replaced mid-plan finds the plan where it left it and looks first. // gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since // novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on. @@ -668,29 +668,6 @@ func askedAny(p inventory.Plan) bool { return false } -// advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built -// and whose gates are applied gives way to the next; the last tier done is the plan done. Called -// after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes. -// -// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two -// controllers — the old and the new while a machine hands its controller over — would each ask a -// tier the other had just asked. Taken without waiting: whoever holds the plans is moving them. -func advancePlans(ctx context.Context, open *stores) { - release, err := open.inventory.HoldPlans(ctx, false) - if err != nil { - if !errors.Is(err, inventory.ErrPlansBusy) { - fmt.Printf("plans: cannot hold them: %v\n", err) - } - return - } - defer release() - // What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353). - if _, err := releaseBacklog(ctx, open, ""); err != nil { - fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err) - } - advanceHeld(ctx, open) -} - // keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold // on the plans so it never lengthens it: measured, never acted on, and an error only said. func keepWalkPhases(ctx context.Context, open *stores) { @@ -699,8 +676,13 @@ func keepWalkPhases(ctx context.Context, open *stores) { } } -// advanceHeld is advancePlans for a caller already holding the plans. -func advanceHeld(ctx context.Context, open *stores) { +// advanceHeld moves every open plan as far as the facts allow, for a caller holding the plans: a tier whose modules +// are all built and whose gates are applied gives way to the next; the last tier done is the plan done. It says +// whether a step failed, which the look at the plans (lookAtPlans) retries. +// +// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two controllers — the +// old and the new while a machine hands its controller over — would each ask a tier the other had just asked. +func advanceHeld(ctx context.Context, open *stores) (failed bool) { inv := open.inventory // A walk that ended lets the next batch be cut at once, not at a window's next expiry (novox/hq ADR 0276, ADR 0407). defer func() { @@ -711,15 +693,15 @@ func advanceHeld(ctx context.Context, open *stores) { plans, err := inv.OpenPlans(ctx) if err != nil { fmt.Printf("plans: cannot read them: %v\n", err) - return + return true } if len(plans) == 0 { - return + return false } edges, err := inv.Dependencies(ctx) if err != nil { fmt.Printf("plans: cannot read the dependencies: %v\n", err) - return + return true } rollsOut := func(module string) bool { u, err := inv.UpgradeOf(ctx, module) @@ -733,10 +715,12 @@ func advanceHeld(ctx context.Context, open *stores) { // stood still for half an hour was written every few seconds, its revision moved, `plan-moved` // said on the bus, and its last save read as when it began building. before := planSnapshot(*p) - moved, err := advanceOnce(ctx, open, p, edges, rollsOut) + sends := &sendsMade{} + moved, err := advanceOnce(withSendsMade(ctx, sends), open, p, edges, rollsOut) if err != nil { + failed = true // Kept in the plan, so `plans` says why it has not moved rather than the log alone; - // the state is left as it was and the step is tried again on the next tick. Said and + // the state is left as it was and the step is tried again by the next look. Said and // kept when it is new: the same refusal on every tick is one fact, not one per tick. p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again" // A refusal for the bus's planned step is a person's to end: S17 says it from its first moment. @@ -757,14 +741,18 @@ func advanceHeld(ctx context.Context, open *stores) { if planSnapshot(*p) != before { if err := inv.SavePlan(ctx, p); err != nil { fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err) + failed = true break } + // Said once the send is kept (ADR 0406 §5, issue 974): what an outsider hears of a tier sent. + saySends(ctx, *p, sends) } if !moved { break } } } + return failed } // planSnapshot is a plan as a save would write it, to compare before and after a step: what the store @@ -1066,6 +1054,16 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, p.Modules[m].SentAt = &now p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat) } + // One `sent` for each machine, with what it was sent (ADR 0406 §5, issue 974). + for _, n := range sent { + var carried []string + for _, m := range rest { + if slices.Contains(restTo[m], n) { + carried = append(carried, m) + } + } + noteSends(ctx, p.Tier, []string{n}, carried, now) + } fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "), strings.Join(sent, ", ")) return true, nil @@ -1338,6 +1336,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string s.GatedBy = lead } } + noteSends(ctx, p.Tier, sent, modules, now) p.State = inventory.PlanRolling p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "), strings.Join(sent, ", ")) @@ -1652,23 +1651,6 @@ func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) { p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `push --behind` sends them" } -// planTicker advances open plans on a timer, for the steps outcomes alone cannot take. -func planTicker(ctx context.Context, open *stores) { - advancePlans(ctx, open) - keepWalkPhases(ctx, open) - tick := time.NewTicker(30 * time.Second) - defer tick.Stop() - for { - select { - case <-ctx.Done(): - return - case <-tick.C: - advancePlans(ctx, open) - keepWalkPhases(ctx, open) - } - } -} - // planLine is one plan as `status` says it, given the least bound of a tier. func planLine(p inventory.Plan, now time.Time) string { return planLineWith(p, now, pauseView{}, tierAtLeast) diff --git a/cmd/mesh-controller/rollout_silent_bus.go b/cmd/mesh-controller/rollout_silent_bus.go index ca21ce66..28a46b9b 100644 --- a/cmd/mesh-controller/rollout_silent_bus.go +++ b/cmd/mesh-controller/rollout_silent_bus.go @@ -49,8 +49,10 @@ func parseSilentKey(key string) (silentWait, bool) { // silentClock is the bus's side of the wait for a machine's report. type silentClock interface { - // Arm puts a wait's key, to live ttl: made again if the machine is sent again. - Arm(ctx context.Context, w silentWait, ttl time.Duration) error + // Arm puts a wait's key, to live ttl: made again if the machine is sent again. The key holds when the send was. + Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error + // Held is every wait that has a key now, with when its send was. + Held(ctx context.Context) (map[silentWait]time.Time, error) // Reported takes a wait's key away: the machine reported on the send, and nothing is to wake the controller. Reported(ctx context.Context, w silentWait) error } @@ -94,13 +96,13 @@ func newBusSilent(ctx context.Context, api jetstream.JetStream, silent func(sile return &busSilent{kv: kv}, nil } -func (s *busSilent) Arm(ctx context.Context, w silentWait, ttl time.Duration) error { +func (s *busSilent) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error { key := silentKey(w) // Made again, not changed: a key's time to live is set when it is made. if err := s.kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { return err } - _, err := s.kv.Create(ctx, key, []byte(time.Now().UTC().Format(time.RFC3339)), jetstream.KeyTTL(leastSecond(ttl))) + _, err := s.kv.Create(ctx, key, []byte(sent.UTC().Format(time.RFC3339Nano)), jetstream.KeyTTL(leastSecond(ttl))) return err } @@ -119,3 +121,30 @@ func leastSecond(d time.Duration) time.Duration { } return (d + time.Second - 1) / time.Second * time.Second } + +func (s *busSilent) Held(ctx context.Context) (map[silentWait]time.Time, error) { + keys, err := s.kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + if err != nil { + return nil, err + } + held := map[silentWait]time.Time{} + for _, key := range keys { + w, ok := parseSilentKey(key) + if !ok { + continue + } + e, err := s.kv.Get(ctx, key) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + continue + } + return nil, err + } + at, _ := time.Parse(time.RFC3339Nano, string(e.Value())) + held[w] = at + } + return held, nil +} diff --git a/cmd/mesh-controller/rollout_silent_bus_test.go b/cmd/mesh-controller/rollout_silent_bus_test.go index b9cc46f0..fef01437 100644 --- a/cmd/mesh-controller/rollout_silent_bus_test.go +++ b/cmd/mesh-controller/rollout_silent_bus_test.go @@ -57,7 +57,7 @@ func hearsSilent(t *testing.T, heard <-chan silentWait, within time.Duration) (s func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { s, heard := silentOnABus(t) want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"} - if err := s.Arm(t.Context(), want, time.Second); err != nil { + if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil { t.Fatal(err) } if got, ok := hearsSilent(t, heard, 10*time.Second); !ok || got != want { @@ -70,7 +70,7 @@ func TestAReportDeletesTheKeyAndNoOneIsNamedSilent(t *testing.T) { s, heard := silentOnABus(t) ctx := t.Context() w := silentWait{Plan: "plan-1", Tier: 0, Node: "g14"} - if err := s.Arm(ctx, w, time.Second); err != nil { + if err := s.Arm(ctx, w, time.Second, time.Now()); err != nil { t.Fatal(err) } if err := s.Reported(ctx, w); err != nil { @@ -90,11 +90,11 @@ func TestASendMadeAgainPutsTheKeyAgain(t *testing.T) { s, heard := silentOnABus(t) ctx := t.Context() w := silentWait{Plan: "plan-3", Tier: 1, Node: "novox"} - if err := s.Arm(ctx, w, 2*time.Second); err != nil { + if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { t.Fatal(err) } time.Sleep(1200 * time.Millisecond) - if err := s.Arm(ctx, w, 2*time.Second); err != nil { + if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { t.Fatal(err) } if got, ok := hearsSilent(t, heard, 1500*time.Millisecond); ok { diff --git a/cmd/mesh-controller/status_summary.go b/cmd/mesh-controller/status_summary.go index c52f3c41..790ae212 100644 --- a/cmd/mesh-controller/status_summary.go +++ b/cmd/mesh-controller/status_summary.go @@ -210,6 +210,11 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e if news { l.summary.nudge() } + // A machine's account of an apply is looked at at once (novox/hq ADR 0406 §5, issue 974): it ends the wait for the + // machine, and the look is what deletes its key, knowing the report was on that send. + if err == nil && report.Declared != "" && report.Superseded == "" && report.Rekey == nil { + askToLook() + } // What it says of its long-running resources (novox/hq ADR 0240): kept, and its modules' conditions // raised or cleared from it. Only from an account of the machine the store took. if err == nil && report.Health != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" && diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 396a353c..bb3f82c1 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -378,10 +378,15 @@ var DegradableGrants = []DegradableGrant{ Reason: "refused: the controller closes a batch's window by a timer of its own for that batch, as the " + "window setting says (ADR 0407 decision 1, issue 973)", }, + { + Subject: "mesh.seat." + ControllerSeat + ".event.sent", + Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " + + "be said is printed and goes on (ADR 0406 §5, issue 974)", + }, { Subject: "$KV." + RolloutSilentBucket + ".>", - Reason: "refused: the controller advances a rollout on the machines' reports alone, and a machine never heard " + - "from is found when the plan is next looked at (ADR 0406 §5, issue 974)", + Reason: "refused: the controller looks at the plans when a wait for a machine's report is up, by a timer of its own " + + "for that wait (ADR 0406 §5, issue 974)", }, } diff --git a/internal/broker/states_agreement_test.go b/internal/broker/states_agreement_test.go index c47b15ff..955d4325 100644 --- a/internal/broker/states_agreement_test.go +++ b/internal/broker/states_agreement_test.go @@ -34,6 +34,8 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) { states = append(states, link.KeyChecked) // And a walk kept in a new state, for the delivery it walks (novox/hq ADR 0239). states = append(states, link.KeyPlanMoved) + // And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974). + states = append(states, link.KeySent) // And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3). states = append(states, link.KeyAssignmentChanged, link.KeyDeclaredConfigChanged, link.KeyPredefinedChanged) for _, event := range states { diff --git a/internal/broker/streams.go b/internal/broker/streams.go index f3384f70..70b76520 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -368,6 +368,8 @@ var ControllerStates = []string{"applied", "refused", "built-before", "checked", // And a walk kept in a new state (novox/hq ADR 0239): what the delivery it walks reads its steps from. "plan-moved", + // And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974). + "sent", // And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and // each registered module's declared config (design 57 F3). "assignment-changed", "declared-config-changed", "predefined-changed"} diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 6178d798..d76e8659 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index 584e7826..31269dfe 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -108,6 +108,8 @@ var defaultSeats = append([]Seat{ "checked", // A walk kept in a new state, for the delivery it walks (novox/hq ADR 0239). "plan-moved", + // A tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974). + "sent", // A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3). "assignment-changed", "declared-config-changed", "predefined-changed"}, Serves: ControllerVerbs}, diff --git a/internal/link/events.go b/internal/link/events.go index 39081146..5d6abee6 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -76,6 +76,10 @@ const ( // KeyChecked: a pull request's merge check was judged (novox/hq to-be 45 §9) — the verdict, its // summary and its report, for the forge's holder to set as the pull request's status. KeyChecked = "checked" + // KeySent: a plan's tier was sent to a machine, and the send kept (novox/hq ADR 0406 §5, issue 974): the plan, the tier, + // the machine, the modules and when. What an outsider hears of a rollout; the controller's own next step does not + // wait for it. + KeySent = "sent" // KeyPlanMoved: a walk — the controller's plan of one trunk commit, tier by tier across the machines — // was kept in a new state (novox/hq ADR 0239): the plan whole, for the delivery it walks to read its // steps from. Said after every save of the serving controller; a delivery's holder also asks for the From 3d192a4857f33d99db47b541403b3af80edb6a15 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 13:30:27 +0200 Subject: [PATCH 03/12] issue 974: every plan is written through the one hook that says plan-moved, proven by reading the sources and by the moves a merge and a cut make --- .../plan_moved_every_writer_test.go | 140 ++++++++++++++++++ testdata/store-tests | 1 + 2 files changed, 141 insertions(+) create mode 100644 cmd/mesh-controller/plan_moved_every_writer_test.go diff --git a/cmd/mesh-controller/plan_moved_every_writer_test.go b/cmd/mesh-controller/plan_moved_every_writer_test.go new file mode 100644 index 00000000..4be2f8aa --- /dev/null +++ b/cmd/mesh-controller/plan_moved_every_writer_test.go @@ -0,0 +1,140 @@ +package main + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan +// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there +// after the commit; so no statement that writes the plan table may stand anywhere else, and writePlan, the one that +// does, may be called from nowhere but SavePlansMoving. Checked by reading the sources, no store needed. +func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) { + write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`) + var found []string + for _, root := range []string{"../../cmd", "../../internal"} { + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + text := string(raw) + if write.MatchString(text) && filepath.ToSlash(path) != "../../internal/inventory/plans.go" { + found = append(found, path+" writes the plan table") + } + if strings.Contains(text, "writePlan(") && filepath.ToSlash(path) != "../../internal/inventory/plans.go" { + found = append(found, path+" calls writePlan") + } + return nil + }) + if err != nil { + t.Fatal(err) + } + } + if len(found) > 0 { + t.Fatalf("a plan is written without plan-moved being said:\n%s", strings.Join(found, "\n")) + } + raw, err := os.ReadFile("../../internal/inventory/plans.go") + if err != nil { + t.Fatal(err) + } + text := string(raw) + if n := len(write.FindAllString(text, -1)); n != 1 { + t.Errorf("plans.go has %d statements that write the plan table; want the one in writePlan", n) + } + calls := regexp.MustCompile(`writePlan\(ctx, tx`).FindAllStringIndex(text, -1) + save := strings.Index(text, "func (i *Inventory) SavePlansMoving") + next := strings.Index(text[save:], "\nfunc ") + if len(calls) != 1 || calls[0][0] < save || calls[0][0] > save+next { + t.Errorf("writePlan is called %d times, and not only from SavePlansMoving", len(calls)) + } + if !strings.Contains(text[save:save+next], "PlanSaved(i, *p)") { + t.Error("SavePlansMoving does not say each plan it wrote") + } +} + +// movesSaid records what the hook hears: for each plan, the revisions it was said at. +type movesSaid struct { + mu sync.Mutex + revs map[string][]int64 +} + +func (s *movesSaid) hear(_ *inventory.Inventory, p inventory.Plan) { + s.mu.Lock() + defer s.mu.Unlock() + s.revs[p.ID] = append(s.revs[p.ID], p.Revision) +} + +// sayingEverySave installs a PlanSaved that records, as the serving controller's says plan-moved. +func sayingEverySave(t *testing.T) *movesSaid { + t.Helper() + s := &movesSaid{revs: map[string][]int64{}} + was := inventory.PlanSaved + inventory.PlanSaved = s.hear + t.Cleanup(func() { inventory.PlanSaved = was }) + return s +} + +// noUnsaid fails the test for a plan whose stored revision is not one the hook heard: a move nobody said. +func (s *movesSaid) noUnsaid(t *testing.T, open *stores) { + t.Helper() + recent, err := open.inventory.RecentPlans(t.Context(), 50) + if err != nil { + t.Fatal(err) + } + s.mu.Lock() + defer s.mu.Unlock() + if len(recent) == 0 { + t.Fatal("no plan was kept") + } + for _, p := range recent { + revs := s.revs[p.ID] + if len(revs) == 0 || revs[len(revs)-1] != p.Revision { + t.Errorf("%s is kept at revision %d and was said at %v: a move nobody said", p.ID, p.Revision, revs) + } + } +} + +// Live, with the store: the moves a merge makes of a plan — the batch it opens, the walk it is cut into, the tier that +// is asked — are each said, and a look that changes nothing says nothing (novox/hq ADR 0406 §2). +func TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksWithPaths(t) + heard := sayingEverySave(t) + hear(t, open, catalogueMerge("553b7191claude", "app", t0), t0.Add(time.Second)) + if _, bs := walks(t, open); len(bs) != 1 { + t.Fatalf("%d batch(es) opened", len(bs)) + } + heard.noUnsaid(t, open) + cutAt(t, open, t0.Add(91*time.Second)) + if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { + t.Fatalf("%d walk(s), %d batch(es) after the cut", len(ws), len(bs)) + } + heard.noUnsaid(t, open) + counted := 0 + for _, r := range heard.revs { + counted += len(r) + } + cutAt(t, open, t0.Add(92*time.Second)) + again := 0 + for _, r := range heard.revs { + again += len(r) + } + if again != counted { + t.Errorf("a look that changed nothing said %d more move(s)", again-counted) + } +} diff --git a/testdata/store-tests b/testdata/store-tests index d2d9f485..3d3dff3a 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -133,3 +133,4 @@ ./cmd/mesh-controller TestAnOwnPathBatchNeverWalksBeside ./cmd/mesh-controller TestMergesHeardWhileAnotherHolderHasThePlansAreRecordedAndCarried ./cmd/mesh-controller TestABatchWindowClosesByTheExpiryOfItsKey +./cmd/mesh-controller TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid From 20c8d9454a7fa52163448e8788049dd95b4c2245 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 13:33:49 +0200 Subject: [PATCH 04/12] issue 974: the step's acceptance test: a rollout advances on events with no ticker, a silent machine is named when its key expires, a replaced controller resumes --- cmd/mesh-controller/plan_looks.go | 12 +- cmd/mesh-controller/plan_looks_test.go | 2 +- cmd/mesh-controller/release.go | 4 +- cmd/mesh-controller/release_plan.go | 4 +- cmd/mesh-controller/rollout_events_test.go | 281 +++++++++++++++++++++ testdata/store-tests | 1 + 6 files changed, 297 insertions(+), 7 deletions(-) create mode 100644 cmd/mesh-controller/rollout_events_test.go diff --git a/cmd/mesh-controller/plan_looks.go b/cmd/mesh-controller/plan_looks.go index 10ad6832..6ae5db79 100644 --- a/cmd/mesh-controller/plan_looks.go +++ b/cmd/mesh-controller/plan_looks.go @@ -156,6 +156,10 @@ func startPlanLooks(ctx context.Context, open *stores) { l.ask() } +// applySilentAfter is how long a machine may say nothing on a send before the plan names it silent: the mesh's +// ApplySilentAfter, a variable so a test of the wait needs no quarter of an hour. +var applySilentAfter = inventory.ApplySilentAfter + // planLookPasses is the most times one look goes over the plans again because the first pass changed which are open. const planLookPasses = 6 @@ -316,7 +320,7 @@ func silentWaitsOf(p inventory.Plan, reports []inventory.Reported, now time.Time // silentTTL is how long a wait's key lives: what is left of ApplySilentAfter from the send, and a second over, so the // look it wakes finds the machine silent and not a fraction early. func silentTTL(sent, now time.Time) time.Duration { - left := sent.Add(inventory.ApplySilentAfter).Sub(now) + time.Second + left := sent.Add(applySilentAfter).Sub(now) + time.Second if left < time.Second { return time.Second } @@ -429,7 +433,11 @@ func startRolloutSilent(ctx context.Context, js *broker.JetStream) error { setSilentClock(clock) go func() { <-ctx.Done() - setSilentClock(nil) + silentMu.Lock() + if silentKeyed == silentClock(clock) { + silentKeyed = nil + } + silentMu.Unlock() }() return nil } diff --git a/cmd/mesh-controller/plan_looks_test.go b/cmd/mesh-controller/plan_looks_test.go index fb3dc659..e6612960 100644 --- a/cmd/mesh-controller/plan_looks_test.go +++ b/cmd/mesh-controller/plan_looks_test.go @@ -101,7 +101,7 @@ func lookerCounting(failures *atomic.Int32, looks *atomic.Int32, block chan stru func eventually(t *testing.T, what string, ok func() bool) { t.Helper() - for end := time.Now().Add(3 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) { + for end := time.Now().Add(15 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) { if ok() { return } diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 5751b1cd..f7939794 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -1023,12 +1023,12 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. outcome, reported = provisionedOutcome(node, sent.Only(carried...), byNode[node], reported, provLookup(ctx)) } if !reported { - if now.Sub(*g.Since) > inventory.ApplySilentAfter { + if now.Sub(*g.Since) > applySilentAfter { g.Silent = true r.Silent = append(r.Silent, node) changed = true notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node, - inventory.ApplySilentAfter)) + applySilentAfter)) continue } judging = append(judging, node+": not yet reported on its send") diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 64e0fa8d..c0b90e63 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -1174,7 +1174,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", ")) } if len(silent) > 0 { - said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", inventory.ApplySilentAfter, + said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", applySilentAfter, strings.Join(silent, ", "))) } said = append(said, recordedSaid(*p)...) @@ -1239,7 +1239,7 @@ func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time, case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed): failed = append(failed, node+" ("+outcome+")") } - case !at.IsZero() && now.Sub(at) > inventory.ApplySilentAfter: + case !at.IsZero() && now.Sub(at) > applySilentAfter: seen[node] = true silent = append(silent, node) default: diff --git a/cmd/mesh-controller/rollout_events_test.go b/cmd/mesh-controller/rollout_events_test.go new file mode 100644 index 00000000..7617a7dd --- /dev/null +++ b/cmd/mesh-controller/rollout_events_test.go @@ -0,0 +1,281 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +// Acceptance of the step (novox/hq ADR 0406 §5, §6, acceptance 7; issue 974), live, with the store and a real bus and no +// ticker: a rollout of two tiers advances on the events alone — the plan kept, the machines' reports, a silent key's +// expiry — a machine that reports in time is never named silent and one that never reports is named silent when its +// key expires, and a controller replaced in the middle of it resumes with no tier asked twice and none skipped. +func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + + // A quarter of an hour of silence is two seconds here. + wasSilent := applySilentAfter + applySilentAfter = 2 * time.Second + t.Cleanup(func() { applySilentAfter = wasSilent }) + + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + legacy, err := js.Conn().JetStream() + if err != nil { + t.Fatal(err) + } + checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} + t.Cleanup(func() { checkEvents = nil }) + sentHeard := make(chan *nats.Msg, 32) + sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + + // Every save of a plan is said and asks the look, as the serving controller's hook does. + saved := &movesSaid{revs: map[string][]int64{}} + wasSaved := inventory.PlanSaved + inventory.PlanSaved = func(i *inventory.Inventory, p inventory.Plan) { saved.hear(i, p); planKept(p) } + t.Cleanup(func() { inventory.PlanSaved = wasSaved }) + + // The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept. + var mu sync.Mutex + var asks []string + var sends [][]string + wasAsk := askABuild + askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) { + mu.Lock() + defer mu.Unlock() + id := link.NewBuildID(time.Now().Add(time.Duration(len(asks)) * time.Millisecond)) + asks = append(asks, source.Repository+"#"+id) + return id, nil + } + t.Cleanup(func() { askABuild = wasAsk }) + digests := 0 + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + mu.Lock() + defer mu.Unlock() + sends = append(sends, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + builds := map[string]string{} + for _, m := range []string{"a", "b"} { + builds[m] = current[m].Commit + } + for _, node := range names { + digests++ + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil { + return nil, err + } + } + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + + register := func(m, commit string, asked time.Time) { + t.Helper() + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, + inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit, + Asked: asked}); err != nil { + t.Fatal(err) + } + } + record := func(id, m, commit string, at time.Time) { + t.Helper() + b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at} + b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + old := time.Now().Add(-2 * time.Hour) + for _, m := range []string{"a", "b"} { + record("build-"+m+"-1", m, "c1", old) + register(m, "c1", old) + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, m); err != nil { + t.Fatal(err) + } + } + } + for _, n := range []string{"anchor", "laptop"} { + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil { + t.Fatal(err) + } + } + record("build-a-2", "a", "c2", time.Now().Add(-time.Minute)) + register("a", "c2", time.Now().Add(-time.Minute)) + + // A machine's report on what it was sent, heard as the controller hears it. + listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} + seq := int64(0) + reports := func(node string) { + t.Helper() + last, found, err := inv.SentTo(ctx, node) + if err != nil || !found { + t.Fatalf("%s was sent nothing: %v", node, err) + } + seq++ + body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest, + "report_sequence": seq}) + var report link.Report + if err := json.Unmarshal(body, &report); err != nil { + t.Fatal(err) + } + if _, err := listener.Heard(ctx, report); err != nil { + t.Fatal(err) + } + } + heldKeys := func() map[silentWait]time.Time { + c := silentClockOf() + if c == nil { + return nil + } + held, err := c.Held(ctx) + if err != nil { + t.Fatal(err) + } + return held + } + plan := func() inventory.Plan { + t.Helper() + p, err := inv.PlanByID(ctx, "plan-974") + if err != nil { + t.Fatal(err) + } + return p + } + + // The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything. + first, stopFirst := context.WithCancel(ctx) + defer stopFirst() + if err := startRolloutSilent(first, js); err != nil { + t.Fatal(err) + } + startPlanLooks(first, open) + built := time.Now().UTC() + p := inventory.Plan{ID: "plan-974", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: built, + State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, + Modules: map[string]*inventory.PlanModule{ + "a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"}, + "b": {}, + }} + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + eventually(t, "tier 0 sent to both machines and tier 1 asked, on the save alone", func() bool { + mu.Lock() + defer mu.Unlock() + return len(sends) == 1 && len(asks) == 1 && plan().Tier == 1 + }) + eventually(t, "a key on the bus for each machine tier 0 was sent to", func() bool { return len(heldKeys()) == 2 }) + + // The controller is replaced: its looks and its watcher end. anchor reports meanwhile, to nobody. + stopFirst() + time.Sleep(300 * time.Millisecond) + reports("anchor") + + second, stopSecond := context.WithCancel(ctx) + defer stopSecond() + if err := startRolloutSilent(second, js); err != nil { + t.Fatal(err) + } + startPlanLooks(second, open) + // Its first look finds anchor reported, so its key goes; laptop's stays, and is the new controller's to hear expire. + eventually(t, "the new controller's first look deletes the key of the machine that reported", func() bool { + h := heldKeys() + _, anchor := h[silentWait{Plan: "plan-974", Tier: 0, Node: "anchor"}] + return !anchor + }) + + // tier 1's build comes in, as the build machine's outcome is taken in. + b := plan().Modules["b"] + if b == nil || b.Build == "" { + t.Fatalf("tier 1 was not asked: %+v", b) + } + askedAt, _ := link.BuildAskedAt(b.Build) + record(b.Build, "b", "c2", askedAt) + register("b", "c2", askedAt) + planBuilt(ctx, open, "b", "c2", "", askedAt, b.Build) + eventually(t, "tier 1 sent to both machines", func() bool { + mu.Lock() + defer mu.Unlock() + return len(sends) == 2 + }) + reports("anchor") + + // laptop never reports: its key expires and the new controller names it silent, which ends the rollout. + eventually(t, "the rollout ended, naming laptop silent and not anchor", func() bool { + p := plan() + return p.State == inventory.PlanDone + }) + p = plan() + if !strings.Contains(p.Note, "laptop") || strings.Contains(p.Note, "anchor") || !strings.Contains(p.Note, "said nothing") { + t.Errorf("the plan ended with the note %q; want laptop named silent and not anchor", p.Note) + } + eventually(t, "no key left on the bus", func() bool { return len(heldKeys()) == 0 }) + + // No tier asked twice and none skipped: one ask for tier 1 (tier 0 was built when the plan was kept), each tier sent + // once to both machines, and said as sent once for each. + mu.Lock() + if len(asks) != 1 || !strings.HasPrefix(asks[0], "novox/b#") { + t.Errorf("asks = %v; want tier 1's module asked once", asks) + } + if len(sends) != 2 || strings.Join(sends[0], ",") != "anchor,laptop" || strings.Join(sends[1], ",") != "anchor,laptop" { + t.Errorf("sends = %v; want both machines sent each tier once", sends) + } + mu.Unlock() + said := map[string]int{} + for deadline := time.After(5 * time.Second); len(said) < 4; { + select { + case m := <-sentHeard: + var body planSentBody + if err := json.Unmarshal(m.Data, &body); err != nil { + t.Fatal(err) + } + said[fmt.Sprintf("%d/%s", body.Tier, body.Node)]++ + case <-deadline: + t.Fatalf("sent was said for %v; want tiers 0 and 1 to both machines", said) + } + } + time.Sleep(300 * time.Millisecond) + for len(sentHeard) > 0 { + var body planSentBody + _ = json.Unmarshal((<-sentHeard).Data, &body) + said[fmt.Sprintf("%d/%s", body.Tier, body.Node)]++ + } + for k, n := range said { + if n != 1 { + t.Errorf("sent for %s was said %d times", k, n) + } + } + // And every move of the plan was said as plan-moved. + saved.noUnsaid(t, open) +} diff --git a/testdata/store-tests b/testdata/store-tests index 3d3dff3a..8a116cec 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -134,3 +134,4 @@ ./cmd/mesh-controller TestMergesHeardWhileAnotherHolderHasThePlansAreRecordedAndCarried ./cmd/mesh-controller TestABatchWindowClosesByTheExpiryOfItsKey ./cmd/mesh-controller TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid +./cmd/mesh-controller TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt From 9ca2c2bfb350dedc6803d7665f3c45857c870d7e Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 13:45:41 +0200 Subject: [PATCH 05/12] issue 974: review 1: a verb's end, a provisioner client's word and a failed key read each ask or retry a look; the silent bucket ages; the acceptance test starts the rollout through the serving wiring --- cmd/mesh-controller/delivery.go | 2 +- cmd/mesh-controller/plan_looks.go | 69 +++++++++++++++++- cmd/mesh-controller/plan_looks_test.go | 58 +++++++++++++++ cmd/mesh-controller/push.go | 18 +---- cmd/mesh-controller/release_plan.go | 2 +- cmd/mesh-controller/rollout_events_test.go | 84 +++++++++++++++++++--- cmd/mesh-controller/seatverbs.go | 14 +++- cmd/mesh-controller/verb_looks_test.go | 54 ++++++++++++++ internal/broker/controller_buckets.go | 6 ++ internal/broker/controller_buckets_test.go | 36 ++++++++++ testdata/store-tests | 1 + 11 files changed, 313 insertions(+), 31 deletions(-) create mode 100644 cmd/mesh-controller/verb_looks_test.go diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index 17ba3c4a..42b0ee7b 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -977,7 +977,7 @@ func deliveryCommand(ctx context.Context, args []string) error { }); err != nil { return err } - fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked at the next pass\n", p.ID, p.Repository, + fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository, short(p.Commit), *by) return nil case "stop": diff --git a/cmd/mesh-controller/plan_looks.go b/cmd/mesh-controller/plan_looks.go index 6ae5db79..6e4f155d 100644 --- a/cmd/mesh-controller/plan_looks.go +++ b/cmd/mesh-controller/plan_looks.go @@ -160,6 +160,33 @@ func startPlanLooks(ctx context.Context, open *stores) { // ApplySilentAfter, a variable so a test of the wait needs no quarter of an hour. var applySilentAfter = inventory.ApplySilentAfter +// startRolloutEvents starts what a rollout advances on, for as long as the context lives (novox/hq ADR 0406 §5, issue 974): +// the wait for each machine's report as a key on the bus that expires (where the bus will not take it a timer of its own +// for that wait looks instead), a provisioner client's words, and the look at the plans, asked for first so a plan left +// by a replaced controller resumes. The serving controller's start and the tests of a rollout call this one. +func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream) { + if err := startRolloutSilent(ctx, js); err != nil { + fmt.Printf("plans: the wait for the machines' reports could not be put on the bus (%v); each is looked at by a timer of its own\n", err) + } + if err := startProvAppliedLooks(ctx, js); err != nil { + fmt.Printf("plans: the provisioner clients' words could not be watched (%v); a walk waits for them until the next look\n", err) + } + startPlanLooks(ctx, open) +} + +// servingPlanSaved is what the serving controller does for each plan it keeps: said as plan-moved, looked at (a build, a +// send kept or a command's move is followed by the look that asks the next tier, in place of the timer that did), and, +// for a walk that ended, its merges' check builds forgotten (novox/hq ADR 0350 §5). +func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inventory.Inventory, inventory.Plan) { + return func(_ *inventory.Inventory, p inventory.Plan) { + sayPlanMoved(ctx, bus, open.inventory, p) + planKept(p) + if p.WalkSaid == inventory.WalkEnded { + go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p) + } + } +} + // planLookPasses is the most times one look goes over the plans again because the first pass changed which are open. const planLookPasses = 6 @@ -193,6 +220,7 @@ func lookAtPlans(ctx context.Context, open *stores) (failed bool) { } if err := syncSilentKeys(ctx, open); err != nil { fmt.Printf("plans: the wait for the machines' reports could not be kept on the bus: %v\n", err) + failed = true // so the look is retried } }() keepWalkPhases(ctx, open) @@ -365,15 +393,25 @@ func syncSilentKeys(ctx context.Context, open *stores) error { want[w] = at } } - if c == nil { - // No bucket to put them in: each wait is looked at by a timer of its own, once. + return keepSilentKeys(ctx, c, want, now) +} + +// keepSilentKeys puts the keys on the bus as the waits are. Where the bus will not take a key — no bucket, the keys +// cannot be read, one cannot be put — the wait is looked at by a timer of its own, once, and the error is given back so +// the look is retried. +func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time.Time, now time.Time) error { + byTimer := func() { for w, sent := range want { silentByTimer(w, sent, silentTTL(sent, now)) } + } + if c == nil { + byTimer() return nil } held, err := c.Held(ctx) if err != nil { + byTimer() return err } arm, drop := silentDiff(want, held) @@ -442,6 +480,33 @@ func startRolloutSilent(ctx context.Context, js *broker.JetStream) error { return nil } +// startProvAppliedLooks has the plans looked at when a machine's provisioner client says a word (`prov.applied..`): +// a send that handed modules to the provisioner is applied only once the client said so too, and without this the walk +// would wait for its silent key (ADR 0362, ADR 0406 §5, issue 974). New words only: the first look reads the rest. +func startProvAppliedLooks(ctx context.Context, js *broker.JetStream) error { + api, err := jetstream.New(js.Conn()) + if err != nil { + return err + } + stream, err := api.Stream(ctx, broker.ProvAppliedStream) + if err != nil { + return fmt.Errorf("the provisioner clients' words: %w", err) + } + consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverNewPolicy}) + if err != nil { + return fmt.Errorf("watching the provisioner clients' words: %w", err) + } + consuming, err := consumer.Consume(func(jetstream.Msg) { askToLook() }) + if err != nil { + return fmt.Errorf("watching the provisioner clients' words: %w", err) + } + go func() { + <-ctx.Done() + consuming.Stop() + }() + return nil +} + // --- the end of a walk whose end is not known --- var walkEnds = struct { diff --git a/cmd/mesh-controller/plan_looks_test.go b/cmd/mesh-controller/plan_looks_test.go index e6612960..3f4544eb 100644 --- a/cmd/mesh-controller/plan_looks_test.go +++ b/cmd/mesh-controller/plan_looks_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "sync/atomic" "testing" "time" @@ -257,3 +258,60 @@ func TestAnExpiredSilentKeyAsksForALookAndAReportsDeleteDoesNot(t *testing.T) { } eventually(t, "the expiry asks for a look", func() bool { return looks.Load() == 1 }) } + +// A provisioner client's word on a module's files asks for a look, as a send that handed it modules is applied only +// once it has said so (issue 974, review 1): no walk waits for its silent key for what the client already said. +func TestAProvisionerClientsWordAsksForALook(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + var failures, looks atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + if err := startProvAppliedLooks(ctx, js); err != nil { + t.Fatal(err) + } + legacy, err := js.Conn().JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := legacy.Publish(broker.ProvisionerAppliedSubject("ace", "plex"), []byte(`{"status":"applied"}`)); err != nil { + t.Fatal(err) + } + eventually(t, "a look on the client's word", func() bool { return looks.Load() >= 1 }) +} + +type brokenClock struct{ silentClock } + +func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) { + return nil, errors.New("the bus is down") +} + +// A bus that will not say which keys it holds does not leave the waits unwatched: each is looked at by a timer of its own +// and the error is given back, so the look is marked failed and retried (issue 974, review 1). +func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) { + w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"} + sent := time.Now().UTC() + t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() }) + err := keepSilentKeys(t.Context(), brokenClock{}, map[silentWait]time.Time{w: sent}, sent) + if err == nil { + t.Fatal("no error given back") + } + silentTimers.mu.Lock() + armed := silentTimers.timers[w].Equal(sent) + silentTimers.mu.Unlock() + if !armed { + t.Error("no timer of its own watches the wait") + } +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index af6d40e1..66099d0a 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -183,12 +183,7 @@ func serve(ctx context.Context) (err error) { // Open plans move on events (novox/hq ADR 0406 §5, issue 974): a tier is asked on the `built` and the send of the // tier before it, a walk ends on its machines' reports, a silent machine is a key that expired, and a plan left // by a replaced controller resumes from the first look, which this asks for. No timer looks every so often. - // The wait for each machine's report is a key on the bus that expires; where the bus will not take it a timer of its - // own for that wait looks instead. - if err := startRolloutSilent(ctx, server.JetStream()); err != nil { - fmt.Printf("plans: the wait for the machines' reports could not be put on the bus (%v); each is looked at by a timer of its own\n", err) - } - startPlanLooks(ctx, open) + startRolloutEvents(ctx, open, server.JetStream()) // And the merge window (novox/hq ADR 0276, ADR 0407): a batch is cut into its walk when the window closes, // which no merge or outcome says. The window is two keys on the bus that expire and a watcher that hears it; // nothing polls (issue 973). @@ -258,16 +253,7 @@ func serve(ctx context.Context) (err error) { // (novox/hq to-be 45 §9). checkEvents = bus // And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239). - inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { - sayPlanMoved(ctx, bus, open.inventory, p) - // And the plan's next step is looked at (ADR 0406 §5): a build, a send kept or a command's move is followed - // by the look that asks the next tier, in place of the timer that did. - planKept(p) - // A walk ended has taken what it reuses: its merges' check builds go (novox/hq ADR 0350 §5). - if p.WalkSaid == inventory.WalkEnded { - go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p) - } - } + inventory.PlanSaved = servingPlanSaved(ctx, bus, open) if err := server.Checks(following{open: open}); err != nil { return err } diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index c0b90e63..e10eb7cf 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -2016,7 +2016,7 @@ func plansCommand(ctx context.Context, args []string) error { }); err != nil { return err } - fmt.Printf("%s (%s at %s) is let go by hand; its first tier is asked at the next pass\n", p.ID, p.Repository, + fmt.Printf("%s (%s at %s) is let go by hand; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository, short(p.Commit)) return nil } diff --git a/cmd/mesh-controller/rollout_events_test.go b/cmd/mesh-controller/rollout_events_test.go index 7617a7dd..f3809bba 100644 --- a/cmd/mesh-controller/rollout_events_test.go +++ b/cmd/mesh-controller/rollout_events_test.go @@ -56,10 +56,11 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { } t.Cleanup(func() { _ = sub.Unsubscribe() }) - // Every save of a plan is said and asks the look, as the serving controller's hook does. + // Every save of a plan is kept, said and looked at by the serving controller's own hook. saved := &movesSaid{revs: map[string][]int64{}} wasSaved := inventory.PlanSaved - inventory.PlanSaved = func(i *inventory.Inventory, p inventory.Plan) { saved.hear(i, p); planKept(p) } + serving := servingPlanSaved(ctx, checkEvents, open) + inventory.PlanSaved = func(i *inventory.Inventory, p inventory.Plan) { saved.hear(i, p); serving(i, p) } t.Cleanup(func() { inventory.PlanSaved = wasSaved }) // The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept. @@ -176,10 +177,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { // The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything. first, stopFirst := context.WithCancel(ctx) defer stopFirst() - if err := startRolloutSilent(first, js); err != nil { - t.Fatal(err) - } - startPlanLooks(first, open) + startRolloutEvents(first, open, js) built := time.Now().UTC() p := inventory.Plan{ID: "plan-974", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, @@ -204,10 +202,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { second, stopSecond := context.WithCancel(ctx) defer stopSecond() - if err := startRolloutSilent(second, js); err != nil { - t.Fatal(err) - } - startPlanLooks(second, open) + startRolloutEvents(second, open, js) // Its first look finds anchor reported, so its key goes; laptop's stays, and is the new controller's to hear expire. eventually(t, "the new controller's first look deletes the key of the machine that reported", func() bool { h := heldKeys() @@ -279,3 +274,72 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { // And every move of the plan was said as plan-moved. saved.noUnsaid(t, open) } + +// Acceptance of the verb path (issue 974, review 1), live, with the store and a bus: a walk the delivery's owner lets +// go by the verb `deliver` after the operator's approval is asked its first tier with no other event. The verb runs as a +// child process, whose saves the serving controller does not hear; the handler asks the look as the verb ends. +func TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + legacy, err := js.Conn().JetStream() + if err != nil { + t.Fatal(err) + } + checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} + t.Cleanup(func() { checkEvents = nil }) + asked := asksRecorded(t) + register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) + if _, err := assign(ctx, open, "anchor", "app"); err != nil { + t.Fatal(err) + } + cut := time.Now().UTC() + waiting := inventory.Plan{ID: "plan-974-deliver", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8", + Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {}}, + Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}, + Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}} + if err := open.inventory.SavePlan(ctx, &waiting); err != nil { + t.Fatal(err) + } + + // The serving controller, as serve starts it. + wasSaved := inventory.PlanSaved + inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open) + t.Cleanup(func() { inventory.PlanSaved = wasSaved }) + started, stop := context.WithCancel(ctx) + defer stop() + startRolloutEvents(started, open, js) + time.Sleep(500 * time.Millisecond) // the first look: the walk waits, and is asked nothing + + // The verb as its child process runs it: the delivery command, whose saves are not the serving controller's. + wasRun := runTheVerb + runTheVerb = func(ctx context.Context, argv []string) (verbAnswer, error) { + was := inventory.PlanSaved + inventory.PlanSaved = func(*inventory.Inventory, inventory.Plan) {} + defer func() { inventory.PlanSaved = was }() + return verbAnswer{}, deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}) + } + t.Cleanup(func() { runTheVerb = wasRun }) + handlers, _, err := seatToolHandlers() + if err != nil { + t.Fatal(err) + } + if _, err := handlers["deliver"](ctx, []byte(`{"plan":"`+waiting.ID+`","why":"its turn"}`)); err != nil { + t.Fatal(err) + } + eventually(t, "the walk's first tier asked, with no event but the verb's end", func() bool { + p, err := open.inventory.PlanByID(ctx, waiting.ID) + return err == nil && p.Modules["app"] != nil && p.Modules["app"].Build != "" && len(*asked) == 1 + }) +} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 101db41c..4a2e0552 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -1398,12 +1398,24 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { // reloading its user list forgets the answer it was about to permit (novox/hq issue 265). link.Acknowledge(ctx) } - return runVerb(ctx, argv) + answer, err := runTheVerb(ctx, argv) + // **A plan the verb moved is looked at as it ends** (novox/hq ADR 0406 §5, issue 974): the verb ran as a + // child process, whose saves this process does not hear, so `deliver` after an approval, a stop, a retry + // and a send from a verb would otherwise wait for an unrelated event. Even when it failed: it may have + // kept a plan before it did. + if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) { + askToLook() + } + return answer, err } } return handlers, behind, nil } +// runTheVerb is how the serving controller runs a verb: as a command of its own. A variable so a test of what follows a +// verb needs no child process. +var runTheVerb = runVerb + // actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them. func actsOnAPlan(args map[string]any) bool { for _, act := range []string{"stop", "close", "retry"} { diff --git a/cmd/mesh-controller/verb_looks_test.go b/cmd/mesh-controller/verb_looks_test.go new file mode 100644 index 00000000..d16e190a --- /dev/null +++ b/cmd/mesh-controller/verb_looks_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "context" + "sync/atomic" + "testing" + "time" +) + +// A plan moved by a seat verb is looked at as the verb ends (issue 974, review 1): the verb runs as a child process +// whose saves the serving controller does not hear, so `deliver`, which lets a walk go after the operator's approval, +// would leave its first tier waiting for an unrelated event. +func TestAVerbThatMayMoveAPlanHasTheServingControllerLookWhenItEnds(t *testing.T) { + var looks, failures atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + + was := runTheVerb + runTheVerb = func(context.Context, []string) (verbAnswer, error) { return verbAnswer{}, nil } + t.Cleanup(func() { runTheVerb = was }) + handlers, _, err := seatToolHandlers() + if err != nil { + t.Fatal(err) + } + for _, c := range []struct { + verb string + args string + look bool + }{ + {"deliver", `{"plan":"p"}`, true}, + {"plans", `{"stop":"plan-1","why":"w"}`, true}, + {"plans", `{}`, false}, + {"nodes", `{}`, false}, + } { + before := looks.Load() + h := handlers[c.verb] + if h == nil { + t.Fatalf("no verb %s", c.verb) + } + if _, err := h(ctx, []byte(c.args)); err != nil { + t.Fatalf("%s %s: %v", c.verb, c.args, err) + } + time.Sleep(100 * time.Millisecond) + if got := looks.Load() > before; got != c.look { + t.Errorf("%s %s: look asked = %v; want %v", c.verb, c.args, got, c.look) + } + } +} diff --git a/internal/broker/controller_buckets.go b/internal/broker/controller_buckets.go index b47d9111..7d8ba4e4 100644 --- a/internal/broker/controller_buckets.go +++ b/internal/broker/controller_buckets.go @@ -97,6 +97,11 @@ const MergeWindowMarkerTTL = time.Minute // RolloutSilentMarkerTTL is how long the marker an expired silent key leaves is kept, for the same reason. const RolloutSilentMarkerTTL = time.Minute +// RolloutSilentKeptFor is the age of the silent keys' bucket: every report deletes a key and the delete leaves a marker, which +// with one value per key would stay for ever and fill the bucket. A key lives a quarter of an hour at most, so an hour +// takes nothing live. +const RolloutSilentKeptFor = time.Hour + // AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own. const AskedKeptFor = 30 * 24 * time.Hour @@ -251,6 +256,7 @@ func (j *JetStream) EnsureControllerBuckets() error { "lives the time a machine may stay silent and is deleted by its report; one that expires names the " + "machine silent; written by the controller alone", History: 1, + TTL: RolloutSilentKeptFor, MaxValueSize: 4 << 10, MaxBytes: 8 << 20, Storage: jetstream.FileStorage, diff --git a/internal/broker/controller_buckets_test.go b/internal/broker/controller_buckets_test.go index e34e0d3e..4b337dc1 100644 --- a/internal/broker/controller_buckets_test.go +++ b/internal/broker/controller_buckets_test.go @@ -389,6 +389,42 @@ func TestTheRolloutsSilentKeysExpireAsAPurgeAndAReportsDeleteIsNot(t *testing.T) } } +// **The silent keys' bucket ages**, or its delete markers would fill it (issue 974, review 1): every report deletes a key, +// the delete leaves a marker, and with one value per key and no age the markers stay for ever. A bucket made before the +// age was set is brought to it by the next start, which the server accepts as an update of an existing bucket. +func TestTheSilentKeysBucketAgesAndAnOlderOneIsBroughtToIt(t *testing.T) { + js, err := Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer js.Close() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + // As the first build made it: no age. + if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: RolloutSilentBucket, History: 1, + MaxValueSize: 4 << 10, MaxBytes: 8 << 20, Storage: jetstream.FileStorage, LimitMarkerTTL: RolloutSilentMarkerTTL}); err != nil { + t.Fatal(err) + } + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatalf("a bucket made without the age is not brought to it: %v", err) + } + kv, err := api.KeyValue(ctx, RolloutSilentBucket) + if err != nil { + t.Fatal(err) + } + status, err := kv.Status(ctx) + if err != nil { + t.Fatal(err) + } + if status.TTL() != RolloutSilentKeptFor { + t.Errorf("the bucket ages at %v; want %v", status.TTL(), RolloutSilentKeptFor) + } +} + // **The merges have a consumer of their own, which a reset of the events consumer leaves alone** (issue 973, novox/hq // ADR 0407 decision 3): healer H4 re-makes the events consumer from now, dropping what it holds pending; a merge // waiting in the merges consumer is not that consumer's, and nothing would catch it up. diff --git a/testdata/store-tests b/testdata/store-tests index 8a116cec..81cc53c1 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -135,3 +135,4 @@ ./cmd/mesh-controller TestABatchWindowClosesByTheExpiryOfItsKey ./cmd/mesh-controller TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid ./cmd/mesh-controller TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt +./cmd/mesh-controller TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent From 96325c295c2cdb391a5ee68daee98a3ef0ae6237 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 13:47:47 +0200 Subject: [PATCH 06/12] issue 974: the serving controller follows its own plan-moved, so a plan moved from the terminal is looked at --- cmd/mesh-controller/plan_looks.go | 33 +++++++++++++++++++++++++ cmd/mesh-controller/plan_looks_test.go | 34 ++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/cmd/mesh-controller/plan_looks.go b/cmd/mesh-controller/plan_looks.go index 6e4f155d..1bfdbf7d 100644 --- a/cmd/mesh-controller/plan_looks.go +++ b/cmd/mesh-controller/plan_looks.go @@ -171,6 +171,9 @@ func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream) if err := startProvAppliedLooks(ctx, js); err != nil { fmt.Printf("plans: the provisioner clients' words could not be watched (%v); a walk waits for them until the next look\n", err) } + if err := startPlanMovedLooks(ctx, js); err != nil { + fmt.Printf("plans: the plans moved by other processes could not be followed (%v)\n", err) + } startPlanLooks(ctx, open) } @@ -507,6 +510,36 @@ func startProvAppliedLooks(ctx context.Context, js *broker.JetStream) error { return nil } +// startPlanMovedLooks has the plans looked at when any process says one moved: the serving controller follows its own +// `plan-moved`, so a plan a person moved with a verb at the control node's terminal — a process this one did not spawn, +// whose saves it does not hear — is looked at all the same (novox/hq ADR 0406 §2, §5, issue 974). New messages only; the +// first look reads the rest. A look its own save asks is the same look: asks are one. +func startPlanMovedLooks(ctx context.Context, js *broker.JetStream) error { + api, err := jetstream.New(js.Conn()) + if err != nil { + return err + } + stream, err := api.Stream(ctx, broker.EventsStream) + if err != nil { + return fmt.Errorf("the events stream: %w", err) + } + consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{ + FilterSubjects: []string{link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved)}, + DeliverPolicy: jetstream.DeliverNewPolicy}) + if err != nil { + return fmt.Errorf("following the plans moved: %w", err) + } + consuming, err := consumer.Consume(func(jetstream.Msg) { askToLook() }) + if err != nil { + return fmt.Errorf("following the plans moved: %w", err) + } + go func() { + <-ctx.Done() + consuming.Stop() + }() + return nil +} + // --- the end of a walk whose end is not known --- var walkEnds = struct { diff --git a/cmd/mesh-controller/plan_looks_test.go b/cmd/mesh-controller/plan_looks_test.go index 3f4544eb..c2076a4b 100644 --- a/cmd/mesh-controller/plan_looks_test.go +++ b/cmd/mesh-controller/plan_looks_test.go @@ -9,6 +9,7 @@ import ( "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/testbus" ) @@ -315,3 +316,36 @@ func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) { t.Error("no timer of its own watches the wait") } } + +// A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is +// looked at: the controller follows its own plan-moved (issue 974, round 1). +func TestAPlanMovedByAnotherProcessAsksForALook(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + var failures, looks atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + if err := startPlanMovedLooks(ctx, js); err != nil { + t.Fatal(err) + } + legacy, err := js.Conn().JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := legacy.Publish(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), []byte(`{}`)); err != nil { + t.Fatal(err) + } + eventually(t, "a look on the plan moved", func() bool { return looks.Load() >= 1 }) +} From f8838309752e47c3dd72c9206ace699d56c649ef Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 13:53:22 +0200 Subject: [PATCH 07/12] issue 974: two work-queue streams for the checks of pull requests (CHECKS_READY, CHECKS_OTHER), their workers, grants and writer --- cmd/mesh-controller/signals.go | 2 + internal/broker/checks.go | 112 +++++++++++++++++++++++++ internal/broker/nats.go | 18 ++++ internal/broker/streams.go | 8 +- internal/broker/streams_test.go | 3 + internal/broker/testdata/composed.conf | 2 +- internal/broker/writers.go | 5 ++ internal/broker/writers_test.go | 1 + internal/catalogue/streams.go | 2 +- internal/link/advisories.go | 4 +- 10 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 internal/broker/checks.go diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index b9e2a576..a0a5b560 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -671,6 +671,8 @@ func consumerWho(stream, consumer string) string { switch { case consumer == broker.ControllerName || consumer == broker.MergesConsumerName: return "the controller" + case broker.IsCheckStream(stream): + return "the holder of " + broker.TheBuildSeat case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(consumer, "_worker"): seat := strings.ToLower(strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(consumer, "SEAT_"), "_worker"), "_", "-")) return "the holder of " + seat diff --git a/internal/broker/checks.go b/internal/broker/checks.go new file mode 100644 index 00000000..e9efd275 --- /dev/null +++ b/internal/broker/checks.go @@ -0,0 +1,112 @@ +package broker + +import "strings" + +// The pull requests' checks have work queues of their own (novox/hq ADR 0406 §3, issue 974). +// +// A delivery build goes on the build seat's own queue, `mesh.seat.node-build-agent.accept.build`. A check of a pull +// request marked ready goes on a second work-queue stream and every other check on a third. A builder takes from the +// first, then the second, then the third; the others are taken in the order asked. **Each check stream has one subject +// per pull request and keeps only the newest request** (a limit of one message a subject, the older discarded), so a +// newer request replaces one still waiting and a pull request has at most one waiting check: the stream's rule, not a +// pass of the controller that moves asks about. Nothing pauses the worker or deletes and republishes a request. + +const ( + // ChecksReadyStream holds the checks of pull requests marked ready; ChecksOtherStream every other check. + ChecksReadyStream = "CHECKS_READY" + ChecksOtherStream = "CHECKS_OTHER" + // ChecksReadyWorker and ChecksOtherWorker are the durable consumers the build seat's holders share on them, as + // they share the seat's worker on its own queue (novox/hq ADR 0190). + ChecksReadyWorker = "CHECKS_READY_worker" + ChecksOtherWorker = "CHECKS_OTHER_worker" + // TheBuildSeat is the role whose checks these streams queue. The retired build role's checks stay on its own queue. + TheBuildSeat = "node-build-agent" +) + +// PullsBucket is mesh-delivery's key per pull request: its head, the trunk commit its current check is asked against and +// its verdict (ADR 0406 §4). A running check watches its key. +var PullsBucket = BucketName("mesh-delivery", "pulls") + +// CheckSubjectPrefix is where a seat's checks of one kind are asked: `mesh.seat..check.ready.` or `.other.`. The +// `check` token keeps the subjects out of the seat's own queue, which takes `accept.>`. +func CheckSubjectPrefix(seat string, ready bool) string { + kind := "other" + if ready { + kind = "ready" + } + return "mesh.seat." + seat + ".check." + kind + "." +} + +// CheckStreams are the two check queues, in the order a builder takes from them. +func CheckStreams() []Stream { + return []Stream{ + { + Name: ChecksReadyStream, + Subjects: []string{CheckSubjectPrefix(TheBuildSeat, true) + ">"}, + Retention: RetentionWorkQueue, + MaxAge: 7 * 24 * 60 * 60, + MaxMsgsPerSubject: 1, + Why: "the checks of pull requests marked ready, taken after a delivery build and before any other check " + + "(novox/hq ADR 0406 §3); one subject a pull request, the newest request kept, so a newer one replaces a " + + "request still waiting", + }, + { + Name: ChecksOtherStream, + Subjects: []string{CheckSubjectPrefix(TheBuildSeat, false) + ">"}, + Retention: RetentionWorkQueue, + MaxAge: 7 * 24 * 60 * 60, + MaxMsgsPerSubject: 1, + Why: "every other check, taken last and in the order asked (novox/hq ADR 0406 §3); one subject a pull " + + "request, the newest request kept; one left behind for a head already judged is dropped when taken", + }, + } +} + +// CheckWorkers are the consumers the build seat's holders pull the check queues through: as the seat's own worker is, +// one for every holder, one ask at a time, acknowledged when the check is done. +func CheckWorkers() []Consumer { + worker := func(name, stream string, ready bool) Consumer { + return Consumer{ + Name: name, + Stream: stream, + Filters: []string{CheckSubjectPrefix(TheBuildSeat, ready) + ">"}, + AckWaitSeconds: 60, + MaxDeliver: 5, + Why: "every holder of " + TheBuildSeat + " pulls one check at a time from this worker and acknowledges after " + + "the check is done, so a crash mid-check redelivers rather than loses (novox/hq ADR 0406 §3)", + } + } + return []Consumer{ + worker(ChecksReadyWorker, ChecksReadyStream, true), + worker(ChecksOtherWorker, ChecksOtherStream, false), + } +} + +// holderCheckGrants are what a holder of the build seat is granted to take checks: ask about the two workers, pull from +// them and acknowledge, and nothing of the subjects themselves. +func holderCheckGrants() []string { + var pub []string + for _, w := range CheckWorkers() { + pub = append(pub, + "$JS.API.CONSUMER.INFO."+w.Stream+"."+w.Name, + "$JS.API.CONSUMER.MSG.NEXT."+w.Stream+"."+w.Name, + "$JS.ACK."+w.Stream+"."+w.Name+".>") + } + return pub +} + +// IsCheckStream says a stream is one of the check queues. +func IsCheckStream(stream string) bool { + return stream == ChecksReadyStream || stream == ChecksOtherStream +} + +// isCheckAsk says a message of a check queue is an ask the controller itself makes. +func isCheckAsk(stream, subject string) bool { + switch stream { + case ChecksReadyStream: + return strings.HasPrefix(subject, CheckSubjectPrefix(TheBuildSeat, true)) + case ChecksOtherStream: + return strings.HasPrefix(subject, CheckSubjectPrefix(TheBuildSeat, false)) + } + return false +} diff --git a/internal/broker/nats.go b/internal/broker/nats.go index bb3f82c1..7507be33 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -217,6 +217,9 @@ var seatsTheControllerAsks = []string{"node-build-agent", "mesh-build-machine"} // original — and no other can (novox/hq issue 334, ADR 0264's consequences: no grant over the seats' // queues). func TheControllersAsk(stream, subject string) bool { + if isCheckAsk(stream, subject) { + return true + } for _, seat := range seatsTheControllerAsks { if stream == seatStreamName(seat) && strings.HasPrefix(subject, "mesh.seat."+seat+".accept.") { return true @@ -378,6 +381,12 @@ var DegradableGrants = []DegradableGrant{ Reason: "refused: the controller closes a batch's window by a timer of its own for that batch, as the " + "window setting says (ADR 0407 decision 1, issue 973)", }, + { + Subject: "mesh.seat." + TheBuildSeat + ".check.>", + Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " + + "first push, and a check asked before that is asked again by the next announcement of its head (ADR 0406 §3, " + + "issue 974)", + }, { Subject: "mesh.seat." + ControllerSeat + ".event.sent", Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " + @@ -575,6 +584,11 @@ func PermissionsFor(p Principal) (Permissions, error) { // the role, and whichever machine holding it is idle takes it. for _, seat := range seatsTheControllerAsks { pub = append(pub, "mesh.seat."+seat+".accept.>") + // **And its checks, on the two queues of their own** (novox/hq ADR 0406 §3, issue 974): one subject a pull + // request on each, under `check`; the retired role's checks stay on its accepts. + if seat == TheBuildSeat { + pub = append(pub, "mesh.seat."+seat+".check.>") + } // **And its holders' verbs, on every machine** (novox/hq ADR 0219): what a holder is // building, kill it, pause it, resume it. The queue is the controller's to show and to // change, and what one machine is doing with an ask it took only that machine can say. @@ -883,6 +897,10 @@ func PermissionsFor(p Principal) (Permissions, error) { "$JS.API.CONSUMER.INFO."+stream+"."+worker, "$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker, "$JS.ACK."+stream+"."+worker+".>") + // And the checks' two workers, for the build role they are queued for (novox/hq ADR 0406 §3). + if s.Name == TheBuildSeat { + pub = append(pub, holderCheckGrants()...) + } // **And whether an ask it took was cancelled** (novox/hq ADR 0219): one key of the // seat's cancelled set, read directly by its id, so a holder that fetched an ask in the // moment the controller cancelled it ends it instead of building it. Read, never written: diff --git a/internal/broker/streams.go b/internal/broker/streams.go index 70b76520..f8377211 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -147,7 +147,7 @@ func OriginalOfAgain(subject string) (string, bool) { // every tool invocation in the mesh, and a tool call must never be persisted (design 25 §3 keeps // tools on core NATS, where a lost call is a timeout the caller already handles). func MeshStreams() []Stream { - return []Stream{ + return append([]Stream{ { Name: "CONTROL", // A build's outcome is no longer here: it is the build-machine seat's own event, so one @@ -221,7 +221,7 @@ func MeshStreams() []Stream { "failed with the reason (novox/hq ADR 0349 §4): what a walk's end and a gate wait for (ADR 0362, " + "ADR 0349 §15), read by subject; a week, as the events are", }, - } + }, CheckStreams()...) } // ProvAppliedStream keeps what each machine's provisioner client last said of each module's files, newest per @@ -465,7 +465,7 @@ const EventsStream = "EVENTS" // held through a store restart — the exact message the stream exists to protect — some minutes // before the controller had finished deciding about it. func MeshConsumers() []Consumer { - return []Consumer{ + return append([]Consumer{ { Name: ControllerName, Stream: "CONTROL", @@ -525,7 +525,7 @@ func MeshConsumers() []Consumer { Why: "the controller copies each message a consumer gave up on into DEAD_LETTERS; no max-deliver, " + "because a notice it gave up on would lose the message it is about", }, - } + }, CheckWorkers()...) } // NoticesConsumer is the controller's consumer on DEAD_LETTER_NOTICES (novox/hq issue 330). diff --git a/internal/broker/streams_test.go b/internal/broker/streams_test.go index 7a52d422..68831f41 100644 --- a/internal/broker/streams_test.go +++ b/internal/broker/streams_test.go @@ -134,6 +134,9 @@ func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) { // What each machine's provisioner client last said of each module's files (design 57 P6): the newest // word per module, what a walk's end reads. "PROV_APPLIED": RetentionLastPerSubject, + // The checks of pull requests, in two work queues of their own (novox/hq ADR 0406 §3, issue 974). + "CHECKS_READY": RetentionWorkQueue, + "CHECKS_OTHER": RetentionWorkQueue, } got := map[string]Retention{} for _, s := range MeshStreams() { diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index d76e8659..dadf2781 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/broker/writers.go b/internal/broker/writers.go index 1b03593f..437612e1 100644 --- a/internal/broker/writers.go +++ b/internal/broker/writers.go @@ -148,6 +148,11 @@ var WritersTable = []WriterRow{ Subjects: []string{"mesh.seat.node-build-agent.event.built", "mesh.seat.mesh-build-machine.event.built"}, Writes: holdsSeatOf, Shared: "every machine holding the build seat answers the asks it took; each outcome names its ask"}, + {State: "a pull request's check asked", Writer: "controller", + KeptIn: "the work queues " + ChecksReadyStream + " and " + ChecksOtherStream + ", one subject a pull request, the newest kept", + Others: "the build seat's holders take it", + Subjects: []string{CheckSubjectPrefix(TheBuildSeat, true) + ">", CheckSubjectPrefix(TheBuildSeat, false) + ">"}, + Writes: isController}, {State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)", KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule}, // A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer. diff --git a/internal/broker/writers_test.go b/internal/broker/writers_test.go index 0777967d..6f4c585f 100644 --- a/internal/broker/writers_test.go +++ b/internal/broker/writers_test.go @@ -27,6 +27,7 @@ var designRows = []string{ "the healers' acts and their brake", "stream definitions and bus permissions", "builds and their outcomes", + "a pull request's check asked", "a merge announced", "a pull request's head announced", "a pull request's merge check", diff --git a/internal/catalogue/streams.go b/internal/catalogue/streams.go index a3e3da68..012178b6 100644 --- a/internal/catalogue/streams.go +++ b/internal/catalogue/streams.go @@ -36,7 +36,7 @@ var ReservedStreamPrefixes = []string{"SEAT_", "LOG_", "KV_", "OBJ_", "DEAD_"} // MeshStreamNames are the mesh's own streams by name. Written here as well as in the broker, which this // package cannot import; a test keeps them agreeing. -var MeshStreamNames = []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS", "DEAD_LETTER_NOTICES", "DEAD_LETTERS", "PROV_APPLIED"} +var MeshStreamNames = []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS", "DEAD_LETTER_NOTICES", "DEAD_LETTERS", "PROV_APPLIED", "CHECKS_READY", "CHECKS_OTHER"} // StreamDeclaration is one stream a module owns. type StreamDeclaration struct { diff --git a/internal/link/advisories.go b/internal/link/advisories.go index ced95ffa..5d0ec818 100644 --- a/internal/link/advisories.go +++ b/internal/link/advisories.go @@ -143,7 +143,7 @@ func MeshNamed(stream, name string) bool { return false // the mesh defines no durable consumer on a bucket's stream, or the memberships' case stream == "NODES": return true - case strings.HasPrefix(stream, "SEAT_"): + case strings.HasPrefix(stream, "SEAT_") || broker.IsCheckStream(stream): return strings.HasSuffix(name, "_worker") case name == broker.ControllerName || name == broker.MergesConsumerName: return true @@ -160,6 +160,8 @@ func ConsumerInWords(stream, name string) string { return "the controller's consumer on " + stream case stream == "NODES": return "how " + name + " hears what it should be (its declaration consumer)" + case broker.IsCheckStream(stream): + return "the worker every holder of " + broker.TheBuildSeat + " takes its checks from (" + stream + ")" case strings.HasPrefix(stream, "SEAT_") && strings.HasSuffix(name, "_worker"): seat := strings.ToLower(strings.ReplaceAll(strings.TrimPrefix(stream, "SEAT_"), "_", "-")) return "the worker every holder of " + seat + " takes its asks from" From 3d09d5235f9f0e97a41d41514aa9ecc5cafdc552 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 14:01:33 +0200 Subject: [PATCH 08/12] issue 974: the checks are queued by pull request on two work-queue streams, taken after a delivery build, dropped when judged, stopped when their key moves; the reordering and the forge-asking end are gone --- cmd/mesh-builder/main.go | 27 +- cmd/mesh-controller/checks.go | 7 +- cmd/mesh-controller/delivery.go | 15 +- cmd/mesh-controller/queue.go | 37 +-- cmd/mesh-controller/seatverbs.go | 9 +- cmd/mesh-controller/superseded.go | 157 --------- cmd/mesh-controller/superseded_test.go | 192 ----------- cmd/mesh-controller/watchdogs.go | 3 - internal/catalogue/verbs.go | 4 +- internal/link/build.go | 6 + internal/link/builds_nats.go | 261 ++++++++++----- internal/link/builds_nats_test.go | 6 + internal/link/checks_nats.go | 170 ++++++++++ internal/link/checks_nats_test.go | 309 +++++++++++++++++ internal/link/events.go | 3 + internal/link/queue.go | 148 ++++++--- internal/link/queue_kind.go | 50 +++ internal/link/queue_order.go | 433 ------------------------ internal/link/queue_order_test.go | 441 ------------------------- internal/link/try_test.go | 22 +- 20 files changed, 888 insertions(+), 1412 deletions(-) delete mode 100644 cmd/mesh-controller/superseded.go delete mode 100644 cmd/mesh-controller/superseded_test.go create mode 100644 internal/link/checks_nats.go create mode 100644 internal/link/checks_nats_test.go create mode 100644 internal/link/queue_kind.go delete mode 100644 internal/link/queue_order.go delete mode 100644 internal/link/queue_order_test.go diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index b23b862a..4216b495 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -27,6 +27,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "syscall" "time" @@ -290,6 +291,26 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri if err := work.Began(ctx); err != nil { fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err) } + // **A running check watches its pull request's key** (novox/hq ADR 0406 §4, issue 974): when the head or the trunk + // commit in it moves, the check is for a state nobody wants any more, and ends itself — nothing is asked of the forge + // (issue 726). A group's check, a try and a build watch nothing. + var superseded atomic.Bool + if w, ok := work.(interface { + WatchPull(context.Context, func(link.PullKey)) (func(), error) + }); ok && request.Check != nil { + stopWatching, err := w.WatchPull(building, func(k link.PullKey) { + superseded.Store(true) + say("superseded", fmt.Sprintf("the pull request's key moved to %.12s on %.12s while this checked %.12s on %.12s: stopped", + k.Head, k.Trunk, request.Ref, request.Check.Trunk)) + cancel() + }) + if err != nil { + fmt.Fprintf(os.Stderr, "the key of %s/%s#%d could not be watched, so its check runs on if the key moves: %v\n", + request.Check.Owner, request.Check.Repo, request.Check.Number, err) + } else { + defer stopWatching() + } + } result := link.BuildResult{ ID: request.ID, Repository: request.Repository, Path: request.Path, @@ -382,7 +403,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri if mine != nil { killed = h.returned(mine) && err != nil } - if killed { + if superseded.Load() && err != nil { + // Ended by its own key moving on (ADR 0406 §4): said as the check's failure, as a kill is. + result.Failed = link.SupersededByKey + say("failed", link.SupersededByKey) + } else if killed { // **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the // error it ended with is the kill's consequence, not a fault of the source. result.Failed = link.KilledByHand diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index 5ddbd7ee..b3ba6d6a 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -191,8 +191,9 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error { if err := ask.Ask(ctx, request); err != nil { return err } - // A new head ends the check of the old one (novox/hq issue 726). - endingSupersededChecks(ctx, seat, p, request.ID, f.oneShot) + // **A newer request replaces the one still waiting, and a newer key stops the one running** (novox/hq ADR 0406 §3, §4, + // issue 974): the check queue keeps one request a pull request, and a running check watches its key. Nothing is asked + // of the forge here (issue 726). what := "its own merge-check.sh alone: " + noModuleTouched if scope.gated() { what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))", @@ -270,7 +271,7 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop Held: heldBy(ctx), Seats: seatBases(ctx), Source: sourceOnSeat(source), - Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, + Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, Trunk: p.BaseSHA, Ready: p.Ready, Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New, Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds), Release: release, Running: runningBy(ctx)}, diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index 42b0ee7b..86dafe7b 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -584,6 +584,9 @@ type orderMember struct { // BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit // a verdict no build judges names when the head is checked again through mesh-delivery. BaseSHA string `json:"base_sha,omitempty"` + // Ready says the member's pull request is marked ready (novox/hq ADR 0406 §3, issue 974): its check is queued before + // every other check. Said by mesh-delivery on a ready mark. + Ready bool `json:"ready,omitempty"` } // pull is the member as the forge announced it. @@ -595,7 +598,7 @@ func (m orderMember) pull() link.PullUpdated { } return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, BaseSHA: m.BaseSHA, CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed, - ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid} + ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid, Ready: m.Ready} } // orderPair is one "before" among a group's members, why, and the rules it won over. @@ -892,6 +895,8 @@ func deliveryCommand(ctx context.Context, args []string) error { removed := set.String("removed", "", "the files it deletes, comma-separated") membersJSON := set.String("members", "", "a group's members, as JSON") group := set.String("group", "", "a delivery group's id") + ready := set.Bool("ready", false, "the head's pull request is marked ready: its check is queued before every other check "+ + "(novox/hq ADR 0406 §3)") by := set.String("by", catalogue.DeliverySeat, "who says it") why := set.String("why", "", "why") without := set.String("without", "", "merges a stop leaves out, @, comma-separated: the walk's "+ @@ -907,6 +912,12 @@ func deliveryCommand(ctx context.Context, args []string) error { if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil { return fmt.Errorf("the members are not readable: %w", err) } + // Marked ready by the verb's word for every member it names, besides any member that says so itself. + if *ready { + for i := range members { + members[i].Ready = true + } + } } open, err := openStores(ctx) if err != nil { @@ -1166,6 +1177,8 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or return "", err } request.Check.Group = group + // A group's check is queued as ready when any member's pull request is marked ready. + request.Check.Ready = slices.ContainsFunc(members, func(m orderMember) bool { return m.Ready }) world, err := theRestOfTheMesh(ctx, open.inventory, shelfOf(entries), "") if err != nil { return "", err diff --git a/cmd/mesh-controller/queue.go b/cmd/mesh-controller/queue.go index e6bd065d..ea2f8fed 100644 --- a/cmd/mesh-controller/queue.go +++ b/cmd/mesh-controller/queue.go @@ -80,30 +80,6 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error { return nil } -// keepingTheQueueInOrder runs the build queue's passes while this controller acts (novox/hq issue 558): -// the one process that puts the queue in order, so no two passes pause and resume it over each other. -// The build seat is read from the catalogue at most every half minute. -func keepingTheQueueInOrder(ctx context.Context, open *stores, js *broker.JetStream) { - if js == nil { - return - } - seat, readAt := link.TheBuildMachine, time.Time{} - seatNow := func() string { - if time.Since(readAt) < watchEvery { - return seat - } - if entries, err := open.inventory.Catalogued(ctx); err == nil { - seat, readAt = buildSeatAmong(entries), time.Now() - } else { - fmt.Printf("the build seat is read as %s: the catalogue cannot be read: %v\n", seat, err) - } - return seat - } - link.KeepingInOrder(ctx, js, seatNow, link.Holding, func(format string, args ...any) { - fmt.Printf(format+"\n", args...) - }) -} - // queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask // waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the // table, written from the same read: the verb answers it as its output and the rest as its answer, so the @@ -303,16 +279,8 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err) } } - worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}}) - // **Where it waits now** (novox/hq issue 558): moved into order since it was read, a waiting ask has a - // later sequence, and the worker handing out what was behind its old one is not it being taken. - if ask.State == link.AskWaiting { - if q, err := link.ReadQueue(ctx, js, seat); err == nil { - if now, ok := q.Find(ask.ID); ok && now.State == link.AskWaiting { - ask.Seq = now.Seq - } - } - } + // The worker of the queue it waits in: the seat's own, or one of the two check queues' (novox/hq ADR 0406 §3). + worker := link.WorkerOf(seat, ask) switch ask.State { case link.AskWaiting: if taken, err := takenSince(js, worker, ask.Seq); err != nil { @@ -334,7 +302,6 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str "ends it there", ask.ID, started, ask.ID) } } - // Found by its id when it was moved into order since it was read (novox/hq issue 558). removed, err := link.DeleteAsk(ctx, js, seat, ask) if err != nil { return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+ diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 4a2e0552..bf64027e 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -413,10 +413,15 @@ func (a *verbArguments) commandLine() ([]string, error) { if err := need("members"); err != nil { return nil, err } + argv := []string{"delivery", "check", "--members", str("members")} if g := str("group"); g != "" { - return []string{"delivery", "check", "--group", g, "--members", str("members")}, nil + argv = []string{"delivery", "check", "--group", g, "--members", str("members")} } - return []string{"delivery", "check", "--members", str("members")}, nil + // Marked ready (novox/hq ADR 0406 §3): the check is queued before every other check. + if r := strings.ToLower(str("ready")); r != "" && r != "false" && r != "no" && r != "0" { + argv = append(argv, "--ready") + } + return argv, nil case "deliver": if err := need("plan"); err != nil { return nil, err diff --git a/cmd/mesh-controller/superseded.go b/cmd/mesh-controller/superseded.go deleted file mode 100644 index e3906713..00000000 --- a/cmd/mesh-controller/superseded.go +++ /dev/null @@ -1,157 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/link" -) - -// A new push to a pull request ends the check of its old head (novox/hq issue 726). -// -// A check judges one head; once the pull request has another, its verdict answers nobody and its run holds a build -// machine for minutes the new head waits behind. So when a check is asked for pull request N of a repository at a -// head, **every check of that pull request at another head ends**: a waiting one is dropped from the queue as -// `cancel` drops it, one in flight is ended on its machine as `kill` ends it. Each is said in the log. -// -// **Never anything else**: a try (`try`, asked for no pull request), a walk's build (no check), a delivery group's -// composed check (it answers several pull requests), another pull request's check, a dead ask (left for `clear -// --dead`), and a check of the same head — a recheck of that head is not another head. - -// **Only for the pull request's current head.** Announcements of a pull request are not ordered, one is heard again -// after a restart, and a recheck asks for the head it recorded: a check asked for a head the pull request no longer -// has must never end the check of the head it has. So the forge is asked first which head the pull request has; only -// when it names this one does anything end, and when it cannot be asked, or names another, nothing ends and that is -// said. - -// forgePullHead asks the forge, through its module's tools on the bus, which commit a pull request's head is. A -// variable so a test needs no forge. -var forgePullHead = func(ctx context.Context, js *broker.JetStream, owner, repo string, number int) (string, error) { - raw, _ := json.Marshal(map[string]any{"path": fmt.Sprintf("/repos/%s/%s/pulls/%d", owner, repo, number)}) - answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second) - if err != nil { - return "", err - } - if answer.Error != "" { - return "", errors.New(answer.Error) - } - var pull struct { - Result struct { - Head struct { - SHA string `json:"sha"` - } `json:"head"` - } `json:"result"` - } - if err := json.Unmarshal(answer.Result, &pull); err != nil { - return "", fmt.Errorf("the forge's answer about the pull request cannot be read: %w", err) - } - if pull.Result.Head.SHA == "" { - return "", errors.New("the forge named no head for the pull request") - } - return pull.Result.Head.SHA, nil -} - -// supersededChecks are the checks in the queue that a check of owner/repo#number at head ends: of the same pull -// request, at another head, waiting or in flight, and not the ask keep. -func supersededChecks(q link.Queue, owner, repo string, number int, head, keep string) []link.QueuedAsk { - if number <= 0 || head == "" { - return nil - } - var out []link.QueuedAsk - for _, a := range q.Asks { - c := a.Request.Check - switch { - case c == nil, link.IsTry(a.Request), c.Group != "", c.Number != number, a.ID == keep, - !strings.EqualFold(c.Owner, owner), !strings.EqualFold(c.Repo, repo), - a.State != link.AskWaiting && a.State != link.AskInFlight: - continue - } - ref := a.Request.Ref - if ref == "" { - ref = a.Ref - } - if ref == "" || ref == head || link.SameHead(ref, head) { - continue - } - out = append(out, a) - } - return out -} - -// endSupersededChecks ends the checks a check of p at its head supersedes, keep being the new check's own ask, and -// says each in the log. A waiting one is cancelled — nothing recorded: a check builds nothing, and the forge's status -// of the old head is no longer the pull request's — and one a machine builds is killed there. -func endSupersededChecks(ctx context.Context, js *broker.JetStream, seat string, p link.PullUpdated, keep string) []string { - if p.Number <= 0 || p.Commit == "" { - return nil - } - current, err := forgePullHead(ctx, js, p.Owner, p.Repo, p.Number) - if err != nil { - return []string{fmt.Sprintf("%s/%s#%d (%.8s): nothing ended — the forge could not say the pull request's "+ - "current head, so checks of its other heads keep running: %v", p.Owner, p.Repo, p.Number, p.Commit, err)} - } - if current != p.Commit && !link.SameHead(current, p.Commit) { - return []string{fmt.Sprintf("%s/%s#%d (%.8s): nothing ended — the pull request's head is %.12s now, not "+ - "this one", p.Owner, p.Repo, p.Number, p.Commit, current)} - } - q, err := link.ReadQueue(ctx, js, seat) - if err != nil { - return []string{fmt.Sprintf("%s/%s#%d (%.8s): could not read the %s queue for checks of its earlier heads: %v", - p.Owner, p.Repo, p.Number, p.Commit, seat, err)} - } - var said []string - for _, a := range supersededChecks(q, p.Owner, p.Repo, p.Number, p.Commit, keep) { - old := a.Request.Ref - if old == "" { - old = a.Ref - } - what := fmt.Sprintf("check of %s/%s#%d at %.12s", p.Owner, p.Repo, p.Number, old) - var err error - if a.State == link.AskInFlight && a.On != "" { - var answer link.Answer - answer, err = link.AskSeatTool(ctx, js.Conn(), seat, "kill", a.On, map[string]string{"id": a.ID}, killAnswer) - if err == nil && answer.Error != "" { - err = fmt.Errorf("%s refused: %s", a.On, answer.Error) - } - } else { - _, err = cancelAsk(ctx, js, nil, seat, a) - } - if err != nil { - said = append(said, fmt.Sprintf("not ended: %s (%s) — superseded by %.12s, but: %v", what, a.ID, p.Commit, err)) - continue - } - said = append(said, fmt.Sprintf("ended: %s — superseded by %.12s (%s, %s)", what, p.Commit, a.ID, a.State)) - } - return said -} - -// endingSupersededChecks ends them beside the announcement, so a kill waiting on its machine never holds the -// handler: on its own connection to the bus, within twice a kill's wait. A one-shot process (oneShot) ends them -// within the call instead, as its exit would cut them off. -func endingSupersededChecks(ctx context.Context, seat string, p link.PullUpdated, keep string, oneShot bool) { - end := func() { - ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*killAnswer) - defer cancel() - js, err := dialTheBus() - if err != nil { - fmt.Fprintf(os.Stderr, "%s/%s#%d (%.8s): could not reach the bus to end checks of its earlier heads: %v\n", - p.Owner, p.Repo, p.Number, p.Commit, err) - return - } - defer js.Close() - for _, line := range endSupersededChecks(ctx, js, seat, p, keep) { - fmt.Println(line) - } - } - if oneShot { - end() - return - } - go end() -} diff --git a/cmd/mesh-controller/superseded_test.go b/cmd/mesh-controller/superseded_test.go deleted file mode 100644 index 104fb78b..00000000 --- a/cmd/mesh-controller/superseded_test.go +++ /dev/null @@ -1,192 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "strings" - "sync" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/link" -) - -// A new push to a pull request ends the check of its old head (novox/hq issue 726): the waiting one is dropped from -// the queue, the one in flight is killed on its machine, and each is said. A try, a walk's build, another pull -// request's check, a group's check and a recheck of the new head are left alone. On the bus alone: no store. -func TestANewHeadEndsTheChecksOfTheOldOnes(t *testing.T) { - js, _ := aBuildQueue(t) - ctx := t.Context() - const ( - oldHead = "1111111111111111111111111111111111111111" - olderHead = "2222222222222222222222222222222222222222" - newHead = "3333333333333333333333333333333333333333" - ) - publish := func(r link.BuildRequest) { - t.Helper() - body, _ := json.Marshal(r) - if _, err := js.Context().Publish(link.BuildWork(), body); err != nil { - t.Fatal(err) - } - } - check := func(id string, number int, head string) link.BuildRequest { - return link.BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head, - Check: &link.CheckRequest{Owner: "novox", Repo: "a", Number: number}} - } - // In flight first: taken by a holder, started on ace. - publish(check("check-running", 7, olderHead)) - worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) - sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil { - t.Fatal(err) - } - started, _ := json.Marshal(link.BuildStart{ID: "check-running", On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) - if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil { - t.Fatal(err) - } - publish(check("check-old", 7, oldHead)) - publish(check("check-other-pull", 8, oldHead)) - publish(check("check-recheck", 7, newHead)) - publish(check("check-new", 7, newHead)) - try := check("try-old", 7, oldHead) - try.Check.Number, try.Check.Try = 0, true - publish(try) - group := check("group-old", 7, oldHead) - group.Check.Group = "g1" - publish(group) - publish(link.BuildRequest{ID: "walk-build", Repository: "https://forge.example/novox/a.git", Ref: oldHead}) - - var mu sync.Mutex - var killed []string - stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", map[string]link.ToolHandler{ - "kill": func(_ context.Context, raw json.RawMessage) (any, error) { - var args struct{ ID string } - _ = json.Unmarshal(raw, &args) - mu.Lock() - killed = append(killed, args.ID) - mu.Unlock() - return map[string]any{"said": "killed " + args.ID}, nil - }}, nil) - if err != nil { - t.Fatal(err) - } - defer stop() - - forgeSays(t, newHead, nil) - said := endSupersededChecks(ctx, js, link.TheBuildMachine, - link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: newHead}, "check-new") - - mu.Lock() - if len(killed) != 1 || killed[0] != "check-running" { - t.Errorf("the holder was asked to kill %v, want only the in-flight check of the older head", killed) - } - mu.Unlock() - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - if _, found := q.Find("check-old"); found { - t.Error("the waiting check of the old head is still in the queue") - } - for _, id := range []string{"check-other-pull", "check-recheck", "check-new", "try-old", "group-old", "walk-build"} { - if _, found := q.Find(id); !found { - t.Errorf("%s was ended; only checks of the same pull request at another head end", id) - } - } - text := strings.Join(said, "\n") - for _, want := range []string{ - "ended: check of novox/a#7 at " + oldHead[:12] + " — superseded by " + newHead[:12], - "ended: check of novox/a#7 at " + olderHead[:12] + " — superseded by " + newHead[:12], - } { - if !strings.Contains(text, want) { - t.Errorf("the log does not say %q:\n%s", want, text) - } - } - if len(said) != 2 { - t.Errorf("said %d lines, want 2:\n%s", len(said), text) - } -} - -// Which asks a check supersedes, read from the queue alone. -func TestSupersededChecksAreOnlyThePullRequestsOtherHeads(t *testing.T) { - ask := func(id, state, ref string, c *link.CheckRequest) link.QueuedAsk { - return link.QueuedAsk{ID: id, State: state, Ref: ref, Request: link.BuildRequest{ID: id, Ref: ref, Check: c}} - } - pr := func(n int) *link.CheckRequest { return &link.CheckRequest{Owner: "Novox", Repo: "A", Number: n} } - head := "abcdefabcdefabcdef0123456789abcdef012345" - q := link.Queue{Asks: []link.QueuedAsk{ - ask("old", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(7)), - ask("short-same", link.AskWaiting, head[:12], pr(7)), - ask("dead", link.AskDead, "0123456789ab0123456789ab0123456789ab0123", pr(7)), - ask("other", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(8)), - ask("build", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", nil), - ask("keep", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(7)), - }} - got := supersededChecks(q, "novox", "a", 7, head, "keep") - if len(got) != 1 || got[0].ID != "old" { - t.Fatalf("superseded %+v, want only old", got) - } - if got := supersededChecks(q, "novox", "a", 0, head, ""); got != nil { - t.Errorf("a check of no pull request superseded %+v", got) - } -} - -// forgeSays makes the forge name head as every pull request's current head, or fail with err. -func forgeSays(t *testing.T, head string, err error) { - t.Helper() - was := forgePullHead - forgePullHead = func(context.Context, *broker.JetStream, string, string, int) (string, error) { return head, err } - t.Cleanup(func() { forgePullHead = was }) -} - -// An older head heard after a newer one — an announcement out of order, one heard again after a restart, a recheck -// of a recorded head — ends nothing: the forge says the pull request's head is the newer one. Nor does a check whose -// pull request's head the forge cannot say. -func TestAnOlderHeadHeardLateEndsNothing(t *testing.T) { - js, _ := aBuildQueue(t) - ctx := t.Context() - const ( - oldHead = "1111111111111111111111111111111111111111" - newHead = "3333333333333333333333333333333333333333" - ) - for _, id := range []string{"check-new", "check-old-late"} { - head := newHead - if id == "check-old-late" { - head = oldHead - } - body, _ := json.Marshal(link.BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head, - Check: &link.CheckRequest{Owner: "novox", Repo: "a", Number: 7}}) - if _, err := js.Context().Publish(link.BuildWork(), body); err != nil { - t.Fatal(err) - } - } - late := link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: oldHead} - forgeSays(t, newHead, nil) - said := endSupersededChecks(ctx, js, link.TheBuildMachine, late, "check-old-late") - if len(said) != 1 || !strings.Contains(said[0], "nothing ended") { - t.Errorf("an older head heard late said %q", said) - } - forgeSays(t, "", errors.New("nothing serves gitea.gitea_api")) - said = endSupersededChecks(ctx, js, link.TheBuildMachine, - link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: newHead}, "check-new") - if len(said) != 1 || !strings.Contains(said[0], "nothing ended") { - t.Errorf("a head the forge could not confirm said %q", said) - } - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - for _, id := range []string{"check-new", "check-old-late"} { - if _, found := q.Find(id); !found { - t.Errorf("%s was ended", id) - } - } -} diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 6927bfe0..fccf15e2 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -759,9 +759,6 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li // and the answer chosen is performed on its warrant. startAsking(watching, open, server, bus.Conn, keeper) go forgettingOldHeals(watching, open.inventory) - // And the build queue's order (novox/hq issue 558): a check that has waited its while goes ahead of the - // deliveries asked after it even when nothing new is asked. - go keepingTheQueueInOrder(watching, open, server.JetStream()) fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+ "and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery, broker.ConditionsBucket, conditions.Seat) diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index a55b1da5..48b62765 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -160,7 +160,9 @@ var ControllerVerbs = []Verb{ "the verdict is said as `checked` with the group's id. With one member and no group, that head is checked " + "again as its pull request is.", Input: schema(map[string]string{"group": "the group's id (none for one head checked again)", - "members": "the members, as JSON, as delivery-order takes them"}, []string{"members"})}, + "members": "the members, as JSON, as delivery-order takes them", + "ready": "\"true\" when the pull request is marked ready: its check is queued before every other check (novox/hq ADR 0406 §3)"}, + []string{"members"})}, {Name: "deliver", Description: "The delivery's word that a walk waiting for it may start (novox/hq ADR 0239): its " + "first tier is asked at the controller's next pass. Refused for a walk that waits for nobody.", Input: schema(map[string]string{"plan": "the walk's id", "why": "what lets it go: its turn, a person's release"}, diff --git a/internal/link/build.go b/internal/link/build.go index acc03bfa..e698f9da 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -144,6 +144,12 @@ type CheckRequest struct { Repo string `json:"repo"` Number int `json:"number,omitempty"` Base string `json:"base,omitempty"` + // Trunk is the base branch's head the check is asked against, as the forge announced it with the head (novox/hq ADR + // 0406 §4): with the head, what the pull request's key says the current check is for, and what a waiting request + // is dropped by and a running check stopped by when the key moves. + Trunk string `json:"trunk,omitempty"` + // Ready says the pull request is marked ready (ADR 0406 §3): its check is queued before every other check. + Ready bool `json:"ready,omitempty"` // Paths are the files the pull request changes, for the width of its rebuild. Paths []string `json:"paths,omitempty"` // Beside are the repositories the check reads next to this one, each cloned at the ref given — the diff --git a/internal/link/builds_nats.go b/internal/link/builds_nats.go index d3147bda..ce33dbec 100644 --- a/internal/link/builds_nats.go +++ b/internal/link/builds_nats.go @@ -76,11 +76,9 @@ func (b *natsBuilds) Ask(ctx context.Context, request BuildRequest) error { } publish, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil { + if _, err := b.js.Context().Publish(WorkSubjectOf(b.role(), request), body, nats.Context(publish)); err != nil { return fmt.Errorf("cannot submit a build: %w", err) } - // The acting controller puts it in order (novox/hq issue 558); this only says an ask joined. - signalAsked() return nil } @@ -112,11 +110,9 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest, // be assumed, because nothing else will ever say so. publish, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil { + if _, err := b.js.Context().Publish(WorkSubjectOf(b.role(), request), body, nats.Context(publish)); err != nil { return BuildResult{}, fmt.Errorf("%w: cannot submit a build: %w", ErrNotHandedOver, err) } - // The acting controller puts it in order (novox/hq issue 558); this only says an ask joined. - signalAsked() waiting, cancelWait := context.WithTimeout(ctx, wait) defer cancelWait() @@ -162,10 +158,29 @@ type natsMachine struct { js *broker.JetStream on string seat string - sub *nats.Subscription + // subs are the queues this machine takes from, in the order it takes from them: the seat's own, where a delivery + // build waits, then the checks of pull requests marked ready, then every other check (novox/hq ADR 0406 §3). + subs []*takeQueue opts MachineOptions } +// takeQueue is one queue a machine pulls from. +type takeQueue struct { + sub *nats.Subscription + // kind is what its asks are: a build, a ready check, or another check. + kind string + // worker is the consumer bound; bound is false until a bind succeeded, for a queue the mesh may not have made yet. + worker broker.Consumer + bound bool +} + +// The kinds of queue a machine takes from, in the order it takes from them. +const ( + queueBuilds = "builds" + queueReady = "ready" + queueOther = "other" +) + // MachineOptions is what a holder tells the taking loop about itself (novox/hq ADR 0219). type MachineOptions struct { // Paused is asked before every fetch: while it says so, nothing new is taken, and a build @@ -207,47 +222,56 @@ const ( ) func (m *natsMachine) Close() { - if m.sub != nil { - _ = m.sub.Unsubscribe() + for _, q := range m.subs { + if q.sub != nil { + _ = q.sub.Unsubscribe() + } } } -// Take binds to the role's worker and pulls one request at a time, handing each over. +// Take binds to the role's workers and pulls one request at a time, handing each over. // -// **Bound, never created.** The work queue and the worker on it are the controller's to define +// **Bound, never created.** The work queues and the workers on them are the controller's to define // (design 25 §3), and a build machine reaches no part of the JetStream API — so a missing one is said // as the mesh's to answer rather than quietly created with whatever this client defaults to. // // **Pulled, one at a time, by whichever holder is free** (novox/hq ADR 0190). Every machine holding -// the role binds this same worker; a machine asks for the next request only when it has finished +// the role binds these same workers; a machine asks for the next request only when it has finished // the last, so a slow machine never holds an ask an idle one could take, and a machine that took // five at once would run five container builds against one runtime and finish all of them slower // than the first. +// +// **In this order** (novox/hq ADR 0406 §3): a delivery build, then a check of a pull request marked ready, then any +// other check, each queue tried without waiting; when all three are empty the first is waited on for a second at most +// and the three are tried again, so a delivery build is never much more than a second behind and nothing spins. func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error { worker, found := broker.HolderConsumerFor(m.on, "build-agent", broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}}) if !found { return fmt.Errorf("%s accepts no work, so there is nothing for this machine to take", m.seat) } - - // **The consumer's own filter, not the one subject this machine cares about.** The client checks - // what is asked for against the consumer's filter and refuses anything that is not the same — - // "subject does not match consumer" — so subscribing `…accept.build` against a consumer filtered - // on `…accept.>` is rejected even though it is narrower. Learned twice now, on two different - // consumers, which is why it is written down here. - filter := worker.Filters[0] - sub, err := m.js.Context().PullSubscribe(filter, worker.Name, - nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) - if err != nil { + m.subs = []*takeQueue{{kind: queueBuilds, worker: worker}} + if m.seat == broker.TheBuildSeat { + for i, w := range broker.CheckWorkers() { + kind := queueReady + if i == 1 { + kind = queueOther + } + m.subs = append(m.subs, &takeQueue{kind: kind, worker: w}) + } + } + // The seat's own queue is bound at once and its absence is this machine's error, as it always was. The check queues + // are bound when they are there: a controller that has not made them yet is no reason to take no build. + if err := m.bind(m.subs[0]); err != nil { return fmt.Errorf( "this machine cannot take work from %s: %w. The mesh creates that queue and this "+ "machine's worker on it, and a build machine may not create one itself — so this is "+ "the mesh's to answer, not this machine's", m.seat, err) } - m.sub = sub if m.opts.Taking != nil { m.opts.Taking() } + var bindAgain time.Time for { if ctx.Err() != nil { @@ -264,29 +288,31 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) } continue } - // One, and wait a while for it; an empty queue is a timeout, which is the normal state of a - // machine with nothing to build, and is asked again. - // Asked for a few seconds at a time when the holder can be paused, so a pause reaches a pull - // already waiting within that, rather than when the client's own wait runs out. - asking, endAsking := ctx, func() {} - if m.opts.Paused != nil { - asking, endAsking = context.WithTimeout(ctx, pausedFetch) + if time.Now().After(bindAgain) { + for _, q := range m.subs[1:] { + if !q.bound { + if err := m.bind(q); err != nil { + fmt.Fprintf(os.Stderr, "the %s check queue is not there to take from yet: %v\n", q.kind, err) + } + } + } + bindAgain = time.Now().Add(30 * time.Second) } - fetched, err := sub.Fetch(1, nats.Context(asking)) - endAsking() + msg, q, err := m.next(ctx) switch { case ctx.Err() != nil: // Ours ended: the machine is being stopped. return nil - case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout): + case msg != nil: + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout), err == nil: // **An empty queue, not the end.** A fetch on a context without a deadline waits the // client's own while and then says the deadline passed — the client's, not ours. Read // as "stop", every idle build machine exited clean every half minute and was started // again by its supervisor, which looked like a crash loop with nothing in the log to // say why (2026-10-03, the first build agents). Asked again. continue - case err != nil: - if sub.IsValid() { + default: + if m.subs[0].sub.IsValid() { // A transient fault in asking — a reconnect, a slow server — is asked past rather // than ending the machine; one that outlasts the ack wait redelivers nothing lost. time.Sleep(time.Second) @@ -294,49 +320,130 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) } return fmt.Errorf("the bus stopped delivering build work: %w", err) } - for _, msg := range fetched { - // **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for - // the one that arrived in that moment. Handed back at once for another holder — counted as - // a delivery, which a pause landing exactly then costs and nothing else does. - if m.opts.Paused != nil && m.opts.Paused() { - _ = msg.Nak() + // **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for + // the one that arrived in that moment. Handed back at once for another holder — counted as + // a delivery, which a pause landing exactly then costs and nothing else does. + if m.opts.Paused != nil && m.opts.Paused() { + _ = msg.Nak() + continue + } + var request BuildRequest + if err := json.Unmarshal(msg.Data, &request); err != nil { + // Unreadable: terminated rather than retried, because the next attempt reads the same + // bytes. Nobody waiting hears an answer, which is right — there was no request. + _ = msg.Term() + continue + } + // **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the + // id into the seat's cancelled set before deleting the ask, so one taken in between is + // ended here, terminated rather than redelivered, and its outcome said as failed — never + // built. A set that cannot be read is said and the ask built: a cancel is a person's + // exception, and a holder that refused every build while its grant was missing would + // stop the mesh building for it. + build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)} + if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil { + fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err) + } else if cancelled { + endCancelled(ctx, build) + continue + } + // **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a + // head and a trunk commit its pull request's key already says were judged is acknowledged and not run. + if q.kind == queueOther { + if why := m.judgedAlready(ctx, request); why != "" { + fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why) + _ = msg.Ack() continue } - var request BuildRequest - if err := json.Unmarshal(msg.Data, &request); err != nil { - // Unreadable: terminated rather than retried, because the next attempt reads the same - // bytes. Nobody waiting hears an answer, which is right — there was no request. - _ = msg.Term() - continue - } - // **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the - // id into the seat's cancelled set before deleting the ask, so one taken in between is - // ended here, terminated rather than redelivered, and its outcome said as failed — never - // built. A set that cannot be read is said and the ask built: a cancel is a person's - // exception, and a holder that refused every build while its grant was missing would - // stop the mesh building for it. - build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)} - if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil { - fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err) - } else if cancelled { - endCancelled(ctx, build) - continue - } - // **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause - // landing then is. - if m.opts.Claim != nil && !m.opts.Claim() { - _ = msg.Nak() - continue - } - // A build outlives the acknowledgement window many times over; said while it runs, - // as the controller says it for its own long handlers, so the server neither hands - // the ask to a second machine nor counts the wait against its deliveries. - working := make(chan struct{}) - go stillWorking(msg, working, heartbeatWhileWorking) - do(ctx, build) - close(working) + } + // **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause + // landing then is. + if m.opts.Claim != nil && !m.opts.Claim() { + _ = msg.Nak() + continue + } + // A build outlives the acknowledgement window many times over; said while it runs, + // as the controller says it for its own long handlers, so the server neither hands + // the ask to a second machine nor counts the wait against its deliveries. + working := make(chan struct{}) + go stillWorking(msg, working, heartbeatWhileWorking) + do(ctx, build) + close(working) + } +} + +// bind binds a queue's worker, which exists and is not created here. +// +// **The consumer's own filter, not the one subject this machine cares about.** The client checks what is asked for +// against the consumer's filter and refuses anything that is not the same — "subject does not match consumer" — so +// subscribing `…accept.build` against a consumer filtered on `…accept.>` is rejected even though it is narrower. Learned +// twice now, on two different consumers, which is why it is written down here. +func (m *natsMachine) bind(q *takeQueue) error { + sub, err := m.js.Context().PullSubscribe(q.worker.Filters[0], q.worker.Name, + nats.Bind(q.worker.Stream, q.worker.Name), nats.ManualAck()) + if err != nil { + return err + } + q.sub, q.bound = sub, true + return nil +} + +// next takes the next request in the order of the queues: each tried for a moment, and when all are empty the first waited +// on for a second. Nothing, with no error, when there was nothing. +func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) { + var last error + for _, q := range m.subs { + if !q.bound { + continue + } + trying, end := context.WithTimeout(ctx, checkTakeWait) + fetched, err := q.sub.Fetch(1, nats.Context(trying)) + end() + if len(fetched) > 0 { + return fetched[0], q, nil + } + if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) && + !errors.Is(err, context.Canceled) { + last = err + } + if ctx.Err() != nil { + return nil, nil, nil } } + if last != nil { + return nil, nil, last + } + waiting, end := context.WithTimeout(ctx, takeIdleWait) + defer end() + fetched, err := m.subs[0].sub.Fetch(1, nats.Context(waiting)) + if len(fetched) > 0 { + return fetched[0], m.subs[0], nil + } + if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) && + !errors.Is(err, context.Canceled) { + return nil, nil, err + } + return nil, nil, nil +} + +// judgedAlready says why a check just taken from the other checks' queue need not run, or nothing: its pull request's key +// holds its head and trunk commit and a verdict. A key that cannot be read, or none, runs the check (the key is +// mesh-delivery's, and a check wrongly dropped is worse than one run twice). +func (m *natsMachine) judgedAlready(ctx context.Context, r BuildRequest) string { + c := r.Check + if c == nil || c.Group != "" || c.Try || c.Number <= 0 { + return "" + } + key, found, err := ReadPullKey(ctx, m.js.Conn(), c.Owner, c.Repo, c.Number) + if err != nil { + fmt.Fprintf(os.Stderr, "could not read the key of %s/%s#%d, so its check runs: %v\n", c.Owner, c.Repo, c.Number, err) + return "" + } + if found && key.Judged(r.Ref, c.Trunk) { + return fmt.Sprintf("%s/%s#%d is judged already at %.12s on %.12s (%s)", c.Owner, c.Repo, c.Number, r.Ref, c.Trunk, + key.Verdict) + } + return "" } type natsBuild struct { @@ -398,6 +505,12 @@ func (b *natsBuild) Announce(ctx context.Context, result BuildResult) error { func (b *natsBuild) Done() error { return b.msg.Ack() } +// WatchPull has moved called once the pull request's key moves on from what this check was asked for; the stop ends the +// watch (see WatchPull). A request that is no pull request's check watches nothing. +func (b *natsBuild) WatchPull(ctx context.Context, moved func(PullKey)) (func(), error) { + return WatchPull(ctx, b.js.Conn(), b.request, moved) +} + // Began publishes that this machine has taken the build, into the stream like the outcome, so a // reader that asks afterwards sees when it started as well as how it ended. func (b *natsBuild) Began(ctx context.Context) error { diff --git a/internal/link/builds_nats_test.go b/internal/link/builds_nats_test.go index ac8a0eb1..cc5931fb 100644 --- a/internal/link/builds_nats_test.go +++ b/internal/link/builds_nats_test.go @@ -44,6 +44,12 @@ func aBusWithTheBuildRole(t *testing.T) *broker.JetStream { }); err != nil { t.Fatal(err) } + // And the two check queues' workers, which the controller makes beside the streams (novox/hq ADR 0406 §3). + for _, w := range broker.CheckWorkers() { + if err := js.EnsureConsumer(w); err != nil { + t.Fatal(err) + } + } clean := func() { _ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT") for _, s := range broker.MeshStreams() { diff --git a/internal/link/checks_nats.go b/internal/link/checks_nats.go new file mode 100644 index 00000000..f0d90cf2 --- /dev/null +++ b/internal/link/checks_nats.go @@ -0,0 +1,170 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" +) + +// A pull request's check is queued by the pull request (novox/hq ADR 0406 §3, §4, issue 974). + +// SupersededByKey is the outcome of a check that stopped because its pull request's key moved on: its head or its trunk +// commit is no longer the one it was asked for. Said as the check's failure, as a kill is; nothing is asked of the forge. +const SupersededByKey = "superseded: the pull request's key moved on" + +// unsafeToken is what may not stand in a subject token or a key. +var unsafeToken = strings.NewReplacer(".", "_", "*", "_", ">", "_", " ", "_", "/", "_", "\t", "_", "\n", "_") + +// tokenOf is a name as one subject token: any dot, wildcard or space replaced by an underscore. +func tokenOf(name string) string { + if name == "" { + return "_" + } + return unsafeToken.Replace(name) +} + +// PullKeyOf is a pull request's key in mesh-delivery's bucket, and the last tokens of the subject its check is asked on: +// `..`, a dot in a name made an underscore. +func PullKeyOf(owner, repo string, number int) string { + return tokenOf(owner) + "." + tokenOf(repo) + "." + strconv.Itoa(number) +} + +// CheckSubjectOf is the subject a check is asked on, and whether it has one of its own: a check of a pull request on +// the ready queue or the other, by the pull request; a delivery group's composed check by `group.`; any other +// check (a branch tried before its pull request) by its own id, so it replaces nothing. +func CheckSubjectOf(seat string, r BuildRequest) (string, bool) { + c := r.Check + if seat != broker.TheBuildSeat || c == nil { + return "", false + } + prefix := broker.CheckSubjectPrefix(seat, c.Ready) + switch { + case c.Group != "": + return prefix + "group." + tokenOf(c.Group), true + case c.Try || c.Number <= 0: + return prefix + "try." + tokenOf(r.ID), true + } + return prefix + PullKeyOf(c.Owner, c.Repo, c.Number), true +} + +// WorkSubjectOf is where an ask is put: a check on the build seat on its own queue, anything else on the seat's. +func WorkSubjectOf(seat string, r BuildRequest) string { + if subject, ok := CheckSubjectOf(seat, r); ok { + return subject + } + return BuildWorkOf(seat) +} + +// PullKey is what mesh-delivery keeps of a pull request (ADR 0406 §4): its head, the trunk commit its current check is +// asked against, and its verdict once judged. +type PullKey struct { + Head string `json:"head"` + Trunk string `json:"trunk"` + Verdict string `json:"verdict,omitempty"` +} + +// sameCommit says two spellings name one commit; either may be short, but not under twelve characters. +func sameCommit(a, b string) bool { + return a == b || SameHead(a, b) +} + +// Moved says a key no longer holds what a check was asked for: its head or its trunk commit differs. +func (k PullKey) Moved(head, trunk string) bool { + return !sameCommit(k.Head, head) || (trunk != "" && k.Trunk != "" && !sameCommit(k.Trunk, trunk)) +} + +// Judged says a key holds the verdict of the check of this head against this trunk commit. +func (k PullKey) Judged(head, trunk string) bool { + return k.Verdict != "" && sameCommit(k.Head, head) && sameCommit(k.Trunk, trunk) && trunk != "" +} + +// ReadPullKey reads a pull request's key from mesh-delivery's bucket: false when it has none. +func ReadPullKey(ctx context.Context, conn *nats.Conn, owner, repo string, number int) (PullKey, bool, error) { + api, err := jetstream.New(conn) + if err != nil { + return PullKey{}, false, err + } + kv, err := api.KeyValue(ctx, broker.PullsBucket) + if err != nil { + return PullKey{}, false, fmt.Errorf("the pull requests' keys: %w", err) + } + e, err := kv.Get(ctx, PullKeyOf(owner, repo, number)) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return PullKey{}, false, nil + } + if err != nil { + return PullKey{}, false, err + } + var key PullKey + if err := json.Unmarshal(e.Value(), &key); err != nil { + return PullKey{}, false, fmt.Errorf("the key of %s/%s#%d is not a pull request's: %w", owner, repo, number, err) + } + return key, true, nil +} + +// WatchPull calls moved once when the pull request's key moves on from what this check was asked for — its head or its +// trunk commit differs — and not for a key that only gains its verdict. It hears updates, not the key as it stands: the +// key and the request are written for one announcement by two writers, so a key found older than the request is not +// a move. The check is then stopped by whoever asked for it, without asking the forge (novox/hq issue 726). The stop +// ends the watch. +func WatchPull(ctx context.Context, conn *nats.Conn, r BuildRequest, moved func(PullKey)) (stop func(), err error) { + c := r.Check + if c == nil || c.Group != "" || c.Try || c.Number <= 0 { + return func() {}, nil + } + api, err := jetstream.New(conn) + if err != nil { + return nil, err + } + kv, err := api.KeyValue(ctx, broker.PullsBucket) + if err != nil { + return nil, fmt.Errorf("the pull requests' keys: %w", err) + } + watching, end := context.WithCancel(ctx) + w, err := kv.Watch(watching, PullKeyOf(c.Owner, c.Repo, c.Number), jetstream.UpdatesOnly()) + if err != nil { + end() + return nil, err + } + go func() { + defer w.Stop() + for { + select { + case <-watching.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + var key PullKey + if json.Unmarshal(e.Value(), &key) != nil { + continue + } + if key.Moved(r.Ref, c.Trunk) { + moved(key) + return + } + } + } + }() + return end, nil +} + +// checkTakeWait is how long a taking loop waits on one queue for an ask that is not there, and takeIdleWait how long it +// waits on the first when all three are empty. +const ( + checkTakeWait = 30 * time.Millisecond + takeIdleWait = time.Second +) diff --git a/internal/link/checks_nats_test.go b/internal/link/checks_nats_test.go new file mode 100644 index 00000000..9326ade6 --- /dev/null +++ b/internal/link/checks_nats_test.go @@ -0,0 +1,309 @@ +package link + +import ( + "context" + "encoding/json" + "reflect" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" +) + +// The checks of pull requests are queued by the pull request (novox/hq ADR 0406 §3, §4, issue 974). + +func aPullCheck(id string, number int, head, trunk string, ready bool) BuildRequest { + return BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head, + Check: &CheckRequest{Owner: "novox", Repo: "a", Number: number, Trunk: trunk, Ready: ready}} +} + +// The subject a check is asked on says its queue and its pull request: ready or other, `..` with +// a dot in a name made an underscore, `group.` for a group's, the id for a try; a build is on the seat's own queue. +func TestACheckIsAskedOnItsQueueBySubject(t *testing.T) { + for _, c := range []struct { + r BuildRequest + want string + }{ + {aPullCheck("b1", 7, "h", "t", false), "mesh.seat.node-build-agent.check.other.novox.a.7"}, + {aPullCheck("b2", 7, "h", "t", true), "mesh.seat.node-build-agent.check.ready.novox.a.7"}, + {BuildRequest{ID: "b3", Check: &CheckRequest{Owner: "no.vox", Repo: "a.b", Number: 12}}, "mesh.seat.node-build-agent.check.other.no_vox.a_b.12"}, + {BuildRequest{ID: "b4", Check: &CheckRequest{Owner: "novox", Repo: "a", Group: "g.1", Ready: true}}, "mesh.seat.node-build-agent.check.ready.group.g_1"}, + {BuildRequest{ID: "b5", Check: &CheckRequest{Owner: "novox", Repo: "a", Try: true}}, "mesh.seat.node-build-agent.check.other.try.b5"}, + {BuildRequest{ID: "b6", Repository: "/r"}, "mesh.seat.node-build-agent.accept.build"}, + {BuildRequest{ID: "b7", Repository: "/r", DryRun: true}, "mesh.seat.node-build-agent.accept.build"}, + } { + if got := WorkSubjectOf(TheBuildMachine, c.r); got != c.want { + t.Errorf("%s is asked on %s; want %s", c.r.ID, got, c.want) + } + } + // The retired role has no check queues: its checks wait with its builds. + if got := WorkSubjectOf(TheBuildMachineBefore, aPullCheck("b8", 7, "h", "t", true)); got != "mesh.seat.mesh-build-machine.accept.build" { + t.Errorf("a check of the retired role is asked on %s", got) + } +} + +func queueOf(t *testing.T, js *broker.JetStream) Queue { + t.Helper() + q, err := ReadQueue(context.Background(), js, TheBuildMachine) + if err != nil { + t.Fatal(err) + } + return q +} + +// A newer request for a pull request replaces the one still waiting, by the stream's rule, and each open pull request is +// left with one queued check (acceptance 5). +func TestANewerCheckRequestReplacesOneStillWaiting(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx := t.Context() + asks := BuildsOn(js, TheBuildMachine) + for _, r := range []BuildRequest{ + aPullCheck("check-7-a", 7, "head-a-0000000", "trunk-0-00000", false), + aPullCheck("check-8-a", 8, "head-b-0000000", "trunk-0-00000", false), + aPullCheck("check-7-b", 7, "head-a-0000000", "trunk-1-00000", false), // a merge moved the trunk + aPullCheck("check-8-b", 8, "head-b-0000000", "trunk-1-00000", false), + aPullCheck("check-7-c", 7, "head-a-0000000", "trunk-2-00000", false), // and a second merge within the minute + } { + if err := asks.Ask(ctx, r); err != nil { + t.Fatal(err) + } + } + var got []string + for _, a := range queueOf(t, js).Asks { + if a.State != AskWaiting { + t.Errorf("%s is %s", a.ID, a.State) + } + got = append(got, a.ID) + } + if want := []string{"check-8-b", "check-7-c"}; !reflect.DeepEqual(got, want) { + t.Fatalf("queued %v; want one check a pull request, the newest: %v", got, want) + } +} + +// A delivery build is taken before a check of a ready pull request, that before any other check, and the others in the +// order they were asked. +func TestADeliveryBuildIsTakenBeforeAReadyCheckAndThatBeforeAnyOther(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + asks := BuildsOn(js, TheBuildMachine) + for _, r := range []BuildRequest{ + aPullCheck("other-1", 1, "h1", "t", false), + aPullCheck("other-2", 2, "h2", "t", false), + aPullCheck("ready-3", 3, "h3", "t", true), + {ID: "delivery-4", Repository: "/d"}, + aPullCheck("other-5", 5, "h5", "t", false), + aPullCheck("ready-6", 6, "h6", "t", true), + } { + if err := asks.Ask(ctx, r); err != nil { + t.Fatal(err) + } + } + took := make(chan string, 8) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + var order []string + for len(order) < 6 { + select { + case id := <-took: + order = append(order, id) + case <-time.After(15 * time.Second): + t.Fatalf("taken so far: %v", order) + } + } + if want := []string{"delivery-4", "ready-3", "ready-6", "other-1", "other-2", "other-5"}; !reflect.DeepEqual(order, want) { + t.Fatalf("taken as %v; want %v", order, want) + } +} + +// aKeyBucket makes mesh-delivery's bucket of pull requests' keys, which the bus here has not been given by that module. +func aKeyBucket(t *testing.T, js *broker.JetStream) jetstream.KeyValue { + t.Helper() + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + kv, err := api.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: broker.PullsBucket}) + if err != nil { + t.Fatal(err) + } + return kv +} + +func putKey(t *testing.T, kv jetstream.KeyValue, number int, k PullKey) { + t.Helper() + body, _ := json.Marshal(k) + if _, err := kv.Put(t.Context(), PullKeyOf("novox", "a", number), body); err != nil { + t.Fatal(err) + } +} + +// A request left on the third queue for a head and a trunk commit its key says were judged is acknowledged and dropped when a +// builder takes it; one for a head the key does not say judged is run. +func TestARequestLeftBehindForAJudgedHeadIsDroppedWhenTaken(t *testing.T) { + js := aBusWithTheBuildRole(t) + kv := aKeyBucket(t, js) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"}) + putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-1-000000"}) // asked, not judged + putKey(t, kv, 9, PullKey{Head: "head-9-000000000", Trunk: "trunk-2-000000", Verdict: "pass"}) // judged against another trunk + asks := BuildsOn(js, TheBuildMachine) + for _, r := range []BuildRequest{ + aPullCheck("left-7", 7, "head-7-000000000", "trunk-1-000000", false), + aPullCheck("fresh-8", 8, "head-8-000000000", "trunk-1-000000", false), + aPullCheck("moved-9", 9, "head-9-000000000", "trunk-1-000000", false), + } { + if err := asks.Ask(ctx, r); err != nil { + t.Fatal(err) + } + } + took := make(chan string, 8) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + var ran []string + for len(ran) < 2 { + select { + case id := <-took: + ran = append(ran, id) + case <-time.After(15 * time.Second): + t.Fatalf("run so far: %v", ran) + } + } + if want := []string{"fresh-8", "moved-9"}; !reflect.DeepEqual(ran, want) { + t.Fatalf("run %v; want %v, the judged one dropped", ran, want) + } + time.Sleep(300 * time.Millisecond) + if n := len(queueOf(t, js).Asks); n != 0 { + t.Errorf("%d ask(s) are left in the queues; the dropped one was to be acknowledged", n) + } +} + +// A running check stops when its key's head moves or its trunk commit does, not when the key only gains its verdict, and a +// group's check, which a member's merge in the group's order writes nothing to the key of, is not stopped at all. +func TestARunningCheckStopsWhenItsKeysHeadMovesAndAGroupsDoesNot(t *testing.T) { + js := aBusWithTheBuildRole(t) + kv := aKeyBucket(t, js) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + moved := make(chan PullKey, 4) + req := aPullCheck("check-7", 7, "head-7-000000000", "trunk-1-000000", false) + end, err := WatchPull(ctx, js.Conn(), req, func(k PullKey) { moved <- k }) + if err != nil { + t.Fatal(err) + } + defer end() + putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000"}) + putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"}) + select { + case k := <-moved: + t.Fatalf("stopped by a key that moved nothing: %+v", k) + case <-time.After(500 * time.Millisecond): + } + putKey(t, kv, 7, PullKey{Head: "head-7-111111111", Trunk: "trunk-1-000000"}) + select { + case k := <-moved: + if k.Head != "head-7-111111111" { + t.Errorf("stopped by %+v", k) + } + case <-time.After(5 * time.Second): + t.Fatal("a key whose head moved did not stop the check") + } + + // And the trunk commit. + req2 := aPullCheck("check-8", 8, "head-8-000000000", "trunk-1-000000", false) + end2, err := WatchPull(ctx, js.Conn(), req2, func(k PullKey) { moved <- k }) + if err != nil { + t.Fatal(err) + } + defer end2() + putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-2-000000"}) + select { + case <-moved: + case <-time.After(5 * time.Second): + t.Fatal("a key whose trunk commit moved did not stop the check") + } + + // A group's check watches nothing: a sibling's merge in the group's order writes nothing to a key, and a key it does + // have is the pull request's own. + group := BuildRequest{ID: "group-1", Ref: "head-7-000000000", + Check: &CheckRequest{Owner: "novox", Repo: "a", Number: 7, Group: "g1"}} + end3, err := WatchPull(ctx, js.Conn(), group, func(k PullKey) { moved <- k }) + if err != nil { + t.Fatal(err) + } + defer end3() + putKey(t, kv, 7, PullKey{Head: "head-7-222222222", Trunk: "trunk-9-000000"}) + select { + case k := <-moved: + t.Fatalf("a group's check was stopped by %+v", k) + case <-time.After(500 * time.Millisecond): + } +} + +func TestAPullKeyIsMovedByHeadOrTrunkAndJudgedByVerdict(t *testing.T) { + k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: "pass"} + if k.Moved("aaaaaaaaaaaa", "tttttttttttt") || !k.Moved("bbbbbbbbbbbbbbbb", "tttttttttttt") || !k.Moved("aaaaaaaaaaaa", "uuuuuuuuuuuuuu") { + t.Error("moved is wrong") + } + if !k.Judged("aaaaaaaaaaaa1111", "tttttttttttt1111") || k.Judged("aaaaaaaaaaaa1111", "") || (PullKey{Head: "a", Trunk: "t"}).Judged("a", "t") { + t.Error("judged is wrong") + } +} + +// The queue is read across the three streams in the order they are handed out, each ask says the stream it waits in, and an +// ask is removed from that stream. +func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx := t.Context() + asks := BuildsOn(js, TheBuildMachine) + for _, r := range []BuildRequest{ + aPullCheck("other-1", 1, "h1", "t", false), + aPullCheck("ready-2", 2, "h2", "t", true), + {ID: "delivery-3", Repository: "/d"}, + } { + if err := asks.Ask(ctx, r); err != nil { + t.Fatal(err) + } + } + q := queueOf(t, js) + var ids, streams []string + for _, a := range q.Asks { + ids, streams = append(ids, a.ID), append(streams, a.Stream) + } + if want := []string{"delivery-3", "ready-2", "other-1"}; !reflect.DeepEqual(ids, want) { + t.Fatalf("read %v; want the order they are handed out, %v", ids, want) + } + if want := []string{"SEAT_NODE_BUILD_AGENT", broker.ChecksReadyStream, broker.ChecksOtherStream}; !reflect.DeepEqual(streams, want) { + t.Fatalf("streams %v; want %v", streams, want) + } + ready, _ := q.Find("ready-2") + if w := WorkerOf(TheBuildMachine, ready); w.Name != broker.ChecksReadyWorker { + t.Errorf("the ready check is handed out by %s", w.Name) + } + if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != ready.Seq { + t.Fatalf("deleted %d, %v", removed, err) + } + if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != 0 { + t.Fatalf("deleted again: %d, %v", removed, err) + } + if n := len(queueOf(t, js).Asks); n != 2 { + t.Errorf("%d asks left; want 2", n) + } +} diff --git a/internal/link/events.go b/internal/link/events.go index 5d6abee6..e545802d 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -264,6 +264,9 @@ type PullUpdated struct { // check's second layer; MergeCheckSaid says the announcer looked. MergeCheck bool `json:"merge_check,omitempty"` MergeCheckSaid bool `json:"merge_check_said,omitempty"` + // Ready says the pull request is marked ready (novox/hq ADR 0406 §3): not the forge's word, mesh-delivery's, said + // to the controller with the head to check again; its check is queued before every other check. + Ready bool `json:"ready,omitempty"` } // Checked is a pull request's merge check, judged: what the controller says as `checked`. diff --git a/internal/link/queue.go b/internal/link/queue.go index e63c8154..c9c3b0fe 100644 --- a/internal/link/queue.go +++ b/internal/link/queue.go @@ -62,11 +62,12 @@ type QueuedAsk struct { Path string `json:"path,omitempty"` Ref string `json:"ref,omitempty"` AskedAt time.Time `json:"asked-at,omitempty"` - // Stored is when the stream first took it, a move into order kept (novox/hq issue 558). + // Stored is when the stream took it. Stored time.Time `json:"stored,omitempty"` State string `json:"state"` - // movedFrom is the MovedFromHeader it carries: the sequence of the message a move copied it from. - movedFrom string + // Stream is the work queue holding it: the seat's own, or one of the two check queues (novox/hq ADR 0406 §3). Its + // Seq is the sequence there. + Stream string `json:"stream,omitempty"` // On and Started are the holder building an in-flight ask and when it started, from its started // event. Was is the holder that started an in-flight ask and is no longer building it — handed // back, to be handed out again — with Started its start there. @@ -201,62 +202,113 @@ func startedAt(s BuildStart) time.Time { return t } -// ReadQueue reads a build seat's work queue: the stream's asks, the worker's position, and what the -// seat's events said about the builds behind it. Through the JetStream API alone (STREAM.INFO, -// CONSUMER.INFO, STREAM.MSG.GET), which only the controller's account reaches. -func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, error) { +// queueStream is one stream a seat's asks wait in, and the worker that hands them out. +type queueStream struct { + Worker broker.Consumer + Filter string +} + +// queueStreamsOf are the streams a seat's asks wait in, in the order a builder takes from them (novox/hq ADR 0406 §3): the +// seat's own queue, where a delivery build waits, then the checks of pull requests marked ready, then every other check. +// Only the current build role has the check queues; the retired one's checks wait on its own. +func queueStreamsOf(seat string) []queueStream { worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}}) + streams := []queueStream{{Worker: worker, Filter: BuildWorkOf(seat)}} + if seat == broker.TheBuildSeat { + for _, w := range broker.CheckWorkers() { + streams = append(streams, queueStream{Worker: w, Filter: w.Filters[0]}) + } + } + return streams +} + +// WorkerOf is the worker that hands out an ask, by the queue it waits in. +func WorkerOf(seat string, ask QueuedAsk) broker.Consumer { + streams := queueStreamsOf(seat) + for _, qs := range streams { + if qs.Worker.Stream == ask.Stream { + return qs.Worker + } + } + return streams[0].Worker +} + +// ReadQueue reads a build seat's work queues: the streams' asks, the workers' positions, and what the seat's events +// said about the builds behind them. Through the JetStream API alone (STREAM.INFO, CONSUMER.INFO, STREAM.MSG.GET), +// which only the controller's account reaches. The asks come in the order they are handed out: the seat's own queue's, +// the ready checks', the other checks'. +func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, error) { q := Queue{Seat: seat} api, err := jetstream.New(js.Conn()) if err != nil { return q, err } - stream, err := api.Stream(ctx, worker.Stream) - if err != nil { - return q, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err) + type read struct { + asks []QueuedAsk + delivered uint64 + ackPending int } - consumer, err := stream.Consumer(ctx, worker.Name) - switch { - case errors.Is(err, jetstream.ErrConsumerNotFound): - // No holder has ever been given a worker: everything in the stream is waiting. - case err != nil: - return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err) - default: - info, err := consumer.Info(ctx) + var reads []read + for i, qs := range queueStreamsOf(seat) { + stream, err := api.Stream(ctx, qs.Worker.Stream) if err != nil { + return q, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, qs.Worker.Stream, err) + } + var r read + consumer, err := stream.Consumer(ctx, qs.Worker.Name) + switch { + case errors.Is(err, jetstream.ErrConsumerNotFound): + // No holder has ever been given a worker: everything in the stream is waiting. + case err != nil: return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err) + default: + info, err := consumer.Info(ctx) + if err != nil { + return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err) + } + r.delivered, r.ackPending = info.Delivered.Stream, info.NumAckPending + if i == 0 { + q.Delivered, q.MaxDeliver = info.Delivered.Stream, info.Config.MaxDeliver + } + q.AckPending += info.NumAckPending + if q.MaxDeliver == 0 { + q.MaxDeliver = info.Config.MaxDeliver + } } - q.Delivered = info.Delivered.Stream - q.AckPending = info.NumAckPending - q.MaxDeliver = info.Config.MaxDeliver + // Every ask, by next-by-subject from the first: the stream holds only what is unsettled, so this + // is a handful of reads, never the history. + var seq uint64 = 1 + for n := 0; n < 10000; n++ { + msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(qs.Filter)) + if errors.Is(err, jetstream.ErrMsgNotFound) { + break + } + if err != nil { + return q, fmt.Errorf("reading the %s work queue: %w", seat, err) + } + a := askOf(msg) + a.Stream = qs.Worker.Stream + r.asks = append(r.asks, a) + seq = msg.Sequence + 1 + } + reads = append(reads, r) } - // Every ask, by next-by-subject from the first: the stream holds only what is unsettled, so this - // is a handful of reads, never the history. - var asks []QueuedAsk - var seq uint64 = 1 - for n := 0; n < 10000; n++ { - msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat))) - if errors.Is(err, jetstream.ErrMsgNotFound) { - break - } - if err != nil { - return q, fmt.Errorf("reading the %s work queue: %w", seat, err) - } - asks = append(asks, askOf(msg)) - seq = msg.Sequence + 1 - } - - // What the events say, from shortly before the oldest ask behind the worker: its start, if any, - // came after it was asked. + // What the events say, from shortly before the oldest ask behind a worker: its start, if any, came after it was asked. var since time.Time - for _, a := range asks { - if a.Seq <= q.Delivered && (since.IsZero() || (!a.AskedAt.IsZero() && a.AskedAt.Before(since))) { - since = a.AskedAt + behind := false + for _, r := range reads { + for _, a := range r.asks { + if a.Seq <= r.delivered { + behind = true + if since.IsZero() || (!a.AskedAt.IsZero() && a.AskedAt.Before(since)) { + since = a.AskedAt + } + } } } heard := BuildHeard{Started: map[string]BuildStart{}, Outcomes: map[string]bool{}} - if len(asks) > 0 && q.Delivered > 0 { + if behind { if since.IsZero() { since = time.Now().Add(-7 * 24 * time.Hour) } @@ -264,17 +316,15 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e return q, err } } - q.Asks = ClassifyQueue(asks, q.Delivered, q.AckPending, heard) + for _, r := range reads { + q.Asks = append(q.Asks, ClassifyQueue(r.asks, r.delivered, r.ackPending, heard)...) + } return q, nil } // askOf is one ask as the stream holds it: what its request says, and when the stream took it. func askOf(msg *jetstream.RawStreamMsg) QueuedAsk { ask := QueuedAsk{Seq: msg.Sequence, Stored: msg.Time} - // Moved into order, it keeps when the stream first took it (novox/hq issue 558). - if at, err := time.Parse(time.RFC3339Nano, msg.Header.Get(QueuedAtHeader)); err == nil { - ask.Stored = at - } if json.Unmarshal(msg.Data, &ask.Request) == nil { r := ask.Request ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref @@ -282,7 +332,7 @@ func askOf(msg *jetstream.RawStreamMsg) QueuedAsk { ask.AskedAt = at } } else { - // Nothing read is nothing kept: a half-read request would pass for a delivery (novox/hq issue 558). + // Nothing read is nothing kept: a half-read request would pass for a delivery. ask.Request = BuildRequest{} } return ask diff --git a/internal/link/queue_kind.go b/internal/link/queue_kind.go new file mode 100644 index 00000000..1662311e --- /dev/null +++ b/internal/link/queue_kind.go @@ -0,0 +1,50 @@ +package link + +import ( + "context" + "errors" + "strings" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" +) + +// The kinds of ask, as the queue says them. **The kind decides the queue** (novox/hq ADR 0406 §3): a delivery build is on +// the seat's own queue, taken before any check; a check of a pull request marked ready is on the second; every other +// check on the third. Nothing orders the asks any more: a builder takes from the queues in that order, and the check +// queues keep one request a pull request. The reordering of one queue by moving asks to its tail, with its pause of the +// worker, its look every two seconds and its second copies, is gone (issue 558 is answered by the streams). +const ( + KindDelivery = "delivery" + KindCheck = "check" +) + +// KindOf is an ask's kind: a check for a pull request's check — a recheck the merge tool asks included — +// a dry run, or an ask nothing could be read from; a delivery for any other build. +func KindOf(r BuildRequest) string { + if r.ID == "" || r.Check != nil || r.DryRun { + return KindCheck + } + return KindDelivery +} + +// DeleteAsk removes a waiting or dead ask from the queue it is in, at its sequence. Says the sequence it removed, 0 when +// it was not there any more: taken and settled, or replaced by a newer request for its pull request. +func DeleteAsk(_ context.Context, js *broker.JetStream, seat string, ask QueuedAsk) (uint64, error) { + stream := ask.Stream + if stream == "" { + worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}}) + stream = worker.Stream + } + err := js.Context().DeleteMsg(stream, ask.Seq) + switch { + case err == nil: + return ask.Seq, nil + case errors.Is(err, nats.ErrMsgNotFound) || errors.Is(err, jetstream.ErrMsgNotFound) || + strings.Contains(err.Error(), "no message found"): + return 0, nil + } + return 0, err +} diff --git a/internal/link/queue_order.go b/internal/link/queue_order.go deleted file mode 100644 index 90e68f50..00000000 --- a/internal/link/queue_order.go +++ /dev/null @@ -1,433 +0,0 @@ -package link - -import ( - "context" - "errors" - "fmt" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" -) - -// The order the build queue hands asks out in (novox/hq issue 558). -// -// **A build a walk waits for goes before a pull request's check.** The queue was first come, first served, -// and a merged change's delivery build waited minutes behind checks while its walk, and every walk after -// it, waited on the build. So: -// -// - a delivery is handed out before any check; -// - a check that has waited CheckWaitsAtMost goes before every delivery asked after it, so no check -// waits for ever behind a stream of deliveries; -// - within a kind, first come, first served, by when it was asked; -// - a branch tried before its pull request goes after everything else, however long it has waited: it is -// never promoted ahead of a delivery, and never ahead of a pull request's check (novox/hq issue 686); -// - an ask whose kind cannot be told counts as a check: a delivery taken late breaks a budget, a check -// taken late breaks nothing. -// -// **Kept in the stream, never beside it.** The queue is the seat's work queue, which the server hands out -// in stream order to whichever holder pulls; nothing else holds the order, so a restarted controller -// finds it as it was left. An ask that must go behind another is moved to the tail — deleted from where -// it was and published again, its body unchanged, under the same id — while the worker is paused, so no -// holder takes it in the moment it moves. Only waiting asks move: what a holder took is never touched. - -// The kinds of ask, as the queue says them. -const ( - KindDelivery = "delivery" - KindCheck = "check" -) - -// CheckWaitsAtMost is how long a check waits before it goes ahead of the deliveries asked after it. -var CheckWaitsAtMost = 10 * time.Minute - -// KindOf is an ask's kind: a check for a pull request's check — a recheck the merge tool asks included — -// a dry run, or an ask nothing could be read from; a delivery for any other build. -func KindOf(r BuildRequest) string { - if r.ID == "" || r.Check != nil || r.DryRun { - return KindCheck - } - return KindDelivery -} - -// askedOrStored is when an ask was asked, as its id says, or else when the queue took it. -func (a QueuedAsk) askedOrStored() time.Time { - if !a.AskedAt.IsZero() { - return a.AskedAt - } - return a.Stored -} - -// QueueOrder is the order the waiting asks are to be handed out in, now: the deliveries and every check -// that has waited CheckWaitsAtMost, by when each was asked, then the other checks, by when each was asked. -func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk { - var first, then, tries []QueuedAsk - for _, a := range waiting { - if IsTry(a.Request) { - tries = append(tries, a) - continue - } - if KindOf(a.Request) == KindCheck && now.Sub(a.askedOrStored()) < CheckWaitsAtMost { - then = append(then, a) - continue - } - first = append(first, a) - } - byAsking := func(s []QueuedAsk) { - sort.SliceStable(s, func(i, j int) bool { - if ti, tj := s[i].askedOrStored(), s[j].askedOrStored(); !ti.Equal(tj) { - return ti.Before(tj) - } - return s[i].Seq < s[j].Seq - }) - } - byAsking(first) - byAsking(then) - byAsking(tries) - return append(append(first, then...), tries...) -} - -// toMove is the asks to move to the tail, in this order, for the queue to stand as order says: everything -// after the longest start of order that the queue already holds in that order. -func toMove(queue, order []QueuedAsk) []QueuedAsk { - at := make(map[uint64]int, len(queue)) - for i, a := range queue { - at[a.Seq] = i - } - kept, last := 0, -1 - for kept < len(order) { - i, ok := at[order[kept].Seq] - if !ok || i < last { - break - } - last = i - kept++ - } - return order[kept:] -} - -// orderPasses keeps one pass at a time. Only the acting controller's loop runs one (KeepingInOrder), so -// there is one; the lock holds a test's two at once to that too. -var orderPasses sync.Mutex - -// pausedFor bounds a pause the pass could not end itself: a controller that stops mid-pass leaves the -// worker paused this long at most. moveMargin is the part of it no move starts in: a pass stops moving -// with at least this left, and the next pass moves the rest, so no move runs past the pause. -const ( - pausedFor = 20 * time.Second - moveMargin = 5 * time.Second - publishWaits = 3 * time.Second -) - -// QueuedAtHeader carries, on a moved ask, when the stream first took it: a move keeps an ask's place in -// time, its id unread or not. -const QueuedAtHeader = "Mesh-Queued-At" - -// MovedFromHeader names, on a move's copy, the sequence of the message it is a copy of: with QueuedAtHeader, -// what proves a second message of one id is a move's copy and not an ask delivered again (ADR 0293), whose -// headers are its original's — and so never name that original's own sequence. -const MovedFromHeader = "Mesh-Moved-From" - -// Pass is what one pass did: the asks it moved, and each second copy it removed, in words. -type Pass struct { - Moved []string - Removed []string -} - -// asked is the signal an ask sends the acting controller's pass loop, in this process. An ask asked in -// another process — a verb run as its own command — is seen by the loop as the queue's last sequence moving. -var asked = make(chan struct{}, 1) - -// signalAsked tells the pass loop an ask joined the queue. Never blocks, never runs a pass. -func signalAsked() { - select { - case asked <- struct{}{}: - default: - } -} - -// PutInOrder moves the seat's waiting asks into the order QueueOrder says, and says each it moved. A queue -// already in order is read and left alone; the worker is paused only while asks move. Run by the acting -// controller alone (KeepingInOrder): two passes in two processes would each resume what the other paused. -func PutInOrder(ctx context.Context, js *broker.JetStream, seat string) (Pass, error) { - var pass Pass - orderPasses.Lock() - defer orderPasses.Unlock() - api, err := jetstream.New(js.Conn()) - if err != nil { - return pass, err - } - worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}}) - waiting, extra, delivered, err := readWaiting(ctx, api, worker, seat) - if err != nil || (len(extra) == 0 && len(toMove(waiting, QueueOrder(waiting, time.Now()))) == 0) { - return pass, err - } - // Paused, then read again: what the worker handed out up to the pause is not waiting any more. - pauseEnds := time.Now().Add(pausedFor) - if delivered >= 0 { - if _, err := api.PauseConsumer(ctx, worker.Stream, worker.Name, pauseEnds); err != nil { - return pass, fmt.Errorf("the %s worker could not be paused to put its queue in order, so it is left "+ - "first come, first served: %w", seat, err) - } - defer func() { - resuming, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) - defer cancel() - if _, err := api.ResumeConsumer(resuming, worker.Stream, worker.Name); err != nil { - fmt.Printf("the %s worker could not be resumed after its queue was put in order; it resumes by "+ - "itself by %s: %v\n", seat, pauseEnds.Format(time.RFC3339), err) - } - }() - if waiting, extra, _, err = readWaiting(ctx, api, worker, seat); err != nil { - return pass, err - } - } - stream, err := api.Stream(ctx, worker.Stream) - if err != nil { - return pass, err - } - // A second copy of one ask — a move that published and stopped before it deleted — goes first. - for _, a := range extra { - if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) { - return pass, fmt.Errorf("a second copy of %s (seq %d) could not be removed: %w", a.ID, a.Seq, err) - } - pass.Removed = append(pass.Removed, fmt.Sprintf("%s at seq %d, a move's second copy of the message at seq %s", - a.ID, a.Seq, a.movedFrom)) - } - for _, a := range toMove(waiting, QueueOrder(waiting, time.Now())) { - if delivered >= 0 && time.Until(pauseEnds) < moveMargin { - // The rest waits for the next pass, under a pause of its own. - break - } - ok, err := moveToTheTail(ctx, js, stream, worker.Name, seat, a) - if err != nil { - return pass, err - } - if ok { - pass.Moved = append(pass.Moved, a.ID) - } - } - return pass, nil -} - -// readWaiting is every ask the worker has not yet handed out, in queue order, but each second copy of one -// ask; the second copies, to remove; and the last sequence the worker handed out, -1 when there is no -// worker, so nothing is handed out to pause. -// -// **A second copy is proven, never guessed from an id**: a waiting message whose MovedFromHeader names an -// earlier message of the same request id still in the queue, and whose QueuedAtHeader is that message's -// first-queued time — a move that published its copy and stopped before it deleted the original. The -// original stays, waiting or handed out (a holder's then, and the copy would build it twice). Any other -// second message of one id — an ask a dead letter delivered again beside its original (ADR 0293) — is a -// real ask and stays. -func readWaiting(ctx context.Context, api jetstream.JetStream, worker broker.Consumer, seat string) ( - waiting, extra []QueuedAsk, delivered int64, err error) { - stream, err := api.Stream(ctx, worker.Stream) - if err != nil { - return nil, nil, 0, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err) - } - delivered = -1 - if consumer, err := stream.Consumer(ctx, worker.Name); err == nil { - info, err := consumer.Info(ctx) - if err != nil { - return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err) - } - delivered = int64(info.Delivered.Stream) - } else if !errors.Is(err, jetstream.ErrConsumerNotFound) { - return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err) - } - // From the first: the stream holds only what is unsettled, so this is a handful of reads. - // Every message by sequence, with its id and first-queued time, to prove a copy against. - type held struct { - id string - stored time.Time - } - bySeq := map[string]held{} - var seq uint64 = 1 - for n := 0; n < 10000; n++ { - msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat))) - if errors.Is(err, jetstream.ErrMsgNotFound) { - break - } - if err != nil { - return nil, nil, 0, fmt.Errorf("reading the %s work queue: %w", seat, err) - } - seq = msg.Sequence + 1 - a := askOf(msg) - a.movedFrom = msg.Header.Get(MovedFromHeader) - from, known := bySeq[a.movedFrom] - again := a.ID != "" && a.movedFrom != "" && known && from.id == a.ID && from.stored.Equal(a.Stored) - bySeq[strconv.FormatUint(a.Seq, 10)] = held{id: a.ID, stored: a.Stored} - switch { - case int64(a.Seq) <= delivered: - case again: - extra = append(extra, a) - default: - waiting = append(waiting, a) - } - } - return waiting, extra, delivered, nil -} - -// moveToTheTail publishes the ask's body again at the queue's tail and then deletes it where it was. **The -// copy first**: a publish that fails deletes nothing, and a pass that stops between the two leaves two -// copies, which the next pass sees by their one id and makes one (readWaiting). A move never loses an ask. -// Looked at once more just before: one the worker has handed out since is a holder's now and stays. False -// when it was not moved. -func moveToTheTail(ctx context.Context, js *broker.JetStream, stream jetstream.Stream, workerName, seat string, - a QueuedAsk) (bool, error) { - if consumer, err := stream.Consumer(ctx, workerName); err == nil { - info, err := consumer.Info(ctx) - if err != nil { - return false, err - } - if info.Delivered.Stream >= a.Seq { - return false, nil - } - } - msg, err := stream.GetMsg(ctx, a.Seq) - if errors.Is(err, jetstream.ErrMsgNotFound) { - return false, nil - } - if err != nil { - return false, err - } - again := &nats.Msg{Subject: msg.Subject, Data: msg.Data, Header: nats.Header{}} - for k, v := range msg.Header { - // The server's own headers stay behind: a message id would be taken for a duplicate and dropped. - if !strings.HasPrefix(k, "Nats-") { - again.Header[k] = v - } - } - // Its place in time goes with it: when the stream first took it, as askOf read it; and the message it - // is a copy of, which proves it one. - again.Header.Set(QueuedAtHeader, askOf(msg).Stored.UTC().Format(time.RFC3339Nano)) - again.Header.Set(MovedFromHeader, strconv.FormatUint(a.Seq, 10)) - // Inside the margin, so the publish ends while the worker is still paused. - publishing, cancel := context.WithTimeout(ctx, publishWaits) - ack, err := js.Context().PublishMsg(again, nats.Context(publishing)) - cancel() - if err != nil { - return false, fmt.Errorf("%s (seq %d) could not be put behind, and stays where it is: %w", a.ID, a.Seq, err) - } - // **Handed out meanwhile** — a bus slow enough that the pause ended: the original is a holder's, and the - // copy goes instead, so it is built once. - if consumer, err := stream.Consumer(ctx, workerName); err == nil { - info, err := consumer.Info(ctx) - if err != nil || info.Delivered.Stream >= a.Seq { - if derr := stream.DeleteMsg(ctx, ack.Sequence); derr != nil && !errors.Is(derr, jetstream.ErrMsgNotFound) { - return false, fmt.Errorf("%s was handed out while it moved and its copy (seq %d) could not be "+ - "removed; the next pass removes it: %w", a.ID, ack.Sequence, derr) - } - if err != nil { - return false, fmt.Errorf("whether %s was handed out while it moved cannot be read, so its copy was "+ - "removed and it stays where it was: %w", a.ID, err) - } - if info.Delivered.Stream >= ack.Sequence { - return false, fmt.Errorf("%s was handed out while it moved, and its copy too: it may be built twice", a.ID) - } - return false, nil - } - } - if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) { - return false, fmt.Errorf("%s was copied behind and its original (seq %d) not removed; the next pass "+ - "removes the second copy: %w", a.ID, a.Seq, err) - } - return true, nil -} - -// KeepingInOrder runs the passes, on the acting controller alone, until ctx ends: when an ask in this -// process says so, when the queue's last sequence moved — an ask from another process — and every -// passEvery, for a check that comes of age with nothing asked. seat says the build seat to order; acting -// whether this controller acts (Holding); said what a pass did. -func KeepingInOrder(ctx context.Context, js *broker.JetStream, seat func() string, acting func() bool, - said func(format string, args ...any)) { - tick := time.NewTicker(orderLooks) - defer tick.Stop() - var lastSeq uint64 - var lastPass time.Time - for { - signalled := false - select { - case <-ctx.Done(): - return - case <-asked: - signalled = true - case <-tick.C: - } - if !acting() { - continue - } - name := seat() - worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: name, Accepts: []string{"build"}}) - info, err := js.Context().StreamInfo(worker.Stream) - if err != nil { - continue - } - if !signalled && info.State.LastSeq == lastSeq && time.Since(lastPass) < passEvery { - continue - } - lastSeq, lastPass = info.State.LastSeq, time.Now() - passing, cancel := context.WithTimeout(ctx, pausedFor+10*time.Second) - pass, err := PutInOrder(passing, js, name) - cancel() - for _, r := range pass.Removed { - said("the %s queue: removed %s", name, r) - } - if len(pass.Moved) > 0 { - said("the %s queue put in order: %s moved behind what goes first", name, strings.Join(pass.Moved, ", ")) - } - if err != nil { - said("the %s queue could not be put in order: %v", name, err) - } - if info, err := js.Context().StreamInfo(worker.Stream); err == nil { - // Its own moves are not an ask. - lastSeq = info.State.LastSeq - } - } -} - -// orderLooks is how often the loop looks whether an ask joined from another process; passEvery how often -// it passes with nothing asked. -var ( - orderLooks = 2 * time.Second - passEvery = 30 * time.Second -) - -// DeleteAsk removes a waiting or dead ask from the queue: at its sequence, or — moved into order since it -// was read — the copy carrying its id. Says the sequence it removed, 0 when neither was there any more. -func DeleteAsk(ctx context.Context, js *broker.JetStream, seat string, ask QueuedAsk) (uint64, error) { - worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}}) - gone := func(err error) bool { - return errors.Is(err, nats.ErrMsgNotFound) || errors.Is(err, jetstream.ErrMsgNotFound) || - (err != nil && strings.Contains(err.Error(), "no message found")) - } - err := js.Context().DeleteMsg(worker.Stream, ask.Seq) - if err == nil { - return ask.Seq, nil - } - if !gone(err) { - return 0, err - } - if ask.ID == "" { - return 0, nil - } - q, err := ReadQueue(ctx, js, seat) - if err != nil { - return 0, err - } - for _, a := range q.Asks { - if a.ID == ask.ID && a.Seq != ask.Seq && a.State != AskInFlight { - if err := js.Context().DeleteMsg(worker.Stream, a.Seq); err != nil && !gone(err) { - return 0, err - } else if err == nil { - return a.Seq, nil - } - } - } - return 0, nil -} diff --git a/internal/link/queue_order_test.go b/internal/link/queue_order_test.go deleted file mode 100644 index 315b7dac..00000000 --- a/internal/link/queue_order_test.go +++ /dev/null @@ -1,441 +0,0 @@ -package link - -import ( - "context" - "encoding/json" - "fmt" - "reflect" - "sync" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" -) - -// The order the build queue hands asks out in (novox/hq issue 558): a build a walk waits for before any -// pull request's check, a check that has waited ten minutes before every delivery asked after it, and -// first come, first served within a kind. - -func anAsk(seq uint64, id string, asked time.Time, check bool) QueuedAsk { - r := BuildRequest{ID: id, Repository: "/r"} - if check { - r.Check = &CheckRequest{Owner: "novox", Repo: "r"} - } - return QueuedAsk{Seq: seq, ID: id, AskedAt: asked, Request: r, State: AskWaiting} -} - -func idsOf(asks []QueuedAsk) []string { - out := []string{} - for _, a := range asks { - out = append(out, a.ID) - } - return out -} - -func TestADeliveryIsHandedOutBeforeTheChecksAskedBeforeIt(t *testing.T) { - now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) - waiting := []QueuedAsk{ - anAsk(1, "check-a", now.Add(-3*time.Minute), true), - anAsk(2, "check-b", now.Add(-2*time.Minute), true), - anAsk(3, "delivery-d", now.Add(-time.Minute), false), - } - want := []string{"delivery-d", "check-a", "check-b"} - if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) { - t.Fatalf("handed out as %v, want %v", got, want) - } - // Only the checks ahead of it move behind it; the delivery stays where it is. - if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"check-a", "check-b"}) { - t.Fatalf("moved %v", got) - } -} - -func TestACheckThatWaitedTenMinutesGoesBeforeALaterDelivery(t *testing.T) { - now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) - // As the queue stands after a delivery put the young checks behind it, ten minutes ago and since. - waiting := []QueuedAsk{ - anAsk(5, "delivery-early", now.Add(-12*time.Minute), false), - anAsk(6, "delivery-later", now.Add(-time.Minute), false), - anAsk(7, "check-old", now.Add(-11*time.Minute), true), - anAsk(8, "check-young", now.Add(-9*time.Minute), true), - } - want := []string{"delivery-early", "check-old", "delivery-later", "check-young"} - if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) { - t.Fatalf("handed out as %v, want %v", got, want) - } - if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"delivery-later", "check-young"}) { - t.Fatalf("moved %v", got) - } -} - -func TestAnAskWhoseKindCannotBeToldIsACheck(t *testing.T) { - for _, r := range []BuildRequest{ - {}, // unreadable: nothing was read from it - {ID: "build-1", Check: &CheckRequest{}}, // a pull request's check, or a recheck the merge tool asked - {ID: "build-2", DryRun: true}, // looked at, never delivered - } { - if got := KindOf(r); got != KindCheck { - t.Errorf("%+v is a %s", r, got) - } - } - if got := KindOf(BuildRequest{ID: "build-3", Repository: "/r"}); got != KindDelivery { - t.Errorf("a build is a %s", got) - } -} - -// Against a real server: two checks wait, a delivery is asked, and the next free holder takes the delivery. -func TestNatsADeliveryIsTakenBeforeTheChecksQueuedAheadOfIt(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - asks := BuildsOn(js, TheBuildMachine) - now := time.Now() - for i, r := range []BuildRequest{ - {ID: NewBuildID(now.Add(-3 * time.Minute)), Repository: "/a", Check: &CheckRequest{Owner: "novox", Repo: "a"}}, - {ID: NewBuildID(now.Add(-2 * time.Minute)), Repository: "/b", Check: &CheckRequest{Owner: "novox", Repo: "b"}}, - {ID: NewBuildID(now), Repository: "/d"}, - } { - if i == 2 { - // And one nobody can read, whose kind cannot be told: it waits with the checks. - if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil { - t.Fatal(err) - } - } - if err := asks.Ask(ctx, r); err != nil { - t.Fatal(err) - } - } - // The acting controller's loop puts it in order, told by the asks. - go KeepingInOrder(ctx, js, func() string { return TheBuildMachine }, func() bool { return true }, - func(string, ...any) {}) - time.Sleep(500 * time.Millisecond) - took := make(chan string, 4) - machine := MachineOverNATS(js, "anchor") - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request().Repository - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) - _ = work.Done() - }) - }() - var order []string - for len(order) < 3 { - select { - case r := <-took: - order = append(order, r) - case <-time.After(10 * time.Second): - t.Fatalf("taken so far: %v", order) - } - } - if want := []string{"/d", "/a", "/b"}; !reflect.DeepEqual(order, want) { - t.Fatalf("taken as %v, want %v", order, want) - } -} - -// --- the moves, against a real server ------------------------------------------------------------ - -func publishAsk(t *testing.T, js *broker.JetStream, r BuildRequest) { - t.Helper() - body, _ := json.Marshal(r) - if _, err := js.Context().Publish(BuildWork(), body); err != nil { - t.Fatal(err) - } -} - -func aCheck(asked time.Time, repository string) BuildRequest { - return BuildRequest{ID: NewBuildID(asked), Repository: repository, Check: &CheckRequest{Owner: "novox", Repo: "r"}} -} - -func aDelivery(asked time.Time, repository string) BuildRequest { - return BuildRequest{ID: NewBuildID(asked), Repository: repository} -} - -// queued is the stream's asks, in queue order. -func queued(t *testing.T, js *broker.JetStream) []QueuedAsk { - t.Helper() - q, err := ReadQueue(context.Background(), js, TheBuildMachine) - if err != nil { - t.Fatal(err) - } - return q.Asks -} - -// takeAll runs a holder that records how often it was handed each repository. -func takeAll(t *testing.T, ctx context.Context, js *broker.JetStream) (chan string, func() map[string]int) { - t.Helper() - took := make(chan string, 100) - var mu sync.Mutex - counts := map[string]int{} - machine := MachineOverNATS(js, "anchor") - t.Cleanup(machine.Close) - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { - mu.Lock() - counts[work.Request().Repository]++ - mu.Unlock() - took <- work.Request().Repository - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) - _ = work.Done() - }) - }() - return took, func() map[string]int { - mu.Lock() - defer mu.Unlock() - out := map[string]int{} - for k, v := range counts { - out[k] = v - } - return out - } -} - -// A move that published its copy and stopped before deleting the original leaves two copies of one ask; -// the next pass makes them one, and a copy of one a holder already took is removed, never built twice. -func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx := t.Context() - now := time.Now() - check := aCheck(now.Add(-2*time.Minute), "/c") - publishAsk(t, js, check) - publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/d")) - moveCopyOf(t, js, check.ID) // the copy, published; the original never deleted - if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Removed) != 1 { - t.Fatalf("removed %v: %v", pass.Removed, err) - } - var got []string - for _, a := range queued(t, js) { - got = append(got, a.Repository) - } - if want := []string{"/d", "/c"}; !reflect.DeepEqual(got, want) { - t.Fatalf("after the pass the queue holds %v, want %v", got, want) - } - - // And one a holder took before the copy was published: the copy goes, the holder's stays. - taken := aCheck(now, "/taken") - publishAsk(t, js, taken) - worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}}) - sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - // Take everything waiting up to and including /taken, and keep /taken unsettled. - for { - msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second)) - if err != nil || len(msgs) == 0 { - t.Fatalf("nothing to take: %v", err) - } - var r BuildRequest - _ = json.Unmarshal(msgs[0].Data, &r) - if r.Repository == "/taken" { - break - } - _ = msgs[0].Ack() - } - moveCopyOf(t, js, taken.ID) - if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { - t.Fatal(err) - } - n := 0 - for _, a := range queued(t, js) { - if a.ID == taken.ID { - n++ - } - } - if n != 1 { - t.Fatalf("%d copies of the ask a holder took are in the queue, want 1", n) - } -} - -// moveCopyOf publishes what a move publishes for the ask with this id, as a pass stopped before its delete -// would leave it. -func moveCopyOf(t *testing.T, js *broker.JetStream, id string) { - t.Helper() - for _, a := range queued(t, js) { - if a.ID != id { - continue - } - body, _ := json.Marshal(a.Request) - msg := &nats.Msg{Subject: BuildWork(), Data: body, Header: nats.Header{}} - msg.Header.Set(QueuedAtHeader, a.Stored.UTC().Format(time.RFC3339Nano)) - msg.Header.Set(MovedFromHeader, fmt.Sprint(a.Seq)) - if _, err := js.Context().PublishMsg(msg); err != nil { - t.Fatal(err) - } - return - } - t.Fatalf("%s is not queued", id) -} - -// A dead letter delivered again whose original stayed beside it (ADR 0293: its removal failed) is a real -// ask with the same id, and is never removed as a second copy — even when the original was itself a move's -// copy, whose headers the delivery carries. -func TestNatsAnAskDeliveredAgainBesideItsOriginalStays(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx := t.Context() - ask := aCheck(time.Now(), "/again") - publishAsk(t, js, aCheck(time.Now().Add(-time.Minute), "/before")) - publishAsk(t, js, ask) - publishAsk(t, js, aDelivery(time.Now(), "/d")) - if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { // /again becomes a move's copy - t.Fatal(err) - } - worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}}) - sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - var original *nats.Msg - for original == nil { - msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second)) - if err != nil || len(msgs) == 0 { - t.Fatalf("nothing to take: %v", err) - } - var r BuildRequest - _ = json.Unmarshal(msgs[0].Data, &r) - if r.ID == ask.ID { - original = msgs[0] // given up on, never settled: dead, and kept - continue - } - _ = msgs[0].Ack() - } - // Delivered again as DeliverAgain does: its headers, and its own. - again := &nats.Msg{Subject: BuildWork(), Data: original.Data, Header: nats.Header{}} - for k, v := range original.Header { - again.Header[k] = v - } - again.Header.Set(AgainHeader, "7") - if _, err := js.Context().PublishMsg(again); err != nil { - t.Fatal(err) - } - pass, err := PutInOrder(ctx, js, TheBuildMachine) - if err != nil || len(pass.Removed) != 0 { - t.Fatalf("removed %v: %v", pass.Removed, err) - } - n := 0 - for _, a := range queued(t, js) { - if a.ID == ask.ID { - n++ - } - } - if n != 2 { - t.Fatalf("%d messages of the ask delivered again are queued, want its original and the delivery", n) - } -} - -// Two passes at once: nothing is built twice, and nothing is lost. -func TestNatsTwoPassesAtOnceBuildNothingTwice(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - took, counts := takeAll(t, ctx, js) - now := time.Now() - const asks = 24 - var passes sync.WaitGroup - for p := 0; p < 2; p++ { - passes.Add(1) - go func() { - defer passes.Done() - for i := 0; i < 15; i++ { - _, _ = PutInOrder(ctx, js, TheBuildMachine) - } - }() - } - for i := 0; i < asks; i++ { - repository := fmt.Sprintf("/%02d", i) - if i%3 == 2 { - publishAsk(t, js, aDelivery(now.Add(time.Duration(i)*time.Millisecond), repository)) - } else { - publishAsk(t, js, aCheck(now.Add(time.Duration(i)*time.Millisecond), repository)) - } - } - passes.Wait() - for got := 0; got < asks; got++ { - select { - case <-took: - case <-time.After(15 * time.Second): - t.Fatalf("taken %d of %d: %v", got, asks, counts()) - } - } - time.Sleep(time.Second) - c := counts() - for i := 0; i < asks; i++ { - if n := c[fmt.Sprintf("/%02d", i)]; n != 1 { - t.Errorf("/%02d was built %d times", i, n) - } - } -} - -// A check that has waited ten minutes goes before a delivery asked after it, even one queued ahead of it. -func TestNatsACheckThatWaitedTenMinutesIsTakenBeforeALaterDelivery(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - now := time.Now() - publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/delivery")) - publishAsk(t, js, aCheck(now.Add(-11*time.Minute), "/old-check")) - if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { - t.Fatal(err) - } - took, _ := takeAll(t, ctx, js) - var order []string - for len(order) < 2 { - select { - case r := <-took: - order = append(order, r) - case <-time.After(10 * time.Second): - t.Fatalf("taken so far: %v", order) - } - } - if want := []string{"/old-check", "/delivery"}; !reflect.DeepEqual(order, want) { - t.Fatalf("taken as %v, want %v", order, want) - } -} - -// A cancel that read an ask before it moved deletes the moved copy, found by its id, and says when -// there was nothing left to delete. -func TestNatsACancelRacingAMoveDeletesTheMovedCopy(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx := t.Context() - now := time.Now() - publishAsk(t, js, aCheck(now.Add(-time.Minute), "/c")) - read := queued(t, js)[0] - publishAsk(t, js, aDelivery(now, "/d")) - if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 { - t.Fatalf("moved %v: %v", pass.Moved, err) - } - removed, err := DeleteAsk(ctx, js, TheBuildMachine, read) - if err != nil || removed == 0 || removed == read.Seq { - t.Fatalf("removed seq %d (read at %d): %v", removed, read.Seq, err) - } - for _, a := range queued(t, js) { - if a.ID == read.ID { - t.Fatalf("the cancelled ask is still queued at seq %d", a.Seq) - } - } - if removed, err := DeleteAsk(ctx, js, TheBuildMachine, read); err != nil || removed != 0 { - t.Fatalf("an ask gone already is said removed at %d: %v", removed, err) - } -} - -// An ask nothing can be read from keeps its place in time when it moves. -func TestNatsAnUnreadableAskKeepsItsAgeWhenItMoves(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx := t.Context() - if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil { - t.Fatal(err) - } - first := queued(t, js)[0].Stored - time.Sleep(50 * time.Millisecond) - publishAsk(t, js, aDelivery(time.Now(), "/d")) - if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 { - t.Fatalf("moved %v: %v", pass.Moved, err) - } - asks := queued(t, js) - if len(asks) != 2 || asks[1].ID != "" || !asks[1].Stored.Equal(first) { - t.Fatalf("after the move the unreadable ask was taken at %v, first at %v: %+v", asks[1].Stored, first, asks) - } -} diff --git a/internal/link/try_test.go b/internal/link/try_test.go index f434dc18..c2b6d8a4 100644 --- a/internal/link/try_test.go +++ b/internal/link/try_test.go @@ -2,7 +2,6 @@ package link import ( "encoding/json" - "reflect" "strings" "testing" "time" @@ -30,25 +29,8 @@ func TestATrysStartNamesNoCommitTheForgeCouldMatch(t *testing.T) { } } -// A try stays behind every delivery and every pull request's check, however long it has waited: the ten -// minutes that put a check ahead of later deliveries never promote a try (novox/hq issue 686). -func TestATryIsNeverHandedOutBeforeADeliveryOrAPullRequestsCheck(t *testing.T) { - now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) - try := func(seq uint64, id string, asked time.Time) QueuedAsk { - a := anAsk(seq, id, asked, true) - a.Request.Check.Try = true - return a - } - waiting := []QueuedAsk{ - try(1, "try-old", now.Add(-40*time.Minute)), - try(2, "try-young", now.Add(-time.Minute)), - anAsk(3, "check-young", now.Add(-2*time.Minute), true), - anAsk(4, "delivery", now.Add(-time.Minute), false), - } - want := []string{"delivery", "check-young", "try-old", "try-young"} - if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) { - t.Fatalf("handed out as %v, want %v", got, want) - } +// An id that is a pattern never passes for a build's id. +func TestAnIdThatIsAPatternIsNoBuildId(t *testing.T) { if !SafeBuildID("build-20261014T000000-1a2b") || SafeBuildID("x.>") || SafeBuildID("*") || SafeBuildID("") { t.Fatal("an id that is a pattern passes for a build's id") } From a83b921af0a7ab27d1d9bd40fac70864b1ffab12 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 14:03:26 +0200 Subject: [PATCH 09/12] issue 974: acceptance 5: two merges within a minute leave each open pull request one queued check --- cmd/mesh-controller/checks_queue_test.go | 95 ++++++++++++++++++++++++ testdata/store-tests | 1 + 2 files changed, 96 insertions(+) create mode 100644 cmd/mesh-controller/checks_queue_test.go diff --git a/cmd/mesh-controller/checks_queue_test.go b/cmd/mesh-controller/checks_queue_test.go new file mode 100644 index 00000000..98dfb31a --- /dev/null +++ b/cmd/mesh-controller/checks_queue_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// Acceptance 5 of milestone 2 (novox/hq ADR 0406 §3, issue 974), live, with the store and a real bus: two merges into one +// repository within a minute, with two open pull requests, leave each pull request with exactly one queued check, by the +// stream's rule. Each merge moves the trunk, and mesh-delivery asks the controller to check both open pull requests again +// against it (`delivery check`, which is the controller's own path for a head announced); nothing moves a request, nothing +// asks the forge, and the queue is read back whole. +func TestTwoMergesWithinAMinuteLeaveEachOpenPullRequestOneQueuedCheck(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + js, _ := aBuildQueue(t) + servingBus.Store(js) + t.Cleanup(func() { servingBus.Store(nil) }) + holdTheDeliverySeat(t, open) + was := askDeliveryOwner + t.Cleanup(func() { askDeliveryOwner = was }) + askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, _ map[string]any) (json.RawMessage, error) { + if verb != "groups" { + t.Fatalf("asked the owner %s", verb) + } + return json.RawMessage(`[]`), nil + } + // The repository the two pull requests are into, as a module the mesh builds from it. + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, + inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + + heads := map[int]string{7: "7777777777777777777777777777777777777777", 8: "8888888888888888888888888888888888888888"} + check := func(number int, trunk string) { + t.Helper() + p := orderMember{ID: fmt.Sprintf("novox/mesh-controller@%.12s", heads[number]), Repository: "novox/mesh-controller", + Base: "main", Head: heads[number], Number: number, Paths: []string{"main.go"}, ModuleDirsSaid: true, + BaseSHA: trunk, Ready: number == 8}.pull() + p.MergeCheck, p.MergeCheckSaid = true, true + if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil { + t.Fatal(err) + } + } + // Announced against the trunk as it stands, then asked again after each of two merges. + for _, trunk := range []string{"1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", + "3333333333333333333333333333333333333333"} { + check(7, trunk) + check(8, trunk) + } + + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + seen := map[int]string{} + for _, a := range q.Asks { + c := a.Request.Check + if c == nil { + t.Fatalf("%s is no check", a.ID) + } + if _, twice := seen[c.Number]; twice { + t.Errorf("pull request %d has two queued checks", c.Number) + } + seen[c.Number] = c.Trunk + if a.State != link.AskWaiting { + t.Errorf("%s is %s", a.ID, a.State) + } + if want := "3333333333333333333333333333333333333333"; c.Trunk != want { + t.Errorf("pull request %d is queued against %.8s; want the newest trunk %.8s", c.Number, c.Trunk, want) + } + } + if len(seen) != 2 { + t.Fatalf("queued checks for %v; want one for each of the two open pull requests", seen) + } + // By the streams' own count: the ready one on its stream, the other on its. + for stream, want := range map[string]uint64{broker.ChecksReadyStream: 1, broker.ChecksOtherStream: 1} { + info, err := js.Context().StreamInfo(stream) + if err != nil { + t.Fatal(err) + } + if info.State.Msgs != want { + t.Errorf("%s holds %d request(s); want %d", stream, info.State.Msgs, want) + } + } +} diff --git a/testdata/store-tests b/testdata/store-tests index 81cc53c1..61113e7e 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -136,3 +136,4 @@ ./cmd/mesh-controller TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid ./cmd/mesh-controller TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt ./cmd/mesh-controller TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent +./cmd/mesh-controller TestTwoMergesWithinAMinuteLeaveEachOpenPullRequestOneQueuedCheck From 357e08a4f6d0f080b91ac20ce74b55af5c3c4eff Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 14:04:27 +0200 Subject: [PATCH 10/12] issue 974: a broker test shows the build agent's reads reach its module principal as a read and watch of the pull requests' keys --- internal/broker/checks_test.go | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 internal/broker/checks_test.go diff --git a/internal/broker/checks_test.go b/internal/broker/checks_test.go new file mode 100644 index 00000000..e778abeb --- /dev/null +++ b/internal/broker/checks_test.go @@ -0,0 +1,39 @@ +package broker + +import "testing" + +// The build agent's credential is the module principal `.build-agent` (Username, KindModule): its manifest's +// `reads: ["mesh-delivery.pulls"]` reaches that principal as a read and a watch of mesh-delivery's bucket of pull requests' +// keys — the direct get of a key, the consumer a watch makes, the stream's info — and as no write and no direct +// subscription of the bucket's subjects (novox/hq ADR 0406 §4, issue 974). +func TestTheBuildAgentReadsAndWatchesThePullRequestsKeys(t *testing.T) { + seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}} + p := Principal{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{seat}, + Reads: []string{"mesh-delivery.pulls"}, PasswordHash: "x"} + if got := p.Username(); got != "one.build-agent" { + t.Fatalf("the build agent's credential is %q", got) + } + perms, err := PermissionsFor(p) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{ + "$JS.API.STREAM.INFO.KV_mesh-delivery_pulls", + "$JS.API.DIRECT.GET.KV_mesh-delivery_pulls.>", + "$JS.API.CONSUMER.CREATE.KV_mesh-delivery_pulls.>", + "$JS.API.CONSUMER.DELETE.KV_mesh-delivery_pulls.>", + // And the two check workers it takes from. + "$JS.API.CONSUMER.MSG.NEXT.CHECKS_READY.CHECKS_READY_worker", + "$JS.API.CONSUMER.MSG.NEXT.CHECKS_OTHER.CHECKS_OTHER_worker", + "$JS.ACK.CHECKS_OTHER.CHECKS_OTHER_worker.>", + } { + has(t, perms.Publish, s) + } + hasNot(t, perms.Publish, "$KV."+PullsBucket+".>") + hasNot(t, perms.Publish, "mesh.seat.node-build-agent.check.>") + for _, s := range perms.Subscribe { + if s == "$KV."+PullsBucket+".>" { + t.Fatalf("the build agent subscribes the bucket's subjects directly: %v", perms.Subscribe) + } + } +} From bd257afbd52d2e8fa94ffb3db5362da4190cd1fe Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 14:10:26 +0200 Subject: [PATCH 11/12] issue 974: the builders say the checks they run in a bucket that ages, and a builder drops a waiting request for a head and trunk commit another is running --- internal/broker/checks.go | 6 + internal/broker/checks_test.go | 3 + internal/broker/controller_buckets.go | 30 ++++- internal/broker/nats.go | 5 + internal/broker/testdata/composed.conf | 2 +- internal/broker/writers.go | 12 ++ internal/broker/writers_test.go | 1 + internal/link/builds_nats.go | 33 ++++- internal/link/checks_nats.go | 120 +++++++++++++++++ internal/link/checks_nats_test.go | 177 +++++++++++++++++++++++++ 10 files changed, 380 insertions(+), 9 deletions(-) diff --git a/internal/broker/checks.go b/internal/broker/checks.go index e9efd275..f9ab8da9 100644 --- a/internal/broker/checks.go +++ b/internal/broker/checks.go @@ -92,6 +92,12 @@ func holderCheckGrants() []string { "$JS.API.CONSUMER.MSG.NEXT."+w.Stream+"."+w.Name, "$JS.ACK."+w.Stream+"."+w.Name+".>") } + // And the bucket of the checks they are running: each writes the key of its own check, reads another's, and the bucket + // is a controller's own bucket the build seat's holders write by design (ADR 0406 §3). + pub = append(pub, + "$KV."+CheckRunningBucket+".>", + "$JS.API.STREAM.INFO.KV_"+CheckRunningBucket, + "$JS.API.DIRECT.GET.KV_"+CheckRunningBucket+".>") return pub } diff --git a/internal/broker/checks_test.go b/internal/broker/checks_test.go index e778abeb..23bb9e84 100644 --- a/internal/broker/checks_test.go +++ b/internal/broker/checks_test.go @@ -26,6 +26,9 @@ func TestTheBuildAgentReadsAndWatchesThePullRequestsKeys(t *testing.T) { "$JS.API.CONSUMER.MSG.NEXT.CHECKS_READY.CHECKS_READY_worker", "$JS.API.CONSUMER.MSG.NEXT.CHECKS_OTHER.CHECKS_OTHER_worker", "$JS.ACK.CHECKS_OTHER.CHECKS_OTHER_worker.>", + // And the bucket of the checks running, which it writes and reads (ADR 0406 §3). + "$KV." + CheckRunningBucket + ".>", + "$JS.API.DIRECT.GET.KV_" + CheckRunningBucket + ".>", } { has(t, perms.Publish, s) } diff --git a/internal/broker/controller_buckets.go b/internal/broker/controller_buckets.go index 7d8ba4e4..8aa61828 100644 --- a/internal/broker/controller_buckets.go +++ b/internal/broker/controller_buckets.go @@ -49,6 +49,13 @@ var ( // controller's watcher hearing the expiry ends the wait and names the machine silent. Written by the lease // holder alone. RolloutSilentBucket = BucketName(ControllerSeat, "rollout-silent") + // CheckRunningBucket holds the pull request checks the builders are running now (novox/hq ADR 0406 §3, issue 974): per + // pull request a key `..` (or `group.`) holding the head and trunk commit being checked, the + // builder and its build. A builder puts it when it starts a check and again on each heartbeat, and deletes it when the + // check ends; a key not renewed within the bucket's age is a builder that died, and is gone. A builder that takes a + // waiting request for a head and trunk commit a key names drops it. **Written by the build seat's holders, several by + // design** — each the key of the check it runs. + CheckRunningBucket = BucketName(ControllerSeat, "checks-running") ) // The controller's record mirrors (novox/hq ADR 0289, design 54, design 57 package F3): a bucket per kind of @@ -102,6 +109,10 @@ const RolloutSilentMarkerTTL = time.Minute // takes nothing live. const RolloutSilentKeptFor = time.Hour +// CheckRunningKeptFor is the age of a running check's key: a few heartbeats (ten seconds each) of a builder, so a builder that +// died is forgotten within two minutes and the check it ran is taken again then. +const CheckRunningKeptFor = 2 * time.Minute + // AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own. const AskedKeptFor = 30 * 24 * time.Hour @@ -130,13 +141,13 @@ func IsControllerBucket(bucket string) bool { return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket || bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket || bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket || - bucket == MergeWindowBucket || bucket == RolloutSilentBucket + bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket } // ControllerBuckets are the controller's own buckets, in the order they are asserted. func ControllerBuckets() []string { return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket, - AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket} + AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket} } // ControllerBucketsAsserter is what raising the controller's buckets needs of a connection. @@ -264,6 +275,21 @@ func (j *JetStream) EnsureControllerBuckets() error { }); err != nil { return fmt.Errorf("asserting bucket %s: %w", RolloutSilentBucket, err) } + // **The checks running: a key that ages** (ADR 0406 §3, issue 974). One message a key, each renewal a new one, so the + // age of the bucket is how long a key outlives the builder that stopped renewing it. + if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: CheckRunningBucket, + Description: "the pull request checks the builders are running now (novox/hq ADR 0406): `` holds the " + + "head and trunk commit, the builder and its build; put by the builder when it starts and at each heartbeat, " + + "deleted when the check ends, gone within two minutes of a builder that died; written by the build seat's holders", + History: 1, + TTL: CheckRunningKeptFor, + MaxValueSize: 4 << 10, + MaxBytes: 8 << 20, + Storage: jetstream.FileStorage, + }); err != nil { + return fmt.Errorf("asserting bucket %s: %w", CheckRunningBucket, err) + } // **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose // marker lives a day; a key that expired would be a record the bus forgot while the store still holds it. for _, m := range []struct{ bucket, why string }{ diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 7507be33..b73d975e 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -387,6 +387,11 @@ var DegradableGrants = []DegradableGrant{ "first push, and a check asked before that is asked again by the next announcement of its head (ADR 0406 §3, " + "issue 974)", }, + { + Subject: "$KV." + CheckRunningBucket + ".>", + Reason: "refused: the controller asserts this bucket through the API and writes none of its keys — the build seat's " + + "holders do (ADR 0406 §3, issue 974)", + }, { Subject: "mesh.seat." + ControllerSeat + ".event.sent", Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " + diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index dadf2781..32fd9212 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { jetstream: enabled users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/broker/writers.go b/internal/broker/writers.go index 437612e1..8dc3c4b0 100644 --- a/internal/broker/writers.go +++ b/internal/broker/writers.go @@ -41,6 +41,14 @@ func ownMachine(p Principal, pattern string) bool { return p.Kind == KindNode && len(tokens) > 2 && tokens[2] == p.Node } +// holdsTheBuildSeat is a principal holding the build seat, or the controller, which asserts the bucket the holders write. +func holdsTheBuildSeat(p Principal, _ string) bool { + if p.Kind == KindController { + return true + } + return holdsSeatOf(p, "mesh.seat."+TheBuildSeat+".x") +} + // holdsSeatOf is a principal holding the seat a `mesh.seat..…` pattern names: its module's own // principal, or the node tools carrying a module that holds it (ADR 0175, the runtime is its modules). func holdsSeatOf(p Principal, pattern string) bool { @@ -153,6 +161,10 @@ var WritersTable = []WriterRow{ Others: "the build seat's holders take it", Subjects: []string{CheckSubjectPrefix(TheBuildSeat, true) + ">", CheckSubjectPrefix(TheBuildSeat, false) + ">"}, Writes: isController}, + {State: "a pull request's check running", Writer: "the build seat's holders — several by design, each the key of the check it runs", + KeptIn: "key-value " + CheckRunningBucket + ", aged out when not renewed", Others: "the build seat's holders read it to drop a duplicate request", + Subjects: kvOf(CheckRunningBucket), Writes: holdsTheBuildSeat, + Shared: "every machine holding the build seat puts the key of the check it runs and deletes it when the check ends"}, {State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)", KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule}, // A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer. diff --git a/internal/broker/writers_test.go b/internal/broker/writers_test.go index 6f4c585f..6f929662 100644 --- a/internal/broker/writers_test.go +++ b/internal/broker/writers_test.go @@ -28,6 +28,7 @@ var designRows = []string{ "stream definitions and bus permissions", "builds and their outcomes", "a pull request's check asked", + "a pull request's check running", "a merge announced", "a pull request's head announced", "a pull request's merge check", diff --git a/internal/link/builds_nats.go b/internal/link/builds_nats.go index ce33dbec..b1985f0d 100644 --- a/internal/link/builds_nats.go +++ b/internal/link/builds_nats.go @@ -367,7 +367,11 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) // the ask to a second machine nor counts the wait against its deliveries. working := make(chan struct{}) go stillWorking(msg, working, heartbeatWhileWorking) + // And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat, + // deleted when it ends, whatever the outcome. + stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking) do(ctx, build) + stopRunning() close(working) } } @@ -431,17 +435,34 @@ func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) { // mesh-delivery's, and a check wrongly dropped is worse than one run twice). func (m *natsMachine) judgedAlready(ctx context.Context, r BuildRequest) string { c := r.Check - if c == nil || c.Group != "" || c.Try || c.Number <= 0 { + if c == nil || c.Try || (c.Group == "" && c.Number <= 0) { return "" } - key, found, err := ReadPullKey(ctx, m.js.Conn(), c.Owner, c.Repo, c.Number) + if c.Group == "" { + key, found, err := ReadPullKey(ctx, m.js.Conn(), c.Owner, c.Repo, c.Number) + if err != nil { + fmt.Fprintf(os.Stderr, "could not read the key of %s/%s#%d, so its check runs: %v\n", c.Owner, c.Repo, c.Number, err) + } else if found && key.Judged(r.Ref, c.Trunk) { + return fmt.Sprintf("%s/%s#%d is judged already at %.12s on %.12s (%s)", c.Owner, c.Repo, c.Number, r.Ref, c.Trunk, + key.Verdict) + } + } + return m.beingChecked(ctx, r) +} + +// beingChecked says why a request need not run, or nothing: another builder's check of the same head and trunk commit is +// running, and says so under its pull request's key in the bucket of the checks running (ADR 0406 §3). A key a builder +// stopped renewing is gone within the bucket's age, and the request is taken then. Never the ask's own id: a redelivered ask +// is its own, and a key an older builder wrote that names no head or trunk is no match. +func (m *natsMachine) beingChecked(ctx context.Context, r BuildRequest) string { + running, ok, err := ReadRunning(ctx, m.js.Conn(), r) if err != nil { - fmt.Fprintf(os.Stderr, "could not read the key of %s/%s#%d, so its check runs: %v\n", c.Owner, c.Repo, c.Number, err) + fmt.Fprintf(os.Stderr, "could not read whether %s is being checked, so its check runs: %v\n", r.ID, err) return "" } - if found && key.Judged(r.Ref, c.Trunk) { - return fmt.Sprintf("%s/%s#%d is judged already at %.12s on %.12s (%s)", c.Owner, c.Repo, c.Number, r.Ref, c.Trunk, - key.Verdict) + if ok && running.Build != r.ID && running.Same(r) { + return fmt.Sprintf("%s is being checked at %.12s on %.12s by %s as %s", r.ID, r.Ref, r.Check.Trunk, running.Builder, + running.Build) } return "" } diff --git a/internal/link/checks_nats.go b/internal/link/checks_nats.go index f0d90cf2..cf09cad0 100644 --- a/internal/link/checks_nats.go +++ b/internal/link/checks_nats.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "os" "strconv" "strings" "time" @@ -168,3 +169,122 @@ const ( checkTakeWait = 30 * time.Millisecond takeIdleWait = time.Second ) + +// CheckRunning is what a builder says of the check it is running, in the bucket of the checks running (novox/hq ADR 0406 +// §3): the head and trunk commit it checks, the builder and its build. The key is the pull request's (RunningKeyOf). +type CheckRunning struct { + Head string `json:"head"` + Trunk string `json:"trunk"` + Builder string `json:"builder"` + Build string `json:"build"` +} + +// RunningKeyOf is the key a check is said running under: its pull request's tokens, or `group.` for a group's +// composed check; none for a try, which no other request replaces. +func RunningKeyOf(r BuildRequest) (string, bool) { + c := r.Check + switch { + case c == nil, c.Try: + return "", false + case c.Group != "": + return "group." + tokenOf(c.Group), true + case c.Number <= 0: + return "", false + } + return PullKeyOf(c.Owner, c.Repo, c.Number), true +} + +// runningBucket opens the bucket of the checks running. +func runningBucket(ctx context.Context, conn *nats.Conn) (jetstream.KeyValue, error) { + api, err := jetstream.New(conn) + if err != nil { + return nil, err + } + kv, err := api.KeyValue(ctx, broker.CheckRunningBucket) + if err != nil { + return nil, fmt.Errorf("the checks running: %w", err) + } + return kv, nil +} + +// ReadRunning reads the key of the check running for a request's pull request: false when there is none. +func ReadRunning(ctx context.Context, conn *nats.Conn, r BuildRequest) (CheckRunning, bool, error) { + key, ok := RunningKeyOf(r) + if !ok { + return CheckRunning{}, false, nil + } + kv, err := runningBucket(ctx, conn) + if err != nil { + return CheckRunning{}, false, err + } + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return CheckRunning{}, false, nil + } + if err != nil { + return CheckRunning{}, false, err + } + var running CheckRunning + if err := json.Unmarshal(e.Value(), &running); err != nil { + return CheckRunning{}, false, nil // a key of a builder from before this means no match + } + return running, true, nil +} + +// Same says a running check is for this request's head and trunk commit: both said, and the same. A key that names neither +// (an older builder's) is no match. +func (c CheckRunning) Same(r BuildRequest) bool { + return r.Check != nil && c.Head != "" && c.Trunk != "" && r.Check.Trunk != "" && sameCommit(c.Head, r.Ref) && + sameCommit(c.Trunk, r.Check.Trunk) +} + +// keepRunning says a check is running: its key put now, put again at each heartbeat, and deleted by the stop it returns, +// whatever the outcome. Said to nobody that cannot be told: a bucket that is not there is no reason not to run the check. +func keepRunning(ctx context.Context, conn *nats.Conn, r BuildRequest, builder string, every time.Duration) (stop func()) { + key, ok := RunningKeyOf(r) + if !ok { + return func() {} + } + kv, err := runningBucket(ctx, conn) + if err != nil { + fmt.Fprintf(os.Stderr, "%s runs without saying so: %v\n", r.ID, err) + return func() {} + } + trunk := "" + if r.Check != nil { + trunk = r.Check.Trunk + } + body, _ := json.Marshal(CheckRunning{Head: r.Ref, Trunk: trunk, Builder: builder, Build: r.ID}) + put := func() { + putting, end := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer end() + if _, err := kv.Put(putting, key, body); err != nil { + fmt.Fprintf(os.Stderr, "%s could not say it is running: %v\n", r.ID, err) + } + } + put() + done := make(chan struct{}) + finished := make(chan struct{}) + go func() { + defer close(finished) + tick := time.NewTicker(every) + defer tick.Stop() + for { + select { + case <-done: + return + case <-tick.C: + put() + } + } + }() + return func() { + close(done) + <-finished + deleting, end := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer end() + if err := kv.Delete(deleting, key); err != nil { + fmt.Fprintf(os.Stderr, "%s could not take its running key away; it ages out: %v\n", r.ID, err) + } + } +} diff --git a/internal/link/checks_nats_test.go b/internal/link/checks_nats_test.go index 9326ade6..34c36447 100644 --- a/internal/link/checks_nats_test.go +++ b/internal/link/checks_nats_test.go @@ -307,3 +307,180 @@ func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testi t.Errorf("%d asks left; want 2", n) } } + +// aRunningBucket makes the bucket the builders say the checks they run in, with an age of its own short enough for a test +// (the mesh's is two minutes): a key not renewed within it is gone. +func aRunningBucket(t *testing.T, js *broker.JetStream, age time.Duration) jetstream.KeyValue { + t.Helper() + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + kv, err := api.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: broker.CheckRunningBucket, History: 1, TTL: age}) + if err != nil { + t.Fatal(err) + } + return kv +} + +// A builder taking a request for a pull request's head and trunk commit that another builder is running a check of drops it: +// the bucket of running checks names the head and the trunk, the builder and the build (issue 974). +func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) { + js := aBusWithTheBuildRole(t) + aKeyBucket(t, js) + aRunningBucket(t, js, 2*time.Minute) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + asks := BuildsOn(js, TheBuildMachine) + running := make(chan string, 4) + release := make(chan struct{}) + a := MachineOverNATS(js, "builder-a") + defer a.Close() + go func() { + _ = a.Take(ctx, func(ctx context.Context, work Build) { + running <- work.Request().ID + <-release + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"}) + _ = work.Done() + }) + }() + if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + select { + case <-running: + case <-time.After(10 * time.Second): + t.Fatal("builder a took nothing") + } + // The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it. + if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + b := MachineOverNATS(js, "builder-b") + defer b.Close() + took := make(chan string, 4) + go func() { + _ = b.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) + _ = work.Done() + }) + }() + select { + case id := <-took: + t.Fatalf("builder b took %s, which builder a is running a check of", id) + case <-time.After(3 * time.Second): + } + if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 { + t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n) + } + // Another trunk commit is another check. + if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + if id != "moved" { + t.Fatalf("builder b took %s", id) + } + case <-time.After(10 * time.Second): + t.Fatal("a check against another trunk commit was dropped too") + } + close(release) +} + +// A key that is not renewed ages out, and the request for its head and trunk commit is then taken: a builder that died +// running a check leaves nothing that holds the next one back for longer than the bucket's age. +func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) { + js := aBusWithTheBuildRole(t) + aKeyBucket(t, js) + kv := aRunningBucket(t, js, 2*time.Second) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"}) + if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil { + t.Fatal(err) + } + asks := BuildsOn(js, TheBuildMachine) + took := make(chan string, 4) + b := MachineOverNATS(js, "builder-b") + defer b.Close() + go func() { + _ = b.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) + _ = work.Done() + }) + }() + if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + t.Fatalf("%s was taken while the key was fresh", id) + case <-time.After(1200 * time.Millisecond): + } + time.Sleep(2500 * time.Millisecond) // the key was not renewed + if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + if id != "after-it-aged" { + t.Fatalf("took %s", id) + } + case <-time.After(10 * time.Second): + t.Fatal("the request was not taken after the key aged out") + } +} + +// A builder running a check renews its key on each heartbeat, so it outlives the bucket's age, and deletes it when the check +// ends, whatever the outcome. +func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) { + js := aBusWithTheBuildRole(t) + kv := aRunningBucket(t, js, 2*time.Second) + was := heartbeatWhileWorking + heartbeatWhileWorking = 400 * time.Millisecond + t.Cleanup(func() { heartbeatWhileWorking = was }) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + running := make(chan struct{}, 1) + release := make(chan struct{}) + m := MachineOverNATS(js, "builder-a") + defer m.Close() + go func() { + _ = m.Take(ctx, func(ctx context.Context, work Build) { + running <- struct{}{} + <-release + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"}) + _ = work.Done() + }) + }() + if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + <-running + key := PullKeyOf("novox", "a", 7) + e, err := kv.Get(ctx, key) + if err != nil { + t.Fatalf("no key while the check runs: %v", err) + } + var got CheckRunning + _ = json.Unmarshal(e.Value(), &got) + if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" { + t.Errorf("the key says %+v", got) + } + time.Sleep(3 * time.Second) // longer than the bucket's age + if _, err := kv.Get(ctx, key); err != nil { + t.Fatalf("the key aged out under a check still running: %v", err) + } + close(release) + for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) { + if _, err := kv.Get(ctx, key); err != nil { + break + } + if time.Now().After(end) { + t.Fatal("the key stays after the check ended") + } + } +} From 0f21fe7888bed2ffaa8523120746d53a8637980e Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 14:12:50 +0200 Subject: [PATCH 12/12] issue 974: review 2: a check stopped by its key says no verdict, an error judges nothing, and an unacknowledged check goes to a second builder --- cmd/mesh-controller/checks.go | 7 ++ cmd/mesh-controller/checks_test.go | 51 +++++++++ internal/link/checks_nats.go | 5 +- internal/link/checks_nats_test.go | 164 +++++++++++++++++++++++++++++ 4 files changed, 226 insertions(+), 1 deletion(-) diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index b3ba6d6a..e0104e34 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -583,6 +583,13 @@ const maxCheckReport = 60 << 10 // checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or // registered: a check builds nothing (issue 240's rule for a dry run, kept for a check). func checked(ctx context.Context, result link.BuildResult) { + // **A check stopped by its key says nothing** (novox/hq ADR 0406 §4, issue 974, review 2): it stopped because the head or + // the trunk commit moved on, and the newer request will judge. Said as an error it would name the live head — red + // statuses on the forge, a delivery refused — and written to the key it would stand in the newer request's way. + if result.Failed == link.SupersededByKey { + fmt.Printf("%s: stopped, its pull request's key moved on; the newer check will judge\n", result.ID) + return + } sayChecked(ctx, checkedOf(result)) } diff --git a/cmd/mesh-controller/checks_test.go b/cmd/mesh-controller/checks_test.go index 8ea4242a..dd6a3b8e 100644 --- a/cmd/mesh-controller/checks_test.go +++ b/cmd/mesh-controller/checks_test.go @@ -1,9 +1,16 @@ package main import ( + "encoding/json" "errors" "strings" "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -288,3 +295,47 @@ func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) { t.Fatalf("a rechecked head's direct verdict names trunk %q", c.Trunk) } } + +// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the +// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error. +func TestACheckStoppedByItsKeyIsAnnouncedAsNoVerdict(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + legacy, err := js.Conn().JetStream() + if err != nil { + t.Fatal(err) + } + was := checkEvents + checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} + t.Cleanup(func() { checkEvents = was }) + heard := make(chan *nats.Msg, 4) + sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyChecked), heard) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3} + checked(t.Context(), link.BuildResult{ID: "stopped", Ref: "abc", Checked: asked, Failed: link.SupersededByKey}) + checked(t.Context(), link.BuildResult{ID: "broken", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"}) + select { + case m := <-heard: + var c link.Checked + _ = json.Unmarshal(m.Data, &c) + if c.ID != "broken" || c.Verdict != "error" { + t.Fatalf("the first verdict said is %+v; want only the broken check's error", c) + } + case <-time.After(5 * time.Second): + t.Fatal("nothing was said") + } + select { + case m := <-heard: + t.Fatalf("a second verdict was said: %s", m.Data) + case <-time.After(300 * time.Millisecond): + } +} diff --git a/internal/link/checks_nats.go b/internal/link/checks_nats.go index cf09cad0..fe900eb2 100644 --- a/internal/link/checks_nats.go +++ b/internal/link/checks_nats.go @@ -85,7 +85,10 @@ func (k PullKey) Moved(head, trunk string) bool { // Judged says a key holds the verdict of the check of this head against this trunk commit. func (k PullKey) Judged(head, trunk string) bool { - return k.Verdict != "" && sameCommit(k.Head, head) && sameCommit(k.Trunk, trunk) && trunk != "" + // A verdict of a check that ran: a pass, a warning or a fail. An error is a check that could not run, and a request + // after it must run (issue 974, review 2). + judgedBy := k.Verdict == "pass" || k.Verdict == "warning" || k.Verdict == "fail" + return judgedBy && sameCommit(k.Head, head) && sameCommit(k.Trunk, trunk) && trunk != "" } // ReadPullKey reads a pull request's key from mesh-delivery's bucket: false when it has none. diff --git a/internal/link/checks_nats_test.go b/internal/link/checks_nats_test.go index 34c36447..b5a2f51c 100644 --- a/internal/link/checks_nats_test.go +++ b/internal/link/checks_nats_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" @@ -484,3 +485,166 @@ func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) { } } } + +// A key whose verdict is "error" — a check that could not run, or was stopped — judged nothing: a request for that head and +// trunk commit must run (issue 974, review 2). +func TestAnErrorVerdictJudgesNothing(t *testing.T) { + for verdict, judged := range map[string]bool{"pass": true, "warning": true, "fail": true, "error": false, "": false, "pending": false} { + k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: verdict} + if got := k.Judged("aaaaaaaaaaaa1111", "tttttttttttt1111"); got != judged { + t.Errorf("a key with verdict %q judged = %v; want %v", verdict, got, judged) + } + } +} + +// A check stopped because the trunk moved leaves the newer request, for the head against the new trunk commit, taken and +// run: the stop is the stopped one's failure, not the key's verdict, and the newer request is not dropped by it. +func TestACheckStoppedByATrunkMoveLeavesTheNewerRequestTakenAndRun(t *testing.T) { + js := aBusWithTheBuildRole(t) + kv := aKeyBucket(t, js) + aRunningBucket(t, js, 2*time.Minute) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + asks := BuildsOn(js, TheBuildMachine) + outcomes := make(chan BuildResult, 8) + watching, err := js.Conn().Subscribe(BuildOutcome(), func(m *nats.Msg) { + var r BuildResult + if json.Unmarshal(m.Data, &r) == nil { + outcomes <- r + } + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = watching.Unsubscribe() }() + + ran := make(chan string, 4) + m := MachineOverNATS(js, "builder-a") + defer m.Close() + go func() { + _ = m.Take(ctx, func(ctx context.Context, work Build) { + building, cancel := context.WithCancel(ctx) + defer cancel() + ran <- work.Request().ID + stopped := make(chan struct{}, 1) + end, err := work.(interface { + WatchPull(context.Context, func(PullKey)) (func(), error) + }).WatchPull(building, func(PullKey) { stopped <- struct{}{}; cancel() }) + if err != nil { + t.Error(err) + return + } + defer end() + if work.Request().ID == "older" { + select { + case <-stopped: + _ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: SupersededByKey}) + case <-time.After(15 * time.Second): + _ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: "never stopped"}) + } + } else { + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Check: &CheckOutcome{Verdict: "pass"}}) + } + _ = work.Done() + }) + }() + if err := asks.Ask(ctx, aPullCheck("older", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + if id := <-ran; id != "older" { + t.Fatalf("took %s", id) + } + time.Sleep(300 * time.Millisecond) // the watch is made + // The trunk moved; the head did not. mesh-delivery writes the key for the new request, and the request is asked. + putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-2-000000"}) + if err := asks.Ask(ctx, aPullCheck("newer", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil { + t.Fatal(err) + } + select { + case id := <-ran: + if id != "newer" { + t.Fatalf("took %s", id) + } + case <-time.After(15 * time.Second): + t.Fatal("the newer request, for the same head on the new trunk commit, was not taken") + } + got := map[string]string{} + for len(got) < 2 { + select { + case r := <-outcomes: + got[r.ID] = r.Failed + case <-time.After(10 * time.Second): + t.Fatalf("outcomes so far: %v", got) + } + } + if got["older"] != SupersededByKey || got["newer"] != "" { + t.Errorf("outcomes %v; want the older stopped and the newer run to a verdict", got) + } +} + +// An unacknowledged check goes to a second builder: taken from a check worker and not acknowledged, it is handed out again +// when the worker's wait is up. +func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) { + for _, ready := range []bool{false, true} { + js := aBusWithTheBuildRole(t) + name, worker := "other", broker.ChecksOtherWorker + if ready { + name, worker = "ready", broker.ChecksReadyWorker + } + // A worker whose wait is a second, as the mesh's is a minute. + for _, w := range broker.CheckWorkers() { + if w.Name == worker { + w.AckWaitSeconds = 1 + if err := js.EnsureConsumer(w); err != nil { + t.Fatal(err) + } + } + } + ctx, stop := context.WithCancel(t.Context()) + if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("dies-"+name, 7, "head-7-000000000", "trunk-1-000000", ready)); err != nil { + t.Fatal(err) + } + // The first builder takes it and dies: its pull is fetched, never acknowledged. + if _, err := fetchOne(js, worker); err != nil { + t.Fatal(err) + } + took := make(chan BuildRequest, 2) + second := MachineOverNATS(js, "builder-b") + go func() { + _ = second.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request() + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) + _ = work.Done() + }) + }() + select { + case r := <-took: + if r.ID != "dies-"+name { + t.Errorf("%s: took %s", name, r.ID) + } + case <-time.After(15 * time.Second): + t.Errorf("%s: the unacknowledged check did not go to a second builder", name) + } + second.Close() + stop() + } +} + +// fetchOne takes the next message of a worker and leaves it unacknowledged, as a builder that died would. +func fetchOne(js *broker.JetStream, worker string) (*nats.Msg, error) { + for _, w := range broker.CheckWorkers() { + if w.Name != worker { + continue + } + sub, err := js.Context().PullSubscribe(w.Filters[0], w.Name, nats.Bind(w.Stream, w.Name), nats.ManualAck()) + if err != nil { + return nil, err + } + got, err := sub.Fetch(1, nats.MaxWait(5*time.Second)) + if err != nil { + return nil, err + } + return got[0], nil + } + return nil, nil +}