Test the capability gate, the sweep's mark re-read and the lock use of a send and of the sweep, each failing under its mutation (issues 1696, 1697; review round 2)

cmd/mesh-controller/declaration_files_integration_test.go

@@ -159,3 +159,88 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T)
 		t.Fatalf("the bucket holds %d objects (%v), want the 3 the declaration names", len(infos), err)
 	}
 }
+
+// 1696/AC8 on a real bucket: the sweep removes an object only if it still carries the mark it judged it by. An object marked
+// unnamed, then put again by a send (which erases the mark), is kept when the sweep deletes it by the old mark.
+func TestTheSweepsDeleteLeavesAnObjectASendPutAgainAfterItWasMarked(t *testing.T) { // 1696/AC8
+	s := testbus.Start(t)
+	conn, err := nats.Connect(s.ClientURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	bucket, err := openBucket(ctx, conn, "novox")
+	if err != nil {
+		t.Fatal(err)
+	}
+	name := strings.Repeat("a", 64)
+	content := map[string][]byte{name: bytes.Repeat([]byte("y"), 100)}
+	if err := putObjects(ctx, bucket, content); err != nil {
+		t.Fatal(err)
+	}
+	b := busSweptBucket{bucket.store}
+	judged := time.Now().Add(-25 * time.Hour).Truncate(time.Second)
+	if err := b.MarkUnnamed(ctx, name, judged); err != nil {
+		t.Fatal(err)
+	}
+	if _, marked, err := bucket.Has(ctx, name); err != nil || !marked {
+		t.Fatalf("the object was not marked: %v %v", marked, err)
+	}
+	// The sweep has read the mark; a send now puts the object again, which erases it.
+	if err := putObjects(ctx, bucket, content); err != nil {
+		t.Fatal(err)
+	}
+	if have, marked, err := bucket.Has(ctx, name); err != nil || !have || marked {
+		t.Fatalf("a send did not put the marked object again: have %v marked %v %v", have, marked, err)
+	}
+	gone, err := b.Delete(ctx, name, judged)
+	if err != nil || gone {
+		t.Fatalf("the sweep deleted by an old mark: gone %v, %v", gone, err)
+	}
+	if have, _, err := bucket.Has(ctx, name); err != nil || !have {
+		t.Fatalf("the object a send relies on is gone: %v %v", have, err)
+	}
+	// And the mark still on it is honoured: marked again and deleted by that mark, it goes.
+	if err := b.MarkUnnamed(ctx, name, judged); err != nil {
+		t.Fatal(err)
+	}
+	if gone, err := b.Delete(ctx, name, judged); err != nil || !gone {
+		t.Fatalf("an object still carrying the mark it was judged by is kept: gone %v, %v", gone, err)
+	}
+}
+
+// 1696/AC9 on a real bus: the sweep of a node's bucket takes the node's lock first, so it waits for a send that has put its objects
+// and not yet published the declaration that names them.
+func TestTheSweepOfANodesBucketHoldsTheNodesLock(t *testing.T) { // 1696/AC9
+	s := testbus.Start(t)
+	conn, err := nats.Connect(s.ClientURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	const node = "sweep-lock-node"
+	if _, err := openBucket(ctx, conn, node); err != nil { // the bucket exists, or the sweep would return before the lock
+		t.Fatal(err)
+	}
+	release := lockFilesOf(node) // a send is between its put and its publication
+	done := make(chan struct{})
+	go func() {
+		_, _ = sweepNode(ctx, conn, node)
+		close(done)
+	}()
+	select {
+	case <-done:
+		t.Fatal("the sweep judged the node's bucket while a send held it")
+	case <-time.After(300 * time.Millisecond):
+	}
+	release()
+	select {
+	case <-done:
+	case <-time.After(20 * time.Second):
+		t.Fatal("the sweep did not go on once the send let go")
+	}
+}