Give each node a bucket of its own for the files that travel by digest, and keep the sweep from removing an object a send relies on (issues 1696, 1697; review of ADR 0486 step 3)

A literal * in a published subject matches the grant's *, so in one shared bucket a node could read every node's files. Each node now
has declaration-files-<node>, which its account reads by name and the controller writes; the sweep judges each bucket against that
node's latest declaration, under the node's lock with the send, and an object the sweep marked is put again by the next send. Putting
that fails send after send raises a condition.

cmd/mesh-controller/busobjects.go

@@ -32,6 +32,13 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
 	if err := broker.Raise(r, names); err != nil {
 		return nil, err
 	}
+	// And each node's bucket for the files of its declaration that travel by digest (novox/hq ADR 0486 §4), so a node finds its own
+	// whenever the bus is up.
+	if files, ok := r.(interface{ EnsureDeclarationFiles(nodes []string) error }); ok {
+		if err := files.EnsureDeclarationFiles(names); err != nil {
+			return nil, err
+		}
+	}
 	// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
 	// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
 	// after something started asking for builds flushes the backlog instead of having lost it.

cmd/mesh-controller/declaration_files.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"sort"
+	"sync"
 
 	"git.novox.be/novox/mesh-sdk/go/compose"
 	sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
@@ -13,6 +14,7 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 
 	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/conditions"
 )
 
 // **A declaration's large files travel by their digest to a node that can read one** (novox/hq ADR 0486 §1, §5, issue 1696).
@@ -112,17 +114,20 @@ func namedObjects(body []byte) (map[string]int64, error) {
 	return out, nil
 }
 
-// objectStore is what putting the objects needs of the bucket.
+// objectStore is what putting the objects needs of a node's bucket.
 type objectStore interface {
-	// Has says whether an object of that name is there, complete.
-	Has(ctx context.Context, name string) (bool, error)
-	// Put writes the object under that name.
+	// Has says whether an object of that name is there, complete, and whether it carries the sweep's mark of being named by no
+	// node's latest declaration.
+	Has(ctx context.Context, name string) (have, marked bool, err error)
+	// Put writes the object under that name, replacing one of the same name (and so erasing its mark).
 	Put(ctx context.Context, name string, content []byte) error
 }
 
-// putObjects makes sure every object of the declaration is in the bucket before the declaration that names it is sent: a
-// node that reads the declaration finds each object. An object already there (the same content, the same name) is not
-// written again. Refuses on the first it cannot put, naming it, so the declaration is not sent.
+// putObjects makes sure every object of the declaration is in the bucket before the declaration that names it is sent: a node
+// that reads the declaration finds each object. An object already there (the same content, the same name) is not written again,
+// **unless the sweep marked it unnamed**: then it is written again, so the mark that would have it removed is gone with the
+// old object (issue 1696, review: a mark read just before a send must not take an object the send relies on). Refuses on the
+// first it cannot put, naming it, so the declaration is not sent.
 func putObjects(ctx context.Context, store objectStore, objects map[string][]byte) error {
 	names := make([]string, 0, len(objects))
 	for name := range objects {
@@ -130,48 +135,115 @@ func putObjects(ctx context.Context, store objectStore, objects map[string][]byt
 	}
 	sort.Strings(names)
 	for _, name := range names {
-		have, err := store.Has(ctx, name)
+		have, marked, err := store.Has(ctx, name)
 		if err != nil {
-			return fmt.Errorf("whether the object %s is in the bucket %s: %w", name[:12], broker.DeclarationFilesBucket, err)
+			return fmt.Errorf("whether the object %s is in the bucket: %w", name[:12], err)
 		}
-		if have {
+		if have && !marked {
 			continue
 		}
 		if err := store.Put(ctx, name, objects[name]); err != nil {
-			return fmt.Errorf("putting the object %s (%d bytes) in the bucket %s: %w", name[:12], len(objects[name]),
-				broker.DeclarationFilesBucket, err)
+			return fmt.Errorf("putting the object %s (%d bytes) in the bucket: %w", name[:12], len(objects[name]), err)
 		}
 	}
 	return nil
 }
 
-// busObjects is the bucket on the controller's own connection.
+// busObjects is a node's bucket on the controller's own connection.
 type busObjects struct{ store jetstream.ObjectStore }
 
-func openBucket(ctx context.Context, conn *nats.Conn) (busObjects, error) {
+// openBucket opens the bucket of a node's files, making it first if it is not there (it is asserted for every node at the
+// controller's start; a node adopted since is covered here).
+func openBucket(ctx context.Context, conn *nats.Conn, node string) (busObjects, error) {
 	js, err := jetstream.New(conn)
 	if err != nil {
 		return busObjects{}, err
 	}
-	store, err := js.ObjectStore(ctx, broker.DeclarationFilesBucket)
+	store, err := js.ObjectStore(ctx, broker.DeclarationFilesBucketOf(node))
+	if errors.Is(err, jetstream.ErrBucketNotFound) {
+		if err = broker.EnsureDeclarationFiles(ctx, js, node); err == nil {
+			store, err = js.ObjectStore(ctx, broker.DeclarationFilesBucketOf(node))
+		}
+	}
 	if err != nil {
-		return busObjects{}, fmt.Errorf("the object bucket %s: %w", broker.DeclarationFilesBucket, err)
+		return busObjects{}, fmt.Errorf("the object bucket %s: %w", broker.DeclarationFilesBucketOf(node), err)
 	}
 	return busObjects{store}, nil
 }
 
-func (b busObjects) Has(ctx context.Context, name string) (bool, error) {
+func (b busObjects) Has(ctx context.Context, name string) (bool, bool, error) {
 	info, err := b.store.GetInfo(ctx, name)
 	switch {
 	case errors.Is(err, jetstream.ErrObjectNotFound):
-		return false, nil
+		return false, false, nil
 	case err != nil:
-		return false, err
+		return false, false, err
 	}
-	return !info.Deleted, nil
+	_, marked := info.Metadata[declarationUnnamed]
+	return !info.Deleted, marked, nil
 }
 
 func (b busObjects) Put(ctx context.Context, name string, content []byte) error {
 	_, err := b.store.PutBytes(ctx, name, content)
 	return err
 }
+
+// filesLocks hold a node's bucket for one act at a time in this process: a send puts its objects and publishes the declaration that
+// names them under the node's lock, and the sweep judges the node's bucket under the same one, so the sweep never removes an
+// object between a send's put and its declaration. Another process (a person's push) is covered by the put erasing a mark and by
+// the sweep looking at an object again just before it removes it.
+var filesLocks sync.Map
+
+func lockFilesOf(node string) func() {
+	l, _ := filesLocks.LoadOrStore(node, &sync.Mutex{})
+	mu := l.(*sync.Mutex)
+	mu.Lock()
+	return mu.Unlock
+}
+
+// kindFilesNotPut is a node whose files that travel by digest could not be put in its bucket, send after send.
+const kindFilesNotPut = "declaration-files-not-put"
+
+// filesNotPutAfter is how many sends in a row may fail to put before the condition is raised: the first send to the machine holding
+// the bus fails for want of the grant that very send delivers, and says so, and is not yet a fault.
+const filesNotPutAfter = 2
+
+var (
+	filesFailMu sync.Mutex
+	filesFailed = map[string]int{}
+)
+
+func filesNotPutObservation(node, summary string) conditions.Observation {
+	return conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindFilesNotPut, Token: kindFilesNotPut,
+		Machine: node, Severity: conditions.Warning, Source: "sends", Summary: summary}
+}
+
+// notePut records how putting a send's objects went: a failure counts, the second in a row raises the condition, a success
+// clears the count and the condition.
+func notePut(ctx context.Context, node string, put error) {
+	filesFailMu.Lock()
+	if put == nil {
+		delete(filesFailed, node)
+	} else {
+		filesFailed[node]++
+	}
+	n := filesFailed[node]
+	filesFailMu.Unlock()
+	keeper := conditionsFrom
+	if keeper == nil {
+		return
+	}
+	key := filesNotPutObservation(node, "").Key()
+	switch {
+	case put == nil:
+		if _, err := keeper.Clear(ctx, key, node+" was sent its files by digest"); err != nil {
+			fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
+		}
+	case n >= filesNotPutAfter:
+		summary := fmt.Sprintf("%s's files that travel by digest could not be put in its bucket %d sends in a row, so it is sent them "+
+			"inline: %v", node, n, put)
+		if _, err := keeper.Observe(ctx, filesNotPutObservation(node, summary)); err != nil {
+			fmt.Printf("the condition %s could not be raised: %v\n", key, err)
+		}
+	}
+}

cmd/mesh-controller/declaration_files_integration_test.go

@@ -48,11 +48,11 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := broker.EnsureDeclarationFiles(ctx, api); err != nil {
+	if err := broker.EnsureDeclarationFiles(ctx, api, "novox"); err != nil {
 		t.Fatal(err)
 	}
 	// Run twice: an assertion of an existing bucket is no error and no change.
-	if err := broker.EnsureDeclarationFiles(ctx, api); err != nil {
+	if err := broker.EnsureDeclarationFiles(ctx, api, "novox"); err != nil {
 		t.Fatalf("asserting the bucket again: %v", err)
 	}
 
@@ -74,7 +74,7 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T)
 		t.Fatalf("the inline declaration of more than 3 MiB: %v", err)
 	}
 
-	bucket, err := openBucket(ctx, conn)
+	bucket, err := openBucket(ctx, conn, "novox")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -101,11 +101,11 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T)
 	if err != nil || len(latest) != 1 {
 		t.Fatalf("the latest declarations: %v %v", len(latest), err)
 	}
-	named, err := namedByLatest(latest)
+	named, err := namedByLatest("novox", latest["novox"])
 	if err != nil || len(named) != 3 {
 		t.Fatalf("what the latest declaration names: %v %v", named, err)
 	}
-	store, err := api.ObjectStore(ctx, broker.DeclarationFilesBucket)
+	store, err := api.ObjectStore(ctx, broker.DeclarationFilesBucketOf("novox"))
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/mesh-controller/declaration_files_test.go

@@ -13,6 +13,7 @@ import (
 
 	sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
 
+	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/link"
 )
 
@@ -202,14 +203,15 @@ func TestAFileOfBytesTravelsByDigestOnItsDecodedLength(t *testing.T) {
 }
 
 type memObjects struct {
-	have map[string][]byte
-	puts []string
-	fail map[string]error
+	have   map[string][]byte
+	marked map[string]bool
+	puts   []string
+	fail   map[string]error
 }
 
-func (m *memObjects) Has(_ context.Context, name string) (bool, error) {
+func (m *memObjects) Has(_ context.Context, name string) (bool, bool, error) {
 	_, ok := m.have[name]
-	return ok, nil
+	return ok, m.marked[name], nil
 }
 func (m *memObjects) Put(_ context.Context, name string, c []byte) error {
 	if err := m.fail[name]; err != nil {
@@ -217,6 +219,7 @@ func (m *memObjects) Put(_ context.Context, name string, c []byte) error {
 	}
 	m.puts = append(m.puts, name)
 	m.have[name] = c
+	delete(m.marked, name)
 	return nil
 }
 
@@ -227,6 +230,12 @@ func TestObjectsAreInTheBucketBeforeTheDeclarationIsSent(t *testing.T) { // 1696
 	if err := putObjects(t.Context(), b, objects); err != nil || len(b.puts) != 1 || b.puts[0] != strings.Repeat("b", 64) {
 		t.Fatalf("put %v (%v), want the one absent", b.puts, err)
 	}
+	// An object the sweep marked unnamed is put again, which erases the mark: a send relies on it.
+	b = &memObjects{have: map[string][]byte{strings.Repeat("a", 64): []byte("one"), strings.Repeat("b", 64): []byte("two")},
+		marked: map[string]bool{strings.Repeat("a", 64): true}}
+	if err := putObjects(t.Context(), b, objects); err != nil || len(b.puts) != 1 || b.puts[0] != strings.Repeat("a", 64) || b.marked[strings.Repeat("a", 64)] {
+		t.Fatalf("a marked object: put %v (%v) marked %v, want it put again and the mark gone", b.puts, err, b.marked)
+	}
 	b = &memObjects{have: map[string][]byte{}, fail: map[string]error{strings.Repeat("a", 64): errors.New("permissions violation")}}
 	err := putObjects(t.Context(), b, objects)
 	if err == nil || !strings.Contains(err.Error(), "aaaaaaaaaaaa") || !strings.Contains(err.Error(), "permissions violation") {
@@ -303,10 +312,13 @@ func (b *sweepBucketFake) ClearUnnamed(_ context.Context, n string) error {
 	b.objects[n] = time.Time{}
 	return nil
 }
-func (b *sweepBucketFake) Delete(_ context.Context, n string) error {
+func (b *sweepBucketFake) Delete(_ context.Context, n string, markedAt time.Time) (bool, error) {
+	if at, ok := b.objects[n]; !ok || !at.Equal(markedAt) {
+		return false, nil // put again since it was judged: its mark is gone
+	}
 	delete(b.objects, n)
 	b.deleted = append(b.deleted, n)
-	return nil
+	return true, nil
 }
 
 // 1696/AC6: an object a latest declaration still names is not removed, however long; one unnamed is marked at the first look and removed
@@ -348,12 +360,15 @@ func TestTheObjectsALatestDeclarationNamesAreReadFromTheBus(t *testing.T) {
 	s := sendable{fileReferences: true, Resources: []map[string]any{aFileResource("a", contentOf("a", 70<<10))}}
 	body, _ := s.Body()
 	msg, _ := link.Envelope(body, []byte("sig"))
-	named, err := namedByLatest(map[string][]byte{"novox": msg, "ace": []byte(`{"declaration":"` + "e30=" + `","signature":""}`)})
+	named, err := namedByLatest("novox", msg)
 	name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(contentOf("a", 70<<10))))
 	if err != nil || len(named) != 1 || !named[name] {
 		t.Fatalf("%v %v", named, err)
 	}
-	if _, err := namedByLatest(map[string][]byte{"novox": []byte("not json")}); err == nil {
+	if none, err := namedByLatest("ace", []byte(`{"declaration":"e30=","signature":""}`)); err != nil || len(none) != 0 {
+		t.Fatalf("a declaration naming none: %v %v", none, err)
+	}
+	if _, err := namedByLatest("novox", []byte("not json")); err == nil {
 		t.Fatal("an unreadable latest declaration was read as naming none")
 	}
 }
@@ -379,3 +394,76 @@ func TestTheControlNodesDeclarationInTheReferenceFormStaysUnderHalfTheBusLimit(t
 }
 
 func stdBase64(b []byte) string { return base64.StdEncoding.EncodeToString(b) }
+
+// A sweep that read an object's mark before a send put it again must not remove it: the object, now without the mark, is left.
+func TestASweepDoesNotRemoveAnObjectASendPutAgainAfterItWasJudged(t *testing.T) { // 1696/AC8
+	n := strings.Repeat("4", 64)
+	t0 := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
+	b := &sweepBucketFake{objects: map[string]time.Time{n: t0}}
+	judged := bucketObject{Name: n, UnnamedSince: t0}
+	b.objects[n] = time.Time{} // the send put it again: no mark
+	if gone, err := b.Delete(t.Context(), judged.Name, judged.UnnamedSince); err != nil || gone || len(b.deleted) != 0 {
+		t.Fatalf("an object put again was removed: %v %v %v", gone, err, b.deleted)
+	}
+}
+
+// One node's bucket is held by one act at a time in this process: the sweep waits for a send that has put its objects and not yet
+// published the declaration that names them (1696/AC9).
+func TestTheSweepOfANodesBucketWaitsForASendThatHasPutItsObjects(t *testing.T) { // 1696/AC9
+	unlock := lockFilesOf("lock-test-node")
+	got := make(chan struct{})
+	go func() {
+		defer lockFilesOf("lock-test-node")()
+		close(got)
+	}()
+	select {
+	case <-got:
+		t.Fatal("the sweep took the node's bucket while a send held it")
+	case <-time.After(50 * time.Millisecond):
+	}
+	other := lockFilesOf("lock-test-other-node") // another node's bucket is not held
+	other()
+	unlock()
+	select {
+	case <-got:
+	case <-time.After(2 * time.Second):
+		t.Fatal("the sweep did not take the node's bucket once the send was done")
+	}
+}
+
+// A put that fails send after send raises a condition; the first failure does not (the grant it waits for travels in that very
+// declaration), and a put that works clears it (1696/AC10).
+func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testing.T) { // 1696/AC10
+	store := conditions.NewInMemory()
+	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
+		Teller: &conditions.Told{}, Now: func() time.Time { return time.Date(2026, 10, 21, 14, 0, 0, 0, time.UTC) }})
+	t.Cleanup(func() { k.Close(t.Context()) })
+	was := conditionsFrom
+	conditionsFrom = k
+	t.Cleanup(func() { conditionsFrom = was })
+	key := filesNotPutObservation("cond-node", "").Key()
+	open := func() bool {
+		_, found, err := conditions.ReadOne(t.Context(), store, key)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return found
+	}
+	boom := errors.New("permissions violation")
+	notePut(t.Context(), "cond-node", boom)
+	if open() {
+		t.Fatal("the first failure raised the condition")
+	}
+	notePut(t.Context(), "cond-node", boom)
+	if !open() {
+		t.Fatal("the second failure in a row did not raise the condition")
+	}
+	notePut(t.Context(), "cond-node", nil)
+	if open() {
+		t.Fatal("a put that worked did not clear the condition")
+	}
+	notePut(t.Context(), "cond-node", boom)
+	if open() {
+		t.Fatal("the count did not start again after a put that worked")
+	}
+}

cmd/mesh-controller/declaration_sweep.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/nats-io/nats.go"
@@ -20,7 +21,8 @@ import (
 // asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest
 // declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else
 // is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
-// found named again, that is erased; found unnamed for a day, it is removed. Objects are immutable and regenerable from the
+// found named again, that is erased; found unnamed for a day, it is removed. Each node has a bucket of its own, judged against that
+// node's latest declaration alone. Objects are immutable and regenerable from the
 // controller's state, so a removal loses nothing.
 
 // declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration.
@@ -41,7 +43,9 @@ type sweptBucket interface {
 	Objects(ctx context.Context) ([]bucketObject, error)
 	MarkUnnamed(ctx context.Context, name string, since time.Time) error
 	ClearUnnamed(ctx context.Context, name string) error
-	Delete(ctx context.Context, name string) error
+	// Delete removes the object if it still carries the mark it was judged by, and says whether it did: a send that put it again
+	// since has erased the mark, and the object is left.
+	Delete(ctx context.Context, name string, markedAt time.Time) (bool, error)
 }
 
 // filesSweep says what a sweep did.
@@ -72,10 +76,15 @@ func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[st
 			}
 			r.Marked++
 		case now.Sub(o.UnnamedSince) >= broker.DeclarationFilesKeptFor:
-			if err := bucket.Delete(ctx, o.Name); err != nil {
+			gone, err := bucket.Delete(ctx, o.Name, o.UnnamedSince)
+			if err != nil {
 				return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
 			}
-			r.Removed++
+			if gone {
+				r.Removed++
+			} else {
+				r.Kept++
+			}
 		default:
 			r.Kept++
 		}
@@ -83,22 +92,20 @@ func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[st
 	return r, nil
 }
 
-// namedByLatest is the name of every object any of these latest declarations names. A body that cannot be read is an error, never
-// "names none": an object a node still needs would be marked for removal.
-func namedByLatest(bodies map[string][]byte) (map[string]bool, error) {
-	named := map[string]bool{}
-	for node, signed := range bodies {
-		var s link.Signed
-		if err := json.Unmarshal(signed, &s); err != nil {
-			return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
-		}
-		names, err := namedObjects(s.Declaration)
-		if err != nil {
-			return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
-		}
-		for name := range names {
-			named[name] = true
-		}
+// namedByLatest is the name of every object this latest declaration of a node names. A body that cannot be read is an error, never
+// "names none": an object the node still needs would be marked for removal.
+func namedByLatest(node string, signed []byte) (map[string]bool, error) {
+	var s link.Signed
+	if err := json.Unmarshal(signed, &s); err != nil {
+		return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
+	}
+	names, err := namedObjects(s.Declaration)
+	if err != nil {
+		return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
+	}
+	named := make(map[string]bool, len(names))
+	for name := range names {
+		named[name] = true
 	}
 	return named, nil
 }
@@ -169,8 +176,18 @@ func (b busSweptBucket) ClearUnnamed(ctx context.Context, name string) error {
 	return b.meta(ctx, name, func(m map[string]string) { delete(m, declarationUnnamed) })
 }
 
-func (b busSweptBucket) Delete(ctx context.Context, name string) error {
-	return b.store.Delete(ctx, name)
+func (b busSweptBucket) Delete(ctx context.Context, name string, markedAt time.Time) (bool, error) {
+	info, err := b.store.GetInfo(ctx, name)
+	if errors.Is(err, jetstream.ErrObjectNotFound) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	if at, perr := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); perr != nil || !at.Equal(markedAt) {
+		return false, nil
+	}
+	return true, b.store.Delete(ctx, name)
 }
 
 // sweepingDeclarationFiles judges the bucket every sweepEvery for as long as this controller serves and holds the lease.
@@ -199,40 +216,54 @@ func sweepingDeclarationFiles(ctx context.Context, inv *inventory.Inventory, con
 	}
 }
 
+// sweepOnce judges the bucket of every node against that node's latest declaration, one bucket at a time under the node's lock, with
+// the declaration read again under it: a send that put an object and published its declaration is seen, and one that has not yet
+// is waiting for the lock.
 func sweepOnce(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) string {
-	working, cancel := context.WithTimeout(ctx, 2*time.Minute)
-	defer cancel()
-	nodes, err := inv.Nodes(working)
+	nodes, err := inv.Nodes(ctx)
 	if err != nil {
 		return fmt.Sprintf("the nodes cannot be read: %v", err)
 	}
-	names := make([]string, 0, len(nodes))
+	var problems []string
+	var total filesSweep
 	for _, n := range nodes {
-		names = append(names, n.Name)
+		r, err := sweepNode(ctx, conn, n.Name)
+		total.Kept, total.Marked, total.Cleared, total.Removed = total.Kept+r.Kept, total.Marked+r.Marked, total.Cleared+r.Cleared, total.Removed+r.Removed
+		if err != nil {
+			problems = append(problems, fmt.Sprintf("%s: %v", n.Name, err))
+		}
 	}
-	bodies, err := latestDeclarations(working, conn, names)
-	if err != nil {
-		return err.Error()
-	}
-	named, err := namedByLatest(bodies)
-	if err != nil {
-		return err.Error()
+	if total.Removed+total.Marked+total.Cleared > 0 {
+		fmt.Printf("the buckets of declaration files: %d object(s) kept, %d found unnamed, %d named again, %d removed\n",
+			total.Kept, total.Marked, total.Cleared, total.Removed)
 	}
+	return strings.Join(problems, "; ")
+}
+
+func sweepNode(ctx context.Context, conn *nats.Conn, node string) (filesSweep, error) {
+	working, cancel := context.WithTimeout(ctx, 2*time.Minute)
+	defer cancel()
 	js, err := jetstream.New(conn)
 	if err != nil {
-		return err.Error()
+		return filesSweep{}, err
 	}
-	store, err := js.ObjectStore(working, broker.DeclarationFilesBucket)
-	if err != nil {
-		return fmt.Sprintf("the bucket cannot be read: %v", err)
-	}
-	r, err := sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
-	if r.Removed+r.Marked+r.Cleared > 0 {
-		fmt.Printf("the bucket of declaration files: %d object(s) kept, %d found unnamed, %d named again, %d removed\n",
-			r.Kept, r.Marked, r.Cleared, r.Removed)
+	store, err := js.ObjectStore(working, broker.DeclarationFilesBucketOf(node))
+	if errors.Is(err, jetstream.ErrBucketNotFound) {
+		return filesSweep{}, nil // a node never sent a reference has no bucket yet
 	}
 	if err != nil {
-		return err.Error()
+		return filesSweep{}, fmt.Errorf("the bucket cannot be read: %w", err)
 	}
-	return ""
+	defer lockFilesOf(node)()
+	latest, err := latestDeclarations(working, conn, []string{node})
+	if err != nil {
+		return filesSweep{}, err
+	}
+	named := map[string]bool{}
+	if body, ok := latest[node]; ok {
+		if named, err = namedByLatest(node, body); err != nil {
+			return filesSweep{}, err
+		}
+	}
+	return sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
 }

cmd/mesh-controller/plain_words.go

@@ -195,6 +195,14 @@ var plainWordings = map[string]func(conditions.Observation) words{
 				"there yet is sealed by the controller until the vault holds it.", m, m),
 			Resolved: "Resolved: the vault gave " + m + " the core's credentials"}
 	}),
+	kindFilesNotPut: worded(func(o conditions.Observation) words {
+		m := machineOr(o, "a machine")
+		return words{Headline: "The big files for " + m + " cannot be put on the bus",
+			Needs: "look at the bus's permissions for the controller; the details give the reason.",
+			Explanation: fmt.Sprintf("The mesh could not put the large files of %s's instructions in their place on the bus, "+
+				"send after send, so %s is sent them inside the instructions instead, which may make them too big.", m, m),
+			Resolved: "Resolved: the big files for " + m + " reach the bus again"}
+	}),
 	kindDeclarationNearLimit: worded(func(o conditions.Observation) words {
 		m := machineOr(o, "a machine")
 		if o.Severity == conditions.Urgent {

cmd/mesh-controller/push.go

@@ -1067,13 +1067,14 @@ func (b overTheBus) putFiles(ctx context.Context, s readyNode) error {
 	if !ok {
 		return fmt.Errorf("%s: the declaration names files by digest and this bus has no object bucket", s.node)
 	}
-	store, err := openBucket(ctx, bus.Conn)
-	if err != nil {
-		return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err)
-	}
 	putting, cancel := context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()
-	if err := putObjects(putting, store, objects); err != nil {
+	store, err := openBucket(putting, bus.Conn, s.node)
+	if err == nil {
+		err = putObjects(putting, store, objects)
+	}
+	notePut(ctx, s.node, err)
+	if err != nil {
 		return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err)
 	}
 	return nil
@@ -1096,11 +1097,17 @@ func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
 func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
 	// The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node
 	// that applies it finds every one. A node reading no references has none.
-	if err := b.putFiles(ctx, s); err != nil {
+	// **Under the node's lock** from the put to the publication, so the sweep of its bucket cannot remove an object between them.
+	unlock := lockFilesOf(s.node)
+	err := b.putFiles(ctx, s)
+	if err != nil {
+		unlock()
 		return "", err
 	}
-	if err := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second); err != nil {
-		return "", err
+	declared := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second)
+	unlock()
+	if declared != nil {
+		return "", declared
 	}
 	// After it is away, not before. A digest recorded for something that failed to send would make
 	// the machine look current for a declaration it never received.