Send files over 64 KiB by digest through the bus's object bucket to nodes that read them, grant it, and warn at 75 percent (issues 1696, 1697, 1698; ADR 0486 step 3)

A node that reports file-references gets the digest and size of each file over 64 KiB that may travel so, and the content is put in
the bucket declaration-files before the declaration is sent; others get every content inline. The controller alone writes the
bucket and node-engines read an object by name (the four grants the node-engine's own fetch makes, no list, no write). An object
goes a day after no node's latest declaration names it. The near-limit condition warns at 75 percent, and a test holds the control
node's declaration, in the reference form, under half the bus's largest message.

internal/broker/controller_buckets.go

@@ -231,6 +231,10 @@ func (j *JetStream) EnsureControllerBuckets() error {
 	if err := EnsureLeaseBucket(ctx, js); err != nil {
 		return err
 	}
+	// And the object bucket a declaration's large files travel in (novox/hq ADR 0486 §4): it exists whenever the bus does.
+	if err := EnsureDeclarationFiles(ctx, js); err != nil {
+		return err
+	}
 	if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
 		Bucket: CallsBucket,
 		Description: "the calls of the mesh's own verbs and what came of each (novox/hq to-be 45 §6, issue " +

internal/broker/declaration_files.go

@@ -0,0 +1,90 @@
+package broker
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// The object bucket a declaration's large files travel in (novox/hq ADR 0486, issues 1696 and 1697).
+//
+// **The controller alone writes it, and a node-engine reads an object by its name** (ADR 0486 §2): its account may
+// neither list the bucket nor write it, and no other account has any access. A bucket of the bus's object store is a
+// stream, OBJ_<bucket>, whose messages are the objects' metadata (`$O.<bucket>.M.<base64 of the name>`) and chunks
+// (`$O.<bucket>.C.<nuid>`).
+
+// DeclarationFilesBucket is the bucket's name, which mesh-sdk fixes for the node-engine and the controller alike.
+const DeclarationFilesBucket = declaration.FileBucket
+
+// DeclarationFilesStream is the stream under the bucket.
+const DeclarationFilesStream = "OBJ_" + DeclarationFilesBucket
+
+// DeclarationFilesMaxBytes is the bucket's size: the controller's objects are immutable and regenerable, and the
+// largest declaration it composes holds well under a hundred megabytes of them.
+const DeclarationFilesMaxBytes = 1 << 30
+
+// DeclarationFilesKeptFor is how long an object stays after no node's latest declaration names it (ADR 0486 §3).
+const DeclarationFilesKeptFor = 24 * time.Hour
+
+// DeclarationFilesWrites is the one pattern the controller may publish to write the bucket: every object's metadata
+// and chunks. The server's API for the stream is the controller's `$JS.API.>`.
+const DeclarationFilesWrites = "$O." + DeclarationFilesBucket + ".>"
+
+// DeclarationFilesReads is what a node-engine's account may publish to read an object by its name, and nothing more.
+// The set is the one node-engine's own call (mesh-host internal/link FileObject) makes, for B the bucket and S its
+// stream (issue 1697, notes 4 and 5):
+//
+//   - the object's metadata, by a direct get of the last message of the one subject that names it. **Only the suffixed
+//     form**: the bare `$JS.API.DIRECT.GET.S` takes a sequence number and would read any message of the stream, every
+//     object's name and chunks.
+//   - the ordered consumer on that object's chunks: `$JS.API.CONSUMER.CREATE.S.<name>.$O.B.C.<nuid>`. **Not `S.>`**:
+//     a consumer filtered on `$O.B.M.>` would read every object's name, the list that is refused. The chunks'
+//     subject carries the nuid the metadata gives, which only a reader of the metadata knows.
+//   - the flow-control replies of that consumer, without which a large object stalls at the flow-control window.
+//   - the consumer's deletion, best effort: the consumer's inactivity threshold removes it if this is refused.
+//
+// **Never granted:** STREAM.INFO (a `subjects_filter` of `>` lists every name), STREAM.MSG.GET (any message by its
+// sequence), a publish to `$O.B.>` (a write).
+func DeclarationFilesReads() []string {
+	return []string{
+		"$JS.API.DIRECT.GET." + DeclarationFilesStream + ".$O." + DeclarationFilesBucket + ".M.*",
+		"$JS.API.CONSUMER.CREATE." + DeclarationFilesStream + ".*.$O." + DeclarationFilesBucket + ".C.*",
+		"$JS.API.CONSUMER.DELETE." + DeclarationFilesStream + ".*",
+		"$JS.FC." + DeclarationFilesStream + ".>",
+	}
+}
+
+// EnsureDeclarationFiles creates the bucket if it is absent and brings its options to match. The bucket exists
+// whenever the bus does, so a node that is starting finds it (ADR 0486 §4). An update, never a delete.
+func (j *JetStream) EnsureDeclarationFiles() error {
+	js, err := jetstream.New(j.conn)
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	return EnsureDeclarationFiles(ctx, js)
+}
+
+// EnsureDeclarationFiles is the same on a JetStream the caller holds.
+func EnsureDeclarationFiles(ctx context.Context, js jetstream.JetStream) error {
+	if _, err := js.CreateOrUpdateObjectStore(ctx, DeclarationFilesConfig()); err != nil {
+		return fmt.Errorf("asserting the object bucket %s: %w", DeclarationFilesBucket, err)
+	}
+	return nil
+}
+
+// DeclarationFilesConfig is the bucket as the controller asserts it.
+func DeclarationFilesConfig() jetstream.ObjectStoreConfig {
+	return jetstream.ObjectStoreConfig{
+		Bucket: DeclarationFilesBucket,
+		Description: "the contents of a declaration's files over 64 KiB, each under the hex of its sha256 digest " +
+			"(novox/hq ADR 0486): written by the controller alone, read by a node-engine by name, removed a day after " +
+			"no node's latest declaration names it",
+		MaxBytes: DeclarationFilesMaxBytes,
+		Storage:  jetstream.FileStorage,
+	}
+}

internal/broker/declaration_files_live_test.go

@@ -0,0 +1,239 @@
+//go:build integration
+
+package broker
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"golang.org/x/crypto/bcrypt"
+)
+
+// nodeEnginesFileObject is the node-engine's own call (mesh-host internal/link files.go, OverNATS.FileObject), as the
+// engine makes it: the object's metadata by a direct get of one subject, then its chunks through an ordered consumer
+// bound to the stream. It is copied here so the grants are judged by the call that has to work, and its subjects are the
+// ones the grant names (issue 1697, notes 4 and 5); the node-engine's own integration test holds the other copy.
+func nodeEnginesFileObject(ctx context.Context, nc *nats.Conn, name string) ([]byte, error) {
+	js, err := nc.JetStream(nats.MaxWait(10 * time.Second))
+	if err != nil {
+		return nil, err
+	}
+	stream := "OBJ_" + DeclarationFilesBucket
+	meta, err := js.GetLastMsg(stream, "$O."+DeclarationFilesBucket+".M."+base64.URLEncoding.EncodeToString([]byte(name)),
+		nats.DirectGet(), nats.Context(ctx))
+	if err != nil {
+		return nil, err
+	}
+	var info struct {
+		NUID string `json:"nuid"`
+		Size uint64 `json:"size"`
+	}
+	if err := json.Unmarshal(meta.Data, &info); err != nil {
+		return nil, err
+	}
+	sub, err := js.SubscribeSync("$O."+DeclarationFilesBucket+".C."+info.NUID, nats.OrderedConsumer(), nats.BindStream(stream))
+	if err != nil {
+		return nil, err
+	}
+	defer sub.Unsubscribe()
+	body := make([]byte, 0, info.Size)
+	for uint64(len(body)) < info.Size {
+		msg, err := sub.NextMsgWithContext(ctx)
+		if err != nil {
+			return nil, err
+		}
+		body = append(body, msg.Data...)
+	}
+	return body, nil
+}
+
+// **The bucket's grants on a real server, judged by the node-engine's own call** (novox/hq ADR 0486 §2, issue 1697,
+// 1697/AC2): the controller's account writes an object, a node-engine's account reads it by its name, a large object
+// included, and is refused a list of the bucket (a stream info asking for every subject), a message get by sequence, a
+// direct get by sequence, a consumer on the names, and a write; another module's account is refused all of them.
+func TestNatsANodeEngineReadsAnObjectByNameAndIsRefusedTheListAndTheWrite(t *testing.T) { // 1697/AC2
+	hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
+	if err != nil {
+		t.Fatal(err)
+	}
+	users, err := Users(Records{Nodes: []string{"alpha"}, Assigned: map[string][]Declared{
+		"alpha": {{Module: RuntimeModule}, {Module: "other-module"}},
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range users {
+		users[i].PasswordHash = string(hash)
+	}
+	accounts, err := ComposeAccounts(users)
+	if err != nil {
+		t.Fatal(err)
+	}
+	conf := filepath.Join(t.TempDir(), "bus.conf")
+	if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
+		`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	opts, err := server.ProcessConfigFile(conf)
+	if err != nil {
+		t.Fatalf("the composed accounts do not parse: %v", err)
+	}
+	opts.NoLog, opts.NoSigs = true, true
+	s, err := server.NewServer(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	go s.Start()
+	if !s.ReadyForConnections(10 * time.Second) {
+		t.Fatal("the bus did not come up")
+	}
+	t.Cleanup(s.Shutdown)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	connect := func(user string) (*nats.Conn, chan error) {
+		t.Helper()
+		refused := make(chan error, 16)
+		nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
+			nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
+		if err != nil {
+			t.Fatalf("%s cannot connect: %v", user, err)
+		}
+		t.Cleanup(nc.Close)
+		return nc, refused
+	}
+
+	// The controller makes the bucket (as it does at its start) and writes two objects: a small one and one of
+	// several chunks, which needs the consumer's flow control.
+	controller, _ := connect(ControllerName)
+	cjs, err := jetstream.New(controller)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := EnsureDeclarationFiles(ctx, cjs); err != nil {
+		t.Fatalf("the controller cannot make %s: %v", DeclarationFilesBucket, err)
+	}
+	store, err := cjs.ObjectStore(ctx, DeclarationFilesBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	put := func(content []byte) string {
+		t.Helper()
+		sum := sha256.Sum256(content)
+		name := hex.EncodeToString(sum[:])
+		if _, err := store.PutBytes(ctx, name, content); err != nil {
+			t.Fatalf("the controller cannot write the object %s: %v", name[:8], err)
+		}
+		return name
+	}
+	small := bytes.Repeat([]byte("small file\n"), 100)
+	large := make([]byte, 900<<10)
+	for i := range large {
+		large[i] = byte(i*7 + i>>8)
+	}
+	smallName, largeName := put(small), put(large)
+
+	node, nodeRefused := connect("node.alpha")
+	for name, want := range map[string][]byte{smallName: small, largeName: large} {
+		got, err := nodeEnginesFileObject(ctx, node, name)
+		if err != nil {
+			t.Fatalf("a node-engine cannot read the object %s by its name: %v", name[:8], err)
+		}
+		if !bytes.Equal(got, want) {
+			t.Fatalf("the node-engine read %d bytes of the object %s, want %d", len(got), name[:8], len(want))
+		}
+	}
+	select {
+	case err := <-nodeRefused:
+		t.Fatalf("a node-engine's own read was refused something: %v", err)
+	default:
+	}
+
+	// What each account is refused, judged by the server.
+	listing := []byte(`{"subjects_filter":">"}`)
+	bySeq := []byte(`{"seq":1}`)
+	refusals := []struct {
+		what string
+		do   func(nc *nats.Conn)
+	}{
+		{"a list of the bucket (stream info asking for every subject)", func(nc *nats.Conn) {
+			_, _ = nc.Request("$JS.API.STREAM.INFO."+DeclarationFilesStream, listing, 300*time.Millisecond)
+		}},
+		{"a message get by sequence", func(nc *nats.Conn) {
+			_, _ = nc.Request("$JS.API.STREAM.MSG.GET."+DeclarationFilesStream, bySeq, 300*time.Millisecond)
+		}},
+		{"a direct get by sequence", func(nc *nats.Conn) {
+			_, _ = nc.Request("$JS.API.DIRECT.GET."+DeclarationFilesStream, bySeq, 300*time.Millisecond)
+		}},
+		{"a consumer on every object's name", func(nc *nats.Conn) {
+			js, _ := nc.JetStream(nats.MaxWait(500 * time.Millisecond))
+			sub, err := js.SubscribeSync("$O."+DeclarationFilesBucket+".M.>", nats.OrderedConsumer(), nats.BindStream(DeclarationFilesStream))
+			if err == nil {
+				sub.Unsubscribe()
+			}
+		}},
+		{"a write of an object's metadata", func(nc *nats.Conn) {
+			_ = nc.Publish("$O."+DeclarationFilesBucket+".M.abc", []byte(`{}`))
+			_ = nc.Flush()
+		}},
+		{"a write of a chunk", func(nc *nats.Conn) {
+			_ = nc.Publish("$O."+DeclarationFilesBucket+".C.abc", []byte("x"))
+			_ = nc.Flush()
+		}},
+	}
+	refusedWith := func(who string, nc *nats.Conn, ch chan error, what string, do func(*nats.Conn)) {
+		t.Helper()
+		for len(ch) > 0 {
+			<-ch
+		}
+		do(nc)
+		_ = nc.Flush()
+		deadline := time.After(3 * time.Second)
+		for {
+			select {
+			case err := <-ch:
+				if errors.Is(err, nats.ErrPermissionViolation) {
+					return
+				}
+			case <-deadline:
+				t.Errorf("the server did not refuse %s %s", who, what)
+				return
+			}
+		}
+	}
+	for _, r := range refusals {
+		nodeNC, nodeCh := connect("node.alpha")
+		refusedWith("a node-engine", nodeNC, nodeCh, r.what, r.do)
+		otherNC, otherCh := connect("alpha.other-module")
+		refusedWith("another module", otherNC, otherCh, r.what, r.do)
+	}
+	// And another module cannot read an object by name either.
+	other, otherCh := connect("alpha.other-module")
+	refusedWith("another module", other, otherCh, "a read of one object by its name", func(nc *nats.Conn) {
+		_, _ = nc.Request("$JS.API.DIRECT.GET."+DeclarationFilesStream+".$O."+DeclarationFilesBucket+".M."+
+			base64.URLEncoding.EncodeToString([]byte(smallName)), nil, 300*time.Millisecond)
+	})
+
+	// Nothing was written by the refused: the bucket holds the two objects.
+	var names int
+	lister, err := store.List(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	names = len(lister)
+	if names != 2 {
+		t.Errorf("the bucket holds %d objects, want the 2 the controller wrote", names)
+	}
+}

internal/broker/declaration_files_test.go

@@ -0,0 +1,80 @@
+package broker
+
+import (
+	"slices"
+	"strings"
+	"testing"
+)
+
+// 1697/AC1: the composed accounts give a node-engine exactly the four reads of one object by its name, the controller the
+// one write, and no other principal anything of the bucket.
+func TestTheDeclarationBucketIsWrittenByTheControllerAndReadByNameByANodeEngine(t *testing.T) { // 1697/AC1
+	node, err := PermissionsFor(Principal{Kind: KindNode, Node: "alpha", PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var mine []string
+	for _, s := range node.Publish {
+		if strings.Contains(s, DeclarationFilesStream) || strings.Contains(s, "$O."+DeclarationFilesBucket) {
+			mine = append(mine, s)
+		}
+	}
+	want := []string{
+		"$JS.API.CONSUMER.CREATE.OBJ_declaration-files.*.$O.declaration-files.C.*",
+		"$JS.API.CONSUMER.DELETE.OBJ_declaration-files.*",
+		"$JS.API.DIRECT.GET.OBJ_declaration-files.$O.declaration-files.M.*",
+		"$JS.FC.OBJ_declaration-files.>",
+	}
+	if !slices.Equal(mine, want) {
+		t.Fatalf("a node-engine may publish %q of the bucket, want exactly %q", mine, want)
+	}
+	// What would make a list or a write: none of it, in any pattern a node holds. (The real server's judgement of
+	// the wildcard forms is the live test's.)
+	for _, refused := range []string{
+		"$JS.API.STREAM.INFO.OBJ_declaration-files",
+		"$JS.API.STREAM.MSG.GET.OBJ_declaration-files",
+		"$JS.API.DIRECT.GET.OBJ_declaration-files",
+		"$JS.API.DIRECT.GET.OBJ_declaration-files.$O.declaration-files.M.abc.def",
+		"$JS.API.CONSUMER.CREATE.OBJ_declaration-files.x.$O.declaration-files.M.YWJj",
+		"$JS.API.CONSUMER.CREATE.OBJ_declaration-files.x.$O.declaration-files.M.YWJj.more",
+		"$JS.API.CONSUMER.CREATE.OBJ_declaration-files",
+		"$O.declaration-files.M.abc",
+		"$O.declaration-files.C.abc",
+	} {
+		if MayPublish(node, refused) {
+			t.Errorf("a node-engine may publish %s, which lists or writes the bucket", refused)
+		}
+	}
+	for _, kept := range []string{
+		"$JS.API.DIRECT.GET.OBJ_declaration-files.$O.declaration-files.M.YWJj",
+		"$JS.API.CONSUMER.CREATE.OBJ_declaration-files.h4sh.$O.declaration-files.C.NUID",
+	} {
+		if !MayPublish(node, kept) {
+			t.Errorf("a node-engine may not publish %s, which reads one object by its name", kept)
+		}
+	}
+
+	controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !MayPublish(controller, "$O.declaration-files.M.abc") || !MayPublish(controller, "$O.declaration-files.C.nuid") {
+		t.Error("the controller may not write the bucket")
+	}
+
+	for _, p := range []Principal{
+		{Kind: KindModule, Module: "mesh-vault", PasswordHash: "x"},
+		{Kind: KindNodeTools, Node: "alpha", PasswordHash: "x"},
+		{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
+	} {
+		perms, err := PermissionsFor(p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, s := range append(append([]string(nil), perms.Publish...), perms.Subscribe...) {
+			if strings.Contains(s, "declaration-files") {
+				t.Errorf("%s holds %s, and no account but the controller's and the node-engines' has any access", p.Username(), s)
+			}
+		}
+	}
+}

internal/broker/nats.go

@@ -392,6 +392,12 @@ var DegradableGrants = []DegradableGrant{
 		Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at " +
 			"its first push, which is when it acknowledges a merge (ADR 0407 decision 3, issue 973)",
 	},
+	{
+		Subject: DeclarationFilesWrites,
+		Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " +
+			"first push, and puts a declaration's large files in the bucket only for a node that reports it reads them, " +
+			"after that (ADR 0486 §1, §2, issue 1697)",
+	},
 	{
 		Subject: "$KV." + MergeWindowBucket + ".>",
 		Reason: "refused: the controller closes a batch's window by a timer of its own for that batch, as the " +
@@ -736,6 +742,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
 			}
 			pub = append(pub, "$KV."+bucket+".>")
 		}
+		// **And the object bucket a declaration's large files travel in** (novox/hq ADR 0486 §2, issue 1697): the
+		// controller alone writes it, as a publish to the bucket's own subjects, which `$JS.API.>` does not cover.
+		pub = append(pub, DeclarationFilesWrites)
 		// **And what the bus says about itself, read-only** (novox/hq to-be 45 §3, S9): a durable
 		// consumer that gave up on a message, or one that was deleted. The server already publishes
 		// both in the mesh's own account; the controller says each as a condition in the mesh's words.
@@ -835,6 +844,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
 			return Permissions{}, err
 		}
 		pub = append(pub, haveKeys...)
+		// And an object of the declaration bucket by its name, and nothing of the bucket's list or writing
+		// (novox/hq ADR 0486 §2, issue 1697).
+		pub = append(pub, DeclarationFilesReads()...)
 		// And, while the vault serves it, the vault's move-in in its own name and nobody else's (issue 676): the
 		// values sealed to this machine, which only it opens, handed in sealed to the vault's key for the call.
 		if p.MovesIntoTheVault {

internal/broker/testdata/composed.conf

@@ -29,7 +29,7 @@ accounts {
     ]
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$O.declaration-files.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
         subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }
@@ -38,7 +38,7 @@ accounts {
         subscribe: { allow: ["_INBOX.enrol.one.>"] }
       } }
       { user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
-        publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.one.>", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.INFO.NODES.one", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.one.>", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$KV.node-engine_have.one.>", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
+        publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.one.>", "$JS.API.CONSUMER.CREATE.OBJ_declaration-files.*.$O.declaration-files.C.*", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.DELETE.OBJ_declaration-files.*", "$JS.API.CONSUMER.INFO.NODES.one", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.one.>", "$JS.API.DIRECT.GET.OBJ_declaration-files.$O.declaration-files.M.*", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$JS.FC.OBJ_declaration-files.>", "$KV.node-engine_have.one.>", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
         subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }

internal/broker/writers.go

@@ -219,6 +219,13 @@ var WritersTable = []WriterRow{
 	{State: "a machine's have", Writer: "the node-engine of the machine, under its own keys `<node>.>`",
 		KeptIn: "key-value " + delivery.HaveBucket + ", twenty revisions of each key",
 		Others: "read through `reads` as node-engine.have", Subjects: kvOf(delivery.HaveBucket), Writes: ownNodeKeys},
+	// The contents of a declaration's files over 64 KiB (novox/hq ADR 0486 §2): the controller alone writes the bucket, each
+	// object under the digest of its content; a node-engine reads one by its name, with no list and no write.
+	{State: "a declaration's large files", Writer: "controller, each object under the sha256 digest of its content",
+		KeptIn: "the object bucket " + DeclarationFilesBucket + " (stream " + DeclarationFilesStream + "), an object removed a " +
+			"day after no node's latest declaration names it",
+		Others:   "a node-engine reads an object by its name and may neither list the bucket nor write it",
+		Subjects: []string{DeclarationFilesWrites}, Writes: isController},
 	{State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change",
 		KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig,
 		Subjects: kvOf(DeclaredConfigBucket), Writes: isController},

internal/broker/writers_test.go

@@ -43,6 +43,8 @@ var designRows = []string{
 	"the rollouts of the chain",
 	// What each machine has of each module, written by its node-engine under its own keys (ADR 0429 §1, issue 1136).
 	"a machine's have",
+	// The contents of a declaration's large files, in the bus's object bucket (ADR 0486 §2, issue 1697).
+	"a declaration's large files",
 	"the declared-config mirror",
 	"the predefined-keys mirror",
 	// The provisioner client's word on what it placed (novox/hq ADR 0349 §4, design 57 P6).

internal/link/declare.go

@@ -41,7 +41,7 @@ func Declare(ctx context.Context, bus Bus, signer Signer, node string,
 		return fmt.Errorf("cannot sign a declaration for %s: %w", node, err)
 	}
 
-	body, err := json.Marshal(Signed{Declaration: declaration, Signature: signature})
+	body, err := Envelope(declaration, signature)
 	if err != nil {
 		return err
 	}
@@ -66,8 +66,16 @@ func Declare(ctx context.Context, bus Bus, signer Signer, node string,
 	return bus.PublishDeclaration(publish, node, body)
 }
 
-// NearTheLimit is the share of the bus's largest message at which a declaration is said to be near it.
-const NearTheLimit = 0.9
+// Envelope is the message a declaration goes on the bus as: the declaration and its signature, each as base64 in one JSON
+// document. Its length is what the bus's largest message bounds.
+func Envelope(declaration, signature []byte) ([]byte, error) {
+	return json.Marshal(Signed{Declaration: declaration, Signature: signature})
+}
+
+// NearTheLimit is the share of the bus's largest message at which a declaration is said to be near it: three quarters, so that
+// a declaration which should stay far from the limit raises the condition while there is still room to act (novox/hq ADR 0486
+// §8; the first fix of issue 1689 warned at nine tenths while every file was still inline).
+const NearTheLimit = 0.75
 
 // DeclarationMeasured is told, at every send of a declaration, its size on the wire and the bus's largest message,
 // so the controller raises a condition when it nears the limit and clears it when it is back under (issue 1689).