Merge pull request 'Keep moved modules' filled paths in the node's own composition, put every file object, grant access-act and the provisioner's have read (issue 1944)' (#458) from fix/r6-node-composition into main

Reviewed-on: #458

cmd/mesh-controller/composed_live.go

@@ -83,7 +83,7 @@ func composeModules(ctx context.Context, open *stores, nodes []string) (map[stri
 			errs = append(errs, fmt.Errorf("%s: %w", node, err))
 			continue
 		}
-		body, err := declared.Body()
+		body, err := compositionBody(declared)
 		if err != nil {
 			errs = append(errs, fmt.Errorf("%s: %w", node, err))
 			continue
@@ -360,3 +360,13 @@ func sliceOfModule(resources []json.RawMessage, module string) []json.RawMessage
 	}
 	return out
 }
+
+// compositionBody is the body a node's compositions are split from: every file's content inline, never by digest, so
+// writeComposed puts each object of a file over 64 KiB in the node's bucket before it writes the composition that names
+// it (ADR 0539 §4). A send's body names those files by digest for a node that reads references, and a send put their
+// objects itself; with sends gone, a composition split from that body named objects nothing put (issue 1945, live on
+// 2026-10-25: novox's nats.bus-users).
+func compositionBody(declared sendable) ([]byte, error) {
+	declared.fileReferences = false
+	return declared.Body()
+}

cmd/mesh-controller/composed_objects_test.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"strings"
+	"testing"
+
+	"encoding/json"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+
+	sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
+)
+
+// A composition's file over 64 KiB reaches the node by its digest, and its object is in the node's bucket before the
+// composition is written (ADR 0539 §4; issue 1945, live on 2026-10-25 at 01:22: novox's node-engine could not fetch
+// nats.bus-users, because the body the compositions were split from already named the file by digest, so the writer
+// found no content to put, and nothing else put it once sends were gone).
+func TestALargeFileOfACompositionHasItsObjectInTheBucket(t *testing.T) {
+	big := strings.Repeat("user = x\n", sdkdecl.FileReferenceThreshold/8)
+	declared := sendable{fileReferences: true, Resources: []map[string]any{{"id": "nats.bus-users", "type": "file",
+		"path": "/var/lib/nats-module/conf/accounts.conf", "mode": "0644", "content": big}}}
+	body, err := compositionBody(declared)
+	if err != nil {
+		t.Fatal(err)
+	}
+	comps, _, err := compositionsOf("novox", body, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	at, _, _, _ := aPlace(t, "novox")
+	if _, err := writeNodeCompositions(context.Background(), at, comps); err != nil {
+		t.Fatal(err)
+	}
+	sum := sha256.Sum256([]byte(big))
+	if _, ok := at.objects.(fakeObjects)[hex.EncodeToString(sum[:])]; !ok {
+		t.Fatalf("the object of nats.bus-users is not in the bucket: %d objects", len(at.objects.(fakeObjects)))
+	}
+}
+
+// A composition that names a file by a digest whose object the bucket does not hold is refused, never written: a node
+// that read it could not apply it.
+func TestACompositionNamingAnObjectTheBucketLacksIsRefused(t *testing.T) {
+	big := strings.Repeat("user = x\n", sdkdecl.FileReferenceThreshold/8)
+	declared := sendable{fileReferences: true, Resources: []map[string]any{{"id": "nats.bus-users", "type": "file",
+		"path": "/var/lib/nats-module/conf/accounts.conf", "mode": "0644", "content": big}}}
+	body, err := declared.Body() // the body as a send named it: by digest, the content gone
+	if err != nil {
+		t.Fatal(err)
+	}
+	comps, _, err := compositionsOf("novox", body, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	at, _, _, _ := aPlace(t, "novox")
+	if _, err := writeNodeCompositions(context.Background(), at, comps); err == nil ||
+		!strings.Contains(err.Error(), "does not hold") {
+		t.Fatalf("a composition naming a missing object was written: %v", err)
+	}
+}
+
+// Every file the provisioner fills stays filled in the compositions, a moved module's too: the node's own composition
+// carries the path, so the node-engine leaves its content to the provisioner (issue 1945, live on 2026-10-25 at 01:22:
+// ace's node-engine wrote claude-code's, node-env's and power's filled files empty, because the compositions dropped the
+// filled paths of every module moved into the node's own, and their resources say no content).
+func TestEveryFilledFileStaysFilledInTheCompositions(t *testing.T) {
+	for _, node := range []string{"ace", "g14"} {
+		body, fps := capturedBody(t, node)
+		var d struct {
+			Filled []string `json:"filled_files"`
+		}
+		if err := json.Unmarshal(body, &d); err != nil || len(d.Filled) == 0 {
+			t.Fatalf("%s: the captured declaration fills nothing (%v)", node, err)
+		}
+		// power, claude-code and node-env have no fingerprint of the new form, so they move into the node's own.
+		for _, m := range []string{"power", "claude-code", "node-env"} {
+			delete(fps, m)
+		}
+		comps, _, err := compositionsOf(node, body, fps, nil, nil)
+		if err != nil {
+			t.Fatalf("%s: %v", node, err)
+		}
+		filled := map[string]bool{}
+		for _, c := range comps {
+			for _, p := range c.Filled {
+				filled[p] = true
+			}
+		}
+		for _, p := range d.Filled {
+			if !filled[p] {
+				t.Errorf("%s: %s is filled by the provisioner, and no composition says so", node, p)
+			}
+		}
+		for key, c := range comps {
+			if _, err := sdkdelivery.EncodeComposed(c); err != nil {
+				t.Errorf("%s: %s: %v", node, key, err)
+			}
+		}
+	}
+}

cmd/mesh-controller/composed_writer.go

@@ -99,11 +99,16 @@ func compositionsOf(node string, body []byte, fps map[string]string, manifests m
 		}
 		resources = append(resources, raw)
 	}
-	// A filled file of a module moved into the node's own composition is no module's file any more: the provisioner
-	// fills it all the same, but a composition carries only its own module's filled paths.
-	var filled []string
+	// A filled file of a module moved into the node's own composition is no module's file any more, and the provisioner
+	// fills it all the same: the node's own composition carries its path, so the node-engine leaves the content to the
+	// provisioner and accepts its write. Dropping the path had the node-engine write those files empty, since their
+	// resources say no content (issue 1945, live on 2026-10-25: claude-code's, node-env's and power's files on ace, g14
+	// and shanks).
+	var filled, ownFilled []string
 	for _, p := range envelope.Filled {
-		if !filledByMoved(envelope.Resources, p, inModule) {
+		if filledByMoved(envelope.Resources, p, inModule) {
+			ownFilled = append(ownFilled, p)
+		} else {
 			filled = append(filled, p)
 		}
 	}
@@ -113,6 +118,15 @@ func compositionsOf(node string, body []byte, fps map[string]string, manifests m
 	if err != nil {
 		return nil, nil, err
 	}
+	if len(ownFilled) > 0 {
+		own, err := sdkdelivery.ComposedNodeKey(node)
+		if err != nil {
+			return nil, nil, err
+		}
+		c := out[own]
+		c.Filled = append(c.Filled, ownFilled...)
+		out[own] = c
+	}
 	// A module the node runs that composes no resource of its own (all its needs met by others) still has its
 	// composition, empty, at its fingerprint: its want names that composition (the assigner's seventh rule), and without
 	// one no want could ever be written.
@@ -178,6 +192,9 @@ func writeComposed(ctx context.Context, at composedPlace, key string, c sdkdeliv
 			return false, err
 		}
 	}
+	if err := objectsHeld(ctx, at, key, c, objects); err != nil {
+		return false, err
+	}
 	digest, err := sdkdelivery.ComposedDigestOf(c)
 	if err != nil {
 		return false, err
@@ -213,6 +230,40 @@ func writeComposed(ctx context.Context, at composedPlace, key string, c sdkdeliv
 	return wrote || written, err
 }
 
+// objectsHeld refuses a composition that names a file by a digest whose object the node's bucket does not hold: a node
+// that read it could not apply it (issue 1945). An object this write just put is held.
+func objectsHeld(ctx context.Context, at composedPlace, key string, c sdkdelivery.Composed, put map[string][]byte) error {
+	for _, raw := range c.Resources {
+		r, err := sdkdecl.ParseResource(raw)
+		if err != nil {
+			return err
+		}
+		f, ok := r.(*sdkdecl.File)
+		if !ok || !f.IsReference() {
+			continue
+		}
+		if _, ok := put[f.Digest]; ok {
+			continue
+		}
+		name, err := sdkdecl.ObjectName(f.Digest)
+		if err != nil {
+			return fmt.Errorf("%s: file %s: %w", key, f.ID, err)
+		}
+		if at.objects == nil {
+			return fmt.Errorf("%s names file %s by digest and %s's object bucket is not open", key, f.ID, at.node)
+		}
+		have, _, err := at.objects.Has(ctx, name)
+		if err != nil {
+			return fmt.Errorf("%s: whether the object %s of file %s is in the bucket: %w", key, name[:12], f.ID, err)
+		}
+		if !have {
+			return fmt.Errorf("%s names file %s by the object %s, which %s's bucket does not hold: not written", key, f.ID,
+				name[:12], at.node)
+		}
+	}
+	return nil
+}
+
 // readComposed is the composition a bucket holds under key and its revision; revision 0 when there is none.
 func readComposed(ctx context.Context, kv composedBucketsKV, key string) (sdkdelivery.Composed, uint64, error) {
 	e, err := kv.Get(ctx, key)

go.mod

@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0

go.sum

@@ -1,7 +1,7 @@
 git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
 git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f h1:lUEyNPPF3w4Q0LIgcUUD6ofkKVhCF0n7dVjMoixQmXo=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/broker/nats.go

@@ -526,6 +526,11 @@ func AskJudgeDirectoriesSubject(node string) string {
 // for the same reason.
 func AskMoveSecretSubject(node string) string { return "mesh.node." + node + ".ask.move-secret" }
 
+// AskAccessActSubject is where the controller asks one machine's node-engine to run an act on one access's folder (novox/hq
+// ADR 0328 §7): a request answered once, signed as a hand-over is (the node-engine's link.AccessActContext). The
+// node-engine has subscribed to it since ADR 0325; the grant was missing, so each connect logged a permissions violation.
+func AskAccessActSubject(node string) string { return "mesh.node." + node + ".ask.access-act" }
+
 // inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25
 // §4): with one account, inbox privacy is the permission list or it is nothing, so each user's
 // inbox is derived from its own identity and its permissions name that prefix and no other.
@@ -841,7 +846,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
 			AskJudgeDirectoriesSubject(p.Node),
 			// And asking it to move a value sealed to it into the vault (design 57's V3, issue 676), answered the
 			// same way.
-			AskMoveSecretSubject(p.Node)}
+			AskMoveSecretSubject(p.Node),
+			// And asking it to run an act on an access's folder (novox/hq ADR 0328 §7), answered the same way.
+			AskAccessActSubject(p.Node)}
 		// The node-engine witnesses the core builds it places (novox/hq to-be 45 §8, ADR 0236; the
 		// contract is lease/witness.go): it asks its own machine's node tools PING, and, where the
 		// machine runs the controller, reads the lease's one key — read, never written.

internal/broker/nats_test.go

@@ -125,6 +125,17 @@ func TestOnlyWhatCanBeAskedMayAnswer(t *testing.T) {
 	}
 }
 
+// The node-engine hears the act on an access's folder on its own ask subject (novox/hq ADR 0328 §7), which it has
+// subscribed to since ADR 0325: without the grant every node-engine logs a permissions violation at every connect, and
+// the act could never reach it. Its own node's, and no other's.
+func TestANodeMayHearTheAccessActAskedOfIt(t *testing.T) {
+	node, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", PasswordHash: "x"})
+	if !slices.Contains(node.Subscribe, "mesh.node.one.ask.access-act") ||
+		slices.Contains(node.Subscribe, "mesh.node.two.ask.access-act") {
+		t.Fatalf("the node's subscriptions: %v", node.Subscribe)
+	}
+}
+
 // A module serves every tool under its own name, and no other module's.
 func TestAModuleServesItsOwnNamespaceAndNoOthers(t *testing.T) {
 	p, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "gitea", PasswordHash: "x"})

internal/broker/provisioner.go

@@ -3,6 +3,7 @@ package broker
 import (
 	"errors"
 	"fmt"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
 	"strings"
 
 	"github.com/nats-io/nats.go"
@@ -131,6 +132,10 @@ func provisionerGrants(p Principal) (pub, sub []string, err error) {
 		pub = append(pub, "$KV."+bucket+"."+p.Node+".>")
 	}
 	pub = append(pub, wholeBucketRead(ProvisionerRegistryBucket)...)
+	// Its own node's own have, `<node>.node`, read and never written, and no other key of the bucket: it fills every
+	// module again when the node-engine moves to a new composition of the node's own (issue 1945), since an apply can
+	// leave a filled file as the composition had it.
+	pub = append(pub, readGrants(delivery.HaveBucket, p.Node+".node")...)
 	pub = append(pub, "prov.applied."+p.Node+".>")
 	token := CallerToken(p.Username())
 	if token == "" {

internal/broker/provisioner_have_test.go

@@ -0,0 +1,29 @@
+package broker
+
+import (
+	"slices"
+	"testing"
+)
+
+// The provisioner client reads its own node's own have, `<node>.node` in node-engine_have, and no other key of it: it
+// fills every module again when the node-engine moves to a new composition of the node's own (issue 1945, live on
+// 2026-10-25: the files a composition had written empty were refilled by nothing).
+func TestTheProvisionerReadsItsNodesOwnHaveAlone(t *testing.T) {
+	p, err := PermissionsFor(Principal{Kind: KindProvisioner, Node: "g14", PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"$JS.API.STREAM.INFO.KV_node-engine_have",
+		"$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.g14.node",
+		"$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.g14.node"} {
+		if !slices.Contains(p.Publish, want) {
+			t.Errorf("the provisioner may not %s", want)
+		}
+	}
+	for _, s := range p.Publish {
+		if s == "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.g14.>" ||
+			s == "$JS.API.DIRECT.GET.KV_node-engine_have.>" || s == "$KV.node-engine_have.g14.node" {
+			t.Errorf("the provisioner is granted %s", s)
+		}
+	}
+}

internal/broker/provisioner_test.go

@@ -93,6 +93,13 @@ func TestTheProvisionerClientIsGrantedItsMachinesKeysAndNothingElse(t *testing.T
 		"$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_registry.>",
 		"$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_registry.>",
 		"$JS.FC.KV_mesh-provisioner_registry.>")
+	// Its node's own have, one key, read (issue 1945).
+	wantPub = append(wantPub,
+		"$JS.API.STREAM.INFO.KV_node-engine_have",
+		"$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.g14.node",
+		"$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.g14.node",
+		"$JS.API.CONSUMER.DELETE.KV_node-engine_have.>",
+		"$JS.FC.KV_node-engine_have.>")
 	wantPub = append(wantPub, "prov.applied.g14.>", "mesh.seat.mesh-vault.call.get.provisioner~g14",
 		"prov.changed.g14.>")
 	slices.Sort(wantPub)

internal/broker/testdata/composed.conf

@@ -39,7 +39,7 @@ accounts {
       } }
       { user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
         publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.CREATE.KV_composed-one.>", "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_wants.*.$KV.mesh-delivery-assigner_wants.want.one.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.one.>", "$JS.API.CONSUMER.CREATE.OBJ_declaration-files-one.*.$O.declaration-files-one.C.*", "$JS.API.CONSUMER.DELETE.KV_composed-one.>", "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_wants.>", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.DELETE.OBJ_declaration-files-one.*", "$JS.API.CONSUMER.INFO.NODES.one", "$JS.API.DIRECT.GET.KV_composed-one.>", "$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_wants.$KV.mesh-delivery-assigner_wants.want.one.>", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.one.>", "$JS.API.DIRECT.GET.OBJ_declaration-files-one.$O.declaration-files-one.M.*", "$JS.API.STREAM.INFO.KV_composed-one", "$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_wants", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_composed-one.>", "$JS.FC.KV_mesh-delivery-assigner_wants.>", "$JS.FC.KV_node-engine_have.>", "$JS.FC.OBJ_declaration-files-one.>", "$KV.node-engine_have.one.>", "$SRV.PING.node-tools.one", "mesh.control.one.>", "mesh.seat.node-backup.call.backed-up.one.node~one", "mesh.seat.node-backup.call.now.one.node~one", "mesh.seat.node-backup.tool.backed-up.one", "mesh.seat.node-backup.tool.now.one"] }
-        subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.mod.restic.event.backed-up", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
+        subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.mod.restic.event.backed-up", "mesh.node.one.ask.access-act", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }
       { user: "one.mesh-delivery-watchman", password: "$2a$11$wwwwwwwwwwwwwwwwwwwwww", permissions: {

testdata/beside/CAPTURED

@@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself.
 
 mesh-catalog  3a1017f5594217092a654dd1e2d72226766bc16a  modules/*/module.json, modules/*/files, modules/*/templates, predefined.json
 mesh-host     9d9e3a46c4f5943cfffb413d962ce48b75e2fd39  examples/foundation-first-node.lock
-mesh-sdk      748c564cdf7cd2cad57073cdf54bd752c1947025  conformance/events/module-event.json
+mesh-sdk      10b6e202320f38301c67a6822b0734e0b870ea30  conformance/events/module-event.json
 
 To move them, from this repository's root, with the two repositories checked out beside it: