diff --git a/cmd/mesh-controller/composed_live.go b/cmd/mesh-controller/composed_live.go index a1ea3ead..436be543 100644 --- a/cmd/mesh-controller/composed_live.go +++ b/cmd/mesh-controller/composed_live.go @@ -83,7 +83,7 @@ func composeModules(ctx context.Context, open *stores, nodes []string) (map[stri errs = append(errs, fmt.Errorf("%s: %w", node, err)) continue } - body, err := declared.Body() + body, err := compositionBody(declared) if err != nil { errs = append(errs, fmt.Errorf("%s: %w", node, err)) continue @@ -360,3 +360,13 @@ func sliceOfModule(resources []json.RawMessage, module string) []json.RawMessage } return out } + +// compositionBody is the body a node's compositions are split from: every file's content inline, never by digest, so +// writeComposed puts each object of a file over 64 KiB in the node's bucket before it writes the composition that names +// it (ADR 0539 §4). A send's body names those files by digest for a node that reads references, and a send put their +// objects itself; with sends gone, a composition split from that body named objects nothing put (issue 1945, live on +// 2026-10-25: novox's nats.bus-users). +func compositionBody(declared sendable) ([]byte, error) { + declared.fileReferences = false + return declared.Body() +} diff --git a/cmd/mesh-controller/composed_objects_test.go b/cmd/mesh-controller/composed_objects_test.go new file mode 100644 index 00000000..7fe9b8d9 --- /dev/null +++ b/cmd/mesh-controller/composed_objects_test.go @@ -0,0 +1,102 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "strings" + "testing" + + "encoding/json" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + + sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration" +) + +// A composition's file over 64 KiB reaches the node by its digest, and its object is in the node's bucket before the +// composition is written (ADR 0539 §4; issue 1945, live on 2026-10-25 at 01:22: novox's node-engine could not fetch +// nats.bus-users, because the body the compositions were split from already named the file by digest, so the writer +// found no content to put, and nothing else put it once sends were gone). +func TestALargeFileOfACompositionHasItsObjectInTheBucket(t *testing.T) { + big := strings.Repeat("user = x\n", sdkdecl.FileReferenceThreshold/8) + declared := sendable{fileReferences: true, Resources: []map[string]any{{"id": "nats.bus-users", "type": "file", + "path": "/var/lib/nats-module/conf/accounts.conf", "mode": "0644", "content": big}}} + body, err := compositionBody(declared) + if err != nil { + t.Fatal(err) + } + comps, _, err := compositionsOf("novox", body, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + at, _, _, _ := aPlace(t, "novox") + if _, err := writeNodeCompositions(context.Background(), at, comps); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256([]byte(big)) + if _, ok := at.objects.(fakeObjects)[hex.EncodeToString(sum[:])]; !ok { + t.Fatalf("the object of nats.bus-users is not in the bucket: %d objects", len(at.objects.(fakeObjects))) + } +} + +// A composition that names a file by a digest whose object the bucket does not hold is refused, never written: a node +// that read it could not apply it. +func TestACompositionNamingAnObjectTheBucketLacksIsRefused(t *testing.T) { + big := strings.Repeat("user = x\n", sdkdecl.FileReferenceThreshold/8) + declared := sendable{fileReferences: true, Resources: []map[string]any{{"id": "nats.bus-users", "type": "file", + "path": "/var/lib/nats-module/conf/accounts.conf", "mode": "0644", "content": big}}} + body, err := declared.Body() // the body as a send named it: by digest, the content gone + if err != nil { + t.Fatal(err) + } + comps, _, err := compositionsOf("novox", body, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + at, _, _, _ := aPlace(t, "novox") + if _, err := writeNodeCompositions(context.Background(), at, comps); err == nil || + !strings.Contains(err.Error(), "does not hold") { + t.Fatalf("a composition naming a missing object was written: %v", err) + } +} + +// Every file the provisioner fills stays filled in the compositions, a moved module's too: the node's own composition +// carries the path, so the node-engine leaves its content to the provisioner (issue 1945, live on 2026-10-25 at 01:22: +// ace's node-engine wrote claude-code's, node-env's and power's filled files empty, because the compositions dropped the +// filled paths of every module moved into the node's own, and their resources say no content). +func TestEveryFilledFileStaysFilledInTheCompositions(t *testing.T) { + for _, node := range []string{"ace", "g14"} { + body, fps := capturedBody(t, node) + var d struct { + Filled []string `json:"filled_files"` + } + if err := json.Unmarshal(body, &d); err != nil || len(d.Filled) == 0 { + t.Fatalf("%s: the captured declaration fills nothing (%v)", node, err) + } + // power, claude-code and node-env have no fingerprint of the new form, so they move into the node's own. + for _, m := range []string{"power", "claude-code", "node-env"} { + delete(fps, m) + } + comps, _, err := compositionsOf(node, body, fps, nil, nil) + if err != nil { + t.Fatalf("%s: %v", node, err) + } + filled := map[string]bool{} + for _, c := range comps { + for _, p := range c.Filled { + filled[p] = true + } + } + for _, p := range d.Filled { + if !filled[p] { + t.Errorf("%s: %s is filled by the provisioner, and no composition says so", node, p) + } + } + for key, c := range comps { + if _, err := sdkdelivery.EncodeComposed(c); err != nil { + t.Errorf("%s: %s: %v", node, key, err) + } + } + } +} diff --git a/cmd/mesh-controller/composed_writer.go b/cmd/mesh-controller/composed_writer.go index 2529e613..a6fec709 100644 --- a/cmd/mesh-controller/composed_writer.go +++ b/cmd/mesh-controller/composed_writer.go @@ -99,11 +99,16 @@ func compositionsOf(node string, body []byte, fps map[string]string, manifests m } resources = append(resources, raw) } - // A filled file of a module moved into the node's own composition is no module's file any more: the provisioner - // fills it all the same, but a composition carries only its own module's filled paths. - var filled []string + // A filled file of a module moved into the node's own composition is no module's file any more, and the provisioner + // fills it all the same: the node's own composition carries its path, so the node-engine leaves the content to the + // provisioner and accepts its write. Dropping the path had the node-engine write those files empty, since their + // resources say no content (issue 1945, live on 2026-10-25: claude-code's, node-env's and power's files on ace, g14 + // and shanks). + var filled, ownFilled []string for _, p := range envelope.Filled { - if !filledByMoved(envelope.Resources, p, inModule) { + if filledByMoved(envelope.Resources, p, inModule) { + ownFilled = append(ownFilled, p) + } else { filled = append(filled, p) } } @@ -113,6 +118,15 @@ func compositionsOf(node string, body []byte, fps map[string]string, manifests m if err != nil { return nil, nil, err } + if len(ownFilled) > 0 { + own, err := sdkdelivery.ComposedNodeKey(node) + if err != nil { + return nil, nil, err + } + c := out[own] + c.Filled = append(c.Filled, ownFilled...) + out[own] = c + } // A module the node runs that composes no resource of its own (all its needs met by others) still has its // composition, empty, at its fingerprint: its want names that composition (the assigner's seventh rule), and without // one no want could ever be written. @@ -178,6 +192,9 @@ func writeComposed(ctx context.Context, at composedPlace, key string, c sdkdeliv return false, err } } + if err := objectsHeld(ctx, at, key, c, objects); err != nil { + return false, err + } digest, err := sdkdelivery.ComposedDigestOf(c) if err != nil { return false, err @@ -213,6 +230,40 @@ func writeComposed(ctx context.Context, at composedPlace, key string, c sdkdeliv return wrote || written, err } +// objectsHeld refuses a composition that names a file by a digest whose object the node's bucket does not hold: a node +// that read it could not apply it (issue 1945). An object this write just put is held. +func objectsHeld(ctx context.Context, at composedPlace, key string, c sdkdelivery.Composed, put map[string][]byte) error { + for _, raw := range c.Resources { + r, err := sdkdecl.ParseResource(raw) + if err != nil { + return err + } + f, ok := r.(*sdkdecl.File) + if !ok || !f.IsReference() { + continue + } + if _, ok := put[f.Digest]; ok { + continue + } + name, err := sdkdecl.ObjectName(f.Digest) + if err != nil { + return fmt.Errorf("%s: file %s: %w", key, f.ID, err) + } + if at.objects == nil { + return fmt.Errorf("%s names file %s by digest and %s's object bucket is not open", key, f.ID, at.node) + } + have, _, err := at.objects.Has(ctx, name) + if err != nil { + return fmt.Errorf("%s: whether the object %s of file %s is in the bucket: %w", key, name[:12], f.ID, err) + } + if !have { + return fmt.Errorf("%s names file %s by the object %s, which %s's bucket does not hold: not written", key, f.ID, + name[:12], at.node) + } + } + return nil +} + // readComposed is the composition a bucket holds under key and its revision; revision 0 when there is none. func readComposed(ctx context.Context, kv composedBucketsKV, key string) (sdkdelivery.Composed, uint64, error) { e, err := kv.Get(ctx, key) diff --git a/go.mod b/go.mod index a6bd6973..9f42d778 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index 1f380c29..8d8e9173 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4= git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f h1:lUEyNPPF3w4Q0LIgcUUD6ofkKVhCF0n7dVjMoixQmXo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 9b93db72..2add5851 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -526,6 +526,11 @@ func AskJudgeDirectoriesSubject(node string) string { // for the same reason. func AskMoveSecretSubject(node string) string { return "mesh.node." + node + ".ask.move-secret" } +// AskAccessActSubject is where the controller asks one machine's node-engine to run an act on one access's folder (novox/hq +// ADR 0328 §7): a request answered once, signed as a hand-over is (the node-engine's link.AccessActContext). The +// node-engine has subscribed to it since ADR 0325; the grant was missing, so each connect logged a permissions violation. +func AskAccessActSubject(node string) string { return "mesh.node." + node + ".ask.access-act" } + // inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25 // §4): with one account, inbox privacy is the permission list or it is nothing, so each user's // inbox is derived from its own identity and its permissions name that prefix and no other. @@ -841,7 +846,9 @@ func PermissionsFor(p Principal) (Permissions, error) { AskJudgeDirectoriesSubject(p.Node), // And asking it to move a value sealed to it into the vault (design 57's V3, issue 676), answered the // same way. - AskMoveSecretSubject(p.Node)} + AskMoveSecretSubject(p.Node), + // And asking it to run an act on an access's folder (novox/hq ADR 0328 §7), answered the same way. + AskAccessActSubject(p.Node)} // The node-engine witnesses the core builds it places (novox/hq to-be 45 §8, ADR 0236; the // contract is lease/witness.go): it asks its own machine's node tools PING, and, where the // machine runs the controller, reads the lease's one key — read, never written. diff --git a/internal/broker/nats_test.go b/internal/broker/nats_test.go index bc2695c2..c2bea94e 100644 --- a/internal/broker/nats_test.go +++ b/internal/broker/nats_test.go @@ -125,6 +125,17 @@ func TestOnlyWhatCanBeAskedMayAnswer(t *testing.T) { } } +// The node-engine hears the act on an access's folder on its own ask subject (novox/hq ADR 0328 §7), which it has +// subscribed to since ADR 0325: without the grant every node-engine logs a permissions violation at every connect, and +// the act could never reach it. Its own node's, and no other's. +func TestANodeMayHearTheAccessActAskedOfIt(t *testing.T) { + node, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", PasswordHash: "x"}) + if !slices.Contains(node.Subscribe, "mesh.node.one.ask.access-act") || + slices.Contains(node.Subscribe, "mesh.node.two.ask.access-act") { + t.Fatalf("the node's subscriptions: %v", node.Subscribe) + } +} + // A module serves every tool under its own name, and no other module's. func TestAModuleServesItsOwnNamespaceAndNoOthers(t *testing.T) { p, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "gitea", PasswordHash: "x"}) diff --git a/internal/broker/provisioner.go b/internal/broker/provisioner.go index a23f749c..75dab599 100644 --- a/internal/broker/provisioner.go +++ b/internal/broker/provisioner.go @@ -3,6 +3,7 @@ package broker import ( "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/delivery" "strings" "github.com/nats-io/nats.go" @@ -131,6 +132,10 @@ func provisionerGrants(p Principal) (pub, sub []string, err error) { pub = append(pub, "$KV."+bucket+"."+p.Node+".>") } pub = append(pub, wholeBucketRead(ProvisionerRegistryBucket)...) + // Its own node's own have, `.node`, read and never written, and no other key of the bucket: it fills every + // module again when the node-engine moves to a new composition of the node's own (issue 1945), since an apply can + // leave a filled file as the composition had it. + pub = append(pub, readGrants(delivery.HaveBucket, p.Node+".node")...) pub = append(pub, "prov.applied."+p.Node+".>") token := CallerToken(p.Username()) if token == "" { diff --git a/internal/broker/provisioner_have_test.go b/internal/broker/provisioner_have_test.go new file mode 100644 index 00000000..76b464a1 --- /dev/null +++ b/internal/broker/provisioner_have_test.go @@ -0,0 +1,29 @@ +package broker + +import ( + "slices" + "testing" +) + +// The provisioner client reads its own node's own have, `.node` in node-engine_have, and no other key of it: it +// fills every module again when the node-engine moves to a new composition of the node's own (issue 1945, live on +// 2026-10-25: the files a composition had written empty were refilled by nothing). +func TestTheProvisionerReadsItsNodesOwnHaveAlone(t *testing.T) { + p, err := PermissionsFor(Principal{Kind: KindProvisioner, Node: "g14", PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"$JS.API.STREAM.INFO.KV_node-engine_have", + "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.g14.node", + "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.g14.node"} { + if !slices.Contains(p.Publish, want) { + t.Errorf("the provisioner may not %s", want) + } + } + for _, s := range p.Publish { + if s == "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.g14.>" || + s == "$JS.API.DIRECT.GET.KV_node-engine_have.>" || s == "$KV.node-engine_have.g14.node" { + t.Errorf("the provisioner is granted %s", s) + } + } +} diff --git a/internal/broker/provisioner_test.go b/internal/broker/provisioner_test.go index c0d47c75..2c8ba1a9 100644 --- a/internal/broker/provisioner_test.go +++ b/internal/broker/provisioner_test.go @@ -93,6 +93,13 @@ func TestTheProvisionerClientIsGrantedItsMachinesKeysAndNothingElse(t *testing.T "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_registry.>", "$JS.FC.KV_mesh-provisioner_registry.>") + // Its node's own have, one key, read (issue 1945). + wantPub = append(wantPub, + "$JS.API.STREAM.INFO.KV_node-engine_have", + "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.g14.node", + "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.g14.node", + "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", + "$JS.FC.KV_node-engine_have.>") wantPub = append(wantPub, "prov.applied.g14.>", "mesh.seat.mesh-vault.call.get.provisioner~g14", "prov.changed.g14.>") slices.Sort(wantPub) diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index b64cf1f5..0cc3eab2 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -39,7 +39,7 @@ accounts { } } { user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: { publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.CREATE.KV_composed-one.>", "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_wants.*.$KV.mesh-delivery-assigner_wants.want.one.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.one.>", "$JS.API.CONSUMER.CREATE.OBJ_declaration-files-one.*.$O.declaration-files-one.C.*", "$JS.API.CONSUMER.DELETE.KV_composed-one.>", "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_wants.>", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.DELETE.OBJ_declaration-files-one.*", "$JS.API.CONSUMER.INFO.NODES.one", "$JS.API.DIRECT.GET.KV_composed-one.>", "$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_wants.$KV.mesh-delivery-assigner_wants.want.one.>", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.one.>", "$JS.API.DIRECT.GET.OBJ_declaration-files-one.$O.declaration-files-one.M.*", "$JS.API.STREAM.INFO.KV_composed-one", "$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_wants", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_composed-one.>", "$JS.FC.KV_mesh-delivery-assigner_wants.>", "$JS.FC.KV_node-engine_have.>", "$JS.FC.OBJ_declaration-files-one.>", "$KV.node-engine_have.one.>", "$SRV.PING.node-tools.one", "mesh.control.one.>", "mesh.seat.node-backup.call.backed-up.one.node~one", "mesh.seat.node-backup.call.now.one.node~one", "mesh.seat.node-backup.tool.backed-up.one", "mesh.seat.node-backup.tool.now.one"] } - subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.mod.restic.event.backed-up", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] } + subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.mod.restic.event.backed-up", "mesh.node.one.ask.access-act", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] } allow_responses: { max: 1, ttl: "1m" } } } { user: "one.mesh-delivery-watchman", password: "$2a$11$wwwwwwwwwwwwwwwwwwwwww", permissions: { diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index 7e778f5a..62c0a6ec 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog 3a1017f5594217092a654dd1e2d72226766bc16a modules/*/module.json, modules/*/files, modules/*/templates, predefined.json mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock -mesh-sdk 748c564cdf7cd2cad57073cdf54bd752c1947025 conformance/events/module-event.json +mesh-sdk 10b6e202320f38301c67a6822b0734e0b870ea30 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: