From 7365890298913499cf41b3042afb9fbdcbfb6cf1 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 27 Oct 2026 21:18:45 +0100 Subject: [PATCH] Prove in H8 that a node reads a placed module's build and a provider's served key through its own session, refused without the grant, and take a failed have naming nothing as no module run inside the node's own; pin mesh-sdk 7a8e9f9 (issue 1126, review round 1) H8 found it: a module the node could not compose yet got a failed have naming no fingerprint, which counted as run inside the node's own composition, so the next composition took it from its manifest past its build. --- cmd/mesh-host/nodeagent_integration_test.go | 134 ++++++++++++++++++-- go.mod | 2 +- go.sum | 2 + internal/selfcompose/selfcompose.go | 9 +- 4 files changed, 134 insertions(+), 13 deletions(-) diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index ad46d3c..841576f 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -47,14 +47,16 @@ type itBed struct { controll ed25519.PrivateKey operator ed25519.PrivateKey said []string + // refusals are the bus's asynchronous errors on the node's connection: each permissions violation it met. + refusals []string mu sync.Mutex } // aServer is an embedded bus with JetStream and two users: the test's, unrestricted, and the node's, granted exactly what // buscompose.PermissionsFor gives a node (KindNode); it answers a connection of each. -func aServer(t *testing.T) (admin, node *nats.Conn) { +func aServer(t *testing.T, principal buscompose.Principal, refused func(error)) (admin, node *nats.Conn) { t.Helper() - perms, err := buscompose.PermissionsFor(buscompose.Principal{Kind: buscompose.KindNode, Node: itNode}) + perms, err := buscompose.PermissionsFor(principal) if err != nil { t.Fatal(err) } @@ -78,7 +80,8 @@ func aServer(t *testing.T) (admin, node *nats.Conn) { t.Fatal(err) } t.Cleanup(admin.Close) - if node, err = nats.Connect(srv.ClientURL(), nats.UserInfo(nodeUser, "node"), nats.CustomInboxPrefix("_INBOX."+nodeUser)); err != nil { + if node, err = nats.Connect(srv.ClientURL(), nats.UserInfo(nodeUser, "node"), nats.CustomInboxPrefix("_INBOX."+nodeUser), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused(err) })); err != nil { t.Fatal(err) } t.Cleanup(node.Close) @@ -86,6 +89,12 @@ func aServer(t *testing.T) (admin, node *nats.Conn) { } func newItBed(t *testing.T) *itBed { + t.Helper() + return newItBedAs(t, buscompose.Principal{Kind: buscompose.KindNode, Node: itNode}) +} + +// newItBedAs is newItBed with the node's connection under the grant of principal, a node's (KindNode). +func newItBedAs(t *testing.T, principal buscompose.Principal) *itBed { t.Helper() // Not skipped off Arch: a package manager that answers is put on PATH, so the apply of files runs on any build node. bin := t.TempDir() @@ -98,7 +107,11 @@ func newItBed(t *testing.T) *itBed { t.Cleanup(func() { builtFor = was }) b := &itBed{t: t, dir: t.TempDir()} - b.conn, b.nodeConn = aServer(t) + b.conn, b.nodeConn = aServer(t, principal, func(err error) { + b.mu.Lock() + b.refusals = append(b.refusals, err.Error()) + b.mu.Unlock() + }) var err error if b.js, err = b.conn.JetStream(); err != nil { t.Fatal(err) @@ -160,6 +173,8 @@ func (b *itBed) say(line string) { func fpIt(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) } +func fpDigest(c string) string { return "sha256:" + strings.Repeat(c, 64) } + // compose writes a module's signed composition of one file into the node's bucket and returns its digest. func (b *itBed) compose(module, fp, content string, data bool, signer ed25519.PrivateKey) string { b.t.Helper() @@ -344,8 +359,64 @@ func TestNatsTheHandPathAppliesACompositionTheOperatorSigned(t *testing.T) { // with no key in the controller's compositions; a changed config value is composed again on its event and applied as a // revision of what runs. And a catalogue pull request's placement candidate equal to its placement is answered "same" // in the node's own key, and nothing is applied for it (ADR 0559 §4, subtask 10 of the milestone). +// +// And (issue 1126, ADR 0560) it reads, through its own session under the grant the grants part composes for it +// (buscompose.Users, so it carries ServedReads), the build of a placed module that declares one and the one config value +// a provider on another node serves from (a mail domain); under a grant without that served key the bus refuses the +// read and the node composes nothing. func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) { - b := newItBed(t) + users, err := buscompose.Users(buscompose.Records{Nodes: []string{itNode, "it2"}, + Assigned: map[string][]buscompose.Declared{"it2": {{Module: "mailer", Served: []string{"domain"}}}}}) + if err != nil { + t.Fatal(err) + } + var principal buscompose.Principal + for _, u := range users { + if u.Kind == buscompose.KindNode && u.Node == itNode { + principal = u + } + } + if len(principal.ServedReads) != 1 { + t.Fatalf("the grants part's principal of %s reads %v of the served keys", itNode, principal.ServedReads) + } + t.Run("without the served key the bus refuses the read", func(t *testing.T) { + b := newItBed(t) + stop, _ := h8(t, b, false) + defer stop() + within(t, 15*time.Second, "the bus did not refuse the served key's read, or the node did not say it", func() bool { + b.mu.Lock() + defer b.mu.Unlock() + return strings.Contains(strings.Join(b.refusals, "\n"), "Permissions Violation for Publish to \"$JS.API.CONSUMER.CREATE.KV_"+ + selfcompose.ResolvedBucket) && strings.Contains(strings.Join(b.refusals, "\n"), ".it2.mailer.domain") && + strings.Contains(strings.Join(b.said, "\n"), "what mailer serves on it2 (domain)") + }) + t.Logf("DEBUG refusals: %v", b.refusals) + if b.file("web") != "" { + t.Errorf("composed and applied under a grant without the served key: %q", b.file("web")) + } + }) + b := newItBedAs(t, principal) + stop, st := h8(t, b, true) + defer stop() + placement, resolved, candidates := st.placement, st.resolved, st.candidates + hkey, place, fp := st.hkey, st.place, st.fp + within(t, 5*time.Second, "the node did not compose and apply its own module once placed", func() bool { return b.file("web") == "greeting=bonjour\n" }) + _ = placement + h8rest(t, b, resolved, candidates, hkey, place, fp) +} + +// h8state is what h8 set up that the rest of H8 writes to. +type h8state struct { + placement, resolved, candidates nats.KeyValue + hkey string + place []byte + fp string +} + +// h8 sets H8's bus up and starts the agent, then places the node. With checkBuild, the placed module's build key is +// first one that does not decode, which the node must read through its session (and say), then a done one. +func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { + t.Helper() kvOf := func(bucket string) nats.KeyValue { kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 5}) if err != nil { @@ -355,7 +426,7 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) { } placement, modules, manifests := kvOf(delivery.PlacementBucket), kvOf(delivery.ModulesBucket), kvOf(delivery.ManifestsBucket) resolved := kvOf(selfcompose.ResolvedBucket) - kvOf(delivery.BuilderBuildsBucket) + builds := kvOf(delivery.BuilderBuildsBucket) candidates := kvOf(delivery.PlacementCandidatesBucket) // The controller's stream the node's facts go into, so the node's key carries the facts it says. if _, err := b.js.AddStream(&nats.StreamConfig{Name: "CONTROL", Subjects: []string{link.ReportSubject(itNode)}}); err != nil { @@ -363,7 +434,8 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) { } path := filepath.Join(b.dir, "web.conf") man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` + - `"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"}]}` + `"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"}],` + + `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}` digest := delivery.ManifestDigest([]byte(man)) mraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "web", JSON: man}) mkey, _ := delivery.ManifestsKey(digest) @@ -373,12 +445,32 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) { rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: itNode, Capabilities: []string{"service-manager"}}}) pkey, _ := delivery.PlacementKey(itNode) hkey, _ := delivery.HaveNodeKey(itNode) + // A provider on it2 serving from its own config value: a mail domain, which it1's composition settles. + mailer := `{"module":"mailer","version":"1","config":{"domain":{"type":"string","description":"the mail domain","required":true}},` + + `"provides":[{"name":"smtp","scope":"mesh"}],"serves":{"smtp":{"port":587,"domain":"${config:domain}"}}}` + mdigest := delivery.ManifestDigest([]byte(mailer)) + mlraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "mailer", JSON: mailer}) + mlkey, _ := delivery.ManifestsKey(mdigest) + mlmod, _ := delivery.EncodeModule(delivery.Module{FP2: fpIt("b"), Manifest: mdigest, Repository: "novox/mailer", Commit: strings.Repeat("2", 40)}) + place2, _ := delivery.EncodePlacement(delivery.Placement{Node: "it2", Modules: []delivery.PlacedModule{{Module: "mailer"}}}) + rec2, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}}) + pkey2, _ := delivery.PlacementKey("it2") + hkey2, _ := delivery.HaveNodeKey("it2") + buildKey, _ := delivery.BuilderBuildKey("web", fp) + done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest, + Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://web/files/blobs/" + fpDigest("f"), Digest: fpDigest("f")}}}) + firstBuild := done + if checkBuild { + firstBuild = []byte(`{"v":1,"state":"no such state"}`) + } for _, w := range []struct { kv nats.KeyValue key string value []byte }{{manifests, mkey, mraw}, {modules, "web", modv}, {b.haves, hkey, rec}, - {resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour"}`)}} { + {resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour"}`)}, + {manifests, mlkey, mlraw}, {modules, "mailer", mlmod}, {b.haves, hkey2, rec2}, {placement, pkey2, place2}, + {resolved, "it2.mailer.domain", []byte(`{"value":"example.org"}`)}, {builds, buildKey, firstBuild}} { if _, err := w.kv.Put(w.key, w.value); err != nil { t.Fatal(err) } @@ -390,7 +482,6 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) { } stop := b.start(false) - defer stop() // Not placed yet: the agent applies the controller's compositions, of which there are none. The catalogue's seed then // places the node, and the agent begins again composing its own (ADR 0549 §4), with no restart of the engine. time.Sleep(300 * time.Millisecond) @@ -400,7 +491,30 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) { if _, err := placement.Put(pkey, place); err != nil { t.Fatal(err) } - within(t, 5*time.Second, "the node did not compose and apply its own module once placed", func() bool { return b.file("web") == "greeting=bonjour\n" }) + if checkBuild { + // The build of the placed module is read through the node's session: one that does not decode is said, and nothing + // is applied; a done one, then a config event, and the node composes. + within(t, 5*time.Second, "the node did not read web's build key through its session", func() bool { + b.mu.Lock() + defer b.mu.Unlock() + return strings.Contains(strings.Join(b.said, "\n"), "the builder's "+buildKey) + }) + if b.file("web") != "" { + t.Fatal("composed with a build key that does not decode") + } + if _, err := builds.Put(buildKey, done); err != nil { + t.Fatal(err) + } + if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"bonjour"}`)); err != nil { + t.Fatal(err) + } + } + return stop, h8state{placement: placement, resolved: resolved, candidates: candidates, hkey: hkey, place: place, fp: fp} +} + +// h8rest is H8 once the node composes its own: the have, a config revision, and the candidate answered "same". +func h8rest(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, hkey string, place []byte, fp string) { + t.Helper() within(t, 2*time.Second, "the have does not name the build", func() bool { h := b.have("web") return h.FP == fp && h.Composed != "" && h.Phase == delivery.PhaseRunning diff --git a/go.mod b/go.mod index e7c79d7..ed71e1e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027200317-538ffbaa085b + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 0b2f46c..07b9c16 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027195758-cebc6519d5b0 h1:9B7B38e3 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027195758-cebc6519d5b0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027200317-538ffbaa085b h1:LuBK5dtBgm978krhj41N2MKA07Yv4zetRPOdHRTB6L8= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027200317-538ffbaa085b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648 h1:XoMsgg9kSgjQWa86NZM8h8OpxJ2kovxcMS2C0i6WSVU= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 9946c9a..135912c 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -326,7 +326,7 @@ func (c *Composer) takeRecordChanged(u agent.Update) bool { } // takeModuleHave takes one of this node's modules' haves, and says whether it changed which it runs inside its own -// composition: one whose have names no fingerprint of the new form and is not removed (nats on novox: a build-... +// composition: one whose have names no fingerprint of the new form, is not removed, and runs something (nats on novox: a build-... // id; mesh-wireguard: none). Read from the haves, which outlast a restart, never from what the agent has applied // this run: a fresh agent composes before it applies anything (review of the fold, round 1). func (c *Composer) takeModuleHave(u agent.Update) bool { @@ -337,7 +337,11 @@ func (c *Composer) takeModuleHave(u agent.Update) bool { in := false if !u.Deleted { if h, err := delivery.DecodeHave(u.Value); err == nil { - in = h.Phase != delivery.PhaseRemoved && !strings.HasPrefix(h.FP, delivery.SourcePrefix2) + // A have that names nothing it runs, failed (the node could not compose a module it never ran: NotComposed), is + // no module run inside the node's own composition: taken as one, it was composed from its manifest past its + // build (issue 1126, review round 1 of the builds). + runsNothing := h.Phase == delivery.PhaseFailed && h.FP == "" && h.Composed == "" + in = h.Phase != delivery.PhaseRemoved && !runsNothing && !strings.HasPrefix(h.FP, delivery.SourcePrefix2) } } c.mu.Lock() @@ -636,6 +640,7 @@ func (r reader) Build(ctx context.Context, module, fp string) (*delivery.Builder return nil, err } raw, err := r.c.env.Get(ctx, delivery.BuilderBuildsBucket, key) + r.c.env.Say(fmt.Sprintf("DEBUG build %s: %d bytes err=%v", key, len(raw), err)) if err != nil || raw == nil { return nil, err }