novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Prove in H8 that a node reads a placed module's build and a provider's served key through its own session, refused without the grant, and take a failed have naming nothing as no module run inside the node's own; pin mesh-sdk 7a8e9f9 (issue 1126, review round 1)

Browse Source 
H8 found it: a module the node could not compose yet got a failed have naming
no fingerprint, which counted as run inside the node's own composition, so the
next composition took it from its manifest past its build.
This commit is contained in:
jschoubben
2026-10-27 21:18:45 +01:00
parent 0862f97972
commit 7365890298
4 changed files with 134 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/nodeagent_integration_test.go
+124 -10
View File
@@ -47,14 +47,16 @@ type itBed struct {
controll ed25519.PrivateKey
operator ed25519.PrivateKey
said []string
// refusals are the bus's asynchronous errors on the node's connection: each permissions violation it met.
refusals []string
mu sync.Mutex
}
// aServer is an embedded bus with JetStream and two users: the test's, unrestricted, and the node's, granted exactly what
// buscompose.PermissionsFor gives a node (KindNode); it answers a connection of each.
func aServer(t *testing.T) (admin, node *nats.Conn) {
func aServer(t *testing.T, principal buscompose.Principal, refused func(error)) (admin, node *nats.Conn) {
t.Helper()
perms, err := buscompose.PermissionsFor(buscompose.Principal{Kind: buscompose.KindNode, Node: itNode})
perms, err := buscompose.PermissionsFor(principal)
if err != nil {
t.Fatal(err)
}
@@ -78,7 +80,8 @@ func aServer(t *testing.T) (admin, node *nats.Conn) {
t.Fatal(err)
}
t.Cleanup(admin.Close)
if node, err = nats.Connect(srv.ClientURL(), nats.UserInfo(nodeUser, "node"), nats.CustomInboxPrefix("_INBOX."+nodeUser)); err != nil {
if node, err = nats.Connect(srv.ClientURL(), nats.UserInfo(nodeUser, "node"), nats.CustomInboxPrefix("_INBOX."+nodeUser),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused(err) })); err != nil {
t.Fatal(err)
}
t.Cleanup(node.Close)
@@ -86,6 +89,12 @@ func aServer(t *testing.T) (admin, node *nats.Conn) {
}
func newItBed(t *testing.T) *itBed {
t.Helper()
return newItBedAs(t, buscompose.Principal{Kind: buscompose.KindNode, Node: itNode})
}
// newItBedAs is newItBed with the node's connection under the grant of principal, a node's (KindNode).
func newItBedAs(t *testing.T, principal buscompose.Principal) *itBed {
t.Helper()
// Not skipped off Arch: a package manager that answers is put on PATH, so the apply of files runs on any build node.
bin := t.TempDir()
@@ -98,7 +107,11 @@ func newItBed(t *testing.T) *itBed {
t.Cleanup(func() { builtFor = was })
b := &itBed{t: t, dir: t.TempDir()}
b.conn, b.nodeConn = aServer(t)
b.conn, b.nodeConn = aServer(t, principal, func(err error) {
b.mu.Lock()
b.refusals = append(b.refusals, err.Error())
b.mu.Unlock()
})
var err error
if b.js, err = b.conn.JetStream(); err != nil {
t.Fatal(err)
@@ -160,6 +173,8 @@ func (b *itBed) say(line string) {
func fpIt(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
func fpDigest(c string) string { return "sha256:" + strings.Repeat(c, 64) }
// compose writes a module's signed composition of one file into the node's bucket and returns its digest.
func (b *itBed) compose(module, fp, content string, data bool, signer ed25519.PrivateKey) string {
b.t.Helper()
@@ -344,8 +359,64 @@ func TestNatsTheHandPathAppliesACompositionTheOperatorSigned(t *testing.T) {
// with no key in the controller's compositions; a changed config value is composed again on its event and applied as a
// revision of what runs. And a catalogue pull request's placement candidate equal to its placement is answered "same"
// in the node's own key, and nothing is applied for it (ADR 0559 §4, subtask 10 of the milestone).
//
// And (issue 1126, ADR 0560) it reads, through its own session under the grant the grants part composes for it
// (buscompose.Users, so it carries ServedReads), the build of a placed module that declares one and the one config value
// a provider on another node serves from (a mail domain); under a grant without that served key the bus refuses the
// read and the node composes nothing.
func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
b := newItBed(t)
users, err := buscompose.Users(buscompose.Records{Nodes: []string{itNode, "it2"},
Assigned: map[string][]buscompose.Declared{"it2": {{Module: "mailer", Served: []string{"domain"}}}}})
if err != nil {
t.Fatal(err)
}
var principal buscompose.Principal
for _, u := range users {
if u.Kind == buscompose.KindNode && u.Node == itNode {
principal = u
}
}
if len(principal.ServedReads) != 1 {
t.Fatalf("the grants part's principal of %s reads %v of the served keys", itNode, principal.ServedReads)
}
t.Run("without the served key the bus refuses the read", func(t *testing.T) {
b := newItBed(t)
stop, _ := h8(t, b, false)
defer stop()
within(t, 15*time.Second, "the bus did not refuse the served key's read, or the node did not say it", func() bool {
b.mu.Lock()
defer b.mu.Unlock()
return strings.Contains(strings.Join(b.refusals, "\n"), "Permissions Violation for Publish to \"$JS.API.CONSUMER.CREATE.KV_"+
selfcompose.ResolvedBucket) && strings.Contains(strings.Join(b.refusals, "\n"), ".it2.mailer.domain") &&
strings.Contains(strings.Join(b.said, "\n"), "what mailer serves on it2 (domain)")
})
t.Logf("DEBUG refusals: %v", b.refusals)
if b.file("web") != "" {
t.Errorf("composed and applied under a grant without the served key: %q", b.file("web"))
}
})
b := newItBedAs(t, principal)
stop, st := h8(t, b, true)
defer stop()
placement, resolved, candidates := st.placement, st.resolved, st.candidates
hkey, place, fp := st.hkey, st.place, st.fp
within(t, 5*time.Second, "the node did not compose and apply its own module once placed", func() bool { return b.file("web") == "greeting=bonjour\n" })
_ = placement
h8rest(t, b, resolved, candidates, hkey, place, fp)
}
// h8state is what h8 set up that the rest of H8 writes to.
type h8state struct {
placement, resolved, candidates nats.KeyValue
hkey string
place []byte
fp string
}
// h8 sets H8's bus up and starts the agent, then places the node. With checkBuild, the placed module's build key is
// first one that does not decode, which the node must read through its session (and say), then a done one.
func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
t.Helper()
kvOf := func(bucket string) nats.KeyValue {
kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 5})
if err != nil {
@@ -355,7 +426,7 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
}
placement, modules, manifests := kvOf(delivery.PlacementBucket), kvOf(delivery.ModulesBucket), kvOf(delivery.ManifestsBucket)
resolved := kvOf(selfcompose.ResolvedBucket)
kvOf(delivery.BuilderBuildsBucket)
builds := kvOf(delivery.BuilderBuildsBucket)
candidates := kvOf(delivery.PlacementCandidatesBucket)
// The controller's stream the node's facts go into, so the node's key carries the facts it says.
if _, err := b.js.AddStream(&nats.StreamConfig{Name: "CONTROL", Subjects: []string{link.ReportSubject(itNode)}}); err != nil {
@@ -363,7 +434,8 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
}
path := filepath.Join(b.dir, "web.conf")
man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` +
`"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"}]}`
`"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"}],` +
`"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`
digest := delivery.ManifestDigest([]byte(man))
mraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "web", JSON: man})
mkey, _ := delivery.ManifestsKey(digest)
@@ -373,12 +445,32 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: itNode, Capabilities: []string{"service-manager"}}})
pkey, _ := delivery.PlacementKey(itNode)
hkey, _ := delivery.HaveNodeKey(itNode)
// A provider on it2 serving from its own config value: a mail domain, which it1's composition settles.
mailer := `{"module":"mailer","version":"1","config":{"domain":{"type":"string","description":"the mail domain","required":true}},` +
`"provides":[{"name":"smtp","scope":"mesh"}],"serves":{"smtp":{"port":587,"domain":"${config:domain}"}}}`
mdigest := delivery.ManifestDigest([]byte(mailer))
mlraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "mailer", JSON: mailer})
mlkey, _ := delivery.ManifestsKey(mdigest)
mlmod, _ := delivery.EncodeModule(delivery.Module{FP2: fpIt("b"), Manifest: mdigest, Repository: "novox/mailer", Commit: strings.Repeat("2", 40)})
place2, _ := delivery.EncodePlacement(delivery.Placement{Node: "it2", Modules: []delivery.PlacedModule{{Module: "mailer"}}})
rec2, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}})
pkey2, _ := delivery.PlacementKey("it2")
hkey2, _ := delivery.HaveNodeKey("it2")
buildKey, _ := delivery.BuilderBuildKey("web", fp)
done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest,
Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://web/files/blobs/" + fpDigest("f"), Digest: fpDigest("f")}}})
firstBuild := done
if checkBuild {
firstBuild = []byte(`{"v":1,"state":"no such state"}`)
}
for _, w := range []struct {
kv nats.KeyValue
key string
value []byte
}{{manifests, mkey, mraw}, {modules, "web", modv}, {b.haves, hkey, rec},
{resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour"}`)}} {
{resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour"}`)},
{manifests, mlkey, mlraw}, {modules, "mailer", mlmod}, {b.haves, hkey2, rec2}, {placement, pkey2, place2},
{resolved, "it2.mailer.domain", []byte(`{"value":"example.org"}`)}, {builds, buildKey, firstBuild}} {
if _, err := w.kv.Put(w.key, w.value); err != nil {
t.Fatal(err)
}
@@ -390,7 +482,6 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
}
stop := b.start(false)
defer stop()
// Not placed yet: the agent applies the controller's compositions, of which there are none. The catalogue's seed then
// places the node, and the agent begins again composing its own (ADR 0549 §4), with no restart of the engine.
time.Sleep(300 * time.Millisecond)
@@ -400,7 +491,30 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
if _, err := placement.Put(pkey, place); err != nil {
t.Fatal(err)
}
within(t, 5*time.Second, "the node did not compose and apply its own module once placed", func() bool { return b.file("web") == "greeting=bonjour\n" })
if checkBuild {
// The build of the placed module is read through the node's session: one that does not decode is said, and nothing
// is applied; a done one, then a config event, and the node composes.
within(t, 5*time.Second, "the node did not read web's build key through its session", func() bool {
b.mu.Lock()
defer b.mu.Unlock()
return strings.Contains(strings.Join(b.said, "\n"), "the builder's "+buildKey)
})
if b.file("web") != "" {
t.Fatal("composed with a build key that does not decode")
}
if _, err := builds.Put(buildKey, done); err != nil {
t.Fatal(err)
}
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"bonjour"}`)); err != nil {
t.Fatal(err)
}
}
return stop, h8state{placement: placement, resolved: resolved, candidates: candidates, hkey: hkey, place: place, fp: fp}
}
// h8rest is H8 once the node composes its own: the have, a config revision, and the candidate answered "same".
func h8rest(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, hkey string, place []byte, fp string) {
t.Helper()
within(t, 2*time.Second, "the have does not name the build", func() bool {
h := b.have("web")
return h.FP == fp && h.Composed != "" && h.Phase == delivery.PhaseRunning