//go:build integration package main import ( "context" "crypto/ed25519" "encoding/json" "errors" "fmt" "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" engcomp "github.com/novox/mesh-host/internal/compositions" "os" "path/filepath" "sort" "strings" "sync" "sync/atomic" "testing" "time" nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/seatlist" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/nodegrant" "github.com/novox/mesh-host/internal/seatwait" "github.com/novox/mesh-host/internal/selfcompose" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" ) // The node agent whole, against a real bus with JetStream started in the test: the wants bucket, the node's composition // bucket and the have bucket, a session on the bus, and the real apply of a module's composition into a directory of the // test. R6-A AC1, AC5 and AC9 (novox/hq ADR 0539). const itNode = "it1" type itBed struct { t *testing.T conn *nats.Conn js nats.JetStreamContext // nodeConn and nodeJS are the node's own, under its real grant (the grants part's, nodegrant.For): the agent runs // on them, so a watch or a read the grant does not allow fails here as it fails on the bus (issue 1126, 2026-10-27 // 09:21: a watch of every have key stopped every node agent). conn is the test's, unrestricted, for setting up. nodeConn *nats.Conn nodeJS nats.JetStreamContext wants nats.KeyValue composed nats.KeyValue haves nats.KeyValue dir string opts options session *link.Session controll ed25519.PrivateKey said []string // refusals are the bus's asynchronous errors on the node's connection: each permissions violation it met. refusals []string mu sync.Mutex // seatSilent makes the stand-in of the seat mesh-catalog refuse the seat list (it is served from the start otherwise). seatSilent atomic.Bool // seatAsked counts the node's asks of the stand-in seat, answered or refused: the node reached the bus and asked. seatAsked atomic.Int32 // seatAway makes the stand-in answer as the bus answers a subject nothing serves (no responders), as while the seat's // holder re-serves (issue 1966). seatAway atomic.Bool findings nats.KeyValue // entries are the entries the stand-in of the seat mesh-catalog answers its verb entry from, by fingerprint (design 47 G §421). entries map[string]meshcatalog.CatalogEntry } // registerEntry makes the stand-in answer the entry of fp: module's manifest and its artifacts (refs `:`). func (b *itBed) registerEntry(module, fp, manifest string, artifacts ...string) { b.t.Helper() var m map[string]any if err := json.Unmarshal([]byte(manifest), &m); err != nil { b.t.Fatal(err) } e := meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: m, ManifestDigest: meshcatalogclient.ManifestDigest([]byte(manifest)), Repository: "novox/" + module, Commit: strings.Repeat("1", 40)} for _, a := range artifacts { e.Artifacts = append(e.Artifacts, meshcatalog.Artifact{Ref: module + ":" + a, Kind: "archive", Language: "go"}) } b.mu.Lock() b.entries[fp] = e b.mu.Unlock() } // running says the seat node-engine's event running of module on node at fp, as that node's engine does. func (b *itBed) running(node, module, fp, manifestDigest string) { b.t.Helper() raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: module, Fingerprint: fp, Manifest: manifestDigest, At: time.Now().UTC()}) if _, err := b.js.Publish(nodeengine.SubjectRunning(node, module), raw); err != nil { b.t.Fatal(err) } } // standInEntry answers the seat mesh-catalog's verb entry from the registered entries, and as the holder refuses an unknown one. func (b *itBed) standInEntry() { sub, err := b.conn.Subscribe("mesh.seat.mesh-catalog.call.entry.*", func(m *nats.Msg) { var in meshcatalog.EntryInput _ = json.Unmarshal(m.Data, &in) b.mu.Lock() e, ok := b.entries[in.Fingerprint] b.mu.Unlock() if !ok { _ = m.Respond([]byte(`{"error":"no entry of fingerprint ` + in.Fingerprint + ` is kept"}`)) return } out, _ := json.Marshal(map[string]any{"result": meshcatalog.EntryAnswer{Entry: e}}) _ = m.Respond(out) }) if err != nil { b.t.Fatal(err) } b.t.Cleanup(func() { _ = sub.Unsubscribe() }) } // aServer is an embedded bus with JetStream and two users: the test's, unrestricted, and the node's, granted exactly what // the grants part gives a node (nodegrant.For); it answers a connection of each. func aServer(t *testing.T, perms nodegrant.Permissions, refused func(error)) (admin, node *nats.Conn) { t.Helper() granted := &server.Permissions{Publish: &server.SubjectPermission{Allow: perms.Publish, Deny: perms.PublishDeny}, Subscribe: &server.SubjectPermission{Allow: perms.Subscribe}} if perms.AllowResponses { granted.Response = &server.ResponsePermission{MaxMsgs: 1, Expires: buscompose.ResponseTTL} } nodeUser := buscompose.Principal{Kind: buscompose.KindNode, Node: itNode}.Username() srv, err := server.NewServer(&server.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true, Users: []*server.User{{Username: "test", Password: "test"}, {Username: nodeUser, Password: "node", Permissions: granted}}}) if err != nil { t.Fatal(err) } go srv.Start() if !srv.ReadyForConnections(10 * time.Second) { t.Fatal("the embedded server did not start") } t.Cleanup(srv.Shutdown) if admin, err = nats.Connect(srv.ClientURL(), nats.UserInfo("test", "test")); err != nil { t.Fatal(err) } t.Cleanup(admin.Close) if node, err = nats.Connect(srv.ClientURL(), nats.UserInfo(nodeUser, "node"), nats.CustomInboxPrefix("_INBOX."+nodeUser), nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused(err) })); err != nil { t.Fatal(err) } t.Cleanup(node.Close) return admin, node } func newItBed(t *testing.T) *itBed { t.Helper() return newItBedAs(t, nodegrant.For(itNode)) } // newItBedAs is newItBed with the node's connection under the grant perms, a node's. func newItBedAs(t *testing.T, perms nodegrant.Permissions) *itBed { t.Helper() // Not skipped off Arch: a package manager that answers is put on PATH, so the apply of files runs on any build node. bin := t.TempDir() if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { t.Fatal(err) } t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) was := builtFor builtFor = "arch" t.Cleanup(func() { builtFor = was }) b := &itBed{t: t, dir: t.TempDir(), entries: map[string]meshcatalog.CatalogEntry{}} b.conn, b.nodeConn = aServer(t, perms, func(err error) { b.mu.Lock() b.refusals = append(b.refusals, err.Error()) b.mu.Unlock() }) var err error if b.js, err = b.conn.JetStream(); err != nil { t.Fatal(err) } if b.nodeJS, err = b.nodeConn.JetStream(); err != nil { t.Fatal(err) } composedBucket, _ := engcomp.BucketFor(itNode) for _, c := range []struct { name string into *nats.KeyValue history uint8 }{{buckets.Wants, &b.wants, 8}, {composedBucket, &b.composed, engcomp.History}, {buckets.Have, &b.haves, buckets.HaveHistory}} { kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: c.name, History: c.history}) if err != nil { t.Fatal(err) } *c.into = kv } if b.findings, err = b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: link.FindingsBucket, History: 1}); err != nil { t.Fatal(err) } // The engine's own finding stream and the keeper's ask stream (design 47 G §383, §390), made as the bus module makes them. for _, c := range []nats.StreamConfig{{Name: "MESH-HOST-FINDINGS", Subjects: []string{"mesh.stream.mesh-host.event.>"}}, {Name: condseat.StreamFindingsAsked, Subjects: []string{condseat.SubjectFindingsAsked}}} { if _, err = b.js.AddStream(&c); err != nil { t.Fatal(err) } } // The stream the seat node-engine's event running is kept in, the last of each subject (design 47 G §199). if _, err = b.js.AddStream(&nats.StreamConfig{Name: nodeengine.StreamRunning, Subjects: []string{nodeengine.SubjectRunningAll}, MaxMsgsPerSubject: 1}); err != nil { t.Fatal(err) } b.standInSeat() b.standInEntry() b.opts = options{state: filepath.Join(b.dir, "state", "state.json"), timeout: 2 * time.Second} if err := os.MkdirAll(filepath.Dir(b.opts.state), 0o755); err != nil { t.Fatal(err) } _, b.controll, _ = ed25519.GenerateKey(nil) return b } // itOwnModule is the node-engine's own module of the bed: what its applied record of its own build says (§527, §634). const itOwnModule = "mesh-host" func itOwn() *link.OwnName { o := link.NewOwnName() o.Set(itOwnModule) return o } // start runs the node agent under a session on the bus. func (b *itBed) start() (stop func()) { b.t.Helper() b.session = &link.Session{Own: itOwn(), Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second, Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }} b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode}) ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { defer close(done) controller := b.controll.Public().(ed25519.PublicKey) nodeAgent(ctx, b.opts, controller, newMeshTrust(), b.session, nil, b.say, func() {}, &agentNow{}, func(context.Context, *agent.Agent) {}) }() return func() { cancel(); <-done } } func (b *itBed) say(line string) { b.mu.Lock() b.said = append(b.said, line) b.mu.Unlock() b.t.Log(line) } func fpIt(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) } func fpDigest(c string) string { return "sha256:" + strings.Repeat(c, 64) } func (b *itBed) want(module, fp, digest, backup string) { b.t.Helper() raw, err := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Composed: digest, Backup: backup}) if err != nil { b.t.Fatal(err) } key := meshwants.WantsByWantNodeModuleKey(itNode, module) if _, err := b.wants.Put(key, raw); err != nil { b.t.Fatal(err) } } func (b *itBed) file(module string) string { got, _ := os.ReadFile(filepath.Join(b.dir, module+".conf")) return string(got) } func (b *itBed) have(module string) nodeengine.HaveByNodeModuleValue { key, _ := nec.HaveKey(itNode, module) e, err := b.haves.Get(key) if err != nil { return nodeengine.HaveByNodeModuleValue{} } h, _ := nec.DecodeHave(e.Value()) return h } func within(t *testing.T, d time.Duration, what string, ok func() bool) { t.Helper() deadline := time.Now().Add(d) for !ok() { if time.Now().After(deadline) { t.Fatal(what) } time.Sleep(10 * time.Millisecond) } } // Issue 1126 (ADR 0547 §2, the milestone's H8): the node agent started whole against a real bus, once the catalogue // places its node (the assignment written after it started), composes a module from its want, the indexer's manifest, its record and its mesh-config value, and applies it, // with no key in the controller's compositions; a changed config value is composed again on its event and applied as a // revision of what runs. And a catalogue pull request's assignment candidate equal to its assignment is answered "same" // in the node's own key, and nothing is applied for it (ADR 0559 §4, subtask 10 of the milestone). // // And (issue 1126, ADR 0560) it reads, through its own session under the grant the grants part composes for it // (nodegrant.For with the served key, as the grants part grants it), the build of a placed module that declares one and the one config value // a provider on another node serves from (a mail domain); under a grant without that served key the bus refuses the // read and the node composes nothing. func TestNatsANodeTheAssignmentsNameComposesAndAppliesItsOwn(t *testing.T) { t.Run("without the served key the bus refuses the read", func(t *testing.T) { b := newItBed(t) stop, _ := h8(t, b, false) defer stop() // 15 s, above link.historyWithin (10 s): the bus's refusal of a publish reaches only the connection's error // handler (nats.go fails no request on it), so the refused read ends at its own bound, then the node says it. within(t, 15*time.Second, "the bus did not refuse the served key's read, or the node did not say it", func() bool { b.mu.Lock() defer b.mu.Unlock() return strings.Contains(strings.Join(b.refusals, "\n"), "Permissions Violation for Publish to \"$JS.API.CONSUMER.CREATE.KV_"+ selfcompose.ResolvedBucket) && strings.Contains(strings.Join(b.refusals, "\n"), ".it2.mailer.domain") && strings.Contains(strings.Join(b.said, "\n"), "what mailer serves on it2 (domain)") }) if b.file("web") != "" { t.Errorf("composed and applied under a grant without the served key: %q", b.file("web")) } }) b := newItBedAs(t, nodegrant.For(itNode, "it2.mailer.domain")) stop, st := h8(t, b, true) defer stop() assignment, resolved, candidates := st.assignment, st.resolved, st.candidates hkey, place, fp := st.hkey, st.place, st.fp within(t, 5*time.Second, "the node did not compose and apply its own module once placed", func() bool { return b.file("web") == "greeting=bonjour\n" }) // ADR 0565 §6 (1953): the node's engine writes its own key of the state packages, under the grant the grants part // composes for it, with each package its compositions declare and the module declaring it; once, while nothing changes. wantPkgs := `{"node":"` + itNode + `","packages":[{"absent":false,"modules":["web"],"package":"web-tools"}]}` within(t, 5*time.Second, "the node did not write its key of the state packages", func() bool { e, err := st.packages.Get(itNode) return err == nil && string(e.Value()) == wantPkgs }) written, _ := st.packages.Get(itNode) h8rest(t, b, resolved, candidates, hkey, place, fp) // web composed again for its changed config value declares the same packages: the key is not written again. if e, _ := st.packages.Get(itNode); e == nil || written == nil || e.Revision() != written.Revision() { t.Errorf("the node's packages key was written again with no package changed: %v, was %v", e, written) } t.Run("one answer per candidate revision", func(t *testing.T) { h8once(t, b, resolved, candidates, place) }) h8back(t, b, assignment, resolved, candidates, hkey) } // h8once is issue 1126, 2026-10-28 (shanks answered probe/1126-seed-7 at revision 84 every 13-17 s, its bus flooded): // a candidate revision is answered once. The node's own answer written, a module's have whose health words alone move // (each look rewrites them) and a config value written again with the value it held answer nothing again; the // candidate's next revision is answered once. func h8once(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, place []byte) { answering := func() int { b.mu.Lock() defer b.mu.Unlock() n := 0 for _, l := range b.said { if strings.HasPrefix(l, "answering assignment candidate feat-x ") { n++ } } return n } time.Sleep(time.Second) settled := answering() if settled == 0 { t.Fatal("feat-x was never answered") } hkey, _ := nec.HaveKey(itNode, "web") for _, detail := range []string{"its tool web_health: no link to the bus is open", "its tool web_health: no answer within 10s"} { h := b.have("web") h.Health, h.Detail = nec.HealthFailed, detail raw, err := nec.EncodeHave(h) if err != nil { t.Fatal(err) } if _, err := b.haves.Put(hkey, raw); err != nil { t.Fatal(err) } if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil { t.Fatal(err) } } time.Sleep(1500 * time.Millisecond) if got := answering(); got != settled { t.Errorf("feat-x answered %d more time(s) at the same revision, with nothing it is composed from changed", got-settled) } same, _ := meshmoduleassignerclient.DecodeAssignments(place) craw, err := bucketstest.Candidate(meshmoduleassigner.CandidatesByProposalNodeValue{Proposal: "feat-x", Assignments: &same}) if err != nil { t.Fatal(err) } ckey := meshmoduleassigner.CandidatesByProposalNodeKey("feat-x", itNode) if _, err := candidates.Put(ckey, craw); err != nil { t.Fatal(err) } within(t, 10*time.Second, "the candidate's next revision was not answered", func() bool { return answering() > settled }) time.Sleep(1500 * time.Millisecond) if got := answering(); got != settled+1 { t.Errorf("the candidate's next revision was answered %d time(s), want 1", got-settled) } } // h8back is A7 of issue 1126: no composition made elsewhere is applied. A candidate deleting the node's assignment is // refused, never answered as a return to the controller's compositions; the assignment key deleted leaves the agent on // its own composer, composing nothing and keeping what runs, said in the journal, the node's have still saying it // composes its own. func h8back(t *testing.T, b *itBed, assignment, resolved, candidates nats.KeyValue, hkey string) { t.Helper() node := func() nodeengine.HaveByNodeNodeValue { e, err := b.haves.Get(hkey) if err != nil { return nodeengine.HaveByNodeNodeValue{} } n, _ := nec.DecodeHaveNode(e.Value()) return n } if n := node(); n.Composes != nec.ComposesOwn { t.Errorf("a node composing its own says it composes %q", n.Composes) } craw, err := bucketstest.Candidate(meshmoduleassigner.CandidatesByProposalNodeValue{Proposal: "revert-x"}) if err != nil { t.Fatal(err) } ckey := meshmoduleassigner.CandidatesByProposalNodeKey("revert-x", itNode) if _, err := candidates.Put(ckey, craw); err != nil { t.Fatal(err) } within(t, 10*time.Second, "the node did not refuse a candidate taking its assignment away", func() bool { a := node().Candidates["revert-x"] return a.Proposal == "revert-x" && a.Returns == "" && a.Verdict == nodeengine.CandidateAnswerVerdictRefused && strings.Contains(a.Refused, "composes nothing") }) ran := b.have("web").Composed pkey, _ := meshmoduleassignerclient.NodeKey(itNode) // A pass in flight when the key goes (issue 1126, 2026-10-28): a config value changed starts one, and the key goes at once. if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil { t.Fatal(err) } if err := assignment.Delete(pkey); err != nil { t.Fatal(err) } within(t, 15*time.Second, "the node did not say it composes nothing once its assignment went", func() bool { b.mu.Lock() said := strings.Join(b.said, "\n") b.mu.Unlock() return strings.Contains(said, "key in the seat's assignment state is gone: it composes nothing of its own") }) time.Sleep(3 * time.Second) // what a pass in flight would mark lands within it b.mu.Lock() said := strings.Join(b.said, "\n") b.mu.Unlock() if n := node(); n.Composes != nec.ComposesOwn || strings.Contains(said, "applying the controller's compositions") { t.Errorf("without its assignment the node turned to the controller's compositions: %q", n.Composes) } if h := b.have("web"); h.Composed != ran || h.Phase == nec.PhaseFailed || b.file("web") != "greeting=salut\n" { t.Errorf("without its assignment what runs moved: %+v, %q", h, b.file("web")) } } // h8state is what h8 set up that the rest of H8 writes to. type h8state struct { assignment, resolved, candidates nats.KeyValue // packages is the engine's state packages (ADR 0565 §6), as the grants part creates it from the engine's manifest. packages nats.KeyValue hkey string place []byte fp string } // h8 sets H8's bus up and starts the agent, then places the node. With checkBuild, the placed module's build key is // first one that does not decode, which the node must read through its session (and say), then a done one. func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { t.Helper() kvOf := func(bucket string) nats.KeyValue { kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 5}) if err != nil { t.Fatal(err) } return kv } assignment, manifests := kvOf(buckets.Assignments), kvOf(buckets.Manifests) resolved := kvOf(selfcompose.ResolvedBucket) candidates := kvOf(buckets.ProposalCandidates) packages := kvOf(link.PackagesBucket) path := filepath.Join(b.dir, "web.conf") // Beside the greeting: a file under `if-config`, a template with `config-lines` from a list, and two files under a // value that makes the first of them fail (a file below a file), the second being left (design 42 zfs-owns-zfs §5). more := func(name string) string { return filepath.Join(b.dir, "web-"+name+".conf") } man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"},` + `"mods":{"type":"list","description":"lines","default":["a","b"]}},` + `"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"},` + `{"id":"mods","type":"file","path":"` + more("mods") + `","mode":"0644","content":"${config-lines:mods:m=}\n"},` + `{"id":"salut","type":"file","path":"` + more("salut") + `","mode":"0644","content":"salut\n","if-config":["greeting=salut"]},` + `{"id":"bad","type":"file","path":"` + path + `/x","mode":"0644","content":"x\n","if-config":["greeting=boom"]},` + `{"id":"after","type":"file","path":"` + more("after") + `","mode":"0644","content":"after\n","if-config":["greeting=boom"]},` + `{"id":"tools","type":"package","package":"web-tools"}]}` digest := meshcatalogclient.ManifestDigest([]byte(man)) mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "web", Manifest: man}) mkey, _ := meshcatalogclient.ManifestsKey(digest) fp := fpIt("a") place, _ := meshmoduleassignerclient.EncodeAssignments(meshmoduleassigner.Assignments{Node: itNode, Modules: []meshmoduleassigner.PlacedModule{{Module: "web"}}}) rec, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: itNode, Capabilities: []string{"service-manager", "package-manager"}}}) // it1 in one key per module and node (research 072 row 37), it2 in the form before: a node reads both. pkey, _ := meshmoduleassignerclient.NodeKey(itNode) places, _ := meshmoduleassignerclient.AssignmentsValues(meshmoduleassigner.Assignments{Node: itNode, Modules: []meshmoduleassigner.PlacedModule{{Module: "web"}}}) place = places[pkey] hkey, _ := nec.HaveNodeKey(itNode) // A provider on it2 serving from its own config value: a mail domain, which it1's composition settles. mailer := `{"module":"mailer","version":"1","config":{"domain":{"type":"string","description":"the mail domain","required":true}},` + `"provides":[{"name":"smtp","scope":"mesh"}],"serves":{"smtp":{"port":587,"domain":"${config:domain}"}}}` mdigest := meshcatalogclient.ManifestDigest([]byte(mailer)) mlraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "mailer", Manifest: mailer}) mlkey, _ := meshcatalogclient.ManifestsKey(mdigest) places2, _ := meshmoduleassignerclient.AssignmentsValues(meshmoduleassigner.Assignments{Node: "it2", Modules: []meshmoduleassigner.PlacedModule{{Module: "mailer"}}}) rec2, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}}) pkey2, _ := meshmoduleassignerclient.NodeKey("it2") hkey2, _ := nec.HaveNodeKey("it2") for _, w := range []struct { kv nats.KeyValue key string value []byte }{{manifests, mkey, mraw}, {b.haves, hkey, rec}, {resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)}, {manifests, mlkey, mlraw}, {b.haves, hkey2, rec2}, {assignment, pkey2, places2[pkey2]}, {assignment, "it2.mailer", places2["it2.mailer"]}, {resolved, "it2.mailer.domain", []byte(`{"value":"example.org","source":"module"}`)}} { if _, err := w.kv.Put(w.key, w.value); err != nil { t.Fatal(err) } } // Design 47 G §421: the node reads the entry its want names, through the seat mesh-catalog's verb entry, and the part's digest // from the want; what it hears another node run (the mailer on it2) is read by the same fingerprint. if !checkBuild { b.registerEntry("web", fp, man) } b.registerEntry("mailer", fpIt("b"), mailer) b.running("it2", "mailer", fpIt("b"), mdigest) want, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true}) wkey := meshwants.WantsByWantNodeModuleKey(itNode, "web") if _, err := b.wants.Put(wkey, want); err != nil { t.Fatal(err) } stop := b.start() // Not placed yet: the agent applies the controller's compositions, of which there are none. The catalogue's seed then // places the node, and the agent begins again composing its own (ADR 0549 §4), with no restart of the engine. time.Sleep(300 * time.Millisecond) if b.file("web") != "" { t.Fatal("a node the catalogue does not place composed its own") } if _, err := assignment.Put(itNode+".web", places[itNode+".web"]); err != nil { t.Fatal(err) } if _, err := assignment.Put(pkey, place); err != nil { t.Fatal(err) } if checkBuild { // The entry of the build its want names is asked of the catalog half through the node's session: one the half holds none // of holds the want (entry-missing, naming the fingerprint) and nothing is applied; once the half holds it and it is // asked again by hand, then a config event, and the node composes. within(t, 5*time.Second, "the have does not say the entry of web's build is missing", func() bool { h := b.have("web") return h.Phase == nec.PhaseHeld && strings.HasPrefix(h.Detail, agent.EntryMissing) && strings.Contains(h.Detail, fp) }) if b.file("web") != "" { t.Fatal("composed with an entry the catalog half does not hold") } b.registerEntry("web", fp, man) if err := composerNow.Load().EntryAgain(context.Background(), fp); err != nil { t.Fatal(err) } if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)); err != nil { t.Fatal(err) } } return stop, h8state{assignment: assignment, resolved: resolved, candidates: candidates, packages: packages, hkey: hkey, place: place, fp: fp} } // h8rest is H8 once the node composes its own: the have, a config revision, and the candidate answered "same". func h8rest(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, hkey string, place []byte, fp string) { t.Helper() within(t, 2*time.Second, "the have does not name the build", func() bool { h := b.have("web") return h.Fp == fp && h.Composed != "" && h.Phase == nec.PhaseRunning }) // ADR 0593 §6: the node publishes what it composed in its composition state, under its own grant: web at the build // its want names, with the digest its have names (the plan left out of it), and the node's own. first := b.have("web").Composed webKey, _ := engcomp.Key(itNode, "web", fp) within(t, 5*time.Second, "the node did not publish web's composition in its composition state", func() bool { e, err := b.composed.Get(webKey) if err != nil { return false } c, err := engcomp.Decode(e.Value()) if err != nil { return false } d, err := engcomp.DigestOf(c) return err == nil && d == first }) if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil { t.Fatal(err) } within(t, 3*time.Second, "a changed config value was not composed and applied again", func() bool { return b.file("web") == "greeting=salut\n" }) // Design 42 zfs-owns-zfs §5: the file under `if-config` appears with its value and goes with it, the template's lines // follow the list, and a failing resource stops its module's apply with the later resource left. other := func(name string) string { got, _ := os.ReadFile(filepath.Join(b.dir, "web-"+name+".conf")) return string(got) } setGreeting := func(value string) { if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"`+value+`","source":"module"}`)); err != nil { t.Fatal(err) } } within(t, 3*time.Second, "the file under if-config did not appear with its value, or the lines do not follow the list", func() bool { return other("salut") == "salut\n" && other("mods") == "m=a\nm=b\n" }) setGreeting("bonjour") within(t, 3*time.Second, "the file under if-config did not go with its value", func() bool { _, err := os.Stat(filepath.Join(b.dir, "web-salut.conf")) return b.file("web") == "greeting=bonjour\n" && errors.Is(err, os.ErrNotExist) }) setGreeting("boom") within(t, 5*time.Second, "the failing resource did not fail its module", func() bool { return b.have("web").Phase == nec.PhaseFailed }) if got := other("after"); got != "" { t.Errorf("a resource after the failed one was applied: %q", got) } setGreeting("salut") within(t, 5*time.Second, "the module did not run again once its value held no failing resource", func() bool { return b.file("web") == "greeting=salut\n" && b.have("web").Phase == nec.PhaseRunning }) within(t, 2*time.Second, "the have does not name the revision", func() bool { h := b.have("web") return h.Fp == fp && h.Composed != first && h.Phase == nec.PhaseRunning }) // The node says its facts (its record, as the test wrote it), then a candidate equal to its assignment is written: the // node answers "same" at the candidate's head and revision in its own key, and applies nothing. b.session.SayFacts(context.Background(), link.Report{Profile: map[string]any{"capabilities": []any{ map[string]any{"name": "service-manager", "present": true}}}}) second := b.have("web").Composed same, _ := meshmoduleassignerclient.DecodeAssignments(place) craw, err := bucketstest.Candidate(meshmoduleassigner.CandidatesByProposalNodeValue{Proposal: "feat-x", Assignments: &same}) if err != nil { t.Fatal(err) } ckey := meshmoduleassigner.CandidatesByProposalNodeKey("feat-x", itNode) crev, err := candidates.Put(ckey, craw) if err != nil { t.Fatal(err) } var answer nodeengine.CandidateAnswer within(t, 10*time.Second, "the node did not answer its candidate", func() bool { e, err := b.haves.Get(hkey) if err != nil { return false } n, err := nec.DecodeHaveNode(e.Value()) wire, _ := json.Marshal(n.Candidates["feat-x"]) _ = json.Unmarshal(wire, &answer) return err == nil && answer.Proposal == "feat-x" && n.Record != nil }) if answer.Verdict != nodeengine.CandidateAnswerVerdictSame || answer.Revision != int64(crev) || len(answer.Same) != 1 || answer.Same[0] != "web" { t.Errorf("the answer to a candidate equal to the assignment: %+v", answer) } if h := b.have("web"); h.Composed != second || b.file("web") != "greeting=salut\n" { t.Errorf("a candidate moved what runs: %+v, %q", h, b.file("web")) } h8candidate(t, b, fp, second) } // h8candidate is ADR 0593 §7: a check's candidate is composed on the node through its seat's verb compose-candidate, // under its own grant: the indexer's candidate changing web's manifest is answered composed, web changed at the // candidate's build, within the answer bound; nothing is applied and the composition state does not move. func h8candidate(t *testing.T, b *itBed, fp, running string) { t.Helper() kvOf := func(bucket string) nats.KeyValue { kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 5}) if err != nil { t.Fatal(err) } return kv } cands, _ := kvOf(buckets.Candidates), kvOf(buckets.Verdicts) manifests, err := b.js.KeyValue(buckets.Manifests) if err != nil { t.Fatal(err) } path := filepath.Join(b.dir, "web.conf") // A description past what one answer carries (900 KiB, and under the bus's 1 MiB for one value), so the composition input this // node sends is over the inline bound (A-237b, A-237c). pad := strings.Repeat("x", 930_000) man := `{"module":"web","version":"2","config":{"greeting":{"type":"string","description":"what the file says ` + pad + `","default":"hello"}},` + `"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0640","content":"greeting=${config:greeting}\n"},` + `{"id":"tools","type":"package","package":"web-tools"}]}` digest := meshcatalogclient.ManifestDigest([]byte(man)) mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "web", Manifest: man}) mkey, _ := meshcatalogclient.ManifestsKey(digest) if _, err := manifests.Put(mkey, mraw); err != nil { t.Fatal(err) } head := meshdeliveryclient.Head{Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40)} heads, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{head}) pulls := []meshcatalog.CandidatesByGroupValuePullsItem{bucketstest.Pull(head, []string{"web"}, map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"web": {Fp2: fpIt("c"), Manifest: digest}})} // The indexer keys the candidate by its group (a branch name, slash allowed); the checker asks by the composition key. const group = "fix/3-web" craw, err := bucketstest.GroupCandidate(meshcatalog.CandidatesByGroupValue{Group: group, Heads: heads, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}) if err != nil { t.Fatal(err) } ckey, _ := buckets.CandidateKey(group) key, _ := nec.CompositionKey(group, heads) if _, err := cands.Put(ckey, craw); err != nil { t.Fatal(err) } before, _ := b.composed.Status() // A-237c: asked with input and no call token (the checker's, until G removes it, G-6 §321) an input past what one answer carries // is refused; asked with a token it is sent in chunks through the engine's call, whatever its size (A-237b). if over, err := composeCandidateFrom(itNode, &shadowNow)(key, true, "", nil); err != nil { t.Fatal(err) } else if a, err := nec.DecodeAnswer(over.(json.RawMessage)); err != nil || a.Verdict != nec.Refused || !strings.Contains(a.Refused, "an answer on the bus carries") || a.Input != nil { t.Errorf("an input over the inline bound asked with no token answered %+v (%v)", a, err) } line := &ackingLine{chunks: map[string][]byte{}, done: make(chan struct{})} const call = "0123456789abcdef0123456789abcdef" acts := &link.HandActs{Node: itNode, ComposeCandidate: composeCandidateFrom(itNode, &shadowNow)} chunked, err := acts.Call(link.EngineCall{Verb: link.ActComposeCandidate, Line: line, Args: map[string]any{"candidate": key, "input": true, "call": call}}) if err != nil { t.Fatal(err) } ca, err := nec.DecodeAnswer(chunked.(json.RawMessage)) if err != nil || ca.Verdict != nec.Composed || ca.Chunks < 2 || ca.Input != nil { t.Fatalf("the input asked with a token answered %+v (%v), want the chunks and no inline input", ca, err) } line.want = int(ca.Chunks) select { case <-line.done: case <-time.After(10 * time.Second): t.Fatalf("%d of %d chunks were sent", line.sent(), ca.Chunks) } var file []byte for n := int64(1); n <= ca.Chunks; n++ { var c buscalls.SeatChunk if err := json.Unmarshal(line.chunks[buscalls.SeatChunks{Seat: nodeengine.Seat, Verb: nodeengine.VerbComposeCandidate, Call: call}.Chunk(n)], &c); err != nil { t.Fatal(err) } file = append(file, c.Data...) } if len(file) <= 900<<10 || !json.Valid(file) { t.Errorf("the chunks held %d bytes, want a JSON input over 900 KiB", len(file)) } got, err := composeCandidateFrom(itNode, &shadowNow)(key, false, "", nil) if err != nil { t.Fatal(err) } raw, _ := got.(json.RawMessage) a, err := nec.DecodeAnswer(raw) if err != nil { t.Fatalf("the answer %s: %v", raw, err) } if a.Verdict != nec.Composed || len(a.Modules) != 1 || a.Modules[0].Module != "web" || a.Modules[0].Fp != fpIt("c") || !a.Modules[0].Changed || a.Took > nec.AnswerBound.Milliseconds() { t.Errorf("the node's answer to a candidate changing web: %+v", a) } after, _ := b.composed.Status() if h := b.have("web"); h.Composed != running || b.file("web") != "greeting=salut\n" || before == nil || after == nil || after.Values() != before.Values() { t.Errorf("composing a candidate moved what runs or the composition state: %+v, %q", h, b.file("web")) } if _, err := b.composed.Get(mustComposedKey(t, "web", fpIt("c"))); err == nil { t.Error("the candidate's composition was written into the composition state") } } func mustComposedKey(t *testing.T, module, fp string) string { t.Helper() key, err := engcomp.Key(itNode, module, fp) if err != nil { t.Fatal(err) } return key } // recordingRuntime is a container runtime that keeps its containers in memory and records every command an apply asks // of it (issue 774): what the bus server's container met is read from it, never from a real runtime. type recordingRuntime struct { mu sync.Mutex containers map[string]string // name: the spec label it was made with asked []string } func (r *recordingRuntime) run(_ context.Context, name string, args ...string) (string, error) { r.mu.Lock() defer r.mu.Unlock() r.asked = append(r.asked, name+" "+strings.Join(args, " ")) if name != "docker" || len(args) == 0 { return "", nil } switch args[0] { case "info": return "27.0\n", nil case "container": if len(args) > 1 && args[1] == "inspect" { spec, ok := r.containers[args[len(args)-1]] if !ok { return "", errors.New("no such container") } return "true\t" + spec + "\n", nil } case "image": if len(args) > 1 && args[1] == "inspect" { return "2026-01-01T00:00:00Z\n", nil } case "rm": delete(r.containers, args[len(args)-1]) case "rename": if spec, ok := r.containers[args[1]]; ok { delete(r.containers, args[1]) r.containers[args[2]] = spec } case "run", "create": var made, spec string for i, a := range args { if a == "--name" && i+1 < len(args) { made = args[i+1] } if a == "--label" && i+1 < len(args) && strings.HasPrefix(args[i+1], "mesh-host.spec=") { spec = strings.TrimPrefix(args[i+1], "mesh-host.spec=") } } r.containers[made] = spec return "made\n", nil } return "", nil } // moved is each command that removed, made, stopped, renamed or restarted the container named. func (r *recordingRuntime) moved(container string) []string { r.mu.Lock() defer r.mu.Unlock() var out []string for _, a := range r.asked { for _, verb := range []string{"rm ", "run ", "create ", "stop ", "restart ", "kill ", "rename ", "start "} { for _, prefix := range []string{"docker ", "docker container "} { if strings.HasPrefix(a, prefix+verb) && strings.Contains(a, container) { out = append(out, a) } } } } return out } // ADR 0599 §6 and §4 (issue 774): the bus server the node's own composition made while the bus module was folded is taken // over by the bus module's first want, the node agent started whole against a real bus: it composes the bus module, writes // its changed file and records the server as the bus module's, and asks the container runtime to remove, make, stop, // rename or restart nothing, since the build does not change the server. A later want whose build changes the server // replaces it once, as any module's, once the want carries the yes (§2: the assigner gives it from the release's Approve). // ADR 0601 §2: the bus module keeps data (its streams), so before each of those applies the node agent asks its node's // backup seat to back it up, as for every module that keeps data, and applies only once the backup says it is done. func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *testing.T) { b := newItBed(t) runtime := &recordingRuntime{containers: map[string]string{}} b.opts.runner = runtime.run const server = "mesh-broker-it" conf := filepath.Join(b.dir, "nats.conf") image := "registry.example/nats@sha256:" + strings.Repeat("a", 64) // What the node's own composition made of the folded bus: the server and its file, recorded as the node's own. built, err := system.For(builtFor) if err != nil { t.Fatal(err) } folded, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` + `{"id":"nats.server-conf","type":"file","path":"` + conf + `","mode":"0644","content":"listen: old\n"},` + `{"id":"nats.server","type":"container","name":"` + server + `","image":"` + image + `",` + // As the node-engine starts every container since the fingerprint handover (47/G W13.1): the fingerprint of the // want it applies, here the first want's, so that want changes the server's spec in nothing. `"env":{"MESH_FINGERPRINT":"` + fpIt("e") + `"}}]}`)) if err != nil { t.Fatal(err) } _, known, err := apply.ApplyModule(context.Background(), built, folded, apply.Scope{Module: store.NodeModule}, store.State{}, store.OriginDeclared, runtime.run, nil, nil, nil, nil) if err != nil { t.Fatal(err) } if err := store.Save(b.opts.state, known); err != nil { t.Fatal(err) } if moved := runtime.moved(server); len(moved) != 1 { t.Fatalf("the folded server was not made once: %v", moved) } made := runtime.containers[server] runtime.mu.Lock() runtime.asked = nil runtime.mu.Unlock() kvOf := func(bucket string) nats.KeyValue { kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 5}) if err != nil { t.Fatal(err) } return kv } assignment, modules, manifests := kvOf(buckets.Assignments), kvOf(buckets.Modules), kvOf(buckets.Manifests) kvOf(selfcompose.ResolvedBucket) kvOf(buckets.ProposalCandidates) kvOf(link.PackagesBucket) // The bus module at a build, done: its manifest written and the indexer's module naming it at fp. publish := func(fp, env string) { man := `{"module":"nats","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}],` + `"data":{"own":[{"id":"jetstream","path":"` + b.dir + `","class":"valuable","why":"the bus's streams"}]},"resources":[` + `{"id":"server-conf","type":"file","path":"` + conf + `","mode":"0644","content":"listen: new\n"},` + `{"id":"server","type":"container","name":"` + server + `","image":"` + image + `"` + env + `}]}` digest := meshcatalogclient.ManifestDigest([]byte(man)) mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "nats", Manifest: man}) mkey, _ := meshcatalogclient.ManifestsKey(digest) modv, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("3", 40)}) b.registerEntry("nats", fp, man) // the node reads the entry its want names (design 47 G §421) if _, err := manifests.Put(mkey, mraw); err != nil { t.Fatal(err) } if _, err := modules.Put("nats", modv); err != nil { t.Fatal(err) } } rec, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: itNode, Capabilities: []string{"service-manager", "package-manager", "container-runtime"}}}) hkey, _ := nec.HaveNodeKey(itNode) places, _ := meshmoduleassignerclient.AssignmentsValues(meshmoduleassigner.Assignments{Node: itNode, Modules: []meshmoduleassigner.PlacedModule{{Module: "nats"}}}) // The have the engine wrote of the folded bus: the old-form build, no fingerprint. natsHave, _ := nec.HaveKey(itNode, "nats") oldHave, _ := nec.EncodeHave(nodeengine.HaveByNodeModuleValue{Build: "build-1792674595610176744", Phase: nec.PhaseRunning, Health: nec.HealthOK}) for key, value := range map[string][]byte{hkey: rec, natsHave: oldHave} { if _, err := b.haves.Put(key, value); err != nil { t.Fatal(err) } } busWant := func(w meshwants.WantsByWantNodeModuleValue) { raw, err := bucketstest.Want(w) if err != nil { t.Fatal(err) } key := meshwants.WantsByWantNodeModuleKey(itNode, "nats") if _, err := b.wants.Put(key, raw); err != nil { t.Fatal(err) } } // The node's backup seat: its holder says the bus module has something to back up, starts a backup on now, and says // when it ended on its event; each ask of now notes what the bus's file held then, so the order is seen. var backups []string var backupsMu sync.Mutex if _, err := b.conn.Subscribe(link.SeatToolSubject("node-backup", "backed-up", itNode), func(m *nats.Msg) { _ = m.Respond([]byte(`{"result":[{"module":"nats","runs":1}]}`)) }); err != nil { t.Fatal(err) } if _, err := b.conn.Subscribe(link.SeatToolSubject("node-backup", "now", itNode), func(m *nats.Msg) { got, _ := os.ReadFile(conf) backupsMu.Lock() backups = append(backups, string(got)) backupsMu.Unlock() // The node agent hears the backup's end before it asks (backupNode), so the event follows the answer at once. _ = m.Respond([]byte(`{"result":{"started":true}}`)) _ = b.conn.Publish(backedUpEvent, backupEvent(itNode, "nats", true, time.Now(), "")) }); err != nil { t.Fatal(err) } backedUp := func() []string { backupsMu.Lock() defer backupsMu.Unlock() return append([]string(nil), backups...) } first := fpIt("e") publish(first, "") busWant(meshwants.WantsByWantNodeModuleValue{Fp: first, Self: true}) stop := b.start() defer stop() for key, value := range places { if _, err := assignment.Put(key, value); err != nil { t.Fatal(err) } } within(t, 10*time.Second, "the bus module's first want was not applied", func() bool { h := b.have("nats") got, _ := os.ReadFile(conf) return h.Fp == first && h.Phase == nec.PhaseRunning && string(got) == "listen: new\n" }) if moved := runtime.moved(server); len(moved) != 0 || runtime.containers[server] != made { t.Fatalf("the first want moved the bus server its build does not change: %v", moved) } if got := backedUp(); len(got) != 1 || got[0] != "listen: old\n" { t.Fatalf("the bus module's first apply was not backed up first: the backups saw %q", got) } state, err := store.Load(b.opts.state) if err != nil { t.Fatal(err) } if rec, ok := state.Find("nats.server"); !ok || rec.Module != "nats" { t.Errorf("the server is recorded %+v, not as the bus module's", rec) } // A build that changes the server: replaced once, as any module's, with the yes the assigner writes from the release's // Approve for a move the node judged to recreate it (ADR 0599 §2, §4). next := fpIt("f") publish(next, `,"env":{"CHANGED":"1"}`) busWant(meshwants.WantsByWantNodeModuleValue{Fp: next, Self: true}) within(t, 10*time.Second, "the build that changes the server was not applied", func() bool { h := b.have("nats") if h.Phase == nec.PhaseWaitingYes && h.Waits != "" { busWant(meshwants.WantsByWantNodeModuleValue{Fp: next, Self: true, Yes: h.Waits}) } return h.Fp == next && h.Phase == nec.PhaseRunning }) var runs int for _, a := range runtime.moved(server) { if strings.HasPrefix(a, "docker run ") && strings.Contains(a, " --name "+server+" ") { runs++ } } if got := backedUp(); len(got) != 2 || got[1] != "listen: new\n" { t.Errorf("the replacing apply of the bus module was not backed up first: the backups saw %q", got) } if runs != 1 || runtime.containers[server] == made { t.Errorf("the build that changes the server made it %d times, its spec still the folded one %v: %v", runs, runtime.containers[server] == made, runtime.moved(server)) } } // standInSeat is the seat mesh-catalog as the node reads the seat list from it (novox/hq ADR 0636 §4): with no seat, every // bootstrapped seat with a digest; with one, its definition. While seatSilent it refuses. func (b *itBed) standInSeat() { defs := map[string]json.RawMessage{} var entries []seatlist.Entry for _, d := range seatlist.Bootstrapped() { raw, err := json.Marshal(d) if err != nil { b.t.Fatal(err) } defs[d.Seat] = raw entries = append(entries, seatlist.Entry{Seat: d.Seat, Source: seatlist.SourceTrunk, Digest: "d-" + d.Seat}) } sub, err := b.conn.Subscribe("mesh.seat."+seatlist.Seat+".call."+seatlist.Verb+".*", func(m *nats.Msg) { var q struct { Seat string `json:"seat"` } _ = json.Unmarshal(m.Data, &q) b.seatAsked.Add(1) if b.seatAway.Load() { _ = m.RespondMsg(&nats.Msg{Header: nats.Header{"Status": []string{"503"}}}) // the bus's no responders return } var result any switch { case b.seatSilent.Load(): _ = m.Respond([]byte(`{"error":"the indexer has not started"}`)) return case q.Seat == "": result = seatlist.Listing{Revision: 1, Seats: entries} default: result = seatlist.Answer{Seat: q.Seat, Found: true, Revision: 1, Entry: &seatlist.Entry{Seat: q.Seat, Source: seatlist.SourceTrunk, Digest: "d-" + q.Seat, Definition: defs[q.Seat]}} } out, _ := json.Marshal(map[string]any{"result": result}) _ = m.Respond(out) }) if err != nil { b.t.Fatal(err) } b.t.Cleanup(func() { _ = sub.Unsubscribe() }) } // fiveMinutes is a clock of the test: the bound of the node's wait fires when the test says so, nothing sleeps. type fiveMinutes struct { mu sync.Mutex timers []*time.Timer fire []func() } func (c *fiveMinutes) Now() time.Time { return time.Now() } func (c *fiveMinutes) AfterFunc(_ time.Duration, f func()) seatwait.Timer { c.mu.Lock() defer c.mu.Unlock() t := &stoppable{} c.fire = append(c.fire, func() { if !t.stopped.Load() { f() } }) return t } type stoppable struct{ stopped atomic.Bool } func (s *stoppable) Stop() bool { return !s.stopped.Swap(true) } func (c *fiveMinutes) pass() { c.mu.Lock() fire := c.fire c.fire = nil c.mu.Unlock() for _, f := range fire { f() } } // findingSubject is the subject of one of the engine's finding events on node (views.FindingSubjects). func findingSubject(event, node string) string { return "mesh.stream." + itOwnModule + ".event." + event + "." + node } // ADR 0636 §1, §8, whole against a real bus and the stand-in of the seat: while the seat refuses, the node composes and // applies nothing new, and keeps running what it ran; after the bound it writes one finding in its state findings; when // the seat names its list the module is applied and the finding is cleared; a node started again with the seat gone // applies the composition it applied last. func TestNatsANodeThatCannotReadTheSeatListActsOnNothingNew(t *testing.T) { clock := &fiveMinutes{} was := seatWaitClock seatWaitClock = clock t.Cleanup(func() { seatWaitClock = was }) b := newItBedAs(t, nodegrant.For(itNode, "it2.mailer.domain")) b.seatSilent.Store(true) stop, st := h8(t, b, false) defer stop() _ = st finding := "machine." + itNode + ".seat-list-unread" within(t, 5*time.Second, "the node did not start its bound while the seat refused", func() bool { clock.mu.Lock() defer clock.mu.Unlock() return len(clock.fire) > 0 }) // A fresh node, nothing applied, reaches the bus and asks the seat while its own composition is held: the read needs // the bus the node composes from and the grant it has from its start, nothing of what the hold keeps back. if b.seatAsked.Load() == 0 { t.Fatal("the fresh node did not ask the seat for its list while its own composition was held") } within(t, 5*time.Second, "the fresh node did not say it holds web and its own composition for the unread list", func() bool { b.mu.Lock() defer b.mu.Unlock() said := strings.Join(b.said, "\n") // web's reason names the bootstrapped seats the node holds (issue 1966). var boot []string for _, d := range seatlist.Bootstrapped() { boot = append(boot, d.Seat) } sort.Strings(boot) return strings.Contains(said, selfcompose.WaitsForTheSeatListOwn) && strings.Contains(said, "web: "+fmt.Sprintf(selfcompose.WaitsForTheSeatListHeld, len(boot), strings.Join(boot, ", "))) }) if b.file("web") != "" { t.Fatalf("the node applied web before it read the seat list: %q", b.file("web")) } if _, err := b.findings.Get(finding); err == nil { t.Fatal("the node wrote its finding before the bound") } clock.pass() within(t, 5*time.Second, "the node did not write its finding after the bound", func() bool { _, err := b.findings.Get(finding) return err == nil }) if b.file("web") != "" { t.Fatalf("the node applied web with its finding written: %q", b.file("web")) } // Design 47 G §383, §384, §390 (test 107): the key is followed by the event on the engine's own stream, the engine stated // its findings at its start, and an ask of the keeper is answered by a new statement. lastEvent := func(event string) (string, bool) { m, err := b.js.GetLastMsg("MESH-HOST-FINDINGS", findingSubject(event, itNode)) if err != nil { return "", false } return string(m.Data), true } within(t, 5*time.Second, "the node did not say finding-written after its key", func() bool { got, ok := lastEvent("finding-written") return ok && strings.Contains(got, `"kind":"seat-list-unread"`) && strings.Contains(got, `"observer":"mesh-host"`) }) within(t, 5*time.Second, "the node did not state its findings at its start", func() bool { _, ok := lastEvent("findings-stated"); return ok }) if _, err := b.js.Publish(condseat.SubjectFindingsAsked, []byte(`{"ask":"a1","at":"2026-11-11T12:00:00Z","by":"test"}`)); err != nil { t.Fatal(err) } within(t, 5*time.Second, "the node did not answer the keeper's ask with a statement naming the finding", func() bool { got, ok := lastEvent("findings-stated") return ok && strings.Contains(got, `"asked":"a1"`) && strings.Contains(got, `"kind":"seat-list-unread"`) }) b.seatSilent.Store(false) // The hand act of the operator and the healers (the verb seat-list-again): a read now, as the seat's event would make. if out, err := seatListAgain(); err != nil || out.(map[string]any)["error"] != nil { t.Fatalf("seat-list-again: %v, %v", out, err) } within(t, 50*time.Second, "the node did not apply web once the seat named its list", func() bool { return b.file("web") == "greeting=bonjour\n" }) within(t, 5*time.Second, "the node did not clear its finding at the first full read", func() bool { _, err := b.findings.Get(finding) return errors.Is(err, nats.ErrKeyNotFound) }) within(t, 5*time.Second, "the node did not say finding-cleared for the seat list", func() bool { for i := uint64(1); i < 40; i++ { if m, err := b.js.GetMsg("MESH-HOST-FINDINGS", i); err == nil && m.Subject == findingSubject("finding-cleared", itNode) && strings.Contains(string(m.Data), `"kind":"seat-list-unread"`) { return true } } return false }) // Started again with the seat refusing: nothing new is composed, and what ran is neither removed nor changed. stop() b.seatSilent.Store(true) stopAgain := b.start() defer stopAgain() within(t, 30*time.Second, "the node did not hold web as it was after a start with the seat gone", func() bool { b.mu.Lock() defer b.mu.Unlock() return strings.Contains(strings.Join(b.said, "\n"), "it1 has read no complete seat list") || b.file("web") != "" }) if b.file("web") != "greeting=bonjour\n" { t.Errorf("web does not stay as it ran after a start with the seat gone: %q", b.file("web")) } } // The first read of the seat list ends the wait whether or not it changed what the reader held: Start returns at once, and // the node waits on the reader's Ready before it says the read is over (ADR 0636 §1). // // Issue 1966, the live case of ace: the seat's holder re-serves while the node starts, so the node's reader is answered as // nothing serves the seat (no responders); once the holder serves again the reader reads the list within the bus's retry // bound, with no event of the seat and no hand act. func TestNatsTheFirstReadOfTheSeatListEndsTheNodesWait(t *testing.T) { clock := &fiveMinutes{} was := seatWaitClock seatWaitClock = clock t.Cleanup(func() { seatWaitClock = was }) b := newItBedAs(t, nodegrant.For(itNode, "it2.mailer.domain")) b.seatAway.Store(true) stop, _ := h8(t, b, false) defer stop() var st *seatListState within(t, 5*time.Second, "the node started no reader of the seat list", func() bool { st = seatListNow.Load(); return st != nil }) asked := b.seatAsked.Load() within(t, 10*time.Second, "the node's reader did not ask the seat while nothing served it", func() bool { now := st.reader.Status() return b.seatAsked.Load() > asked || (!now.Reading && !now.Heard) }) b.seatAway.Store(false) within(t, 30*time.Second, "the node did not end its wait at the first read of the seat list", func() bool { st := seatListNow.Load() if st == nil { return false } <-st.reader.Ready() waiting, _ := st.waiter.Waiting() return st.reader.Status().Heard && !waiting }) if seatListNow.Load() != st { t.Error("the list was read by the reader of a run begun since, not by the one that met no responders") } } // ackingLine is a bus connection that acknowledges every chunk published on it, as a caller does, and keeps the chunks. type ackingLine struct { mu sync.Mutex onAck nats.MsgHandler chunks map[string][]byte done chan struct{} want int closed bool } func (l *ackingLine) Subscribe(_ string, cb nats.MsgHandler) (*nats.Subscription, error) { l.mu.Lock() l.onAck = cb l.mu.Unlock() return &nats.Subscription{}, nil } func (l *ackingLine) sent() int { l.mu.Lock(); defer l.mu.Unlock(); return len(l.chunks) } func (l *ackingLine) Publish(subject string, data []byte) error { l.mu.Lock() l.chunks[subject] = data cb := l.onAck if l.want > 0 && len(l.chunks) >= l.want && !l.closed { l.closed = true close(l.done) } l.mu.Unlock() go cb(&nats.Msg{Subject: strings.Replace(subject, ".chunk.", ".ack.", 1)}) return nil }