From 8ee4fdfb8ef15d75a8d11dc792d3a56c3d0cd2f0 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:45:50 +0100 Subject: [PATCH] H1 integration bound: a reload's stand-in serves no repairs; stop waits its Say MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 1. A ledger binding heard while a module was loaded again made refreshRepairs serve the repairs on the reload's stand-in, whose subscriptions nobody stopped: the repairs were served twice, and still answered after the runtime stopped. The stand-in is now marked as reloading and the refresh leaves it alone; the reload serves the module's repairs as its binding is when the load ends. The reload test holds a bundle's start, issues the binding inside the reload, and asks a repair after the runtime stopped: without the change it still answered (6 of 6 runs, GOMAXPROCS=1 and 24). No test failed without keep's stop waiting for the heartbeat's goroutine. The heartbeat test now holds one Say and stops meanwhile: stop must not return before that Say has; without the wait it failed 50 of 50 runs at GOMAXPROCS=1 and at 24. Builds: design 47 H ยง774, H-A-160; principle 11 --- node-tools/internal/alive/alive_test.go | 42 ++++++++++++++++++- node-tools/internal/runtime/reload_test.go | 49 ++++++++++++++++++++-- node-tools/internal/runtime/runtime.go | 14 ++++++- 3 files changed, 100 insertions(+), 5 deletions(-) diff --git a/node-tools/internal/alive/alive_test.go b/node-tools/internal/alive/alive_test.go index 888605e..d0397af 100644 --- a/node-tools/internal/alive/alive_test.go +++ b/node-tools/internal/alive/alive_test.go @@ -3,7 +3,9 @@ package alive import ( "encoding/json" "errors" + "runtime" "sync" + "sync/atomic" "testing" "time" ) @@ -15,13 +17,27 @@ type said struct { fail error // tried is told every Say, said or not, so a test waits on the heartbeat itself and never on the clock. tried chan struct{} + // held, when set, keeps each Say from returning until it is closed; inSay says one is held there now. + held chan struct{} + inSay atomic.Bool } func newSaid(fail error) *said { return &said{fail: fail, tried: make(chan struct{}, 16)} } func (s *said) Say(subject string, body []byte) error { + if s.held != nil { + s.inSay.Store(true) + s.tried <- struct{}{} + <-s.held + defer s.inSay.Store(false) + } s.mu.Lock() - defer func() { s.mu.Unlock(); s.tried <- struct{}{} }() + defer func() { + s.mu.Unlock() + if s.held == nil { + s.tried <- struct{}{} + } + }() if s.fail != nil { return s.fail } @@ -65,6 +81,30 @@ func TestTheRuntimeSaysItIsThereEveryInterval(t *testing.T) { if s.count() != 3 { t.Fatal("said after it was stopped") } + // Stopped while a beat is being said, stop returns only once that Say has returned: nothing is said after it. + slow := newSaid(nil) + slow.held = make(chan struct{}) + stopSlow := keep(slow, "anchor", 20*time.Millisecond, make(chan time.Time), t.Logf) + <-slow.tried + returned := make(chan bool) + go func() { + stopSlow() + returned <- slow.inSay.Load() + }() + // The stop is given its turn before the Say is let go (no clock: a returned stop is seen at once). + for i := 0; i < 1000; i++ { + select { + case stillSaying := <-returned: + close(slow.held) + t.Fatalf("stop returned while a beat was still being said (held: %v)", stillSaying) + default: + runtime.Gosched() + } + } + close(slow.held) + if stillSaying := <-returned; stillSaying { + t.Fatal("stop returned while a beat was still being said") + } // Keep itself keeps the interval on a ticker of its own. kept := newSaid(nil) stopKept := Keep(kept, "anchor", time.Hour, t.Logf) diff --git a/node-tools/internal/runtime/reload_test.go b/node-tools/internal/runtime/reload_test.go index 454c0fb..ee14d3c 100644 --- a/node-tools/internal/runtime/reload_test.go +++ b/node-tools/internal/runtime/reload_test.go @@ -2,6 +2,7 @@ package runtime import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -10,6 +11,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/membership" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/micro" @@ -154,13 +157,17 @@ func newReloadBed(t *testing.T) *reloadBed { } } -func (b *reloadBed) run(t *testing.T) { +// run starts the runtime, stopped at the test's end or once by what it returns. +func (b *reloadBed) run(t *testing.T) func() { t.Helper() stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf) if err != nil { t.Fatal(err) } - t.Cleanup(stop) + var once sync.Once + stopOnce := func() { once.Do(stop) } + t.Cleanup(stopOnce) + return stopOnce } func (b *reloadBed) answers(t *testing.T, tool string) { @@ -269,7 +276,7 @@ func TestAReplacedBundleIsReloaded(t *testing.T) { func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) { b := newReloadBed(t) place(t, b.entry, "first") - b.run(t) + stop := b.run(t) b.answers(t, "first") place(t, b.entry, "second") b.answers(t, "second") @@ -287,6 +294,42 @@ func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) { if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil { t.Fatalf("ledger.first is served again after the membership was reissued, and answered %s\n%s", got, b.logs.All()) } + + // A ledger bound while the bundle is loaded again: its repairs are served once, by the module loaded, and none is + // left behind on the stand-in the reload served nothing on (design 47 H, H-A-160). The bundle placed holds its + // start until the test lets it go, so the membership is heard inside the reload. + held, release := filepath.Join(t.TempDir(), "held"), filepath.Join(t.TempDir(), "release") + placeScript(t, b.entry, strings.Replace(bundleServing("third"), "import json, os, sys\n", + "import json, os, sys, time\nopen("+fmt.Sprintf("%q", held)+", \"w\").close()\nwhile not os.path.exists("+ + fmt.Sprintf("%q", release)+"):\n time.sleep(0.01)\n", 1)) + mt.Until(t, func() error { _, err := os.Stat(held); return err }) + bound := mt.MembershipOf("ledger", "anchor", false, nil) + bound.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, + History: "LEDGER_ledger_records", Subject: "mesh.ledger.ledger.records.history", State: "ledger_records", + Snapshot: "mesh.ledger.ledger.records.snapshot"}} + b.mesh.Issue(t, bound) + mt.Until(t, func() error { + if m := b.nodeTools.Membership("ledger"); m == nil || len(m.Bindings) == 0 { + return errors.New("the ledger's binding is not heard yet") + } + return nil + }) + if err := os.WriteFile(release, nil, 0o644); err != nil { + t.Fatal(err) + } + b.answers(t, "third") + mt.Until(t, func() error { + _, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{}) + if errors.Is(err, bus.ErrNoResponders) || (err != nil && err.Error() == "timeout") { + return fmt.Errorf("the ledger's repairs are not served after its reload: %v", err) + } + return nil + }) + stop() + if _, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{}); !errors.Is(err, bus.ErrNoResponders) { + t.Fatalf("the runtime stopped, and ledger.%s still answered (%v): served on the reload's stand-in and never stopped\n%s", + RepairLag, err, b.logs.All()) + } } // 1945/AC21: a bundle removed while served is no longer served, and its module says why. diff --git a/node-tools/internal/runtime/runtime.go b/node-tools/internal/runtime/runtime.go index 5ade9af..3cd8a7e 100644 --- a/node-tools/internal/runtime/runtime.go +++ b/node-tools/internal/runtime/runtime.go @@ -177,6 +177,11 @@ func (r *runner) refreshRepairs() { changed := false for _, m := range r.modules { l := r.of[m] + // A module being loaded again is left to its reload, which serves its repairs as its binding is then: served + // here on the stand-in, they would be served twice and never stopped (design 47 H, H-A-160). + if l.reloading { + continue + } served := false for _, reg := range l.own { served = served || reg.repairs @@ -205,6 +210,8 @@ type loadedModule struct { failed string names []string // the tools it serves, as module.tool serving map[string]func() // its tools' subscriptions, by module.tool; serve alone changes it + // reloading marks the stand-in a reload puts in place while it loads the module again: it serves nothing. + reloading bool } // unload stops serving the module's tools and stops its bundles. @@ -631,7 +638,7 @@ func (r *runner) reconcile() { func (r *runner) reload(module, what string) { r.mu.Lock() old := r.of[module] - r.of[module] = &loadedModule{served: old.served, seen: old.seen} + r.of[module] = &loadedModule{served: old.served, seen: old.seen, reloading: true} r.mu.Unlock() // Its events are refused first, so one heard while the old bundle stops is refused at once. r.events.dropModule(module) @@ -646,6 +653,11 @@ func (r *runner) reload(module, what string) { r.logf("[mesh-tools] %s's bundle is refused: %v; its tools are not served here", module, err) } r.mu.Lock() + // Its repairs as its binding is now, not as it was when the load began: the refresh left the stand-in alone. + if fresh.failed == "" { + repairs := r.repairsOf(module) + fresh.own, fresh.regs = withRepairs(fresh.own, repairs), withRepairs(fresh.regs, repairs) + } r.of[module] = fresh r.serve(fresh) r.bindVerbs()