From eb174d853ff12541ad65cc85539b6c9b48e0d266 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:17:51 +0100 Subject: [PATCH 1/4] H1 integration bound: the runner follows a membership without dropping a call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On every membership of a module, HandleCalled stopped each subscription of a tool and subscribed again on the same subjects; a call the bus routed in that window was dropped by the client in silence, and its caller waited the runner's whole bound (30 s). On the build node this was the 30 s of TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership and of TestTheLedgerRepairsAreToolsOfEveryLedgerOwner, and most of the integration level's 72 s. Now a membership keeps every subscription whose subject and queue it still names, subscribes the new ones, and only then stops those it no longer names (make before break). The test asks the served tool once after each of five memberships, with the tests' own call bound of 2 s (principle 11); it fails without the change. The runtime tests' call helper and the ledger listing wait that bound, not the runner's 30 s. Builds: design 47 H §774, H-A-158; principle 11 --- node-tools/internal/bus/bus.go | 28 ++++++++++++++------- node-tools/internal/runtime/ledgers_test.go | 2 +- node-tools/internal/runtime/reload_test.go | 13 +++++++--- node-tools/internal/runtime/runtime_test.go | 6 ++++- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/node-tools/internal/bus/bus.go b/node-tools/internal/bus/bus.go index 9997853..726dc0c 100644 --- a/node-tools/internal/bus/bus.go +++ b/node-tools/internal/bus/bus.go @@ -740,25 +740,35 @@ func (c *Conn) HandleCalled(key string, h CalledHandler) (func(), error) { "those of the modules it follows", c.self, key) } var mu sync.Mutex - var stops []func() + stops := map[Served]func(){} stopped := false + // **Make before break** (novox/hq design 47 H, H-A-158): a membership issued again keeps every subscription + // whose subject and queue it still names, subscribes those it newly names, and only then stops those it no + // longer names — a call the bus routes while the membership is followed is never dropped in silence. serve := func() { mu.Lock() defer mu.Unlock() if stopped { return } - for _, s := range stops { - s() - } - stops = nil + named := map[Served]bool{} for _, s := range c.servedOn(module, tool) { + named[s] = true + if _, kept := stops[s]; kept { + continue + } if stop, err := c.answerBoth(s.Subject, s.Queue, h); err == nil { - stops = append(stops, stop) + stops[s] = stop } else { c.Logf("[mesh-tools] cannot serve %s on %s: %v", key, s.Subject, err) } } + for s, stop := range stops { + if !named[s] { + stop() + delete(stops, s) + } + } } serve() // Its stop drops the membership handler too: a handler left behind would bind the stopped tool again @@ -773,10 +783,10 @@ func (c *Conn) HandleCalled(key string, h CalledHandler) (func(), error) { mu.Lock() defer mu.Unlock() stopped = true - for _, s := range stops { - s() + for s, stop := range stops { + stop() + delete(stops, s) } - stops = nil }, nil } diff --git a/node-tools/internal/runtime/ledgers_test.go b/node-tools/internal/runtime/ledgers_test.go index ff586ef..412809b 100644 --- a/node-tools/internal/runtime/ledgers_test.go +++ b/node-tools/internal/runtime/ledgers_test.go @@ -39,7 +39,7 @@ func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) { defer stop() asker := connect(t, "console", "workstation") listed := func(module string) (string, error) { - answered, err := asker.Ask(module+"."+ToolsVerb, map[string]any{}, "") + answered, err := asker.AskWithin(module+"."+ToolsVerb, map[string]any{}, "", callWithin) if err != nil { return "", err } diff --git a/node-tools/internal/runtime/reload_test.go b/node-tools/internal/runtime/reload_test.go index 7375d77..454c0fb 100644 --- a/node-tools/internal/runtime/reload_test.go +++ b/node-tools/internal/runtime/reload_test.go @@ -263,7 +263,9 @@ func TestAReplacedBundleIsReloaded(t *testing.T) { // A replaced bundle's old tool stays gone when the module's membership is issued again: a stopped tool // leaves no membership handler behind to bind it once more, and a call on it starts no unmanaged second -// copy of the bundle (issue 1945, review round 2). +// copy of the bundle (issue 1945, review round 2). And the tool still served answers the first call asked +// after each membership, within the test's own bound: the runner keeps the subscriptions the membership +// still names, so no call is dropped while it follows one (design 47 H, H-A-158). func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) { b := newReloadBed(t) place(t, b.entry, "first") @@ -273,8 +275,13 @@ func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) { b.answers(t, "second") b.silent(t, "first") - b.mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil)) - b.answers(t, "second") + for round := 1; round <= 5; round++ { + b.mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil)) + if got, err := call(t, b.asker, "ledger.second@anchor", map[string]any{}); err != nil || !strings.Contains(got, `"tool":"second"`) { + t.Fatalf("membership %d issued again: the first call of ledger.second answered %s, %v — dropped while the runner "+ + "followed the membership\n%s", round, got, err, b.logs.All()) + } + } // Long enough for the reissued membership to have been heard, before the old tool is asked. time.Sleep(300 * time.Millisecond) if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil { diff --git a/node-tools/internal/runtime/runtime_test.go b/node-tools/internal/runtime/runtime_test.go index 2d07444..32bc75a 100644 --- a/node-tools/internal/runtime/runtime_test.go +++ b/node-tools/internal/runtime/runtime_test.go @@ -26,9 +26,13 @@ func connect(t *testing.T, module, node string) *bus.Conn { return c } +// callWithin is the tests' own bound on a call (principle 11, design 47 H, H-A-158): a call that is never answered +// fails the try it is part of within it, never after the tool runner's own 30 s. +const callWithin = 2 * time.Second + func call(t *testing.T, asker *bus.Conn, key string, args any) (string, error) { t.Helper() - got, err := asker.Ask(key, args, "") + got, err := asker.AskWithin(key, args, "", callWithin) return string(got.Result), err } From 3850eb8eac0efac9b97e92d02664ae95752014a4 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:23:59 +0100 Subject: [PATCH 2/4] H1 integration bound: the heartbeat and the counter tests hold under load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestTheRuntimeSaysItIsThereEveryInterval slept 70 ms and wanted three beats at 20 ms: a loaded machine that did not schedule the heartbeat in time failed it (main.20's integration step on the build node); and its stop returned before the heartbeat's goroutine ended, so a beat could be said after it ("said after it was stopped"). The ticks are now the caller's (keep, beside Keep's own ticker), the test drives them and waits on each beat, and stop returns once nothing more is said. Pinned to one processor at idle priority beside six busy loops, the old test failed 13 of 100 runs; the new passes 100 of 100. TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate bounded a writer's tries at 1000, a guess: on one processor the scheduler lets one writer win every round and the other lost its 1000 tries every time. A write is lost only to the other writer's write, so the bound is the other's count plus one. Builds: design 47 H §774; principle 11 --- node-tools/internal/alive/alive.go | 20 +++++-- node-tools/internal/alive/alive_test.go | 52 ++++++++++++++----- .../internal/runtime/state_counter_test.go | 7 ++- 3 files changed, 59 insertions(+), 20 deletions(-) diff --git a/node-tools/internal/alive/alive.go b/node-tools/internal/alive/alive.go index 843b57f..fc10987 100644 --- a/node-tools/internal/alive/alive.go +++ b/node-tools/internal/alive/alive.go @@ -49,13 +49,20 @@ func Version() string { // Keep says the runtime is there now and every interval until the returned function is called. A // heartbeat that cannot be said is logged when that starts and when it stops — not every minute. func Keep(bus Sayer, node string, every time.Duration, logf func(string, ...any)) func() { + tick := time.NewTicker(every) + stop := keep(bus, node, every, tick.C, logf) + return func() { stop(); tick.Stop() } +} + +// keep is Keep saying it again on every tick: the interval is told in the beat, the ticks are the caller's. +// Its stop returns once nothing more is said. +func keep(bus Sayer, node string, every time.Duration, ticks <-chan time.Time, logf func(string, ...any)) func() { body, _ := json.Marshal(Beat{Node: node, IntervalSeconds: int(every / time.Second), Version: Version()}) - done := make(chan struct{}) + done, ended := make(chan struct{}), make(chan struct{}) var once sync.Once go func() { + defer close(ended) failing := "" - tick := time.NewTicker(every) - defer tick.Stop() for { why := "" if err := bus.Say(Subject(node), body); err != nil { @@ -72,9 +79,12 @@ func Keep(bus Sayer, node string, every time.Duration, logf func(string, ...any) select { case <-done: return - case <-tick.C: + case <-ticks: } } }() - return func() { once.Do(func() { close(done) }) } + return func() { + once.Do(func() { close(done) }) + <-ended + } } diff --git a/node-tools/internal/alive/alive_test.go b/node-tools/internal/alive/alive_test.go index f7cd680..888605e 100644 --- a/node-tools/internal/alive/alive_test.go +++ b/node-tools/internal/alive/alive_test.go @@ -13,11 +13,15 @@ type said struct { subs []string body [][]byte fail error + // tried is told every Say, said or not, so a test waits on the heartbeat itself and never on the clock. + tried chan struct{} } +func newSaid(fail error) *said { return &said{fail: fail, tried: make(chan struct{}, 16)} } + func (s *said) Say(subject string, body []byte) error { s.mu.Lock() - defer s.mu.Unlock() + defer func() { s.mu.Unlock(); s.tried <- struct{}{} }() if s.fail != nil { return s.fail } @@ -32,39 +36,61 @@ func (s *said) count() int { } // **The runtime says it is there at once and every interval, as its machine, with the interval**: the -// controller's bound is three of them (to-be 45 S11). +// controller's bound is three of them (to-be 45 S11). The ticks are the test's own, so a loaded machine +// cannot make it miss one (principle 11; design 47 H §774); stopped, it says nothing more. func TestTheRuntimeSaysItIsThereEveryInterval(t *testing.T) { - s := &said{} - stop := Keep(s, "anchor", 20*time.Millisecond, t.Logf) - time.Sleep(70 * time.Millisecond) + s := newSaid(nil) + ticks := make(chan time.Time) + stop := keep(s, "anchor", 20*time.Millisecond, ticks, t.Logf) + <-s.tried + for i := 0; i < 2; i++ { + ticks <- time.Time{} + <-s.tried + } stop() stop() - if n := s.count(); n < 3 { - t.Fatalf("said %d times in three intervals", n) + if n := s.count(); n != 3 { + t.Fatalf("said %d times at once and on two ticks", n) } var beat Beat if err := json.Unmarshal(s.body[0], &beat); err != nil || beat.Node != "anchor" || beat.IntervalSeconds != 0 || s.subs[0] != "mesh.control.anchor.tools-alive" { t.Fatalf("%s on %s: %v", s.body[0], s.subs[0], err) } - n := s.count() - time.Sleep(50 * time.Millisecond) - if s.count() != n { + select { + case ticks <- time.Time{}: + t.Fatal("a tick was taken after it was stopped") + default: + } + if s.count() != 3 { t.Fatal("said after it was stopped") } + // Keep itself keeps the interval on a ticker of its own. + kept := newSaid(nil) + stopKept := Keep(kept, "anchor", time.Hour, t.Logf) + <-kept.tried + stopKept() + if kept.count() != 1 { + t.Fatalf("Keep said %d times before its first interval", kept.count()) + } } // **A heartbeat the bus will not take is said once, and again when it is taken**: not a line a minute. func TestAHeartbeatThatCannotBeSaidIsLoggedOnce(t *testing.T) { - s := &said{fail: errors.New("permissions violation")} + s := newSaid(errors.New("permissions violation")) var mu sync.Mutex var lines []string - stop := Keep(s, "anchor", 10*time.Millisecond, func(f string, a ...any) { + ticks := make(chan time.Time) + stop := keep(s, "anchor", 10*time.Millisecond, ticks, func(f string, a ...any) { mu.Lock() lines = append(lines, f) mu.Unlock() }) - time.Sleep(60 * time.Millisecond) + <-s.tried + for i := 0; i < 4; i++ { + ticks <- time.Time{} + <-s.tried + } stop() mu.Lock() defer mu.Unlock() diff --git a/node-tools/internal/runtime/state_counter_test.go b/node-tools/internal/runtime/state_counter_test.go index 3cbbfd9..b8594eb 100644 --- a/node-tools/internal/runtime/state_counter_test.go +++ b/node-tools/internal/runtime/state_counter_test.go @@ -78,7 +78,10 @@ func increment(b launch.Bus, key string, attempts int) (n uint64, lost int, err // Two writers incrementing one counter a thousand times each, against a real bus, end at two thousand: // every number handed out once and none lost, because the runtime writes only at the revision read -// (novox/hq research 042: an issue's number is handed out once). +// (novox/hq research 042: an issue's number is handed out once). A write is lost only to the other writer's +// write, so one increment loses at most as many times as the other writer increments: that, not a guess, +// is the bound on its tries, and a machine whose scheduler lets one writer win every round still passes +// (one processor starved a writer of a thousand tries; principle 11, design 47 H §774). func TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate(t *testing.T) { b := counterBus(t) const each = 1000 @@ -91,7 +94,7 @@ func TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate(t *testing.T) { go func(w int) { defer wg.Done() for i := 0; i < each; i++ { - n, l, err := increment(b, "issue", 1000) + n, l, err := increment(b, "issue", each+1) lost[w] += l if err != nil { errs[w] = err From 05deea5952ff3fd4bcf83823340e4226c1810233 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:28:34 +0100 Subject: [PATCH 3/4] H1 integration bound: the repairs' refresh keeps the tools it still serves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a module's ledger binding came or went, refreshRepairs stopped every subscription of its tools and served them all again, so a call in that gap was dropped the same way as on a membership (H-A-158). serve is now the one routine that changes a module's subscriptions, by tool address: it keeps those still named, subscribes the new, then stops those no longer named; refreshRepairs and the first serve both go through it, so no tool is served twice or not at all. The ledger test's later binding now comes and goes ten times, and the module's own tool must answer the first call after each, within the tests' 2 s bound; it failed without the change at GOMAXPROCS=1 and 24. Builds: design 47 H §774, H-A-158, H-A-160; principle 11 --- node-tools/internal/runtime/ledgers_test.go | 12 +++++ node-tools/internal/runtime/runtime.go | 60 +++++++++++++-------- 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/node-tools/internal/runtime/ledgers_test.go b/node-tools/internal/runtime/ledgers_test.go index 412809b..3315484 100644 --- a/node-tools/internal/runtime/ledgers_test.go +++ b/node-tools/internal/runtime/ledgers_test.go @@ -67,6 +67,18 @@ func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) { }) } t.Run("a ledger bound later brings them", func(t *testing.T) { + // Its own tool answers the first call after each binding comes or goes: the repairs' refresh keeps the + // subscriptions it still names (design 47 H, H-A-160). + for round := 1; round <= 10; round++ { + plain.Bindings = nil + if round%2 == 1 { + plain.Bindings = owner.Bindings + } + mesh.Issue(t, plain) + if _, err := asker.AskWithin("plain.who@anchor", map[string]any{}, "", callWithin); err != nil { + t.Fatalf("binding change %d: the first call of plain.who failed: %v — dropped while its tools were served again", round, err) + } + } plain.Bindings = owner.Bindings mesh.Issue(t, plain) mt.Until(t, func() error { diff --git a/node-tools/internal/runtime/runtime.go b/node-tools/internal/runtime/runtime.go index ff6e68d..5ade9af 100644 --- a/node-tools/internal/runtime/runtime.go +++ b/node-tools/internal/runtime/runtime.go @@ -185,10 +185,6 @@ func (r *runner) refreshRepairs() { if served == (repairs != nil) { continue } - for i := len(l.serving) - 1; i >= 0; i-- { - l.serving[i]() - } - l.serving, l.names = nil, nil l.own, l.regs = withRepairs(l.own, repairs), withRepairs(l.regs, repairs) r.serve(l) changed = true @@ -207,14 +203,14 @@ type loadedModule struct { regs []registration // every registration its bundles made own []registration // those under the name of a module served here failed string - names []string // the tools it serves, as module.tool - serving []func() // its tools' subscriptions + names []string // the tools it serves, as module.tool + serving map[string]func() // its tools' subscriptions, by module.tool; serve alone changes it } // unload stops serving the module's tools and stops its bundles. func (l *loadedModule) unload() { - for i := len(l.serving) - 1; i >= 0; i-- { - l.serving[i]() + for _, stop := range l.serving { + stop() } l.serving, l.names = nil, nil for i := len(l.children) - 1; i >= 0; i-- { @@ -493,25 +489,47 @@ func refusal(own []registration) error { return nil } -// serve binds a loaded module's tools. r.mu is held. +// serve binds a loaded module's tools as its registrations name them now, and is the one routine that changes its +// subscriptions (design 47 H, H-A-158, H-A-160): a tool still named keeps its subscription, a newly named one is +// subscribed, and only then is one no longer named stopped — no tool is served twice, and none is dropped while the +// module's tools are served again. r.mu is held. func (r *runner) serve(l *loadedModule) { + if l.serving == nil { + l.serving = map[string]func(){} + } + named := map[string]bool{} + l.names = nil for _, reg := range l.own { for _, t := range reg.tools { t := t - // Told who called, from the subject the call arrived on, never from its body (novox/hq issue 365). - // Through the approval gate, before the bundle or the runner's own tool (design 46 7-approval-mark §20). - module, own := reg.module, reg.repairs - stop, err := r.conn.HandleCalled(reg.module+"."+t.Name, func(caller string, body json.RawMessage) (any, error) { - return r.events.gated(approval.Call{Address: module + "." + t.Name, Module: module, Name: t.Name, Own: own, - Fingerprint: r.fingerprintOf(module), Env: r.envOf(module), Caller: caller, Args: argsOf(body), - Description: t.Description}, t, caller) - }) - if err != nil { - r.logf("[mesh-tools] cannot serve %s.%s: %v", reg.module, t.Name, err) + address := reg.module + "." + t.Name + if named[address] { continue } - l.names = append(l.names, reg.module+"."+t.Name) - l.serving = append(l.serving, stop) + named[address] = true + if _, kept := l.serving[address]; !kept { + // Told who called, from the subject the call arrived on, never from its body (novox/hq issue 365). + // Through the approval gate, before the bundle or the runner's own tool (design 46 7-approval-mark §20). + module, own := reg.module, reg.repairs + stop, err := r.conn.HandleCalled(address, func(caller string, body json.RawMessage) (any, error) { + return r.events.gated(approval.Call{Address: module + "." + t.Name, Module: module, Name: t.Name, Own: own, + Fingerprint: r.fingerprintOf(module), Env: r.envOf(module), Caller: caller, Args: argsOf(body), + Description: t.Description}, t, caller) + }) + if err != nil { + r.logf("[mesh-tools] cannot serve %s.%s: %v", reg.module, t.Name, err) + delete(named, address) + continue + } + l.serving[address] = stop + } + l.names = append(l.names, address) + } + } + for address, stop := range l.serving { + if !named[address] { + stop() + delete(l.serving, address) } } } From 8ee4fdfb8ef15d75a8d11dc792d3a56c3d0cd2f0 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 17:45:50 +0100 Subject: [PATCH 4/4] H1 integration bound: a reload's stand-in serves no repairs; stop waits its Say MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 1. A ledger binding heard while a module was loaded again made refreshRepairs serve the repairs on the reload's stand-in, whose subscriptions nobody stopped: the repairs were served twice, and still answered after the runtime stopped. The stand-in is now marked as reloading and the refresh leaves it alone; the reload serves the module's repairs as its binding is when the load ends. The reload test holds a bundle's start, issues the binding inside the reload, and asks a repair after the runtime stopped: without the change it still answered (6 of 6 runs, GOMAXPROCS=1 and 24). No test failed without keep's stop waiting for the heartbeat's goroutine. The heartbeat test now holds one Say and stops meanwhile: stop must not return before that Say has; without the wait it failed 50 of 50 runs at GOMAXPROCS=1 and at 24. Builds: design 47 H §774, H-A-160; principle 11 --- node-tools/internal/alive/alive_test.go | 42 ++++++++++++++++++- node-tools/internal/runtime/reload_test.go | 49 ++++++++++++++++++++-- node-tools/internal/runtime/runtime.go | 14 ++++++- 3 files changed, 100 insertions(+), 5 deletions(-) diff --git a/node-tools/internal/alive/alive_test.go b/node-tools/internal/alive/alive_test.go index 888605e..d0397af 100644 --- a/node-tools/internal/alive/alive_test.go +++ b/node-tools/internal/alive/alive_test.go @@ -3,7 +3,9 @@ package alive import ( "encoding/json" "errors" + "runtime" "sync" + "sync/atomic" "testing" "time" ) @@ -15,13 +17,27 @@ type said struct { fail error // tried is told every Say, said or not, so a test waits on the heartbeat itself and never on the clock. tried chan struct{} + // held, when set, keeps each Say from returning until it is closed; inSay says one is held there now. + held chan struct{} + inSay atomic.Bool } func newSaid(fail error) *said { return &said{fail: fail, tried: make(chan struct{}, 16)} } func (s *said) Say(subject string, body []byte) error { + if s.held != nil { + s.inSay.Store(true) + s.tried <- struct{}{} + <-s.held + defer s.inSay.Store(false) + } s.mu.Lock() - defer func() { s.mu.Unlock(); s.tried <- struct{}{} }() + defer func() { + s.mu.Unlock() + if s.held == nil { + s.tried <- struct{}{} + } + }() if s.fail != nil { return s.fail } @@ -65,6 +81,30 @@ func TestTheRuntimeSaysItIsThereEveryInterval(t *testing.T) { if s.count() != 3 { t.Fatal("said after it was stopped") } + // Stopped while a beat is being said, stop returns only once that Say has returned: nothing is said after it. + slow := newSaid(nil) + slow.held = make(chan struct{}) + stopSlow := keep(slow, "anchor", 20*time.Millisecond, make(chan time.Time), t.Logf) + <-slow.tried + returned := make(chan bool) + go func() { + stopSlow() + returned <- slow.inSay.Load() + }() + // The stop is given its turn before the Say is let go (no clock: a returned stop is seen at once). + for i := 0; i < 1000; i++ { + select { + case stillSaying := <-returned: + close(slow.held) + t.Fatalf("stop returned while a beat was still being said (held: %v)", stillSaying) + default: + runtime.Gosched() + } + } + close(slow.held) + if stillSaying := <-returned; stillSaying { + t.Fatal("stop returned while a beat was still being said") + } // Keep itself keeps the interval on a ticker of its own. kept := newSaid(nil) stopKept := Keep(kept, "anchor", time.Hour, t.Logf) diff --git a/node-tools/internal/runtime/reload_test.go b/node-tools/internal/runtime/reload_test.go index 454c0fb..ee14d3c 100644 --- a/node-tools/internal/runtime/reload_test.go +++ b/node-tools/internal/runtime/reload_test.go @@ -2,6 +2,7 @@ package runtime import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -10,6 +11,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/membership" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/micro" @@ -154,13 +157,17 @@ func newReloadBed(t *testing.T) *reloadBed { } } -func (b *reloadBed) run(t *testing.T) { +// run starts the runtime, stopped at the test's end or once by what it returns. +func (b *reloadBed) run(t *testing.T) func() { t.Helper() stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf) if err != nil { t.Fatal(err) } - t.Cleanup(stop) + var once sync.Once + stopOnce := func() { once.Do(stop) } + t.Cleanup(stopOnce) + return stopOnce } func (b *reloadBed) answers(t *testing.T, tool string) { @@ -269,7 +276,7 @@ func TestAReplacedBundleIsReloaded(t *testing.T) { func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) { b := newReloadBed(t) place(t, b.entry, "first") - b.run(t) + stop := b.run(t) b.answers(t, "first") place(t, b.entry, "second") b.answers(t, "second") @@ -287,6 +294,42 @@ func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) { if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil { t.Fatalf("ledger.first is served again after the membership was reissued, and answered %s\n%s", got, b.logs.All()) } + + // A ledger bound while the bundle is loaded again: its repairs are served once, by the module loaded, and none is + // left behind on the stand-in the reload served nothing on (design 47 H, H-A-160). The bundle placed holds its + // start until the test lets it go, so the membership is heard inside the reload. + held, release := filepath.Join(t.TempDir(), "held"), filepath.Join(t.TempDir(), "release") + placeScript(t, b.entry, strings.Replace(bundleServing("third"), "import json, os, sys\n", + "import json, os, sys, time\nopen("+fmt.Sprintf("%q", held)+", \"w\").close()\nwhile not os.path.exists("+ + fmt.Sprintf("%q", release)+"):\n time.sleep(0.01)\n", 1)) + mt.Until(t, func() error { _, err := os.Stat(held); return err }) + bound := mt.MembershipOf("ledger", "anchor", false, nil) + bound.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, + History: "LEDGER_ledger_records", Subject: "mesh.ledger.ledger.records.history", State: "ledger_records", + Snapshot: "mesh.ledger.ledger.records.snapshot"}} + b.mesh.Issue(t, bound) + mt.Until(t, func() error { + if m := b.nodeTools.Membership("ledger"); m == nil || len(m.Bindings) == 0 { + return errors.New("the ledger's binding is not heard yet") + } + return nil + }) + if err := os.WriteFile(release, nil, 0o644); err != nil { + t.Fatal(err) + } + b.answers(t, "third") + mt.Until(t, func() error { + _, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{}) + if errors.Is(err, bus.ErrNoResponders) || (err != nil && err.Error() == "timeout") { + return fmt.Errorf("the ledger's repairs are not served after its reload: %v", err) + } + return nil + }) + stop() + if _, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{}); !errors.Is(err, bus.ErrNoResponders) { + t.Fatalf("the runtime stopped, and ledger.%s still answered (%v): served on the reload's stand-in and never stopped\n%s", + RepairLag, err, b.logs.All()) + } } // 1945/AC21: a bundle removed while served is no longer served, and its module says why. diff --git a/node-tools/internal/runtime/runtime.go b/node-tools/internal/runtime/runtime.go index 5ade9af..3cd8a7e 100644 --- a/node-tools/internal/runtime/runtime.go +++ b/node-tools/internal/runtime/runtime.go @@ -177,6 +177,11 @@ func (r *runner) refreshRepairs() { changed := false for _, m := range r.modules { l := r.of[m] + // A module being loaded again is left to its reload, which serves its repairs as its binding is then: served + // here on the stand-in, they would be served twice and never stopped (design 47 H, H-A-160). + if l.reloading { + continue + } served := false for _, reg := range l.own { served = served || reg.repairs @@ -205,6 +210,8 @@ type loadedModule struct { failed string names []string // the tools it serves, as module.tool serving map[string]func() // its tools' subscriptions, by module.tool; serve alone changes it + // reloading marks the stand-in a reload puts in place while it loads the module again: it serves nothing. + reloading bool } // unload stops serving the module's tools and stops its bundles. @@ -631,7 +638,7 @@ func (r *runner) reconcile() { func (r *runner) reload(module, what string) { r.mu.Lock() old := r.of[module] - r.of[module] = &loadedModule{served: old.served, seen: old.seen} + r.of[module] = &loadedModule{served: old.served, seen: old.seen, reloading: true} r.mu.Unlock() // Its events are refused first, so one heard while the old bundle stops is refused at once. r.events.dropModule(module) @@ -646,6 +653,11 @@ func (r *runner) reload(module, what string) { r.logf("[mesh-tools] %s's bundle is refused: %v; its tools are not served here", module, err) } r.mu.Lock() + // Its repairs as its binding is now, not as it was when the load began: the refresh left the stand-in alone. + if fresh.failed == "" { + repairs := r.repairsOf(module) + fresh.own, fresh.regs = withRepairs(fresh.own, repairs), withRepairs(fresh.regs, repairs) + } r.of[module] = fresh r.serve(fresh) r.bindVerbs()