H1 integration bound: a reload's stand-in serves no repairs; stop waits its Say

Review round 1. A ledger binding heard while a module was loaded again
made refreshRepairs serve the repairs on the reload's stand-in, whose
subscriptions nobody stopped: the repairs were served twice, and still
answered after the runtime stopped. The stand-in is now marked as
reloading and the refresh leaves it alone; the reload serves the module's
repairs as its binding is when the load ends. The reload test holds a
bundle's start, issues the binding inside the reload, and asks a repair
after the runtime stopped: without the change it still answered (6 of 6
runs, GOMAXPROCS=1 and 24).

No test failed without keep's stop waiting for the heartbeat's goroutine.
The heartbeat test now holds one Say and stops meanwhile: stop must not
return before that Say has; without the wait it failed 50 of 50 runs at
GOMAXPROCS=1 and at 24.

Builds: design 47 H §774, H-A-160; principle 11
This commit is contained in:
2026-11-20 17:45:50 +01:00
parent 05deea5952
commit 8ee4fdfb8e
3 changed files with 100 additions and 5 deletions
+41 -1
View File
@@ -3,7 +3,9 @@ package alive
import ( import (
"encoding/json" "encoding/json"
"errors" "errors"
"runtime"
"sync" "sync"
"sync/atomic"
"testing" "testing"
"time" "time"
) )
@@ -15,13 +17,27 @@ type said struct {
fail error fail error
// tried is told every Say, said or not, so a test waits on the heartbeat itself and never on the clock. // tried is told every Say, said or not, so a test waits on the heartbeat itself and never on the clock.
tried chan struct{} tried chan struct{}
// held, when set, keeps each Say from returning until it is closed; inSay says one is held there now.
held chan struct{}
inSay atomic.Bool
} }
func newSaid(fail error) *said { return &said{fail: fail, tried: make(chan struct{}, 16)} } func newSaid(fail error) *said { return &said{fail: fail, tried: make(chan struct{}, 16)} }
func (s *said) Say(subject string, body []byte) error { func (s *said) Say(subject string, body []byte) error {
if s.held != nil {
s.inSay.Store(true)
s.tried <- struct{}{}
<-s.held
defer s.inSay.Store(false)
}
s.mu.Lock() s.mu.Lock()
defer func() { s.mu.Unlock(); s.tried <- struct{}{} }() defer func() {
s.mu.Unlock()
if s.held == nil {
s.tried <- struct{}{}
}
}()
if s.fail != nil { if s.fail != nil {
return s.fail return s.fail
} }
@@ -65,6 +81,30 @@ func TestTheRuntimeSaysItIsThereEveryInterval(t *testing.T) {
if s.count() != 3 { if s.count() != 3 {
t.Fatal("said after it was stopped") t.Fatal("said after it was stopped")
} }
// Stopped while a beat is being said, stop returns only once that Say has returned: nothing is said after it.
slow := newSaid(nil)
slow.held = make(chan struct{})
stopSlow := keep(slow, "anchor", 20*time.Millisecond, make(chan time.Time), t.Logf)
<-slow.tried
returned := make(chan bool)
go func() {
stopSlow()
returned <- slow.inSay.Load()
}()
// The stop is given its turn before the Say is let go (no clock: a returned stop is seen at once).
for i := 0; i < 1000; i++ {
select {
case stillSaying := <-returned:
close(slow.held)
t.Fatalf("stop returned while a beat was still being said (held: %v)", stillSaying)
default:
runtime.Gosched()
}
}
close(slow.held)
if stillSaying := <-returned; stillSaying {
t.Fatal("stop returned while a beat was still being said")
}
// Keep itself keeps the interval on a ticker of its own. // Keep itself keeps the interval on a ticker of its own.
kept := newSaid(nil) kept := newSaid(nil)
stopKept := Keep(kept, "anchor", time.Hour, t.Logf) stopKept := Keep(kept, "anchor", time.Hour, t.Logf)
+46 -3
View File
@@ -2,6 +2,7 @@ package runtime
import ( import (
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
@@ -10,6 +11,8 @@ import (
"testing" "testing"
"time" "time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/membership"
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/micro" "github.com/nats-io/nats.go/micro"
@@ -154,13 +157,17 @@ func newReloadBed(t *testing.T) *reloadBed {
} }
} }
func (b *reloadBed) run(t *testing.T) { // run starts the runtime, stopped at the test's end or once by what it returns.
func (b *reloadBed) run(t *testing.T) func() {
t.Helper() t.Helper()
stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf) stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
t.Cleanup(stop) var once sync.Once
stopOnce := func() { once.Do(stop) }
t.Cleanup(stopOnce)
return stopOnce
} }
func (b *reloadBed) answers(t *testing.T, tool string) { func (b *reloadBed) answers(t *testing.T, tool string) {
@@ -269,7 +276,7 @@ func TestAReplacedBundleIsReloaded(t *testing.T) {
func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) { func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) {
b := newReloadBed(t) b := newReloadBed(t)
place(t, b.entry, "first") place(t, b.entry, "first")
b.run(t) stop := b.run(t)
b.answers(t, "first") b.answers(t, "first")
place(t, b.entry, "second") place(t, b.entry, "second")
b.answers(t, "second") b.answers(t, "second")
@@ -287,6 +294,42 @@ func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) {
if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil { if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil {
t.Fatalf("ledger.first is served again after the membership was reissued, and answered %s\n%s", got, b.logs.All()) t.Fatalf("ledger.first is served again after the membership was reissued, and answered %s\n%s", got, b.logs.All())
} }
// A ledger bound while the bundle is loaded again: its repairs are served once, by the module loaded, and none is
// left behind on the stand-in the reload served nothing on (design 47 H, H-A-160). The bundle placed holds its
// start until the test lets it go, so the membership is heard inside the reload.
held, release := filepath.Join(t.TempDir(), "held"), filepath.Join(t.TempDir(), "release")
placeScript(t, b.entry, strings.Replace(bundleServing("third"), "import json, os, sys\n",
"import json, os, sys, time\nopen("+fmt.Sprintf("%q", held)+", \"w\").close()\nwhile not os.path.exists("+
fmt.Sprintf("%q", release)+"):\n time.sleep(0.01)\n", 1))
mt.Until(t, func() error { _, err := os.Stat(held); return err })
bound := mt.MembershipOf("ledger", "anchor", false, nil)
bound.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger,
History: "LEDGER_ledger_records", Subject: "mesh.ledger.ledger.records.history", State: "ledger_records",
Snapshot: "mesh.ledger.ledger.records.snapshot"}}
b.mesh.Issue(t, bound)
mt.Until(t, func() error {
if m := b.nodeTools.Membership("ledger"); m == nil || len(m.Bindings) == 0 {
return errors.New("the ledger's binding is not heard yet")
}
return nil
})
if err := os.WriteFile(release, nil, 0o644); err != nil {
t.Fatal(err)
}
b.answers(t, "third")
mt.Until(t, func() error {
_, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{})
if errors.Is(err, bus.ErrNoResponders) || (err != nil && err.Error() == "timeout") {
return fmt.Errorf("the ledger's repairs are not served after its reload: %v", err)
}
return nil
})
stop()
if _, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{}); !errors.Is(err, bus.ErrNoResponders) {
t.Fatalf("the runtime stopped, and ledger.%s still answered (%v): served on the reload's stand-in and never stopped\n%s",
RepairLag, err, b.logs.All())
}
} }
// 1945/AC21: a bundle removed while served is no longer served, and its module says why. // 1945/AC21: a bundle removed while served is no longer served, and its module says why.
+13 -1
View File
@@ -177,6 +177,11 @@ func (r *runner) refreshRepairs() {
changed := false changed := false
for _, m := range r.modules { for _, m := range r.modules {
l := r.of[m] l := r.of[m]
// A module being loaded again is left to its reload, which serves its repairs as its binding is then: served
// here on the stand-in, they would be served twice and never stopped (design 47 H, H-A-160).
if l.reloading {
continue
}
served := false served := false
for _, reg := range l.own { for _, reg := range l.own {
served = served || reg.repairs served = served || reg.repairs
@@ -205,6 +210,8 @@ type loadedModule struct {
failed string failed string
names []string // the tools it serves, as module.tool names []string // the tools it serves, as module.tool
serving map[string]func() // its tools' subscriptions, by module.tool; serve alone changes it serving map[string]func() // its tools' subscriptions, by module.tool; serve alone changes it
// reloading marks the stand-in a reload puts in place while it loads the module again: it serves nothing.
reloading bool
} }
// unload stops serving the module's tools and stops its bundles. // unload stops serving the module's tools and stops its bundles.
@@ -631,7 +638,7 @@ func (r *runner) reconcile() {
func (r *runner) reload(module, what string) { func (r *runner) reload(module, what string) {
r.mu.Lock() r.mu.Lock()
old := r.of[module] old := r.of[module]
r.of[module] = &loadedModule{served: old.served, seen: old.seen} r.of[module] = &loadedModule{served: old.served, seen: old.seen, reloading: true}
r.mu.Unlock() r.mu.Unlock()
// Its events are refused first, so one heard while the old bundle stops is refused at once. // Its events are refused first, so one heard while the old bundle stops is refused at once.
r.events.dropModule(module) r.events.dropModule(module)
@@ -646,6 +653,11 @@ func (r *runner) reload(module, what string) {
r.logf("[mesh-tools] %s's bundle is refused: %v; its tools are not served here", module, err) r.logf("[mesh-tools] %s's bundle is refused: %v; its tools are not served here", module, err)
} }
r.mu.Lock() r.mu.Lock()
// Its repairs as its binding is now, not as it was when the load began: the refresh left the stand-in alone.
if fresh.failed == "" {
repairs := r.repairsOf(module)
fresh.own, fresh.regs = withRepairs(fresh.own, repairs), withRepairs(fresh.regs, repairs)
}
r.of[module] = fresh r.of[module] = fresh
r.serve(fresh) r.serve(fresh)
r.bindVerbs() r.bindVerbs()