H1 integration bound: a reload's stand-in serves no repairs; stop waits its Say
Review round 1. A ledger binding heard while a module was loaded again made refreshRepairs serve the repairs on the reload's stand-in, whose subscriptions nobody stopped: the repairs were served twice, and still answered after the runtime stopped. The stand-in is now marked as reloading and the refresh leaves it alone; the reload serves the module's repairs as its binding is when the load ends. The reload test holds a bundle's start, issues the binding inside the reload, and asks a repair after the runtime stopped: without the change it still answered (6 of 6 runs, GOMAXPROCS=1 and 24). No test failed without keep's stop waiting for the heartbeat's goroutine. The heartbeat test now holds one Say and stops meanwhile: stop must not return before that Say has; without the wait it failed 50 of 50 runs at GOMAXPROCS=1 and at 24. Builds: design 47 H §774, H-A-160; principle 11
This commit is contained in:
@@ -3,7 +3,9 @@ package alive
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"runtime"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -15,13 +17,27 @@ type said struct {
|
|||||||
fail error
|
fail error
|
||||||
// tried is told every Say, said or not, so a test waits on the heartbeat itself and never on the clock.
|
// tried is told every Say, said or not, so a test waits on the heartbeat itself and never on the clock.
|
||||||
tried chan struct{}
|
tried chan struct{}
|
||||||
|
// held, when set, keeps each Say from returning until it is closed; inSay says one is held there now.
|
||||||
|
held chan struct{}
|
||||||
|
inSay atomic.Bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func newSaid(fail error) *said { return &said{fail: fail, tried: make(chan struct{}, 16)} }
|
func newSaid(fail error) *said { return &said{fail: fail, tried: make(chan struct{}, 16)} }
|
||||||
|
|
||||||
func (s *said) Say(subject string, body []byte) error {
|
func (s *said) Say(subject string, body []byte) error {
|
||||||
|
if s.held != nil {
|
||||||
|
s.inSay.Store(true)
|
||||||
|
s.tried <- struct{}{}
|
||||||
|
<-s.held
|
||||||
|
defer s.inSay.Store(false)
|
||||||
|
}
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer func() { s.mu.Unlock(); s.tried <- struct{}{} }()
|
defer func() {
|
||||||
|
s.mu.Unlock()
|
||||||
|
if s.held == nil {
|
||||||
|
s.tried <- struct{}{}
|
||||||
|
}
|
||||||
|
}()
|
||||||
if s.fail != nil {
|
if s.fail != nil {
|
||||||
return s.fail
|
return s.fail
|
||||||
}
|
}
|
||||||
@@ -65,6 +81,30 @@ func TestTheRuntimeSaysItIsThereEveryInterval(t *testing.T) {
|
|||||||
if s.count() != 3 {
|
if s.count() != 3 {
|
||||||
t.Fatal("said after it was stopped")
|
t.Fatal("said after it was stopped")
|
||||||
}
|
}
|
||||||
|
// Stopped while a beat is being said, stop returns only once that Say has returned: nothing is said after it.
|
||||||
|
slow := newSaid(nil)
|
||||||
|
slow.held = make(chan struct{})
|
||||||
|
stopSlow := keep(slow, "anchor", 20*time.Millisecond, make(chan time.Time), t.Logf)
|
||||||
|
<-slow.tried
|
||||||
|
returned := make(chan bool)
|
||||||
|
go func() {
|
||||||
|
stopSlow()
|
||||||
|
returned <- slow.inSay.Load()
|
||||||
|
}()
|
||||||
|
// The stop is given its turn before the Say is let go (no clock: a returned stop is seen at once).
|
||||||
|
for i := 0; i < 1000; i++ {
|
||||||
|
select {
|
||||||
|
case stillSaying := <-returned:
|
||||||
|
close(slow.held)
|
||||||
|
t.Fatalf("stop returned while a beat was still being said (held: %v)", stillSaying)
|
||||||
|
default:
|
||||||
|
runtime.Gosched()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
close(slow.held)
|
||||||
|
if stillSaying := <-returned; stillSaying {
|
||||||
|
t.Fatal("stop returned while a beat was still being said")
|
||||||
|
}
|
||||||
// Keep itself keeps the interval on a ticker of its own.
|
// Keep itself keeps the interval on a ticker of its own.
|
||||||
kept := newSaid(nil)
|
kept := newSaid(nil)
|
||||||
stopKept := Keep(kept, "anchor", time.Hour, t.Logf)
|
stopKept := Keep(kept, "anchor", time.Hour, t.Logf)
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package runtime
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -10,6 +11,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nats-io/nats.go/micro"
|
"github.com/nats-io/nats.go/micro"
|
||||||
|
|
||||||
@@ -154,13 +157,17 @@ func newReloadBed(t *testing.T) *reloadBed {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *reloadBed) run(t *testing.T) {
|
// run starts the runtime, stopped at the test's end or once by what it returns.
|
||||||
|
func (b *reloadBed) run(t *testing.T) func() {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf)
|
stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Cleanup(stop)
|
var once sync.Once
|
||||||
|
stopOnce := func() { once.Do(stop) }
|
||||||
|
t.Cleanup(stopOnce)
|
||||||
|
return stopOnce
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *reloadBed) answers(t *testing.T, tool string) {
|
func (b *reloadBed) answers(t *testing.T, tool string) {
|
||||||
@@ -269,7 +276,7 @@ func TestAReplacedBundleIsReloaded(t *testing.T) {
|
|||||||
func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) {
|
func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) {
|
||||||
b := newReloadBed(t)
|
b := newReloadBed(t)
|
||||||
place(t, b.entry, "first")
|
place(t, b.entry, "first")
|
||||||
b.run(t)
|
stop := b.run(t)
|
||||||
b.answers(t, "first")
|
b.answers(t, "first")
|
||||||
place(t, b.entry, "second")
|
place(t, b.entry, "second")
|
||||||
b.answers(t, "second")
|
b.answers(t, "second")
|
||||||
@@ -287,6 +294,42 @@ func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) {
|
|||||||
if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil {
|
if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil {
|
||||||
t.Fatalf("ledger.first is served again after the membership was reissued, and answered %s\n%s", got, b.logs.All())
|
t.Fatalf("ledger.first is served again after the membership was reissued, and answered %s\n%s", got, b.logs.All())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A ledger bound while the bundle is loaded again: its repairs are served once, by the module loaded, and none is
|
||||||
|
// left behind on the stand-in the reload served nothing on (design 47 H, H-A-160). The bundle placed holds its
|
||||||
|
// start until the test lets it go, so the membership is heard inside the reload.
|
||||||
|
held, release := filepath.Join(t.TempDir(), "held"), filepath.Join(t.TempDir(), "release")
|
||||||
|
placeScript(t, b.entry, strings.Replace(bundleServing("third"), "import json, os, sys\n",
|
||||||
|
"import json, os, sys, time\nopen("+fmt.Sprintf("%q", held)+", \"w\").close()\nwhile not os.path.exists("+
|
||||||
|
fmt.Sprintf("%q", release)+"):\n time.sleep(0.01)\n", 1))
|
||||||
|
mt.Until(t, func() error { _, err := os.Stat(held); return err })
|
||||||
|
bound := mt.MembershipOf("ledger", "anchor", false, nil)
|
||||||
|
bound.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger,
|
||||||
|
History: "LEDGER_ledger_records", Subject: "mesh.ledger.ledger.records.history", State: "ledger_records",
|
||||||
|
Snapshot: "mesh.ledger.ledger.records.snapshot"}}
|
||||||
|
b.mesh.Issue(t, bound)
|
||||||
|
mt.Until(t, func() error {
|
||||||
|
if m := b.nodeTools.Membership("ledger"); m == nil || len(m.Bindings) == 0 {
|
||||||
|
return errors.New("the ledger's binding is not heard yet")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err := os.WriteFile(release, nil, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b.answers(t, "third")
|
||||||
|
mt.Until(t, func() error {
|
||||||
|
_, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{})
|
||||||
|
if errors.Is(err, bus.ErrNoResponders) || (err != nil && err.Error() == "timeout") {
|
||||||
|
return fmt.Errorf("the ledger's repairs are not served after its reload: %v", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
stop()
|
||||||
|
if _, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{}); !errors.Is(err, bus.ErrNoResponders) {
|
||||||
|
t.Fatalf("the runtime stopped, and ledger.%s still answered (%v): served on the reload's stand-in and never stopped\n%s",
|
||||||
|
RepairLag, err, b.logs.All())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1945/AC21: a bundle removed while served is no longer served, and its module says why.
|
// 1945/AC21: a bundle removed while served is no longer served, and its module says why.
|
||||||
|
|||||||
@@ -177,6 +177,11 @@ func (r *runner) refreshRepairs() {
|
|||||||
changed := false
|
changed := false
|
||||||
for _, m := range r.modules {
|
for _, m := range r.modules {
|
||||||
l := r.of[m]
|
l := r.of[m]
|
||||||
|
// A module being loaded again is left to its reload, which serves its repairs as its binding is then: served
|
||||||
|
// here on the stand-in, they would be served twice and never stopped (design 47 H, H-A-160).
|
||||||
|
if l.reloading {
|
||||||
|
continue
|
||||||
|
}
|
||||||
served := false
|
served := false
|
||||||
for _, reg := range l.own {
|
for _, reg := range l.own {
|
||||||
served = served || reg.repairs
|
served = served || reg.repairs
|
||||||
@@ -205,6 +210,8 @@ type loadedModule struct {
|
|||||||
failed string
|
failed string
|
||||||
names []string // the tools it serves, as module.tool
|
names []string // the tools it serves, as module.tool
|
||||||
serving map[string]func() // its tools' subscriptions, by module.tool; serve alone changes it
|
serving map[string]func() // its tools' subscriptions, by module.tool; serve alone changes it
|
||||||
|
// reloading marks the stand-in a reload puts in place while it loads the module again: it serves nothing.
|
||||||
|
reloading bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// unload stops serving the module's tools and stops its bundles.
|
// unload stops serving the module's tools and stops its bundles.
|
||||||
@@ -631,7 +638,7 @@ func (r *runner) reconcile() {
|
|||||||
func (r *runner) reload(module, what string) {
|
func (r *runner) reload(module, what string) {
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
old := r.of[module]
|
old := r.of[module]
|
||||||
r.of[module] = &loadedModule{served: old.served, seen: old.seen}
|
r.of[module] = &loadedModule{served: old.served, seen: old.seen, reloading: true}
|
||||||
r.mu.Unlock()
|
r.mu.Unlock()
|
||||||
// Its events are refused first, so one heard while the old bundle stops is refused at once.
|
// Its events are refused first, so one heard while the old bundle stops is refused at once.
|
||||||
r.events.dropModule(module)
|
r.events.dropModule(module)
|
||||||
@@ -646,6 +653,11 @@ func (r *runner) reload(module, what string) {
|
|||||||
r.logf("[mesh-tools] %s's bundle is refused: %v; its tools are not served here", module, err)
|
r.logf("[mesh-tools] %s's bundle is refused: %v; its tools are not served here", module, err)
|
||||||
}
|
}
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
|
// Its repairs as its binding is now, not as it was when the load began: the refresh left the stand-in alone.
|
||||||
|
if fresh.failed == "" {
|
||||||
|
repairs := r.repairsOf(module)
|
||||||
|
fresh.own, fresh.regs = withRepairs(fresh.own, repairs), withRepairs(fresh.regs, repairs)
|
||||||
|
}
|
||||||
r.of[module] = fresh
|
r.of[module] = fresh
|
||||||
r.serve(fresh)
|
r.serve(fresh)
|
||||||
r.bindVerbs()
|
r.bindVerbs()
|
||||||
|
|||||||
Reference in New Issue
Block a user