diff --git a/node-tools/internal/alive/alive.go b/node-tools/internal/alive/alive.go index 843b57f..fc10987 100644 --- a/node-tools/internal/alive/alive.go +++ b/node-tools/internal/alive/alive.go @@ -49,13 +49,20 @@ func Version() string { // Keep says the runtime is there now and every interval until the returned function is called. A // heartbeat that cannot be said is logged when that starts and when it stops — not every minute. func Keep(bus Sayer, node string, every time.Duration, logf func(string, ...any)) func() { + tick := time.NewTicker(every) + stop := keep(bus, node, every, tick.C, logf) + return func() { stop(); tick.Stop() } +} + +// keep is Keep saying it again on every tick: the interval is told in the beat, the ticks are the caller's. +// Its stop returns once nothing more is said. +func keep(bus Sayer, node string, every time.Duration, ticks <-chan time.Time, logf func(string, ...any)) func() { body, _ := json.Marshal(Beat{Node: node, IntervalSeconds: int(every / time.Second), Version: Version()}) - done := make(chan struct{}) + done, ended := make(chan struct{}), make(chan struct{}) var once sync.Once go func() { + defer close(ended) failing := "" - tick := time.NewTicker(every) - defer tick.Stop() for { why := "" if err := bus.Say(Subject(node), body); err != nil { @@ -72,9 +79,12 @@ func Keep(bus Sayer, node string, every time.Duration, logf func(string, ...any) select { case <-done: return - case <-tick.C: + case <-ticks: } } }() - return func() { once.Do(func() { close(done) }) } + return func() { + once.Do(func() { close(done) }) + <-ended + } } diff --git a/node-tools/internal/alive/alive_test.go b/node-tools/internal/alive/alive_test.go index f7cd680..d0397af 100644 --- a/node-tools/internal/alive/alive_test.go +++ b/node-tools/internal/alive/alive_test.go @@ -3,7 +3,9 @@ package alive import ( "encoding/json" "errors" + "runtime" "sync" + "sync/atomic" "testing" "time" ) @@ -13,11 +15,29 @@ type said struct { subs []string body [][]byte fail error + // tried is told every Say, said or not, so a test waits on the heartbeat itself and never on the clock. + tried chan struct{} + // held, when set, keeps each Say from returning until it is closed; inSay says one is held there now. + held chan struct{} + inSay atomic.Bool } +func newSaid(fail error) *said { return &said{fail: fail, tried: make(chan struct{}, 16)} } + func (s *said) Say(subject string, body []byte) error { + if s.held != nil { + s.inSay.Store(true) + s.tried <- struct{}{} + <-s.held + defer s.inSay.Store(false) + } s.mu.Lock() - defer s.mu.Unlock() + defer func() { + s.mu.Unlock() + if s.held == nil { + s.tried <- struct{}{} + } + }() if s.fail != nil { return s.fail } @@ -32,39 +52,85 @@ func (s *said) count() int { } // **The runtime says it is there at once and every interval, as its machine, with the interval**: the -// controller's bound is three of them (to-be 45 S11). +// controller's bound is three of them (to-be 45 S11). The ticks are the test's own, so a loaded machine +// cannot make it miss one (principle 11; design 47 H §774); stopped, it says nothing more. func TestTheRuntimeSaysItIsThereEveryInterval(t *testing.T) { - s := &said{} - stop := Keep(s, "anchor", 20*time.Millisecond, t.Logf) - time.Sleep(70 * time.Millisecond) + s := newSaid(nil) + ticks := make(chan time.Time) + stop := keep(s, "anchor", 20*time.Millisecond, ticks, t.Logf) + <-s.tried + for i := 0; i < 2; i++ { + ticks <- time.Time{} + <-s.tried + } stop() stop() - if n := s.count(); n < 3 { - t.Fatalf("said %d times in three intervals", n) + if n := s.count(); n != 3 { + t.Fatalf("said %d times at once and on two ticks", n) } var beat Beat if err := json.Unmarshal(s.body[0], &beat); err != nil || beat.Node != "anchor" || beat.IntervalSeconds != 0 || s.subs[0] != "mesh.control.anchor.tools-alive" { t.Fatalf("%s on %s: %v", s.body[0], s.subs[0], err) } - n := s.count() - time.Sleep(50 * time.Millisecond) - if s.count() != n { + select { + case ticks <- time.Time{}: + t.Fatal("a tick was taken after it was stopped") + default: + } + if s.count() != 3 { t.Fatal("said after it was stopped") } + // Stopped while a beat is being said, stop returns only once that Say has returned: nothing is said after it. + slow := newSaid(nil) + slow.held = make(chan struct{}) + stopSlow := keep(slow, "anchor", 20*time.Millisecond, make(chan time.Time), t.Logf) + <-slow.tried + returned := make(chan bool) + go func() { + stopSlow() + returned <- slow.inSay.Load() + }() + // The stop is given its turn before the Say is let go (no clock: a returned stop is seen at once). + for i := 0; i < 1000; i++ { + select { + case stillSaying := <-returned: + close(slow.held) + t.Fatalf("stop returned while a beat was still being said (held: %v)", stillSaying) + default: + runtime.Gosched() + } + } + close(slow.held) + if stillSaying := <-returned; stillSaying { + t.Fatal("stop returned while a beat was still being said") + } + // Keep itself keeps the interval on a ticker of its own. + kept := newSaid(nil) + stopKept := Keep(kept, "anchor", time.Hour, t.Logf) + <-kept.tried + stopKept() + if kept.count() != 1 { + t.Fatalf("Keep said %d times before its first interval", kept.count()) + } } // **A heartbeat the bus will not take is said once, and again when it is taken**: not a line a minute. func TestAHeartbeatThatCannotBeSaidIsLoggedOnce(t *testing.T) { - s := &said{fail: errors.New("permissions violation")} + s := newSaid(errors.New("permissions violation")) var mu sync.Mutex var lines []string - stop := Keep(s, "anchor", 10*time.Millisecond, func(f string, a ...any) { + ticks := make(chan time.Time) + stop := keep(s, "anchor", 10*time.Millisecond, ticks, func(f string, a ...any) { mu.Lock() lines = append(lines, f) mu.Unlock() }) - time.Sleep(60 * time.Millisecond) + <-s.tried + for i := 0; i < 4; i++ { + ticks <- time.Time{} + <-s.tried + } stop() mu.Lock() defer mu.Unlock() diff --git a/node-tools/internal/bus/bus.go b/node-tools/internal/bus/bus.go index 9997853..726dc0c 100644 --- a/node-tools/internal/bus/bus.go +++ b/node-tools/internal/bus/bus.go @@ -740,25 +740,35 @@ func (c *Conn) HandleCalled(key string, h CalledHandler) (func(), error) { "those of the modules it follows", c.self, key) } var mu sync.Mutex - var stops []func() + stops := map[Served]func(){} stopped := false + // **Make before break** (novox/hq design 47 H, H-A-158): a membership issued again keeps every subscription + // whose subject and queue it still names, subscribes those it newly names, and only then stops those it no + // longer names — a call the bus routes while the membership is followed is never dropped in silence. serve := func() { mu.Lock() defer mu.Unlock() if stopped { return } - for _, s := range stops { - s() - } - stops = nil + named := map[Served]bool{} for _, s := range c.servedOn(module, tool) { + named[s] = true + if _, kept := stops[s]; kept { + continue + } if stop, err := c.answerBoth(s.Subject, s.Queue, h); err == nil { - stops = append(stops, stop) + stops[s] = stop } else { c.Logf("[mesh-tools] cannot serve %s on %s: %v", key, s.Subject, err) } } + for s, stop := range stops { + if !named[s] { + stop() + delete(stops, s) + } + } } serve() // Its stop drops the membership handler too: a handler left behind would bind the stopped tool again @@ -773,10 +783,10 @@ func (c *Conn) HandleCalled(key string, h CalledHandler) (func(), error) { mu.Lock() defer mu.Unlock() stopped = true - for _, s := range stops { - s() + for s, stop := range stops { + stop() + delete(stops, s) } - stops = nil }, nil } diff --git a/node-tools/internal/runtime/ledgers_test.go b/node-tools/internal/runtime/ledgers_test.go index ff586ef..3315484 100644 --- a/node-tools/internal/runtime/ledgers_test.go +++ b/node-tools/internal/runtime/ledgers_test.go @@ -39,7 +39,7 @@ func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) { defer stop() asker := connect(t, "console", "workstation") listed := func(module string) (string, error) { - answered, err := asker.Ask(module+"."+ToolsVerb, map[string]any{}, "") + answered, err := asker.AskWithin(module+"."+ToolsVerb, map[string]any{}, "", callWithin) if err != nil { return "", err } @@ -67,6 +67,18 @@ func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) { }) } t.Run("a ledger bound later brings them", func(t *testing.T) { + // Its own tool answers the first call after each binding comes or goes: the repairs' refresh keeps the + // subscriptions it still names (design 47 H, H-A-160). + for round := 1; round <= 10; round++ { + plain.Bindings = nil + if round%2 == 1 { + plain.Bindings = owner.Bindings + } + mesh.Issue(t, plain) + if _, err := asker.AskWithin("plain.who@anchor", map[string]any{}, "", callWithin); err != nil { + t.Fatalf("binding change %d: the first call of plain.who failed: %v — dropped while its tools were served again", round, err) + } + } plain.Bindings = owner.Bindings mesh.Issue(t, plain) mt.Until(t, func() error { diff --git a/node-tools/internal/runtime/reload_test.go b/node-tools/internal/runtime/reload_test.go index 7375d77..ee14d3c 100644 --- a/node-tools/internal/runtime/reload_test.go +++ b/node-tools/internal/runtime/reload_test.go @@ -2,6 +2,7 @@ package runtime import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -10,6 +11,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/membership" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/micro" @@ -154,13 +157,17 @@ func newReloadBed(t *testing.T) *reloadBed { } } -func (b *reloadBed) run(t *testing.T) { +// run starts the runtime, stopped at the test's end or once by what it returns. +func (b *reloadBed) run(t *testing.T) func() { t.Helper() stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf) if err != nil { t.Fatal(err) } - t.Cleanup(stop) + var once sync.Once + stopOnce := func() { once.Do(stop) } + t.Cleanup(stopOnce) + return stopOnce } func (b *reloadBed) answers(t *testing.T, tool string) { @@ -263,23 +270,66 @@ func TestAReplacedBundleIsReloaded(t *testing.T) { // A replaced bundle's old tool stays gone when the module's membership is issued again: a stopped tool // leaves no membership handler behind to bind it once more, and a call on it starts no unmanaged second -// copy of the bundle (issue 1945, review round 2). +// copy of the bundle (issue 1945, review round 2). And the tool still served answers the first call asked +// after each membership, within the test's own bound: the runner keeps the subscriptions the membership +// still names, so no call is dropped while it follows one (design 47 H, H-A-158). func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) { b := newReloadBed(t) place(t, b.entry, "first") - b.run(t) + stop := b.run(t) b.answers(t, "first") place(t, b.entry, "second") b.answers(t, "second") b.silent(t, "first") - b.mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil)) - b.answers(t, "second") + for round := 1; round <= 5; round++ { + b.mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil)) + if got, err := call(t, b.asker, "ledger.second@anchor", map[string]any{}); err != nil || !strings.Contains(got, `"tool":"second"`) { + t.Fatalf("membership %d issued again: the first call of ledger.second answered %s, %v — dropped while the runner "+ + "followed the membership\n%s", round, got, err, b.logs.All()) + } + } // Long enough for the reissued membership to have been heard, before the old tool is asked. time.Sleep(300 * time.Millisecond) if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil { t.Fatalf("ledger.first is served again after the membership was reissued, and answered %s\n%s", got, b.logs.All()) } + + // A ledger bound while the bundle is loaded again: its repairs are served once, by the module loaded, and none is + // left behind on the stand-in the reload served nothing on (design 47 H, H-A-160). The bundle placed holds its + // start until the test lets it go, so the membership is heard inside the reload. + held, release := filepath.Join(t.TempDir(), "held"), filepath.Join(t.TempDir(), "release") + placeScript(t, b.entry, strings.Replace(bundleServing("third"), "import json, os, sys\n", + "import json, os, sys, time\nopen("+fmt.Sprintf("%q", held)+", \"w\").close()\nwhile not os.path.exists("+ + fmt.Sprintf("%q", release)+"):\n time.sleep(0.01)\n", 1)) + mt.Until(t, func() error { _, err := os.Stat(held); return err }) + bound := mt.MembershipOf("ledger", "anchor", false, nil) + bound.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, + History: "LEDGER_ledger_records", Subject: "mesh.ledger.ledger.records.history", State: "ledger_records", + Snapshot: "mesh.ledger.ledger.records.snapshot"}} + b.mesh.Issue(t, bound) + mt.Until(t, func() error { + if m := b.nodeTools.Membership("ledger"); m == nil || len(m.Bindings) == 0 { + return errors.New("the ledger's binding is not heard yet") + } + return nil + }) + if err := os.WriteFile(release, nil, 0o644); err != nil { + t.Fatal(err) + } + b.answers(t, "third") + mt.Until(t, func() error { + _, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{}) + if errors.Is(err, bus.ErrNoResponders) || (err != nil && err.Error() == "timeout") { + return fmt.Errorf("the ledger's repairs are not served after its reload: %v", err) + } + return nil + }) + stop() + if _, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{}); !errors.Is(err, bus.ErrNoResponders) { + t.Fatalf("the runtime stopped, and ledger.%s still answered (%v): served on the reload's stand-in and never stopped\n%s", + RepairLag, err, b.logs.All()) + } } // 1945/AC21: a bundle removed while served is no longer served, and its module says why. diff --git a/node-tools/internal/runtime/runtime.go b/node-tools/internal/runtime/runtime.go index ff6e68d..3cd8a7e 100644 --- a/node-tools/internal/runtime/runtime.go +++ b/node-tools/internal/runtime/runtime.go @@ -177,6 +177,11 @@ func (r *runner) refreshRepairs() { changed := false for _, m := range r.modules { l := r.of[m] + // A module being loaded again is left to its reload, which serves its repairs as its binding is then: served + // here on the stand-in, they would be served twice and never stopped (design 47 H, H-A-160). + if l.reloading { + continue + } served := false for _, reg := range l.own { served = served || reg.repairs @@ -185,10 +190,6 @@ func (r *runner) refreshRepairs() { if served == (repairs != nil) { continue } - for i := len(l.serving) - 1; i >= 0; i-- { - l.serving[i]() - } - l.serving, l.names = nil, nil l.own, l.regs = withRepairs(l.own, repairs), withRepairs(l.regs, repairs) r.serve(l) changed = true @@ -207,14 +208,16 @@ type loadedModule struct { regs []registration // every registration its bundles made own []registration // those under the name of a module served here failed string - names []string // the tools it serves, as module.tool - serving []func() // its tools' subscriptions + names []string // the tools it serves, as module.tool + serving map[string]func() // its tools' subscriptions, by module.tool; serve alone changes it + // reloading marks the stand-in a reload puts in place while it loads the module again: it serves nothing. + reloading bool } // unload stops serving the module's tools and stops its bundles. func (l *loadedModule) unload() { - for i := len(l.serving) - 1; i >= 0; i-- { - l.serving[i]() + for _, stop := range l.serving { + stop() } l.serving, l.names = nil, nil for i := len(l.children) - 1; i >= 0; i-- { @@ -493,25 +496,47 @@ func refusal(own []registration) error { return nil } -// serve binds a loaded module's tools. r.mu is held. +// serve binds a loaded module's tools as its registrations name them now, and is the one routine that changes its +// subscriptions (design 47 H, H-A-158, H-A-160): a tool still named keeps its subscription, a newly named one is +// subscribed, and only then is one no longer named stopped — no tool is served twice, and none is dropped while the +// module's tools are served again. r.mu is held. func (r *runner) serve(l *loadedModule) { + if l.serving == nil { + l.serving = map[string]func(){} + } + named := map[string]bool{} + l.names = nil for _, reg := range l.own { for _, t := range reg.tools { t := t - // Told who called, from the subject the call arrived on, never from its body (novox/hq issue 365). - // Through the approval gate, before the bundle or the runner's own tool (design 46 7-approval-mark §20). - module, own := reg.module, reg.repairs - stop, err := r.conn.HandleCalled(reg.module+"."+t.Name, func(caller string, body json.RawMessage) (any, error) { - return r.events.gated(approval.Call{Address: module + "." + t.Name, Module: module, Name: t.Name, Own: own, - Fingerprint: r.fingerprintOf(module), Env: r.envOf(module), Caller: caller, Args: argsOf(body), - Description: t.Description}, t, caller) - }) - if err != nil { - r.logf("[mesh-tools] cannot serve %s.%s: %v", reg.module, t.Name, err) + address := reg.module + "." + t.Name + if named[address] { continue } - l.names = append(l.names, reg.module+"."+t.Name) - l.serving = append(l.serving, stop) + named[address] = true + if _, kept := l.serving[address]; !kept { + // Told who called, from the subject the call arrived on, never from its body (novox/hq issue 365). + // Through the approval gate, before the bundle or the runner's own tool (design 46 7-approval-mark §20). + module, own := reg.module, reg.repairs + stop, err := r.conn.HandleCalled(address, func(caller string, body json.RawMessage) (any, error) { + return r.events.gated(approval.Call{Address: module + "." + t.Name, Module: module, Name: t.Name, Own: own, + Fingerprint: r.fingerprintOf(module), Env: r.envOf(module), Caller: caller, Args: argsOf(body), + Description: t.Description}, t, caller) + }) + if err != nil { + r.logf("[mesh-tools] cannot serve %s.%s: %v", reg.module, t.Name, err) + delete(named, address) + continue + } + l.serving[address] = stop + } + l.names = append(l.names, address) + } + } + for address, stop := range l.serving { + if !named[address] { + stop() + delete(l.serving, address) } } } @@ -613,7 +638,7 @@ func (r *runner) reconcile() { func (r *runner) reload(module, what string) { r.mu.Lock() old := r.of[module] - r.of[module] = &loadedModule{served: old.served, seen: old.seen} + r.of[module] = &loadedModule{served: old.served, seen: old.seen, reloading: true} r.mu.Unlock() // Its events are refused first, so one heard while the old bundle stops is refused at once. r.events.dropModule(module) @@ -628,6 +653,11 @@ func (r *runner) reload(module, what string) { r.logf("[mesh-tools] %s's bundle is refused: %v; its tools are not served here", module, err) } r.mu.Lock() + // Its repairs as its binding is now, not as it was when the load began: the refresh left the stand-in alone. + if fresh.failed == "" { + repairs := r.repairsOf(module) + fresh.own, fresh.regs = withRepairs(fresh.own, repairs), withRepairs(fresh.regs, repairs) + } r.of[module] = fresh r.serve(fresh) r.bindVerbs() diff --git a/node-tools/internal/runtime/runtime_test.go b/node-tools/internal/runtime/runtime_test.go index 2d07444..32bc75a 100644 --- a/node-tools/internal/runtime/runtime_test.go +++ b/node-tools/internal/runtime/runtime_test.go @@ -26,9 +26,13 @@ func connect(t *testing.T, module, node string) *bus.Conn { return c } +// callWithin is the tests' own bound on a call (principle 11, design 47 H, H-A-158): a call that is never answered +// fails the try it is part of within it, never after the tool runner's own 30 s. +const callWithin = 2 * time.Second + func call(t *testing.T, asker *bus.Conn, key string, args any) (string, error) { t.Helper() - got, err := asker.Ask(key, args, "") + got, err := asker.AskWithin(key, args, "", callWithin) return string(got.Result), err } diff --git a/node-tools/internal/runtime/state_counter_test.go b/node-tools/internal/runtime/state_counter_test.go index 3cbbfd9..b8594eb 100644 --- a/node-tools/internal/runtime/state_counter_test.go +++ b/node-tools/internal/runtime/state_counter_test.go @@ -78,7 +78,10 @@ func increment(b launch.Bus, key string, attempts int) (n uint64, lost int, err // Two writers incrementing one counter a thousand times each, against a real bus, end at two thousand: // every number handed out once and none lost, because the runtime writes only at the revision read -// (novox/hq research 042: an issue's number is handed out once). +// (novox/hq research 042: an issue's number is handed out once). A write is lost only to the other writer's +// write, so one increment loses at most as many times as the other writer increments: that, not a guess, +// is the bound on its tries, and a machine whose scheduler lets one writer win every round still passes +// (one processor starved a writer of a thousand tries; principle 11, design 47 H §774). func TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate(t *testing.T) { b := counterBus(t) const each = 1000 @@ -91,7 +94,7 @@ func TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate(t *testing.T) { go func(w int) { defer wg.Done() for i := 0; i < each; i++ { - n, l, err := increment(b, "issue", 1000) + n, l, err := increment(b, "issue", each+1) lost[w] += l if err != nil { errs[w] = err