Merge pull request 'Design 47 H1: the tool runner re-serves a membership without dropping a call; its integration level fits its bound (H-A-158, H-A-160)' (#80) from fix/47-H1-node-tools-integration-bound into main

This commit was merged in pull request #80.
This commit is contained in:
2026-11-20 16:49:43 +00:00
8 changed files with 244 additions and 59 deletions
+15 -5
View File
@@ -49,13 +49,20 @@ func Version() string {
// Keep says the runtime is there now and every interval until the returned function is called. A
// heartbeat that cannot be said is logged when that starts and when it stops — not every minute.
func Keep(bus Sayer, node string, every time.Duration, logf func(string, ...any)) func() {
tick := time.NewTicker(every)
stop := keep(bus, node, every, tick.C, logf)
return func() { stop(); tick.Stop() }
}
// keep is Keep saying it again on every tick: the interval is told in the beat, the ticks are the caller's.
// Its stop returns once nothing more is said.
func keep(bus Sayer, node string, every time.Duration, ticks <-chan time.Time, logf func(string, ...any)) func() {
body, _ := json.Marshal(Beat{Node: node, IntervalSeconds: int(every / time.Second), Version: Version()})
done := make(chan struct{})
done, ended := make(chan struct{}), make(chan struct{})
var once sync.Once
go func() {
defer close(ended)
failing := ""
tick := time.NewTicker(every)
defer tick.Stop()
for {
why := ""
if err := bus.Say(Subject(node), body); err != nil {
@@ -72,9 +79,12 @@ func Keep(bus Sayer, node string, every time.Duration, logf func(string, ...any)
select {
case <-done:
return
case <-tick.C:
case <-ticks:
}
}
}()
return func() { once.Do(func() { close(done) }) }
return func() {
once.Do(func() { close(done) })
<-ended
}
}
+79 -13
View File
@@ -3,7 +3,9 @@ package alive
import (
"encoding/json"
"errors"
"runtime"
"sync"
"sync/atomic"
"testing"
"time"
)
@@ -13,11 +15,29 @@ type said struct {
subs []string
body [][]byte
fail error
// tried is told every Say, said or not, so a test waits on the heartbeat itself and never on the clock.
tried chan struct{}
// held, when set, keeps each Say from returning until it is closed; inSay says one is held there now.
held chan struct{}
inSay atomic.Bool
}
func newSaid(fail error) *said { return &said{fail: fail, tried: make(chan struct{}, 16)} }
func (s *said) Say(subject string, body []byte) error {
if s.held != nil {
s.inSay.Store(true)
s.tried <- struct{}{}
<-s.held
defer s.inSay.Store(false)
}
s.mu.Lock()
defer s.mu.Unlock()
defer func() {
s.mu.Unlock()
if s.held == nil {
s.tried <- struct{}{}
}
}()
if s.fail != nil {
return s.fail
}
@@ -32,39 +52,85 @@ func (s *said) count() int {
}
// **The runtime says it is there at once and every interval, as its machine, with the interval**: the
// controller's bound is three of them (to-be 45 S11).
// controller's bound is three of them (to-be 45 S11). The ticks are the test's own, so a loaded machine
// cannot make it miss one (principle 11; design 47 H §774); stopped, it says nothing more.
func TestTheRuntimeSaysItIsThereEveryInterval(t *testing.T) {
s := &said{}
stop := Keep(s, "anchor", 20*time.Millisecond, t.Logf)
time.Sleep(70 * time.Millisecond)
s := newSaid(nil)
ticks := make(chan time.Time)
stop := keep(s, "anchor", 20*time.Millisecond, ticks, t.Logf)
<-s.tried
for i := 0; i < 2; i++ {
ticks <- time.Time{}
<-s.tried
}
stop()
stop()
if n := s.count(); n < 3 {
t.Fatalf("said %d times in three intervals", n)
if n := s.count(); n != 3 {
t.Fatalf("said %d times at once and on two ticks", n)
}
var beat Beat
if err := json.Unmarshal(s.body[0], &beat); err != nil || beat.Node != "anchor" || beat.IntervalSeconds != 0 ||
s.subs[0] != "mesh.control.anchor.tools-alive" {
t.Fatalf("%s on %s: %v", s.body[0], s.subs[0], err)
}
n := s.count()
time.Sleep(50 * time.Millisecond)
if s.count() != n {
select {
case ticks <- time.Time{}:
t.Fatal("a tick was taken after it was stopped")
default:
}
if s.count() != 3 {
t.Fatal("said after it was stopped")
}
// Stopped while a beat is being said, stop returns only once that Say has returned: nothing is said after it.
slow := newSaid(nil)
slow.held = make(chan struct{})
stopSlow := keep(slow, "anchor", 20*time.Millisecond, make(chan time.Time), t.Logf)
<-slow.tried
returned := make(chan bool)
go func() {
stopSlow()
returned <- slow.inSay.Load()
}()
// The stop is given its turn before the Say is let go (no clock: a returned stop is seen at once).
for i := 0; i < 1000; i++ {
select {
case stillSaying := <-returned:
close(slow.held)
t.Fatalf("stop returned while a beat was still being said (held: %v)", stillSaying)
default:
runtime.Gosched()
}
}
close(slow.held)
if stillSaying := <-returned; stillSaying {
t.Fatal("stop returned while a beat was still being said")
}
// Keep itself keeps the interval on a ticker of its own.
kept := newSaid(nil)
stopKept := Keep(kept, "anchor", time.Hour, t.Logf)
<-kept.tried
stopKept()
if kept.count() != 1 {
t.Fatalf("Keep said %d times before its first interval", kept.count())
}
}
// **A heartbeat the bus will not take is said once, and again when it is taken**: not a line a minute.
func TestAHeartbeatThatCannotBeSaidIsLoggedOnce(t *testing.T) {
s := &said{fail: errors.New("permissions violation")}
s := newSaid(errors.New("permissions violation"))
var mu sync.Mutex
var lines []string
stop := Keep(s, "anchor", 10*time.Millisecond, func(f string, a ...any) {
ticks := make(chan time.Time)
stop := keep(s, "anchor", 10*time.Millisecond, ticks, func(f string, a ...any) {
mu.Lock()
lines = append(lines, f)
mu.Unlock()
})
time.Sleep(60 * time.Millisecond)
<-s.tried
for i := 0; i < 4; i++ {
ticks <- time.Time{}
<-s.tried
}
stop()
mu.Lock()
defer mu.Unlock()
+19 -9
View File
@@ -740,25 +740,35 @@ func (c *Conn) HandleCalled(key string, h CalledHandler) (func(), error) {
"those of the modules it follows", c.self, key)
}
var mu sync.Mutex
var stops []func()
stops := map[Served]func(){}
stopped := false
// **Make before break** (novox/hq design 47 H, H-A-158): a membership issued again keeps every subscription
// whose subject and queue it still names, subscribes those it newly names, and only then stops those it no
// longer names — a call the bus routes while the membership is followed is never dropped in silence.
serve := func() {
mu.Lock()
defer mu.Unlock()
if stopped {
return
}
for _, s := range stops {
s()
}
stops = nil
named := map[Served]bool{}
for _, s := range c.servedOn(module, tool) {
named[s] = true
if _, kept := stops[s]; kept {
continue
}
if stop, err := c.answerBoth(s.Subject, s.Queue, h); err == nil {
stops = append(stops, stop)
stops[s] = stop
} else {
c.Logf("[mesh-tools] cannot serve %s on %s: %v", key, s.Subject, err)
}
}
for s, stop := range stops {
if !named[s] {
stop()
delete(stops, s)
}
}
}
serve()
// Its stop drops the membership handler too: a handler left behind would bind the stopped tool again
@@ -773,10 +783,10 @@ func (c *Conn) HandleCalled(key string, h CalledHandler) (func(), error) {
mu.Lock()
defer mu.Unlock()
stopped = true
for _, s := range stops {
s()
for s, stop := range stops {
stop()
delete(stops, s)
}
stops = nil
}, nil
}
+13 -1
View File
@@ -39,7 +39,7 @@ func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) {
defer stop()
asker := connect(t, "console", "workstation")
listed := func(module string) (string, error) {
answered, err := asker.Ask(module+"."+ToolsVerb, map[string]any{}, "")
answered, err := asker.AskWithin(module+"."+ToolsVerb, map[string]any{}, "", callWithin)
if err != nil {
return "", err
}
@@ -67,6 +67,18 @@ func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) {
})
}
t.Run("a ledger bound later brings them", func(t *testing.T) {
// Its own tool answers the first call after each binding comes or goes: the repairs' refresh keeps the
// subscriptions it still names (design 47 H, H-A-160).
for round := 1; round <= 10; round++ {
plain.Bindings = nil
if round%2 == 1 {
plain.Bindings = owner.Bindings
}
mesh.Issue(t, plain)
if _, err := asker.AskWithin("plain.who@anchor", map[string]any{}, "", callWithin); err != nil {
t.Fatalf("binding change %d: the first call of plain.who failed: %v — dropped while its tools were served again", round, err)
}
}
plain.Bindings = owner.Bindings
mesh.Issue(t, plain)
mt.Until(t, func() error {
+56 -6
View File
@@ -2,6 +2,7 @@ package runtime
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
@@ -10,6 +11,8 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/membership"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/micro"
@@ -154,13 +157,17 @@ func newReloadBed(t *testing.T) *reloadBed {
}
}
func (b *reloadBed) run(t *testing.T) {
// run starts the runtime, stopped at the test's end or once by what it returns.
func (b *reloadBed) run(t *testing.T) func() {
t.Helper()
stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf)
if err != nil {
t.Fatal(err)
}
t.Cleanup(stop)
var once sync.Once
stopOnce := func() { once.Do(stop) }
t.Cleanup(stopOnce)
return stopOnce
}
func (b *reloadBed) answers(t *testing.T, tool string) {
@@ -263,23 +270,66 @@ func TestAReplacedBundleIsReloaded(t *testing.T) {
// A replaced bundle's old tool stays gone when the module's membership is issued again: a stopped tool
// leaves no membership handler behind to bind it once more, and a call on it starts no unmanaged second
// copy of the bundle (issue 1945, review round 2).
// copy of the bundle (issue 1945, review round 2). And the tool still served answers the first call asked
// after each membership, within the test's own bound: the runner keeps the subscriptions the membership
// still names, so no call is dropped while it follows one (design 47 H, H-A-158).
func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) {
b := newReloadBed(t)
place(t, b.entry, "first")
b.run(t)
stop := b.run(t)
b.answers(t, "first")
place(t, b.entry, "second")
b.answers(t, "second")
b.silent(t, "first")
b.mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
b.answers(t, "second")
for round := 1; round <= 5; round++ {
b.mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
if got, err := call(t, b.asker, "ledger.second@anchor", map[string]any{}); err != nil || !strings.Contains(got, `"tool":"second"`) {
t.Fatalf("membership %d issued again: the first call of ledger.second answered %s, %v — dropped while the runner "+
"followed the membership\n%s", round, got, err, b.logs.All())
}
}
// Long enough for the reissued membership to have been heard, before the old tool is asked.
time.Sleep(300 * time.Millisecond)
if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil {
t.Fatalf("ledger.first is served again after the membership was reissued, and answered %s\n%s", got, b.logs.All())
}
// A ledger bound while the bundle is loaded again: its repairs are served once, by the module loaded, and none is
// left behind on the stand-in the reload served nothing on (design 47 H, H-A-160). The bundle placed holds its
// start until the test lets it go, so the membership is heard inside the reload.
held, release := filepath.Join(t.TempDir(), "held"), filepath.Join(t.TempDir(), "release")
placeScript(t, b.entry, strings.Replace(bundleServing("third"), "import json, os, sys\n",
"import json, os, sys, time\nopen("+fmt.Sprintf("%q", held)+", \"w\").close()\nwhile not os.path.exists("+
fmt.Sprintf("%q", release)+"):\n time.sleep(0.01)\n", 1))
mt.Until(t, func() error { _, err := os.Stat(held); return err })
bound := mt.MembershipOf("ledger", "anchor", false, nil)
bound.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger,
History: "LEDGER_ledger_records", Subject: "mesh.ledger.ledger.records.history", State: "ledger_records",
Snapshot: "mesh.ledger.ledger.records.snapshot"}}
b.mesh.Issue(t, bound)
mt.Until(t, func() error {
if m := b.nodeTools.Membership("ledger"); m == nil || len(m.Bindings) == 0 {
return errors.New("the ledger's binding is not heard yet")
}
return nil
})
if err := os.WriteFile(release, nil, 0o644); err != nil {
t.Fatal(err)
}
b.answers(t, "third")
mt.Until(t, func() error {
_, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{})
if errors.Is(err, bus.ErrNoResponders) || (err != nil && err.Error() == "timeout") {
return fmt.Errorf("the ledger's repairs are not served after its reload: %v", err)
}
return nil
})
stop()
if _, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{}); !errors.Is(err, bus.ErrNoResponders) {
t.Fatalf("the runtime stopped, and ledger.%s still answered (%v): served on the reload's stand-in and never stopped\n%s",
RepairLag, err, b.logs.All())
}
}
// 1945/AC21: a bundle removed while served is no longer served, and its module says why.
+52 -22
View File
@@ -177,6 +177,11 @@ func (r *runner) refreshRepairs() {
changed := false
for _, m := range r.modules {
l := r.of[m]
// A module being loaded again is left to its reload, which serves its repairs as its binding is then: served
// here on the stand-in, they would be served twice and never stopped (design 47 H, H-A-160).
if l.reloading {
continue
}
served := false
for _, reg := range l.own {
served = served || reg.repairs
@@ -185,10 +190,6 @@ func (r *runner) refreshRepairs() {
if served == (repairs != nil) {
continue
}
for i := len(l.serving) - 1; i >= 0; i-- {
l.serving[i]()
}
l.serving, l.names = nil, nil
l.own, l.regs = withRepairs(l.own, repairs), withRepairs(l.regs, repairs)
r.serve(l)
changed = true
@@ -207,14 +208,16 @@ type loadedModule struct {
regs []registration // every registration its bundles made
own []registration // those under the name of a module served here
failed string
names []string // the tools it serves, as module.tool
serving []func() // its tools' subscriptions
names []string // the tools it serves, as module.tool
serving map[string]func() // its tools' subscriptions, by module.tool; serve alone changes it
// reloading marks the stand-in a reload puts in place while it loads the module again: it serves nothing.
reloading bool
}
// unload stops serving the module's tools and stops its bundles.
func (l *loadedModule) unload() {
for i := len(l.serving) - 1; i >= 0; i-- {
l.serving[i]()
for _, stop := range l.serving {
stop()
}
l.serving, l.names = nil, nil
for i := len(l.children) - 1; i >= 0; i-- {
@@ -493,25 +496,47 @@ func refusal(own []registration) error {
return nil
}
// serve binds a loaded module's tools. r.mu is held.
// serve binds a loaded module's tools as its registrations name them now, and is the one routine that changes its
// subscriptions (design 47 H, H-A-158, H-A-160): a tool still named keeps its subscription, a newly named one is
// subscribed, and only then is one no longer named stopped — no tool is served twice, and none is dropped while the
// module's tools are served again. r.mu is held.
func (r *runner) serve(l *loadedModule) {
if l.serving == nil {
l.serving = map[string]func(){}
}
named := map[string]bool{}
l.names = nil
for _, reg := range l.own {
for _, t := range reg.tools {
t := t
// Told who called, from the subject the call arrived on, never from its body (novox/hq issue 365).
// Through the approval gate, before the bundle or the runner's own tool (design 46 7-approval-mark §20).
module, own := reg.module, reg.repairs
stop, err := r.conn.HandleCalled(reg.module+"."+t.Name, func(caller string, body json.RawMessage) (any, error) {
return r.events.gated(approval.Call{Address: module + "." + t.Name, Module: module, Name: t.Name, Own: own,
Fingerprint: r.fingerprintOf(module), Env: r.envOf(module), Caller: caller, Args: argsOf(body),
Description: t.Description}, t, caller)
})
if err != nil {
r.logf("[mesh-tools] cannot serve %s.%s: %v", reg.module, t.Name, err)
address := reg.module + "." + t.Name
if named[address] {
continue
}
l.names = append(l.names, reg.module+"."+t.Name)
l.serving = append(l.serving, stop)
named[address] = true
if _, kept := l.serving[address]; !kept {
// Told who called, from the subject the call arrived on, never from its body (novox/hq issue 365).
// Through the approval gate, before the bundle or the runner's own tool (design 46 7-approval-mark §20).
module, own := reg.module, reg.repairs
stop, err := r.conn.HandleCalled(address, func(caller string, body json.RawMessage) (any, error) {
return r.events.gated(approval.Call{Address: module + "." + t.Name, Module: module, Name: t.Name, Own: own,
Fingerprint: r.fingerprintOf(module), Env: r.envOf(module), Caller: caller, Args: argsOf(body),
Description: t.Description}, t, caller)
})
if err != nil {
r.logf("[mesh-tools] cannot serve %s.%s: %v", reg.module, t.Name, err)
delete(named, address)
continue
}
l.serving[address] = stop
}
l.names = append(l.names, address)
}
}
for address, stop := range l.serving {
if !named[address] {
stop()
delete(l.serving, address)
}
}
}
@@ -613,7 +638,7 @@ func (r *runner) reconcile() {
func (r *runner) reload(module, what string) {
r.mu.Lock()
old := r.of[module]
r.of[module] = &loadedModule{served: old.served, seen: old.seen}
r.of[module] = &loadedModule{served: old.served, seen: old.seen, reloading: true}
r.mu.Unlock()
// Its events are refused first, so one heard while the old bundle stops is refused at once.
r.events.dropModule(module)
@@ -628,6 +653,11 @@ func (r *runner) reload(module, what string) {
r.logf("[mesh-tools] %s's bundle is refused: %v; its tools are not served here", module, err)
}
r.mu.Lock()
// Its repairs as its binding is now, not as it was when the load began: the refresh left the stand-in alone.
if fresh.failed == "" {
repairs := r.repairsOf(module)
fresh.own, fresh.regs = withRepairs(fresh.own, repairs), withRepairs(fresh.regs, repairs)
}
r.of[module] = fresh
r.serve(fresh)
r.bindVerbs()
+5 -1
View File
@@ -26,9 +26,13 @@ func connect(t *testing.T, module, node string) *bus.Conn {
return c
}
// callWithin is the tests' own bound on a call (principle 11, design 47 H, H-A-158): a call that is never answered
// fails the try it is part of within it, never after the tool runner's own 30 s.
const callWithin = 2 * time.Second
func call(t *testing.T, asker *bus.Conn, key string, args any) (string, error) {
t.Helper()
got, err := asker.Ask(key, args, "")
got, err := asker.AskWithin(key, args, "", callWithin)
return string(got.Result), err
}
@@ -78,7 +78,10 @@ func increment(b launch.Bus, key string, attempts int) (n uint64, lost int, err
// Two writers incrementing one counter a thousand times each, against a real bus, end at two thousand:
// every number handed out once and none lost, because the runtime writes only at the revision read
// (novox/hq research 042: an issue's number is handed out once).
// (novox/hq research 042: an issue's number is handed out once). A write is lost only to the other writer's
// write, so one increment loses at most as many times as the other writer increments: that, not a guess,
// is the bound on its tries, and a machine whose scheduler lets one writer win every round still passes
// (one processor starved a writer of a thousand tries; principle 11, design 47 H §774).
func TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate(t *testing.T) {
b := counterBus(t)
const each = 1000
@@ -91,7 +94,7 @@ func TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate(t *testing.T) {
go func(w int) {
defer wg.Done()
for i := 0; i < each; i++ {
n, l, err := increment(b, "issue", 1000)
n, l, err := increment(b, "issue", each+1)
lost[w] += l
if err != nil {
errs[w] = err