From 4e338436dadda5a80c6891a90ba2eb3b8bd01546 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 02:21:53 +0100 Subject: [PATCH] Kept-call pace read once and atomically: no data race with the kept calls reading it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReadPace wrote the pace at every Connect and Serve while kept calls on other goroutines read it. The pace is now held atomically and read once at the start: a later read of the same pace changes nothing, one of another pace is refused naming both. KeptPace becomes a function. Builds: design 47 H2 §782, §775 --- go/busretry/busretry.go | 50 ++++++++++++++++++--------- go/busretry/busretry_test.go | 65 ++++++++++++++++++++++++++---------- go/process/call.go | 2 +- 3 files changed, 83 insertions(+), 34 deletions(-) diff --git a/go/busretry/busretry.go b/go/busretry/busretry.go index 2c9a1693..52daccef 100644 --- a/go/busretry/busretry.go +++ b/go/busretry/busretry.go @@ -23,6 +23,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "time" "github.com/nats-io/nats.go" @@ -43,10 +44,9 @@ var ( // It sets the kept-call pace (KeptPace) to first and most too, so a test of a kept call waits milliseconds. func Waits(bound, first, most time.Duration) (restore func()) { was := [3]time.Duration{Bound, FirstWait, MostWait} - wasPace := KeptPace + wasPace := keptPace.Swap(&Pace{First: first, Max: most}) Bound, FirstWait, MostWait = bound, first, most - KeptPace = Pace{First: first, Max: most} - return func() { Bound, FirstWait, MostWait, KeptPace = was[0], was[1], was[2], wasPace } + return func() { Bound, FirstWait, MostWait = was[0], was[1], was[2]; keptPace.Store(wasPace) } } // Pace is the kept-call form's pace (design 47 H2 §775): on a transport fault the call is made again after First, @@ -63,28 +63,45 @@ const ( // between two lives (a restart takes 20 to 50 s) is met within one minute without a call a second. var DefaultPace = Pace{First: time.Second, Max: 30 * time.Second} +// keptPace is the pace ReadPace read, nil until it read one. It is set once and read by every kept call at once, so it +// is held atomically: a kept call on one goroutine never races a Connect or a Serve on another (§782). +var keptPace atomic.Pointer[Pace] + // KeptPace is the pace of every kept call this process makes: DefaultPace until the process reads its settings -// (ReadPace, at process.Connect and at a bundle's stdio.Serve). -var KeptPace = DefaultPace +// (ReadPace, at process.Connect and at a bundle's stdio.Serve). Safe for concurrent use. +func KeptPace() Pace { + if p := keptPace.Load(); p != nil { + return *p + } + return DefaultPace +} // ReadPace sets KeptPace from the module's settings PaceFirstSetting and PaceMaxSetting, read through read (the one // settings reader, stdio.SettingDuration, which answers its default for a setting not composed), so a module's // declared pace paces its kept calls (§775). A setting that is no duration, or a most below the first, is an error -// naming it, and KeptPace stays. +// naming it, and KeptPace stays. The pace is read once, at the process's start: a later call reading the same pace +// changes nothing, and one reading another pace is refused, naming both, and KeptPace stays (§782). Safe for +// concurrent use. func ReadPace(read func(key string, def time.Duration) (time.Duration, error)) error { - first, err := read(PaceFirstSetting, KeptPace.First) + held := KeptPace() + first, err := read(PaceFirstSetting, held.First) if err != nil { return err } - most, err := read(PaceMaxSetting, KeptPace.Max) + most, err := read(PaceMaxSetting, held.Max) if err != nil { return err } if most < first { return fmt.Errorf("the setting %s (%s) is below %s (%s)", PaceMaxSetting, most, PaceFirstSetting, first) } - if read := (Pace{First: first, Max: most}); read != KeptPace { - KeptPace = read + got := Pace{First: first, Max: most} + if keptPace.CompareAndSwap(nil, &got) { + return nil + } + if held = KeptPace(); got != held { + return fmt.Errorf("the kept-call pace is read once, at the start: it is %s to %s, and the settings now say %s to %s", + held.First, held.Max, got.First, got.Max) } return nil } @@ -199,11 +216,14 @@ func DoPaced[T any](k Kept, try func() (T, error)) (T, error) { if bound <= 0 { bound = Bound } - if pace.First <= 0 { - pace.First = KeptPace.First - } - if pace.Max <= 0 { - pace.Max = KeptPace.Max + if pace.First <= 0 || pace.Max <= 0 { + held := KeptPace() + if pace.First <= 0 { + pace.First = held.First + } + if pace.Max <= 0 { + pace.Max = held.Max + } } logf := k.Logf if logf == nil { diff --git a/go/busretry/busretry_test.go b/go/busretry/busretry_test.go index 423f5f9e..d805b115 100644 --- a/go/busretry/busretry_test.go +++ b/go/busretry/busretry_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strings" + "sync" "testing" "time" @@ -186,32 +187,60 @@ func subAKeptCallIsPaced(t *testing.T) { } }) t.Run("the pace is the module's settings", func(t *testing.T) { - was := KeptPace - t.Cleanup(func() { KeptPace = was }) - for _, c := range []struct { - set map[string]time.Duration - bad bool - want Pace - }{ - {set: nil, want: was}, - {set: map[string]time.Duration{PaceFirstSetting: 2 * time.Second, PaceMaxSetting: time.Minute}, want: Pace{First: 2 * time.Second, Max: time.Minute}}, - {set: map[string]time.Duration{PaceFirstSetting: time.Minute, PaceMaxSetting: time.Second}, bad: true, want: was}, - {set: map[string]time.Duration{PaceFirstSetting: -1}, bad: true, want: was}, - } { - KeptPace = was - err := ReadPace(func(key string, def time.Duration) (time.Duration, error) { - if d, ok := c.set[key]; ok { + was := keptPace.Load() + t.Cleanup(func() { keptPace.Store(was) }) + reader := func(set map[string]time.Duration) func(string, time.Duration) (time.Duration, error) { + return func(key string, def time.Duration) (time.Duration, error) { + if d, ok := set[key]; ok { if d < 0 { return 0, fmt.Errorf("the setting %s is no duration", key) } return d, nil } return def, nil - }) - if (err != nil) != c.bad || KeptPace != c.want { - t.Errorf("settings %v: pace %+v, %v", c.set, KeptPace, err) } } + two := map[string]time.Duration{PaceFirstSetting: 2 * time.Second, PaceMaxSetting: time.Minute} + for _, c := range []struct { + name string + held *Pace // the pace read before, nil none + set map[string]time.Duration + bad string + want Pace + }{ + {name: "none set", want: DefaultPace}, + {name: "both set", set: two, want: Pace{First: 2 * time.Second, Max: time.Minute}}, + {name: "most below first", set: map[string]time.Duration{PaceFirstSetting: time.Minute, PaceMaxSetting: time.Second}, bad: "is below", want: DefaultPace}, + {name: "no duration", set: map[string]time.Duration{PaceFirstSetting: -1}, bad: "no duration", want: DefaultPace}, + // Design 47 H2 §782: the pace is read once; a later read of the same pace changes nothing, of another is refused. + {name: "read again, the same", held: &Pace{First: 2 * time.Second, Max: time.Minute}, set: two, want: Pace{First: 2 * time.Second, Max: time.Minute}}, + {name: "read again, none set", held: &Pace{First: 2 * time.Second, Max: time.Minute}, want: Pace{First: 2 * time.Second, Max: time.Minute}}, + {name: "read again, another", held: &DefaultPace, set: two, bad: "read once", want: DefaultPace}, + } { + keptPace.Store(c.held) + err := ReadPace(reader(c.set)) + if (err != nil) != (c.bad != "") || (err != nil && !strings.Contains(err.Error(), c.bad)) || KeptPace() != c.want { + t.Errorf("%s: pace %+v, %v", c.name, KeptPace(), err) + } + } + // Design 47 H2 §782, under -race: a Connect or a Serve reading the pace while kept calls read it is no data race. + keptPace.Store(nil) + var wg sync.WaitGroup + for i := range 8 { + wg.Add(2) + go func() { defer wg.Done(); _ = ReadPace(reader(two)) }() + go func() { + defer wg.Done() + if _, err := DoPaced(Kept{Seat: "s", Verb: fmt.Sprint(i), Logf: func(string, ...any) {}}, func() (int, error) { return 1, nil }); err != nil { + t.Error(err) + } + _ = KeptPace() + }() + } + wg.Wait() + if KeptPace() != (Pace{First: 2 * time.Second, Max: time.Minute}) { + t.Errorf("after concurrent reads the pace is %+v", KeptPace()) + } }) t.Run("one line per seat a minute", func(t *testing.T) { fakeClock(t) diff --git a/go/process/call.go b/go/process/call.go index 5d48c856..8cf84bd9 100644 --- a/go/process/call.go +++ b/go/process/call.go @@ -54,7 +54,7 @@ func (c *Conn) CallOn(node, key string, body any) (json.RawMessage, error) { // CallKept is Call for a call whose answer the caller keeps and which is safe to make again (a read, or a write the // callee deduplicates): mesh-sdk's kept-call form (design 47 H2 §775), asked again through a transport fault, a timeout -// or no responders at the kept-call pace (busretry.KeptPace, jittered), whether or not the bus is blacked out, until it +// or no responders at the kept-call pace (busretry.KeptPace(), jittered), whether or not the bus is blacked out, until it // is answered or busretry.Bound has passed (novox/hq ADR 0668, extending ADR 0615 §6; issue 1966). A refusal returns at // once. A caller with an ask id and a bound of its own uses CallKeptAsk. func (c *Conn) CallKept(key string, body any) (json.RawMessage, error) {