Files
mesh-tools/node-tools/internal/runtime/state_counter_test.go
T
jschoubben 3850eb8eac H1 integration bound: the heartbeat and the counter tests hold under load
TestTheRuntimeSaysItIsThereEveryInterval slept 70 ms and wanted three
beats at 20 ms: a loaded machine that did not schedule the heartbeat in
time failed it (main.20's integration step on the build node); and its
stop returned before the heartbeat's goroutine ended, so a beat could be
said after it ("said after it was stopped"). The ticks are now the
caller's (keep, beside Keep's own ticker), the test drives them and waits
on each beat, and stop returns once nothing more is said. Pinned to one
processor at idle priority beside six busy loops, the old test failed 13
of 100 runs; the new passes 100 of 100.

TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate bounded a
writer's tries at 1000, a guess: on one processor the scheduler lets one
writer win every round and the other lost its 1000 tries every time. A
write is lost only to the other writer's write, so the bound is the
other's count plus one.

Builds: design 47 H §774; principle 11
2026-11-20 17:23:59 +01:00

273 lines
9.6 KiB
Go

package runtime
import (
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
"github.com/novox/mesh-tools/node-tools/internal/bus"
"github.com/novox/mesh-tools/node-tools/internal/launch"
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
)
// counterBus is a module, issues, that keeps a state numbers in a bucket of a real bus, reached as a bundle
// reaches it: mesh/state.<verb> with its JSON, through the module's bus.
func counterBus(t *testing.T) launch.Bus {
t.Helper()
mesh := mt.New(t)
mesh.Bucket(t, "issues_numbers")
mesh.BucketKeyTTL(t, "issues_wakes")
m := mt.MembershipOf("issues", "anchor", false, nil)
m.State = []bus.StateIssued{{Name: "numbers", Bucket: "issues_numbers", Writes: true},
{Name: "wakes", Bucket: "issues_wakes", Writes: true}}
mesh.Issue(t, m)
conn := connect(t, "node-tools", "anchor")
conn.Follow("issues")
mt.Until(t, func() error {
if conn.Membership("issues") == nil {
return errors.New("not issued yet")
}
return nil
})
return (&consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}).forModule("issues")
}
func ask(b launch.Bus, verb string, params map[string]any) (json.RawMessage, error) {
raw, _ := json.Marshal(params)
return b.State(verb, raw)
}
// increment is what a bundle's counter does over the wire: get, then create (no value) or update at the
// revision read; a lost write — told by its code alone — is read again and tried again.
func increment(b launch.Bus, key string, attempts int) (n uint64, lost int, err error) {
for i := 0; i < attempts; i++ {
raw, err := ask(b, "get", map[string]any{"state": "numbers", "key": key})
if err != nil {
return 0, lost, err
}
var e *bus.StateEntry
if err := json.Unmarshal(raw, &e); err != nil {
return 0, lost, err
}
if e == nil {
n = 1
_, err = ask(b, "create", map[string]any{"state": "numbers", "key": key, "value": n})
} else {
if err := json.Unmarshal(e.Value, &n); err != nil {
return 0, lost, err
}
n++
_, err = ask(b, "update", map[string]any{"state": "numbers", "key": key, "value": n, "revision": e.Revision})
}
var coded interface{ ErrorCode() int }
switch {
case err == nil:
return n, lost, nil
case errors.As(err, &coded) && coded.ErrorCode() == launch.CodeChanged:
lost++
default:
return 0, lost, err
}
}
return 0, lost, fmt.Errorf("%s lost %d times", key, attempts)
}
// Two writers incrementing one counter a thousand times each, against a real bus, end at two thousand:
// every number handed out once and none lost, because the runtime writes only at the revision read
// (novox/hq research 042: an issue's number is handed out once). A write is lost only to the other writer's
// write, so one increment loses at most as many times as the other writer increments: that, not a guess,
// is the bound on its tries, and a machine whose scheduler lets one writer win every round still passes
// (one processor starved a writer of a thousand tries; principle 11, design 47 H §774).
func TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate(t *testing.T) {
b := counterBus(t)
const each = 1000
var wg sync.WaitGroup
numbers := make([][]uint64, 2)
errs := make([]error, 2)
lost := make([]int, 2)
for w := range numbers {
wg.Add(1)
go func(w int) {
defer wg.Done()
for i := 0; i < each; i++ {
n, l, err := increment(b, "issue", each+1)
lost[w] += l
if err != nil {
errs[w] = err
return
}
numbers[w] = append(numbers[w], n)
}
}(w)
}
wg.Wait()
if err := errors.Join(errs...); err != nil {
t.Fatal(err)
}
seen := map[uint64]bool{}
for w, ns := range numbers {
for i, n := range ns {
if seen[n] {
t.Errorf("the number %d was handed out twice", n)
}
seen[n] = true
if i > 0 && n <= ns[i-1] {
t.Errorf("writer %d was handed %d after %d", w, n, ns[i-1])
}
}
}
raw, err := ask(b, "get", map[string]any{"state": "numbers", "key": "issue"})
if err != nil {
t.Fatal(err)
}
var e bus.StateEntry
_ = json.Unmarshal(raw, &e)
if string(e.Value) != "2000" || len(seen) != 2*each {
t.Fatalf("the counter ended at %s with %d numbers handed out, not 2000: an increment was lost", e.Value, len(seen))
}
if lost[0]+lost[1] == 0 {
t.Fatal("the two writers never collided: the test proved nothing about contention")
}
t.Logf("2000 increments, %d writes lost to the other writer and tried again", lost[0]+lost[1])
}
// A write at a revision that is no longer the key's is refused with the lost-write code and writes
// nothing; create is refused for a key with a value and taken for one deleted; an update names the
// revision it read, never zero.
// A key's own time to live (issue 1952, mesh-sdk process.State.CreateTTL and RenewTTL): created once, renewed whole by a
// put, its end heard by a watch as an expiry; refused, writing nothing, on a bucket that takes none, under a second, and
// on an update.
func TestAStaleWriteIsRefusedAndWritesNothing(t *testing.T) {
b := counterBus(t)
state := func(verb string, extra map[string]any) (uint64, error) {
p := map[string]any{"state": "numbers", "key": "issue"}
for k, v := range extra {
p[k] = v
}
raw, err := ask(b, verb, p)
if err != nil {
return 0, err
}
var r struct {
Revision uint64 `json:"revision"`
}
_ = json.Unmarshal(raw, &r)
return r.Revision, nil
}
isLost := func(err error) bool {
var coded interface{ ErrorCode() int }
return errors.As(err, &coded) && coded.ErrorCode() == launch.CodeChanged && errors.Is(err, bus.ErrStateChanged)
}
value := func() string {
raw, err := ask(b, "get", map[string]any{"state": "numbers", "key": "issue"})
if err != nil {
t.Fatal(err)
}
var e *bus.StateEntry
_ = json.Unmarshal(raw, &e)
if e == nil {
return "none"
}
return string(e.Value)
}
if _, err := state("update", map[string]any{"value": 1, "revision": 0}); err == nil || isLost(err) {
t.Fatalf("an update at revision zero: %v", err)
}
if value() != "none" {
t.Fatal("an update at revision zero wrote")
}
first, err := state("create", map[string]any{"value": 1})
if err != nil || first == 0 {
t.Fatalf("create of a key with no value: %d %v", first, err)
}
if _, err := state("create", map[string]any{"value": 9}); !isLost(err) {
t.Fatalf("a second create: %v", err)
}
second, err := state("update", map[string]any{"value": 2, "revision": first})
if err != nil || second <= first {
t.Fatalf("an update at the revision read: %d %v", second, err)
}
if _, err := state("update", map[string]any{"value": 9, "revision": first}); !isLost(err) {
t.Fatalf("an update at a stale revision: %v", err)
}
if _, err := state("update", map[string]any{"value": 9, "revision": second + 100}); !isLost(err) {
t.Fatalf("an update at a revision not yet written: %v", err)
}
if v := value(); v != "2" {
t.Fatalf("refused writes changed the value to %s", v)
}
if _, err := ask(b, "delete", map[string]any{"state": "numbers", "key": "issue"}); err != nil {
t.Fatal(err)
}
if _, err := state("create", map[string]any{"value": 1}); err != nil {
t.Fatalf("create of a deleted key: %v", err)
}
if v := value(); v != "1" {
t.Fatalf("after delete and create: %s", v)
}
// A time to live per key: refused where it cannot be kept, writing nothing.
for _, c := range []struct {
verb, state, why string
ttl int
}{
{"create", "numbers", "does not take keys with their own time to live", 60},
{"put", "numbers", "does not take keys with their own time to live", 60},
{"create", "wakes", "shorter than the bus keeps", -1},
{"update", "wakes", "gives no time to live", 60},
} {
_, err := ask(b, c.verb, map[string]any{"state": c.state, "key": "late", "value": 1, "revision": 1, "ttl_seconds": c.ttl})
if err == nil || !strings.Contains(err.Error(), c.why) {
t.Errorf("%s with ttl %d on %s: want refused for %q, got %v", c.verb, c.ttl, c.state, c.why, err)
}
if raw, _ := ask(b, "get", map[string]any{"state": c.state, "key": "late"}); string(raw) != "null" {
t.Errorf("%s with ttl %d on %s wrote %s", c.verb, c.ttl, c.state, raw)
}
}
// The bucket's fill against its caps, for a bundle's tools.
if raw, err := ask(b, "info", map[string]any{"state": "wakes"}); err != nil || !strings.Contains(string(raw),
`"bucket":"issues_wakes","bytes":0,"keys":0,"max_bytes":-1,"max_value_bytes":262144,"history":1,"ttl_seconds":0,"key_ttl":true`) {
t.Errorf("info of a key-ttl bucket answered %s, %v", raw, err)
}
// Created once, renewed whole, and its end heard: a wake with no timer.
heard := make(chan bus.StateChange, 8)
stop, err := b.Watch(json.RawMessage(`{"state":"wakes"}`), func(raw json.RawMessage) error {
var c bus.StateChange
_ = json.Unmarshal(raw, &c)
heard <- c
return nil
})
if err != nil {
t.Fatal(err)
}
defer stop()
if _, err := ask(b, "create", map[string]any{"state": "wakes", "key": "settle", "value": 1, "ttl_seconds": 60}); err != nil {
t.Fatalf("create with a time to live: %v", err)
}
if _, err := ask(b, "create", map[string]any{"state": "wakes", "key": "settle", "value": 2, "ttl_seconds": 60}); !isLost(err) {
t.Fatalf("a second create with a time to live: %v", err)
}
if _, err := ask(b, "put", map[string]any{"state": "wakes", "key": "settle", "value": 3, "ttl_seconds": 1}); err != nil {
t.Fatalf("a put renewing the time to live: %v", err)
}
var ops []string
for want := "put delete put expire"; strings.Join(ops, " ") != want; {
select {
case c := <-heard:
ops = append(ops, c.Op)
if c.Op == "put" && len(ops) == 3 && string(c.Value) != "3" {
t.Fatalf("the renewed key holds %s", c.Value)
}
case <-time.After(5 * time.Second):
t.Fatalf("the watch heard %q, want %q", strings.Join(ops, " "), want)
}
}
}