jschoubben
1583fa85d9
mesh/state.create and mesh/state.put take ttl_seconds (mesh-sdk 32f5ce3's CreateTTL/RenewTTL contract); a bucket not declared key-ttl is refused, naming it; a watch hands an expired key as op "expire". Pins mesh-sdk 32f5ce3.
264 lines
8.9 KiB
Go
264 lines
8.9 KiB
Go
package runtime
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
|
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
|
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
|
)
|
|
|
|
// counterBus is a module, issues, that keeps a state numbers in a bucket of a real bus, reached as a bundle
|
|
// reaches it: mesh/state.<verb> with its JSON, through the module's bus.
|
|
func counterBus(t *testing.T) launch.Bus {
|
|
t.Helper()
|
|
mesh := mt.New(t)
|
|
mesh.Bucket(t, "issues_numbers")
|
|
mesh.BucketKeyTTL(t, "issues_wakes")
|
|
m := mt.MembershipOf("issues", "anchor", false, nil)
|
|
m.State = []bus.StateIssued{{Name: "numbers", Bucket: "issues_numbers", Writes: true},
|
|
{Name: "wakes", Bucket: "issues_wakes", Writes: true}}
|
|
mesh.Issue(t, m)
|
|
conn := connect(t, "node-tools", "anchor")
|
|
conn.Follow("issues")
|
|
mt.Until(t, func() error {
|
|
if conn.Membership("issues") == nil {
|
|
return errors.New("not issued yet")
|
|
}
|
|
return nil
|
|
})
|
|
return (&consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}).forModule("issues")
|
|
}
|
|
|
|
func ask(b launch.Bus, verb string, params map[string]any) (json.RawMessage, error) {
|
|
raw, _ := json.Marshal(params)
|
|
return b.State(verb, raw)
|
|
}
|
|
|
|
// increment is what a bundle's counter does over the wire: get, then create (no value) or update at the
|
|
// revision read; a lost write — told by its code alone — is read again and tried again.
|
|
func increment(b launch.Bus, key string, attempts int) (n uint64, lost int, err error) {
|
|
for i := 0; i < attempts; i++ {
|
|
raw, err := ask(b, "get", map[string]any{"state": "numbers", "key": key})
|
|
if err != nil {
|
|
return 0, lost, err
|
|
}
|
|
var e *bus.StateEntry
|
|
if err := json.Unmarshal(raw, &e); err != nil {
|
|
return 0, lost, err
|
|
}
|
|
if e == nil {
|
|
n = 1
|
|
_, err = ask(b, "create", map[string]any{"state": "numbers", "key": key, "value": n})
|
|
} else {
|
|
if err := json.Unmarshal(e.Value, &n); err != nil {
|
|
return 0, lost, err
|
|
}
|
|
n++
|
|
_, err = ask(b, "update", map[string]any{"state": "numbers", "key": key, "value": n, "revision": e.Revision})
|
|
}
|
|
var coded interface{ ErrorCode() int }
|
|
switch {
|
|
case err == nil:
|
|
return n, lost, nil
|
|
case errors.As(err, &coded) && coded.ErrorCode() == launch.CodeChanged:
|
|
lost++
|
|
default:
|
|
return 0, lost, err
|
|
}
|
|
}
|
|
return 0, lost, fmt.Errorf("%s lost %d times", key, attempts)
|
|
}
|
|
|
|
// Two writers incrementing one counter a thousand times each, against a real bus, end at two thousand:
|
|
// every number handed out once and none lost, because the runtime writes only at the revision read
|
|
// (novox/hq research 042: an issue's number is handed out once).
|
|
func TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate(t *testing.T) {
|
|
b := counterBus(t)
|
|
const each = 1000
|
|
var wg sync.WaitGroup
|
|
numbers := make([][]uint64, 2)
|
|
errs := make([]error, 2)
|
|
lost := make([]int, 2)
|
|
for w := range numbers {
|
|
wg.Add(1)
|
|
go func(w int) {
|
|
defer wg.Done()
|
|
for i := 0; i < each; i++ {
|
|
n, l, err := increment(b, "issue", 1000)
|
|
lost[w] += l
|
|
if err != nil {
|
|
errs[w] = err
|
|
return
|
|
}
|
|
numbers[w] = append(numbers[w], n)
|
|
}
|
|
}(w)
|
|
}
|
|
wg.Wait()
|
|
if err := errors.Join(errs...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
seen := map[uint64]bool{}
|
|
for w, ns := range numbers {
|
|
for i, n := range ns {
|
|
if seen[n] {
|
|
t.Errorf("the number %d was handed out twice", n)
|
|
}
|
|
seen[n] = true
|
|
if i > 0 && n <= ns[i-1] {
|
|
t.Errorf("writer %d was handed %d after %d", w, n, ns[i-1])
|
|
}
|
|
}
|
|
}
|
|
raw, err := ask(b, "get", map[string]any{"state": "numbers", "key": "issue"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var e bus.StateEntry
|
|
_ = json.Unmarshal(raw, &e)
|
|
if string(e.Value) != "2000" || len(seen) != 2*each {
|
|
t.Fatalf("the counter ended at %s with %d numbers handed out, not 2000: an increment was lost", e.Value, len(seen))
|
|
}
|
|
if lost[0]+lost[1] == 0 {
|
|
t.Fatal("the two writers never collided: the test proved nothing about contention")
|
|
}
|
|
t.Logf("2000 increments, %d writes lost to the other writer and tried again", lost[0]+lost[1])
|
|
}
|
|
|
|
// A write at a revision that is no longer the key's is refused with the lost-write code and writes
|
|
// nothing; create is refused for a key with a value and taken for one deleted; an update names the
|
|
// revision it read, never zero.
|
|
// A key's own time to live (issue 1952, mesh-sdk process.State.CreateTTL and RenewTTL): created once, renewed whole by a
|
|
// put, its end heard by a watch as an expiry; refused, writing nothing, on a bucket that takes none, under a second, and
|
|
// on an update.
|
|
func TestAStaleWriteIsRefusedAndWritesNothing(t *testing.T) {
|
|
b := counterBus(t)
|
|
state := func(verb string, extra map[string]any) (uint64, error) {
|
|
p := map[string]any{"state": "numbers", "key": "issue"}
|
|
for k, v := range extra {
|
|
p[k] = v
|
|
}
|
|
raw, err := ask(b, verb, p)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
var r struct {
|
|
Revision uint64 `json:"revision"`
|
|
}
|
|
_ = json.Unmarshal(raw, &r)
|
|
return r.Revision, nil
|
|
}
|
|
isLost := func(err error) bool {
|
|
var coded interface{ ErrorCode() int }
|
|
return errors.As(err, &coded) && coded.ErrorCode() == launch.CodeChanged && errors.Is(err, bus.ErrStateChanged)
|
|
}
|
|
value := func() string {
|
|
raw, err := ask(b, "get", map[string]any{"state": "numbers", "key": "issue"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var e *bus.StateEntry
|
|
_ = json.Unmarshal(raw, &e)
|
|
if e == nil {
|
|
return "none"
|
|
}
|
|
return string(e.Value)
|
|
}
|
|
|
|
if _, err := state("update", map[string]any{"value": 1, "revision": 0}); err == nil || isLost(err) {
|
|
t.Fatalf("an update at revision zero: %v", err)
|
|
}
|
|
if value() != "none" {
|
|
t.Fatal("an update at revision zero wrote")
|
|
}
|
|
first, err := state("create", map[string]any{"value": 1})
|
|
if err != nil || first == 0 {
|
|
t.Fatalf("create of a key with no value: %d %v", first, err)
|
|
}
|
|
if _, err := state("create", map[string]any{"value": 9}); !isLost(err) {
|
|
t.Fatalf("a second create: %v", err)
|
|
}
|
|
second, err := state("update", map[string]any{"value": 2, "revision": first})
|
|
if err != nil || second <= first {
|
|
t.Fatalf("an update at the revision read: %d %v", second, err)
|
|
}
|
|
if _, err := state("update", map[string]any{"value": 9, "revision": first}); !isLost(err) {
|
|
t.Fatalf("an update at a stale revision: %v", err)
|
|
}
|
|
if _, err := state("update", map[string]any{"value": 9, "revision": second + 100}); !isLost(err) {
|
|
t.Fatalf("an update at a revision not yet written: %v", err)
|
|
}
|
|
if v := value(); v != "2" {
|
|
t.Fatalf("refused writes changed the value to %s", v)
|
|
}
|
|
if _, err := ask(b, "delete", map[string]any{"state": "numbers", "key": "issue"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := state("create", map[string]any{"value": 1}); err != nil {
|
|
t.Fatalf("create of a deleted key: %v", err)
|
|
}
|
|
if v := value(); v != "1" {
|
|
t.Fatalf("after delete and create: %s", v)
|
|
}
|
|
|
|
// A time to live per key: refused where it cannot be kept, writing nothing.
|
|
for _, c := range []struct {
|
|
verb, state, why string
|
|
ttl int
|
|
}{
|
|
{"create", "numbers", "does not take keys with their own time to live", 60},
|
|
{"put", "numbers", "does not take keys with their own time to live", 60},
|
|
{"create", "wakes", "shorter than the bus keeps", -1},
|
|
{"update", "wakes", "gives no time to live", 60},
|
|
} {
|
|
_, err := ask(b, c.verb, map[string]any{"state": c.state, "key": "late", "value": 1, "revision": 1, "ttl_seconds": c.ttl})
|
|
if err == nil || !strings.Contains(err.Error(), c.why) {
|
|
t.Errorf("%s with ttl %d on %s: want refused for %q, got %v", c.verb, c.ttl, c.state, c.why, err)
|
|
}
|
|
if raw, _ := ask(b, "get", map[string]any{"state": c.state, "key": "late"}); string(raw) != "null" {
|
|
t.Errorf("%s with ttl %d on %s wrote %s", c.verb, c.ttl, c.state, raw)
|
|
}
|
|
}
|
|
|
|
// Created once, renewed whole, and its end heard: a wake with no timer.
|
|
heard := make(chan bus.StateChange, 8)
|
|
stop, err := b.Watch(json.RawMessage(`{"state":"wakes"}`), func(raw json.RawMessage) error {
|
|
var c bus.StateChange
|
|
_ = json.Unmarshal(raw, &c)
|
|
heard <- c
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer stop()
|
|
if _, err := ask(b, "create", map[string]any{"state": "wakes", "key": "settle", "value": 1, "ttl_seconds": 60}); err != nil {
|
|
t.Fatalf("create with a time to live: %v", err)
|
|
}
|
|
if _, err := ask(b, "create", map[string]any{"state": "wakes", "key": "settle", "value": 2, "ttl_seconds": 60}); !isLost(err) {
|
|
t.Fatalf("a second create with a time to live: %v", err)
|
|
}
|
|
if _, err := ask(b, "put", map[string]any{"state": "wakes", "key": "settle", "value": 3, "ttl_seconds": 1}); err != nil {
|
|
t.Fatalf("a put renewing the time to live: %v", err)
|
|
}
|
|
var ops []string
|
|
for want := "put delete put expire"; strings.Join(ops, " ") != want; {
|
|
select {
|
|
case c := <-heard:
|
|
ops = append(ops, c.Op)
|
|
if c.Op == "put" && len(ops) == 3 && string(c.Value) != "3" {
|
|
t.Fatalf("the renewed key holds %s", c.Value)
|
|
}
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatalf("the watch heard %q, want %q", strings.Join(ops, " "), want)
|
|
}
|
|
}
|
|
}
|