262 lines
10 KiB
Go
262 lines
10 KiB
Go
//go:build integration
|
|
|
|
package runtime
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"git.novox.be/novox/mesh-sdk/go/membership"
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
|
"github.com/novox/mesh-tools/node-tools/internal/ledger"
|
|
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
|
)
|
|
|
|
// racing is a history whose append lets another writer apply first, once: before its own append, or after it.
|
|
type racing struct {
|
|
ledger.History
|
|
before, after func()
|
|
}
|
|
|
|
func (r *racing) Append(key string, op []byte, expect uint64) (uint64, error) {
|
|
if f := r.before; f != nil {
|
|
r.before = nil
|
|
f()
|
|
}
|
|
seq, err := r.History.Append(key, op, expect)
|
|
if f := r.after; f != nil && err == nil {
|
|
r.after = nil
|
|
f()
|
|
}
|
|
return seq, err
|
|
}
|
|
|
|
// cut is a state whose next write is lost, as a writer stopped between its history and its state would lose it.
|
|
type cut struct {
|
|
ledger.State
|
|
once bool
|
|
}
|
|
|
|
func (c *cut) Write(key string, value []byte, revision uint64) (uint64, error) {
|
|
if c.once {
|
|
c.once = false
|
|
return 0, errors.New("the writer stopped")
|
|
}
|
|
return c.State.Write(key, value, revision)
|
|
}
|
|
|
|
// 1966/AC19: the ledger keeps its promises against the real bus (ADR 0597's How it is checked; design 25 § 12.7), one
|
|
// case each, and every apply leaves the snapshot equal to the state.
|
|
func TestTheLedgerKeepsItsPromisesOnTheBus(t *testing.T) {
|
|
mesh := mt.New(t)
|
|
nc, err := nats.Connect(mt.URL(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
js, _ := jetstream.New(nc)
|
|
ctx := context.Background()
|
|
for _, cfg := range []jetstream.StreamConfig{
|
|
{Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.history.>"}},
|
|
{Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger.m.records.snapshot.>"}, MaxMsgsPerSubject: 1},
|
|
} {
|
|
if _, err := js.CreateStream(ctx, cfg); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, b := range []string{"m_records", "m_records_by_title"} {
|
|
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
types := []catalogue.LedgerType{{Name: "decision", Schema: map[string]catalogue.LedgerField{"title": {Type: "string"}},
|
|
Operations: []catalogue.LedgerOperation{
|
|
{Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}, Author: true},
|
|
{Name: "note", Effect: catalogue.EffectAppend, Into: "notes", Fields: []string{"says"}},
|
|
},
|
|
Indexes: []catalogue.LedgerIndex{{Name: "by-title", On: []string{"title"}}}}}
|
|
binding := membership.BusBinding{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger,
|
|
History: "LEDGER_m_records", Subject: "mesh.ledger.m.records.history", State: "m_records",
|
|
Indexes: map[string]string{"by-title": "m_records_by_title"}, Snapshot: "mesh.ledger.m.records.snapshot"}
|
|
mem := mt.MembershipOf("m", "anchor", false, nil)
|
|
binding.Types = types
|
|
mem.Bindings = []membership.BusBinding{binding}
|
|
mesh.Issue(t, mem)
|
|
conn := connect(t, "node-tools", "anchor")
|
|
conn.Follow("m")
|
|
mt.Until(t, func() error {
|
|
if conn.Membership("m") == nil {
|
|
return errors.New("not issued yet")
|
|
}
|
|
return nil
|
|
})
|
|
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
|
WithLedgerWriters(func(string) (*bus.Conn, error) { return conn, nil })(all)
|
|
m := all.forModule("m").(*moduleBus)
|
|
|
|
open := func(t *testing.T) *ledger.Ledger {
|
|
t.Helper()
|
|
l, err := all.ledger("m", "records")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return l
|
|
}
|
|
apply := func(t *testing.T, l *ledger.Ledger, instance, operation, body string) (buscalls.Applied, error) {
|
|
t.Helper()
|
|
return l.Apply(buscalls.ApplyRequest{Ledger: "records", Instance: instance, Operation: operation, Body: json.RawMessage(body)},
|
|
buscalls.Author{Account: "m"})
|
|
}
|
|
must := func(t *testing.T, l *ledger.Ledger, instance, operation, body string) buscalls.Applied {
|
|
t.Helper()
|
|
got, err := apply(t, l, instance, operation, body)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return got
|
|
}
|
|
held := func(t *testing.T, key string) ledger.Record {
|
|
t.Helper()
|
|
raw, _, err := open(t).State.Get(key)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var r ledger.Record
|
|
if raw != nil {
|
|
_ = json.Unmarshal(raw, &r)
|
|
}
|
|
return r
|
|
}
|
|
snapshotIsState := func(t *testing.T, key string) {
|
|
t.Helper()
|
|
l := open(t)
|
|
state, _, _ := l.State.Get(key)
|
|
snap, _, _ := l.Snapshots.Get(key)
|
|
if string(snap) != string(state) {
|
|
t.Fatalf("the snapshot of %s is %s, its state %s", key, snap, state)
|
|
}
|
|
}
|
|
|
|
cases := []struct {
|
|
name, key string
|
|
run func(t *testing.T)
|
|
}{
|
|
{"two writers race and one is refused", "decision.race", func(t *testing.T) {
|
|
l := open(t)
|
|
l.History = &racing{History: l.History, before: func() { must(t, open(t), "decision/race", "note", `{"says":"first"}`) }}
|
|
_, err := apply(t, l, "decision/race", "note", `{"says":"second"}`)
|
|
if !errors.Is(err, ledger.ErrRaced) || !strings.Contains(err.Error(), "not taken") {
|
|
t.Fatalf("the racing writer was answered %v", err)
|
|
}
|
|
if got := held(t, "decision.race"); string(got.State) != `{"notes":[{"says":"first"}]}` {
|
|
t.Fatalf("the state is %s", got.State)
|
|
}
|
|
}},
|
|
{"two interleaved applies leave the state at the higher sequence", "decision.interleave", func(t *testing.T) {
|
|
var inner buscalls.Applied
|
|
l := open(t)
|
|
l.History = &racing{History: l.History, after: func() { inner = must(t, open(t), "decision/interleave", "note", `{"says":"later"}`) }}
|
|
outer := must(t, l, "decision/interleave", "note", `{"says":"earlier"}`)
|
|
got := held(t, "decision.interleave")
|
|
if inner.Sequence <= outer.Sequence || got.Sequence != inner.Sequence ||
|
|
string(got.State) != `{"notes":[{"says":"earlier"},{"says":"later"}]}` {
|
|
t.Fatalf("applied %d then %d, the state at %d: %s", outer.Sequence, inner.Sequence, got.Sequence, got.State)
|
|
}
|
|
}},
|
|
{"a write cut between history and state is repaired by the next apply", "decision.cut", func(t *testing.T) {
|
|
must(t, open(t), "decision/cut", "note", `{"says":"one"}`)
|
|
l := open(t)
|
|
l.State = &cut{State: l.State, once: true}
|
|
if got := must(t, l, "decision/cut", "note", `{"says":"cut"}`); !got.Taken || !got.StateBehind {
|
|
t.Fatalf("the cut apply was answered %+v", got)
|
|
}
|
|
must(t, open(t), "decision/cut", "note", `{"says":"three"}`)
|
|
if got := held(t, "decision.cut"); string(got.State) != `{"notes":[{"says":"one"},{"says":"cut"},{"says":"three"}]}` {
|
|
t.Fatalf("the state is %s", got.State)
|
|
}
|
|
}},
|
|
{"a full state bucket answers taken with its sequence and catches up", "decision.full", func(t *testing.T) {
|
|
must(t, open(t), "decision/full", "note", `{"says":"small"}`)
|
|
kv, _ := js.KeyValue(ctx, "m_records")
|
|
status, _ := kv.Status(ctx)
|
|
if _, err := js.UpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "m_records", MaxBytes: int64(status.Bytes()) + 64}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := must(t, open(t), "decision/full", "note", fmt.Sprintf(`{"says":%q}`, strings.Repeat("x", 512)))
|
|
if !got.Taken || got.Sequence == 0 || !got.StateBehind {
|
|
t.Fatalf("the full bucket's apply was answered %+v", got)
|
|
}
|
|
if _, err := js.UpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "m_records", MaxBytes: -1}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
after := must(t, open(t), "decision/full", "note", `{"says":"room"}`)
|
|
if st := held(t, "decision.full"); after.StateBehind || st.Sequence != after.Sequence || !strings.Contains(string(st.State), strings.Repeat("x", 512)) {
|
|
t.Fatalf("after room: %+v, the state at %d", after, st.Sequence)
|
|
}
|
|
}},
|
|
{"an instance whose operations expired rebuilds from its snapshot", "decision.expired", func(t *testing.T) {
|
|
must(t, open(t), "decision/expired", "title", `{"title":"Kept"}`)
|
|
want := held(t, "decision.expired")
|
|
s, _ := js.Stream(ctx, "LEDGER_m_records")
|
|
if err := s.Purge(ctx, jetstream.WithPurgeSubject("mesh.ledger.m.records.history.decision.expired")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kv, _ := js.KeyValue(ctx, "m_records")
|
|
_ = kv.Purge(ctx, "decision.expired")
|
|
got, err := open(t).Rebuild(buscalls.RebuildRequest{Instance: "decision/expired", From: buscalls.RebuildFromSnapshot})
|
|
if st := held(t, "decision.expired"); err != nil || got.Written != 1 || string(st.State) != string(want.State) {
|
|
t.Fatalf("rebuilt %+v, %v: the state %s, want %s", got, err, st.State, want.State)
|
|
}
|
|
}},
|
|
{"rebuild, reindex and lag repair by hand", "decision.repair", func(t *testing.T) {
|
|
must(t, open(t), "decision/repair", "title", `{"title":"Repaired"}`)
|
|
kv, _ := js.KeyValue(ctx, "m_records")
|
|
_ = kv.Purge(ctx, "decision.repair")
|
|
l := open(t)
|
|
lag, err := l.Lag(buscalls.LedgerRequest{})
|
|
if err != nil || !strings.Contains(fmt.Sprint(lag), "decision/repair") || len(lag.Indexes) == 0 {
|
|
t.Fatalf("lag says %+v, %v", lag, err)
|
|
}
|
|
if got, err := l.Rebuild(buscalls.RebuildRequest{Instance: "decision/repair"}); err != nil || got.Written != 1 {
|
|
t.Fatalf("rebuilt %+v, %v", got, err)
|
|
}
|
|
if _, err := l.Reindex(buscalls.ReindexRequest{Index: "by-title"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if lag, err := l.Lag(buscalls.LedgerRequest{}); err != nil || len(lag.Instances) != 0 || len(lag.Indexes) != 0 {
|
|
t.Fatalf("lag after the repairs says %+v, %v", lag, err)
|
|
}
|
|
q, err := l.Query(buscalls.QueryRequest{Index: "by-title", Key: "Repaired.>"})
|
|
if err != nil || len(q.Entries) != 1 {
|
|
t.Fatalf("queried %+v, %v", q, err)
|
|
}
|
|
}},
|
|
{"a forged author is refused", "decision.forged", func(t *testing.T) {
|
|
_, err := m.BusState(buscalls.LedgerApply, json.RawMessage(`{"ledger":"records","instance":"decision/forged","operation":"title","body":{"title":"T","author":"op@shanks"},"call":"forged"}`))
|
|
if err == nil || !strings.Contains(err.Error(), "the call forged, which is not a tool call in flight") {
|
|
t.Fatalf("a forged call was answered %v", err)
|
|
}
|
|
if last, _ := open(t).History.Last("decision.forged"); last != 0 {
|
|
t.Fatalf("the forged apply was appended at %d", last)
|
|
}
|
|
}},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
c.run(t)
|
|
if c.key != "decision.forged" {
|
|
snapshotIsState(t, c.key)
|
|
}
|
|
})
|
|
}
|
|
}
|