Serve a ledger owner its repairs rebuild, reindex, snapshot and lag as tools, its own ledgers only, marked acts or reads (issue 1966 T4)
This commit is contained in:
@@ -8,8 +8,8 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// 1966/AC15 and the other reads of a ledger: get, query (ADR 0597 §10; design 25 § 12.5).
|
||||
func TestReadsOfALedger(t *testing.T) {
|
||||
// 1966/AC15, the other reads of a ledger (get, query) and its repairs (ADR 0597 §8, §10; design 25 § 12.5).
|
||||
func TestReadsAndRepairsOfALedger(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
run func(t *testing.T, l *Ledger, f fakes)
|
||||
@@ -57,6 +57,59 @@ func TestReadsOfALedger(t *testing.T) {
|
||||
t.Fatalf("an undeclared index: %v", err)
|
||||
}
|
||||
}},
|
||||
{"a lost state rebuilds from the history, an expired one from its snapshot, and lag says what is behind", func(t *testing.T, l *Ledger, f fakes) {
|
||||
applied(t, l, "decision/1", "title", `{"title":"One"}`, owner)
|
||||
applied(t, l, "decision/1", "note", `{"says":"kept"}`, owner)
|
||||
want, _, _ := f.state.Get("decision.1")
|
||||
f.state.kv = nil
|
||||
lag, _ := l.Lag(buscalls.LedgerRequest{})
|
||||
if len(lag.Instances) != 1 || lag.Instances[0].History != 2 || lag.Instances[0].State != 0 || lag.Instances[0].Snapshot != 2 {
|
||||
t.Fatalf("lag says %+v", lag)
|
||||
}
|
||||
if got, err := l.Rebuild(buscalls.RebuildRequest{}); err != nil || got.Written != 1 {
|
||||
t.Fatalf("rebuilt %+v, %v", got, err)
|
||||
}
|
||||
if got, _, _ := f.state.Get("decision.1"); string(record(t, got).State) != string(record(t, want).State) {
|
||||
t.Fatalf("rebuilt from the history %s, want %s", got, want)
|
||||
}
|
||||
f.state.kv = nil
|
||||
f.history.Expire()
|
||||
if got, err := l.Rebuild(buscalls.RebuildRequest{Instance: "decision/1", From: buscalls.RebuildFromSnapshot}); err != nil || got.Written != 1 {
|
||||
t.Fatalf("rebuilt %+v, %v", got, err)
|
||||
}
|
||||
if got, _, _ := f.state.Get("decision.1"); string(got) != string(want) {
|
||||
t.Fatalf("rebuilt from the snapshot %s, want %s", got, want)
|
||||
}
|
||||
}},
|
||||
{"reindex builds a declared index from the states, and lag says an index behind until it does", func(t *testing.T, l *Ledger, f fakes) {
|
||||
applied(t, l, "decision/1", "title", `{"title":"One ledger"}`, owner)
|
||||
applied(t, l, "decision/2", "note", `{"says":"untitled"}`, owner)
|
||||
_ = f.index.Put("stale.decision.9", []byte(`{}`))
|
||||
if lag, _ := l.Lag(buscalls.LedgerRequest{}); len(lag.Indexes) != 1 || lag.Indexes[0].Behind != 1 {
|
||||
t.Fatalf("lag says %+v", lag)
|
||||
}
|
||||
if got, err := l.Reindex(buscalls.ReindexRequest{Index: "by-title"}); err != nil || got.Written != 1 {
|
||||
t.Fatalf("reindexed %+v, %v", got, err)
|
||||
}
|
||||
entries, _ := f.index.Entries("", 0)
|
||||
if len(entries) != 1 || entries[0].Key != "One_ledger.decision.1" || string(entries[0].Value) != `{"instance":"decision/1","sequence":1}` {
|
||||
t.Fatalf("the index holds %+v", entries)
|
||||
}
|
||||
if lag, _ := l.Lag(buscalls.LedgerRequest{}); len(lag.Indexes) != 0 || len(lag.Instances) != 0 {
|
||||
t.Fatalf("lag after reindex says %+v", lag)
|
||||
}
|
||||
}},
|
||||
{"snapshot rewrites a snapshot behind its state", func(t *testing.T, l *Ledger, f fakes) {
|
||||
applied(t, l, "decision/1", "title", `{"title":"One"}`, owner)
|
||||
f.snapshots.kv = nil
|
||||
if got, err := l.Snapshot(buscalls.InstanceRequest{}); err != nil || got.Written != 1 {
|
||||
t.Fatalf("snapshot %+v, %v", got, err)
|
||||
}
|
||||
state, _, _ := f.state.Get("decision.1")
|
||||
if snap, _, _ := f.snapshots.Get("decision.1"); string(snap) != string(state) {
|
||||
t.Fatalf("the snapshot %s, the state %s", snap, state)
|
||||
}
|
||||
}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
package ledger
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// The ledger's repairs (ADR 0597 §8, §10; design 25 § 12.5; issue 1966 T4): rebuild an instance's state from its history
|
||||
// or its snapshot, rewrite a snapshot behind its state, build a declared index again, and say what is behind its history.
|
||||
// The tool runner serves them as tools of every module owning a ledger, its own ledgers only (the runtime's repairs.go).
|
||||
|
||||
// instances is every instance the ledger knows, by key: those its state holds and those its history names.
|
||||
func (l *Ledger) instances() ([]string, error) {
|
||||
seen := map[string]bool{}
|
||||
keys, err := l.State.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, k := range keys {
|
||||
seen[k] = true
|
||||
}
|
||||
kept, err := l.History.Read(">", 1, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ops, err := operationsOf(kept)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, o := range ops {
|
||||
if k, err := KeyOf(o.Instance); err == nil {
|
||||
seen[k] = true
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for k := range seen {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// keysOf is the one instance a request names, or every instance when it names none.
|
||||
func (l *Ledger) keysOf(instance string) ([]string, error) {
|
||||
if instance == "" {
|
||||
return l.instances()
|
||||
}
|
||||
k, err := KeyOf(instance)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{k}, nil
|
||||
}
|
||||
|
||||
// Rebuild derives the state of an instance (every instance when it names none) again, from its whole history or from its
|
||||
// snapshot and the operations after it, and writes it, and its snapshot when that is behind. A derivation further back
|
||||
// than the state holds is not written: the state never goes back (ADR 0597 §5).
|
||||
func (l *Ledger) Rebuild(r buscalls.RebuildRequest) (buscalls.Repaired, error) {
|
||||
from := r.From
|
||||
if from == "" {
|
||||
from = buscalls.RebuildFromHistory
|
||||
}
|
||||
if from != buscalls.RebuildFromHistory && from != buscalls.RebuildFromSnapshot {
|
||||
return buscalls.Repaired{}, fmt.Errorf("a rebuild is from %q or %q, not %q", buscalls.RebuildFromHistory, buscalls.RebuildFromSnapshot, from)
|
||||
}
|
||||
keys, err := l.keysOf(r.Instance)
|
||||
if err != nil {
|
||||
return buscalls.Repaired{}, err
|
||||
}
|
||||
var out buscalls.Repaired
|
||||
for _, key := range keys {
|
||||
base := Record{}
|
||||
if from == buscalls.RebuildFromSnapshot {
|
||||
if base, _, err = l.readSnapshot(key); err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
derived, err := l.replay(key, base, 0)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
held, rev, err := l.readState(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if derived.Sequence < held.Sequence {
|
||||
continue
|
||||
}
|
||||
now, _, err := l.settle(key, derived, rev)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if err := l.writeSnapshot(key, now); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Written++
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Snapshot rewrites the snapshot of an instance (every instance when it names none) that is behind its state.
|
||||
func (l *Ledger) Snapshot(r buscalls.InstanceRequest) (buscalls.Repaired, error) {
|
||||
keys, err := l.keysOf(r.Instance)
|
||||
if err != nil {
|
||||
return buscalls.Repaired{}, err
|
||||
}
|
||||
var out buscalls.Repaired
|
||||
for _, key := range keys {
|
||||
held, _, err := l.readState(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
snap, at, err := l.readSnapshot(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if held.Sequence == 0 || (at != 0 && snap.Sequence >= held.Sequence) {
|
||||
continue
|
||||
}
|
||||
if err := l.writeSnapshot(key, held); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Written++
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// unsafe is what a key part cannot hold.
|
||||
var unsafe = regexp.MustCompile(`[^-_A-Za-z0-9]`)
|
||||
|
||||
// IndexEntryOf is an instance's entry in a declared index, as reindex writes it: its key the values of the fields the
|
||||
// index is on, in their order, then the instance's key, each part with what a key cannot hold written as `_`; its value
|
||||
// the instance and the sequence of the state it was taken from. False when the state holds none of those fields.
|
||||
func IndexEntryOf(on []string, key string, rec Record) (string, []byte, bool) {
|
||||
var state map[string]any
|
||||
if json.Unmarshal(rec.State, &state) != nil || state == nil {
|
||||
return "", nil, false
|
||||
}
|
||||
parts := make([]string, 0, len(on)+1)
|
||||
found := false
|
||||
for _, f := range on {
|
||||
v, has := state[f]
|
||||
if !has {
|
||||
parts = append(parts, "_")
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
raw, _ := json.Marshal(v)
|
||||
s = string(raw)
|
||||
}
|
||||
if s = unsafe.ReplaceAllString(s, "_"); s == "" {
|
||||
s = "_"
|
||||
}
|
||||
parts = append(parts, s)
|
||||
}
|
||||
if !found {
|
||||
return "", nil, false
|
||||
}
|
||||
value, _ := json.Marshal(map[string]any{"instance": InstanceOf(key), "sequence": rec.Sequence})
|
||||
return strings.Join(append(parts, key), "."), value, true
|
||||
}
|
||||
|
||||
// declared is the fields an index is on and the types declaring it.
|
||||
func (l *Ledger) declared(index string) ([]string, map[string]bool, error) {
|
||||
types := map[string]bool{}
|
||||
var on []string
|
||||
for _, t := range l.Types {
|
||||
for _, ix := range t.Indexes {
|
||||
if ix.Name == index {
|
||||
types[t.Name] = true
|
||||
on = ix.On
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(types) == 0 {
|
||||
return nil, nil, fmt.Errorf("%s's ledger %s declares no index %q", l.Module, l.Name, index)
|
||||
}
|
||||
return on, types, nil
|
||||
}
|
||||
|
||||
// wanted is every entry an index should hold, from the states of the types declaring it.
|
||||
func (l *Ledger) wanted(index string) (map[string][]byte, map[string]uint64, error) {
|
||||
on, types, err := l.declared(index)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
keys, err := l.instances()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
want, seqs := map[string][]byte{}, map[string]uint64{}
|
||||
for _, key := range keys {
|
||||
name, _, _ := strings.Cut(key, ".")
|
||||
if !types[name] {
|
||||
continue
|
||||
}
|
||||
held, _, err := l.readState(key)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if k, v, ok := IndexEntryOf(on, key, held); ok {
|
||||
want[k], seqs[k] = v, held.Sequence
|
||||
}
|
||||
}
|
||||
return want, seqs, nil
|
||||
}
|
||||
|
||||
// Reindex builds a declared index again from the ledger's states, derived from its history: every entry it should hold
|
||||
// written, every other removed.
|
||||
func (l *Ledger) Reindex(r buscalls.ReindexRequest) (buscalls.Repaired, error) {
|
||||
ix, err := l.index(r.Index)
|
||||
if err != nil {
|
||||
return buscalls.Repaired{}, err
|
||||
}
|
||||
want, _, err := l.wanted(r.Index)
|
||||
if err != nil {
|
||||
return buscalls.Repaired{}, err
|
||||
}
|
||||
held, err := ix.Entries("", 0)
|
||||
if err != nil {
|
||||
return buscalls.Repaired{}, err
|
||||
}
|
||||
var out buscalls.Repaired
|
||||
for _, e := range held {
|
||||
if _, keep := want[e.Key]; !keep {
|
||||
if err := ix.Delete(e.Key); err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(want))
|
||||
for k := range want {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if err := ix.Put(k, want[k]); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Written++
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Lag is every instance whose state or snapshot is behind its history, and how many entries each declared index lacks
|
||||
// or holds from an older state; nothing when nothing is behind.
|
||||
func (l *Ledger) Lag(buscalls.LedgerRequest) (buscalls.Lagged, error) {
|
||||
keys, err := l.instances()
|
||||
if err != nil {
|
||||
return buscalls.Lagged{}, err
|
||||
}
|
||||
var out buscalls.Lagged
|
||||
for _, key := range keys {
|
||||
last, err := l.History.Last(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
held, _, err := l.readState(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
snap, _, err := l.readSnapshot(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if held.Sequence < last || snap.Sequence < last {
|
||||
out.Instances = append(out.Instances, buscalls.InstanceLag{Instance: InstanceOf(key), History: last,
|
||||
State: held.Sequence, Snapshot: snap.Sequence})
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(l.Indexes))
|
||||
for name := range l.Indexes {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
if _, _, err := l.declared(name); err != nil {
|
||||
// A bucket bound for an index no type declares has nothing it should hold.
|
||||
continue
|
||||
}
|
||||
want, seqs, err := l.wanted(name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
held, err := l.Indexes[name].Entries("", 0)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
at := map[string]uint64{}
|
||||
for _, e := range held {
|
||||
var v struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
}
|
||||
_ = json.Unmarshal(e.Value, &v)
|
||||
at[e.Key] = v.Sequence
|
||||
}
|
||||
behind := uint64(0)
|
||||
for k := range want {
|
||||
if s, has := at[k]; !has || s < seqs[k] {
|
||||
behind++
|
||||
}
|
||||
}
|
||||
if behind > 0 {
|
||||
out.Indexes = append(out.Indexes, buscalls.IndexLag{Index: name, Behind: behind})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in New Issue
Block a user