diff --git a/go/process/prune.go b/go/process/prune.go new file mode 100644 index 00000000..042a39d1 --- /dev/null +++ b/go/process/prune.go @@ -0,0 +1,137 @@ +package process + +import ( + "context" + "encoding/json" + "errors" + "log" + "time" + + "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// sayPrune writes a prune's one journal line (design 47 H2 §812). +var sayPrune = func(line string) { log.Print(line) } + +// PrunedState is a state a process prunes at its own writes (design 47 H2 §812; H-A-380): the entry's figures are read +// once when it is bound and once after each prune, and between those a running sum of the bytes each write puts +// (stdio.PutBytes, an upper bound) is the judgement; past prune-mark of the cap the oldest keys by their current +// revision are purged down to prune-target, skipping a key kept under stdio.KeepKey. No read per write, no poll, no timer. +// Its reads are the state's own. +type PrunedState struct { + State + pruner *stdio.Pruner +} + +// Pruned binds the state for a prune by p, reading its figures once. +func (s State) Pruned(p stdio.Prune) (*PrunedState, error) { + ps := &PrunedState{State: s} + ps.pruner = &stdio.Pruner{State: s.Module + "'s " + s.Name, Prune: p, Say: func(line string) { sayPrune(line) }, + Figures: s.figures, Entries: s.entries, Remove: s.purge} + if err := ps.pruner.Bind(); err != nil { + return nil, err + } + return ps, nil +} + +// figures is the bucket's bytes and its cap, zero for none. +func (s State) figures() (uint64, uint64, error) { + ctx, cancel := context.WithTimeout(context.Background(), stateTimeout) + defer cancel() + kv, err := s.store(ctx) + if err != nil { + return 0, 0, err + } + st, err := kv.Status(ctx) + if err != nil { + return 0, 0, err + } + return st.Bytes(), uint64(max(st.Config().MaxBytes, 0)), nil +} + +// entries is every current value of the bucket with its revision, keeps included, read once. +func (s State) entries() ([]stdio.PruneEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), stateTimeout) + defer cancel() + kv, err := s.store(ctx) + if err != nil { + return nil, err + } + w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes()) + if err != nil { + return nil, err + } + defer w.Stop() + var out []stdio.PruneEntry + for { + select { + case e, open := <-w.Updates(): + if !open { + return nil, errors.New("the read of the current values ended") + } + if e == nil { + return out, nil + } + if e.Operation() == jetstream.KeyValuePut { + out = append(out, stdio.PruneEntry{Key: e.Key(), Revision: e.Revision(), Bytes: stdio.PutBytes(e.Key(), e.Value()), Value: e.Value()}) + } + case <-ctx.Done(): + return nil, ctx.Err() + } + } +} + +// purge removes a key with its past values, so its bytes leave the bucket. +func (s State) purge(key string) error { + ctx, cancel := context.WithTimeout(context.Background(), stateTimeout) + defer cancel() + kv, err := s.store(ctx) + if err != nil { + return err + } + return kv.Purge(ctx, key) +} + +// wrote counts a write and prunes past the mark. +func (s *PrunedState) wrote(key string, value any, rev uint64, err error) (uint64, error) { + if err != nil { + return rev, err + } + data, _ := json.Marshal(value) + if _, err := s.pruner.Wrote(stdio.PutBytes(key, data)); err != nil { + return rev, err + } + return rev, nil +} + +// Put writes a key whole, counted toward the prune. +func (s *PrunedState) Put(key string, value any) (uint64, error) { + rev, err := s.State.Put(key, value) + return s.wrote(key, value, rev, err) +} + +// Create writes a key only when it has none, counted toward the prune. +func (s *PrunedState) Create(key string, value any) (uint64, error) { + rev, err := s.State.Create(key, value) + return s.wrote(key, value, rev, err) +} + +// Update writes a key only when it is still at revision, counted toward the prune. +func (s *PrunedState) Update(key string, value any, revision uint64) (uint64, error) { + rev, err := s.State.Update(key, value, revision) + return s.wrote(key, value, rev, err) +} + +// PruneNow prunes the state to its target now. +func (s *PrunedState) PruneNow() (stdio.Pruned, error) { return s.pruner.PruneNow() } + +// Keep keeps key from the prune until until, written under stdio.KeepKey. +func (s *PrunedState) Keep(key string, until time.Time) error { + if err := stdio.KeepProblem(key, until); err != nil { + return err + } + _, err := s.Put(stdio.KeepKey(key), stdio.Keep{Until: until.UTC()}) + return err +} diff --git a/go/process/prune_test.go b/go/process/prune_test.go new file mode 100644 index 00000000..638980c9 --- /dev/null +++ b/go/process/prune_test.go @@ -0,0 +1,117 @@ +package process + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// pruneKV is a fakeKV with a cap: its bytes are what its current values put (stdio.PutBytes), its Status says them +// with the cap, WatchAll hands every current value, and Purge removes a key. +type pruneKV struct { + *fakeKV + cap int64 + statuss int +} + +type pruneStatus struct { + jetstream.KeyValueStatus + bytes uint64 + cap int64 +} + +func (s pruneStatus) Bytes() uint64 { return s.bytes } +func (s pruneStatus) Config() jetstream.KeyValueConfig { + return jetstream.KeyValueConfig{MaxBytes: s.cap} +} + +func (f *pruneKV) held() uint64 { + f.mu.Lock() + defer f.mu.Unlock() + var n uint64 + for k, v := range f.values { + n += stdio.PutBytes(k, v) + } + return n +} + +func (f *pruneKV) Status(context.Context) (jetstream.KeyValueStatus, error) { + f.statuss++ + return pruneStatus{bytes: f.held(), cap: f.cap}, nil +} + +func (f *pruneKV) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + f.mu.Lock() + defer f.mu.Unlock() + w := &fakeWatcher{updates: make(chan jetstream.KeyValueEntry, len(f.values)+1)} + for k, v := range f.values { + w.updates <- putEntry{fakeEntry{key: k, value: v, rev: f.revs[k]}} + } + w.updates <- nil + return w, nil +} + +func (f *pruneKV) Purge(ctx context.Context, key string, _ ...jetstream.KVDeleteOpt) error { + return f.Delete(ctx, key) +} + +type putEntry struct{ fakeEntry } + +func (putEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut } + +// Design 47 H2 §812 (H-A-380, H-A-386): a process's write that takes its state past prune-mark prunes it, oldest first +// by revision, to prune-target, in one journal line, reading the figures only at the bind and after the prune; a key +// kept under _keep/ survives, and no reader counts the keep. +func TestAProcessWritePastItsMarkPrunesToItsTargetAndAKeptKeySurvives(t *testing.T) { + var lines []string + sayPrune = func(line string) { lines = append(lines, line) } + t.Cleanup(func() { sayPrune = func(string) {} }) + kv := &pruneKV{fakeKV: newFakeKV(), cap: 10_000} + s, err := stateOn(kv).Pruned(stdio.Prune{Mark: 70, Target: 50}) + if err != nil { + t.Fatal(err) + } + value := strings.Repeat("x", 300) + if _, err := s.Put("k00", value); err != nil { + t.Fatal(err) + } + if err := s.Keep("k00", time.Now().Add(time.Hour)); err != nil { + t.Fatal(err) + } + if err := s.Keep("k01", time.Now().Add(-time.Hour)); err == nil { + t.Fatal("a keep until a time past was taken") + } + for i := 1; i < 18; i++ { + if _, err := s.Put(fmt.Sprintf("k%02d", i), value); err != nil { + t.Fatal(err) + } + if len(lines) == 0 && kv.held()*100 > 70*10_000 { + t.Fatalf("the state holds %d of 10000 bytes after k%02d and was not pruned", kv.held(), i) + } + } + if len(lines) != 1 || !strings.Contains(lines[0], "pruned the state mesh-mail's sends") { + t.Fatalf("the prunes said %q, want one line", lines) + } + if kv.statuss != 2 { + t.Fatalf("the figures were read %d times, want 2: at the bind and after the prune", kv.statuss) + } + if held := kv.held(); held*100 > 50*10_000 { + t.Fatalf("the prune left %d of 10000 bytes, above its target of 50 %%", held) + } + keys, err := s.Keys() + if err != nil { + t.Fatal(err) + } + if keys[0] != "k00" || strings.Contains(strings.Join(keys, " "), "_keep") || keys[1] == "k01" || keys[len(keys)-1] != "k17" { + t.Fatalf("after the prune the readers see %v: want k00 kept, the oldest others gone, k17 there, no keep", keys) + } + if e, _ := s.Get(stdio.KeepKey("k00")); e == nil { + t.Fatal("the keep itself was pruned") + } +} diff --git a/go/process/state.go b/go/process/state.go index 6be76193..d923260b 100644 --- a/go/process/state.go +++ b/go/process/state.go @@ -257,7 +257,7 @@ func (s State) put(key string, value any) (uint64, error) { return rev, nil } -// Keys is every key with a current value, sorted; none is an empty list. +// Keys is every key with a current value, sorted; none is an empty list. A keep (stdio.KeepPrefix) is not an entry and is left out. // Keys waits out a bus that answers nothing for a few seconds (busretry): only a request that reached no one is asked again. func (s State) Keys() ([]string, error) { return busretry.Do(s.what("keys"), func() ([]string, error) { return s.keys() }, nil) @@ -277,8 +277,14 @@ func (s State) keys() ([]string, error) { if err != nil { return nil, fmt.Errorf("%s's state %s, keys: %w", s.Module, s.Name, err) } - sort.Strings(keys) - return keys, nil + entries := keys[:0] + for _, k := range keys { + if !stdio.IsKeep(k) { + entries = append(entries, k) + } + } + sort.Strings(entries) + return entries, nil } // watchPattern is a key narrowing as the bus names it: `*` stays one name, `**` is the rest, empty is every key. @@ -334,6 +340,9 @@ func (s State) Watch(key string, handle func(stdio.StateChange) error) error { } } deliver := func(e jetstream.KeyValueEntry, current bool) error { + if stdio.IsKeep(e.Key()) { + return nil // a keep (stdio.KeepPrefix) is not an entry of the state: no watcher hears it + } c := stdio.StateChange{State: s.Name, Key: e.Key(), Op: "put", Revision: e.Revision(), Current: current} if e.Operation() == jetstream.KeyValuePut { c.Value = json.RawMessage(e.Value()) diff --git a/go/prune.go b/go/prune.go new file mode 100644 index 00000000..dbf818d3 --- /dev/null +++ b/go/prune.go @@ -0,0 +1,373 @@ +package stdio + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// A kept store's owner prunes it toward its cap, judged at its own writes (design 47 H2 §812; H-A-380, H-A-386): when +// the fill passes prune-mark of the cap, the oldest keys by their current revision are removed until it is down to +// prune-target, one journal line per prune, with no extra read, no poll and no timer. A key kept by a person +// (`_state_keep`) is never pruned before its until: the keep lives in the pruned state itself under KeepPrefix. + +// The prune's settings and their defaults: prune-mark below the bus module's raise mark of 75 %. +const ( + PruneMarkSetting = "prune-mark" + PruneTargetSetting = "prune-target" + PruneMarkDefault = 70 + PruneTargetDefault = 50 + pruneSettingPercent = "a percentage of the cap between 1 and 100" +) + +// KeepPrefix is the reserved prefix a keep lives under in the state it keeps a key of (H-A-386): `_keep/`, with +// its until. A keep is never pruned and never counted as an entry by the readers (Keys, Page, Watch). +const KeepPrefix = "_keep/" + +// KeepKey is the key the keep of key lives under. +func KeepKey(key string) string { return KeepPrefix + key } + +// IsKeep is whether key is a keep rather than an entry of its state. +func IsKeep(key string) bool { return strings.HasPrefix(key, KeepPrefix) } + +// Keep is a keep's value: the key it keeps is never pruned before Until. +type Keep struct { + Until time.Time `json:"until"` +} + +// SettingPercent is the config value key read as a percentage ("70" or "70%"): def when none is composed or it is +// empty, an error naming the key for a value that is no whole number from 1 to 100. +func SettingPercent(key string, def int) (int, error) { + v, ok := Setting(key) + if v = strings.TrimSpace(v); !ok || v == "" { + return def, nil + } + n, err := strconv.Atoi(strings.TrimSpace(strings.TrimSuffix(v, "%"))) + if err != nil || n < 1 || n > 100 { + return 0, fmt.Errorf("the setting %s (%s) holds %q, which is not %s", key, SettingWord(key), v, pruneSettingPercent) + } + return n, nil +} + +// Prune is where a prune starts and where it ends, in percent of the cap. +type Prune struct { + Mark, Target int +} + +// PruneSettings is the module's prune-mark and prune-target, read through the settings reader with their defaults; a +// target not below the mark is refused, naming both. +func PruneSettings() (Prune, error) { + mark, err := SettingPercent(PruneMarkSetting, PruneMarkDefault) + if err != nil { + return Prune{}, err + } + target, err := SettingPercent(PruneTargetSetting, PruneTargetDefault) + if err != nil { + return Prune{}, err + } + if target >= mark { + return Prune{}, fmt.Errorf("the setting %s (%d %%) is not below %s (%d %%): a prune would never end below its mark", + PruneTargetSetting, target, PruneMarkSetting, mark) + } + return Prune{Mark: mark, Target: target}, nil +} + +// Passed is whether bytes of a cap of maxBytes passes the mark; an entry with no cap never does. +func (p Prune) Passed(bytes, maxBytes uint64) bool { + return maxBytes > 0 && bytes*100 > uint64(p.Mark)*maxBytes +} + +// PruneEntry is one current value of a pruned state as the prune reads it: its key, its current revision and the +// bytes it holds on the bus. +type PruneEntry struct { + Key string + Revision uint64 + Bytes uint64 + Value []byte +} + +// Choose is the keys a prune removes, oldest first by their current revision, until bytes is down to the target of +// maxBytes, and how many it skipped for a keep whose until is still to come. A keep itself is never chosen; a keep that +// cannot be read keeps its key (a prune never removes what a person may have kept). +func (p Prune) Choose(entries []PruneEntry, now time.Time, bytes, maxBytes uint64) (drop []string, kept int) { + keeps := map[string]bool{} + for _, e := range entries { + if !IsKeep(e.Key) { + continue + } + var k Keep + if err := json.Unmarshal(e.Value, &k); err != nil || k.Until.After(now) { + keeps[strings.TrimPrefix(e.Key, KeepPrefix)] = true + } + } + sorted := append([]PruneEntry(nil), entries...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Revision < sorted[j].Revision }) + target := uint64(p.Target) * maxBytes / 100 + for _, e := range sorted { + if bytes <= target { + break + } + if IsKeep(e.Key) { + continue + } + if keeps[e.Key] { + kept++ + continue + } + drop = append(drop, e.Key) + bytes -= min(bytes, e.Bytes) + } + return drop, kept +} + +// Pruned is what one prune did: the state, how many keys it removed and skipped for a keep, and its fill before and +// after against the cap. +type Pruned struct { + State string `json:"state"` + Removed int `json:"removed"` + Kept int `json:"kept"` + From uint64 `json:"from-bytes"` + To uint64 `json:"to-bytes"` + MaxBytes uint64 `json:"max-bytes"` + Mark int `json:"mark"` + Target int `json:"target"` +} + +// Line is the prune's one journal line. +func (r Pruned) Line() string { + return fmt.Sprintf("pruned the state %s: removed %d keys oldest first (%d kept by a keep), from %d to %d bytes of its cap of %d (mark %d %%, target %d %%)", + r.State, r.Removed, r.Kept, r.From, r.To, r.MaxBytes, r.Mark, r.Target) +} + +// Prunable is a state its owner prunes: the generic tools `_state_prune_now` and `_state_keep` act on it. +type Prunable interface { + // PruneNow prunes the state to its target now, whatever its fill. + PruneNow() (Pruned, error) + // Keep keeps key from the prune until the time given. + Keep(key string, until time.Time) error +} + +// Pruner is the prune helper of one state, the same for a process and a bundle: Entries reads the state's current +// values (keeps included) once per prune, Remove removes one key, Figures reads the entry's bytes and cap. A process +// gives Wrote the bytes each write puts and Figures is read at Bind and after each prune; a bundle gives Filled the +// fill its write's answer carries. +type Pruner struct { + State string + Prune Prune + Entries func() ([]PruneEntry, error) + Remove func(key string) error + Figures func() (bytes, maxBytes uint64, err error) + // Say writes the prune's journal line. + Say func(line string) + Now func() time.Time + + mu sync.Mutex + bytes, limit uint64 +} + +// Bind reads the entry's figures once, the base of the running sum Wrote keeps. +func (p *Pruner) Bind() error { + bytes, limit, err := p.Figures() + if err != nil { + return fmt.Errorf("the state %s's fill could not be read for its prune: %w", p.State, err) + } + p.mu.Lock() + p.bytes, p.limit = bytes, limit + p.mu.Unlock() + return nil +} + +// Wrote adds the bytes a write put to the running sum and prunes when it passes the mark: the answer is the prune it +// ran, or nil. +func (p *Pruner) Wrote(n uint64) (*Pruned, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.bytes += n + if !p.Prune.Passed(p.bytes, p.limit) { + return nil, nil + } + r, err := p.pruneLocked(p.bytes, p.limit, true) + return &r, err +} + +// Filled judges the fill a write's answer carries and prunes when it passes the mark: the answer is the prune it ran, +// or nil. +func (p *Pruner) Filled(f buscalls.Fill) (*Pruned, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.bytes, p.limit = f.Bytes, f.MaxBytes + if !p.Prune.Passed(f.Bytes, f.MaxBytes) { + return nil, nil + } + r, err := p.pruneLocked(f.Bytes, f.MaxBytes, false) + return &r, err +} + +// PruneNow reads the figures and prunes to the target now. +func (p *Pruner) PruneNow() (Pruned, error) { + bytes, limit, err := p.Figures() + if err != nil { + return Pruned{State: p.State}, fmt.Errorf("the state %s's fill could not be read for its prune: %w", p.State, err) + } + p.mu.Lock() + defer p.mu.Unlock() + if limit == 0 { + return Pruned{State: p.State}, fmt.Errorf("the state %s names no cap, so it has no target to prune to", p.State) + } + return p.pruneLocked(bytes, limit, true) +} + +// pruneLocked removes the oldest keys from bytes down to the target of limit; reread reads the figures after it, the +// new base of the running sum. +func (p *Pruner) pruneLocked(bytes, limit uint64, reread bool) (Pruned, error) { + r := Pruned{State: p.State, From: bytes, MaxBytes: limit, Mark: p.Prune.Mark, Target: p.Prune.Target} + entries, err := p.Entries() + if err != nil { + return r, fmt.Errorf("the state %s could not be read for its prune: %w", p.State, err) + } + // The running sum is an upper bound (an overwrite's past value is the bucket's history to bound), so what the + // current values hold is the start when it is lower. + var held uint64 + for _, e := range entries { + held += e.Bytes + } + start := min(bytes, held) + now := time.Now + if p.Now != nil { + now = p.Now + } + drop, kept := p.Prune.Choose(entries, now(), start, limit) + r.Kept, r.To = kept, start + sizes := map[string]uint64{} + for _, e := range entries { + sizes[e.Key] = e.Bytes + } + var errs []error + for _, key := range drop { + if err := p.Remove(key); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", key, err)) + continue + } + r.Removed++ + r.To -= min(r.To, sizes[key]) + } + if reread { + if b, l, err := p.Figures(); err == nil { + r.To, r.MaxBytes = b, l + } else { + errs = append(errs, fmt.Errorf("the fill after the prune: %w", err)) + } + } + p.bytes, p.limit = r.To, r.MaxBytes + if p.Say != nil { + p.Say(r.Line()) + } + if len(errs) > 0 { + return r, fmt.Errorf("the prune of the state %s: %w", p.State, errors.Join(errs...)) + } + return r, nil +} + +// PutOverhead is what a write puts on the bus beside its key and its value, an upper bound: the bucket's subject +// prefix, the message's headers and its record. +const PutOverhead = 96 + +// PutBytes is what a write of value under key puts on the bus, as the prune helper counts it. +func PutBytes(key string, value []byte) uint64 { return uint64(len(key) + len(value) + PutOverhead) } + +// PrunedState is a state a bundle prunes at its writes (§812): Put judges the fill the runtime's put answer carries +// (DMC-14, H-A-386) and prunes when it passes the mark. Its other calls are the state's own. +type PrunedState struct { + KeptState + pruner *Pruner + unsaid sync.Once +} + +// Pruned is the state pruned by p at its writes. +func (s KeptState) Pruned(p Prune) *PrunedState { + ps := &PrunedState{KeptState: s} + ps.pruner = &Pruner{State: s.name, Prune: p, Say: func(line string) { sayf("%s", line) }, + Remove: s.Delete, + Figures: func() (uint64, uint64, error) { + i, err := s.Info() + if err != nil { + return 0, 0, err + } + return i.Bytes, uint64(max(i.MaxBytes, 0)), nil + }, + Entries: func() ([]PruneEntry, error) { + var out []PruneEntry + cursor := "" + for { + page, err := s.page(cursor, 768<<10, stateTimeout/2) + if err != nil { + return nil, err + } + for _, e := range page.Entries { + out = append(out, PruneEntry{Key: e.Key, Revision: e.Revision, Bytes: PutBytes(e.Key, e.Value), Value: e.Value}) + } + if page.Next == "" || page.Next == cursor { + return out, nil + } + cursor = page.Next + } + }} + return ps +} + +// Put writes a key whole and returns the revision written; the fill its answer carries is judged, and past the mark +// the state is pruned to its target before Put returns. A runtime whose answer carries no fill is said once, as a +// state that is never pruned. +func (s *PrunedState) Put(key string, value any) (uint64, error) { + raw, err := s.ask("put", map[string]any{"key": key, "value": value}) + if err != nil { + return 0, err + } + var put struct { + Revision uint64 `json:"revision"` + Fill *buscalls.Fill `json:"fill"` + } + if err := json.Unmarshal(raw, &put); err != nil { + return 0, err + } + if put.Fill == nil { + s.unsaid.Do(func() { + sayf("the state %s is never pruned: the runtime's put answer carries no fill (design 47 H2 §812)", s.name) + }) + return put.Revision, nil + } + if _, err := s.pruner.Filled(*put.Fill); err != nil { + return put.Revision, err + } + return put.Revision, nil +} + +// PruneNow prunes the state to its target now. +func (s *PrunedState) PruneNow() (Pruned, error) { return s.pruner.PruneNow() } + +// Keep keeps key from the prune until until, written under KeepKey. +func (s *PrunedState) Keep(key string, until time.Time) error { + if err := KeepProblem(key, until); err != nil { + return err + } + _, err := s.Put(KeepKey(key), Keep{Until: until.UTC()}) + return err +} + +// KeepProblem is why key cannot be kept until until, or nil: a process and a bundle check a keep alike. +func KeepProblem(key string, until time.Time) error { + switch { + case key == "" || IsKeep(key): + return fmt.Errorf("%q is not a key a keep keeps", key) + case !until.After(time.Now()): + return fmt.Errorf("a keep until %s keeps nothing: until is a time still to come", until.Format(time.RFC3339)) + } + return nil +} diff --git a/go/prune_test.go b/go/prune_test.go new file mode 100644 index 00000000..13907e01 --- /dev/null +++ b/go/prune_test.go @@ -0,0 +1,177 @@ +package stdio + +import ( + "bufio" + "encoding/json" + "io" + "sort" + "strings" + "sync" + "testing" + "time" +) + +// Design 47 H2 §812 (H-A-380, H-A-386): prune-mark and prune-target are read through the settings reader with their +// defaults, and a bundle's Put judges the fill the runtime's put answer carries: past 70 % the oldest keys by revision +// are deleted to 50 %, a key kept under _keep/ survives, and no reader (Keys, Page) counts the keep. +func TestABundleWritePastItsMarkPrunesToItsTargetAndAKeptKeySurvives(t *testing.T) { + t.Run("settings", func(t *testing.T) { + if p, err := PruneSettings(); err != nil || p != (Prune{Mark: 70, Target: 50}) { + t.Fatalf("the defaults are %+v, %v", p, err) + } + t.Setenv(SettingWord(PruneMarkSetting), "80%") + t.Setenv(SettingWord(PruneTargetSetting), "60") + if p, err := PruneSettings(); err != nil || p != (Prune{Mark: 80, Target: 60}) { + t.Fatalf("the settings read %+v, %v", p, err) + } + t.Setenv(SettingWord(PruneTargetSetting), "90") + if _, err := PruneSettings(); err == nil || !strings.Contains(err.Error(), "prune-target") { + t.Fatalf("a target above the mark was taken: %v", err) + } + t.Setenv(SettingWord(PruneTargetSetting), "half") + if _, err := PruneSettings(); err == nil || !strings.Contains(err.Error(), "MESH_CONFIG_PRUNE_TARGET") { + t.Fatalf("a target that is no percentage was taken: %v", err) + } + }) + + servesAlone(t) + t.Setenv(ServedAs, "bridge") + inR, inW := io.Pipe() + outR, outW := io.Pipe() + pruned := State("performed").Pruned(Prune{Mark: 70, Target: 50}) + go func() { + _ = ServeOn("", []Tool{ + {Name: "write", Run: func(a map[string]any) (any, error) { return pruned.Put(a["key"].(string), a["value"]) }}, + {Name: "keep", Run: func(a map[string]any) (any, error) { + return nil, pruned.Keep(a["key"].(string), time.Now().Add(time.Hour)) + }}, + {Name: "keys", Run: func(map[string]any) (any, error) { return State("performed").Keys() }}, + {Name: "page", Run: func(map[string]any) (any, error) { return State("performed").Page("", 1<<20, time.Second) }}, + }, inR, outW) + }() + // The runtime: a state of a cap of 10000 bytes whose fill is what its current values put. + const limit = 10_000 + var mu sync.Mutex + values, revs, next := map[string]json.RawMessage{}, map[string]uint64{}, uint64(0) + held := func() uint64 { + var n uint64 + for k, v := range values { + n += PutBytes(k, v) + } + return n + } + var mostHeld uint64 + answers := make(chan map[string]any, 1) + send := func(m map[string]any) { b, _ := json.Marshal(m); _, _ = inW.Write(append(b, '\n')) } + go func() { + lines := bufio.NewScanner(outR) + lines.Buffer(make([]byte, 1<<20), 1<<22) + for lines.Scan() { + var m struct { + ID any `json:"id"` + Method string `json:"method"` + Params map[string]any `json:"params"` + Result any `json:"result"` + Error any `json:"error"` + } + _ = json.Unmarshal(lines.Bytes(), &m) + if m.Method == "" { + answers <- map[string]any{"result": m.Result, "error": m.Error} + continue + } + mu.Lock() + var result any = map[string]any{} + key, _ := m.Params["key"].(string) + switch m.Method { + case "mesh/state.put": + v, _ := json.Marshal(m.Params["value"]) + next++ + values[key], revs[key] = v, next + mostHeld = max(mostHeld, held()) + result = map[string]any{"revision": next, "fill": map[string]any{"count": len(values), "bytes": held(), "max_bytes": limit}} + case "mesh/state.delete": + delete(values, key) + case "mesh/state.keys": + keys := []string{} + for k := range values { + keys = append(keys, k) + } + sort.Strings(keys) + result = keys + case "mesh/state.page": + var entries []StateEntry + for k, v := range values { + entries = append(entries, StateEntry{Key: k, Value: v, Revision: revs[k]}) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Revision < entries[j].Revision }) + result = StatePage{Entries: entries} + } + mu.Unlock() + if m.ID != nil { + send(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": result}) + } + } + }() + call := func(name string, args map[string]any) any { + t.Helper() + send(map[string]any{"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args}}) + select { + case a := <-answers: + r, _ := a["result"].(map[string]any) + if a["error"] != nil || r == nil || r["isError"] == true { + t.Fatalf("%s answered %v %v", name, a["error"], r) + } + var out any + _ = json.Unmarshal([]byte(r["content"].([]any)[0].(map[string]any)["text"].(string)), &out) + return out + case <-time.After(3 * time.Second): + t.Fatalf("%s answered nothing", name) + return nil + } + } + value := strings.Repeat("x", 300) + call("write", map[string]any{"key": "k00", "value": value}) + call("keep", map[string]any{"key": "k00"}) + for _, k := range []string{"k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18", "k19"} { + call("write", map[string]any{"key": k, "value": value}) + mu.Lock() + over := held()*100 > 70*limit + mu.Unlock() + if over { + t.Fatalf("the state holds more than 70 %% of its cap after %s: no prune", k) + } + } + mu.Lock() + _, k00 := values["k00"] + _, k01 := values["k01"] + _, k19 := values["k19"] + _, keep := values[KeepKey("k00")] + after := held() + mu.Unlock() + if mostHeld*100 <= 70*limit { + t.Fatalf("the test never took the state past its mark (%d bytes at most)", mostHeld) + } + if !k00 || !keep || k01 || !k19 { + t.Fatalf("after the prunes k00 %v, its keep %v, k01 %v, k19 %v: want the kept key and its keep, the oldest other gone, the newest there", k00, keep, k01, k19) + } + if after*100 > 70*limit { + t.Fatalf("the state holds %d of %d bytes", after, limit) + } + if keys := call("keys", nil); strings.Contains(strings.Join(strs(keys), " "), KeepPrefix) { + t.Fatalf("Keys counted the keep: %v", keys) + } + page := call("page", nil).(map[string]any) + for _, e := range page["entries"].([]any) { + if IsKeep(e.(map[string]any)["key"].(string)) { + t.Fatalf("Page counted the keep: %v", e) + } + } +} + +func strs(v any) []string { + var out []string + for _, s := range v.([]any) { + out = append(out, s.(string)) + } + return out +} diff --git a/go/state.go b/go/state.go index 103a7e62..bfcd6fe9 100644 --- a/go/state.go +++ b/go/state.go @@ -122,9 +122,22 @@ type StatePage struct { // start), as many as fit maxBytes (keys and values; the runtime holds a page to 768 KB at most) or the time budget // allows, at least one: runtime verb `mesh/state.page`, read from an ordered consumer after the cursor's sequence, so a // state is read once however many pages it takes (design 47 H2 §779). A key written again between two pages comes -// again in a later page with SeenAgain set. Hand Next back for the next page. A runtime that does not answer it gives an +// again in a later page with SeenAgain set. A keep (KeepPrefix) is not an entry and is left out. Hand Next back for the next page. A runtime that does not answer it gives an // error satisfying errors.Is(err, errors.ErrUnsupported), and the reader asks Get per key instead. func (s KeptState) Page(cursor string, maxBytes int, budget time.Duration) (StatePage, error) { + p, err := s.page(cursor, maxBytes, budget) + kept := p.Entries[:0] + for _, e := range p.Entries { + if !IsKeep(e.Key) { + kept = append(kept, e) + } + } + p.Entries = kept + return p, err +} + +// page is one page as the runtime answers it, keeps included (KeepPrefix). +func (s KeptState) page(cursor string, maxBytes int, budget time.Duration) (StatePage, error) { raw, err := s.ask("page", map[string]any{"cursor": cursor, "max_bytes": maxBytes, "budget_ms": budget.Milliseconds()}) if err != nil { return StatePage{}, err @@ -264,14 +277,23 @@ func (s KeptState) Increment(key string, attempts int) (uint64, error) { return 0, fmt.Errorf("%s was written by another writer first %d times: %w", key, attempts, ErrChanged) } -// Keys is every key with a current value, sorted. +// Keys is every key with a current value, sorted; a keep (KeepPrefix) is not an entry and is left out. func (s KeptState) Keys() ([]string, error) { raw, err := s.ask("keys", map[string]any{}) if err != nil { return nil, err } - var keys []string - return keys, json.Unmarshal(raw, &keys) + var all []string + if err := json.Unmarshal(raw, &all); err != nil { + return nil, err + } + keys := all[:0] + for _, k := range all { + if !IsKeep(k) { + keys = append(keys, k) + } + } + return keys, nil } // Watch hands handle every current value — none that is deleted — and then every change, in order, @@ -286,7 +308,14 @@ func (s KeptState) Watch(key string, handle func(StateChange) error) error { return err } id := c.newID() - f := newFollower("the watch "+id+" of the state "+s.name, handle, func(ch StateChange) string { return ch.Op + " " + ch.Key }) + // A keep (KeepPrefix) is not an entry of the state: no watcher hears it. + entries := func(ch StateChange) error { + if IsKeep(ch.Key) { + return nil + } + return handle(ch) + } + f := newFollower("the watch "+id+" of the state "+s.name, entries, func(ch StateChange) string { return ch.Op + " " + ch.Key }) c.mu.Lock() c.watches[id] = f c.mu.Unlock() diff --git a/go/stateobjects/prune.go b/go/stateobjects/prune.go new file mode 100644 index 00000000..c4e48db5 --- /dev/null +++ b/go/stateobjects/prune.go @@ -0,0 +1,109 @@ +package stateobjects + +import ( + "errors" + "fmt" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// Pruning is one state a module prunes toward its cap (design 47 H2 §812): its local name in the manifest, and the +// pruned state (a process's process.PrunedState or a bundle's stdio.PrunedState). +type Pruning struct { + Name string + State stdio.Prunable +} + +// PruneTools are the two generic tools of a module that prunes its states (design 47 H2 §812; H-A-380 (3)), named for +// the module as Tools names its own: `_state_prune_now` (entry, why) prunes an entry to its target now, and +// `_state_keep` (entry, key, until, why) keeps a key from the prune until a time. Each is written in the hand-act +// log before anything is done, and refused when it cannot be. +func PruneTools(module string, entries []Pruning, acts views.HandActs) []stdio.Tool { + if len(entries) == 0 { + return nil + } + var names []string + for _, e := range entries { + names = append(names, e.Name) + } + entryArg := map[string]any{"type": "string", "description": "the state pruned: its local name in the manifest, one of " + strings.Join(names, ", ")} + whyArg := map[string]any{"type": "string", "description": "why, in plain words"} + pick := func(args map[string]any) (Pruning, string, error) { + why, _ := args["why"].(string) + if why = strings.TrimSpace(why); why == "" { + return Pruning{}, "", errors.New("a prune or a keep names why: nothing was done") + } + name, _ := args["entry"].(string) + name = strings.TrimSpace(name) + for _, e := range entries { + if e.Name == name || (name == "" && len(entries) == 1) { + return e, why, nil + } + } + return Pruning{}, "", fmt.Errorf("entry names one of %s, not %q: nothing was done", strings.Join(names, ", "), name) + } + recorded := func(tool, what, why, cause string) error { + if acts == nil { + return errors.New("no way to record a hand act is given, so nothing was done") + } + if err := acts.RecordHandAct(views.HandAct{What: what, Why: why, Cause: cause, Verb: tool}); err != nil { + return fmt.Errorf("the hand act was not recorded, so nothing was done: %w", err) + } + return nil + } + pruneNow, keep := module+"_state_prune_now", module+"_state_keep" + return []stdio.Tool{ + { + Name: pruneNow, + Description: "Prune one of " + module + "'s states to its prune-target now, oldest keys first, whatever its fill (design 47 §812). " + + "A key kept by " + keep + " stays. Written in the hand-act log first; needs why. Answers the keys removed and the fill before and after.", + Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{"entry": entryArg, "why": whyArg}}, + Run: func(args map[string]any) (any, error) { + e, why, err := pick(args) + if err != nil { + return nil, err + } + if err := recorded(pruneNow, "pruned "+module+"'s state "+e.Name+" to its target now", why, "state-pruned"); err != nil { + return nil, err + } + return e.State.PruneNow() + }, + }, + { + Name: keep, + Description: "Keep one key of one of " + module + "'s states from the prune until a time (design 47 §812): the keep is kept in the state " + + "itself under " + stdio.KeepPrefix + ", never pruned and never counted as an entry. Written in the hand-act log first; needs key, until and why.", + Input: map[string]any{"type": "object", "required": []string{"key", "until", "why"}, "properties": map[string]any{ + "entry": entryArg, + "key": map[string]any{"type": "string", "description": "the key kept"}, + "until": map[string]any{"type": "string", "description": "until when, an RFC 3339 time still to come"}, + "why": whyArg, + }}, + Run: func(args map[string]any) (any, error) { + e, why, err := pick(args) + if err != nil { + return nil, err + } + key, _ := args["key"].(string) + text, _ := args["until"].(string) + until, err := time.Parse(time.RFC3339, strings.TrimSpace(text)) + if err != nil { + return nil, fmt.Errorf("until %q is not an RFC 3339 time: nothing was kept", text) + } + if err := stdio.KeepProblem(key, until); err != nil { + return nil, fmt.Errorf("%w: nothing was kept", err) + } + if err := recorded(keep, "kept the key "+key+" of "+module+"'s state "+e.Name+" from the prune until "+until.UTC().Format(time.RFC3339), why, "state-kept"); err != nil { + return nil, err + } + if err := e.State.Keep(key, until); err != nil { + return nil, err + } + return map[string]any{"entry": e.Name, "key": key, "kept-as": stdio.KeepKey(key), "until": until.UTC()}, nil + }, + }, + } +} diff --git a/go/stateobjects/prune_test.go b/go/stateobjects/prune_test.go new file mode 100644 index 00000000..77c1bdda --- /dev/null +++ b/go/stateobjects/prune_test.go @@ -0,0 +1,59 @@ +package stateobjects + +import ( + "errors" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +type prunable struct { + pruned int + kept map[string]time.Time +} + +func (p *prunable) PruneNow() (stdio.Pruned, error) { + p.pruned++ + return stdio.Pruned{State: "performed"}, nil +} +func (p *prunable) Keep(key string, until time.Time) error { + p.kept[key] = until + return nil +} + +// Design 47 H2 §812 (H-A-380 (3)): `_state_prune_now` (entry, why) and `_state_keep` (entry, key, until, +// why) are written in the hand-act log before they act, and do nothing without why or when the act is not recorded. +func subThePruneToolsPruneAndKeepAsHandActs(t *testing.T) { + p := &prunable{kept: map[string]time.Time{}} + a := &acts{} + tools := map[string]stdio.Tool{} + for _, tool := range PruneTools("mesh-forge-bridge", []Pruning{{Name: "performed", State: p}}, a) { + tools[tool.Name] = tool + } + now, keep := tools["mesh-forge-bridge_state_prune_now"], tools["mesh-forge-bridge_state_keep"] + if now.Run == nil || keep.Run == nil { + t.Fatalf("the tools are %v", tools) + } + if _, err := now.Run(map[string]any{"entry": "performed"}); err == nil || p.pruned != 0 { + t.Fatalf("a prune without why ran: %v", err) + } + if _, err := now.Run(map[string]any{"entry": "performed", "why": "full before a delivery"}); err != nil || p.pruned != 1 { + t.Fatalf("prune now: %v, pruned %d", err, p.pruned) + } + until := time.Now().Add(24 * time.Hour).UTC().Format(time.RFC3339) + if _, err := keep.Run(map[string]any{"entry": "performed", "key": "pr.42", "until": until, "why": "the proof reads it"}); err != nil || p.kept["pr.42"].IsZero() { + t.Fatalf("keep: %v, kept %v", err, p.kept) + } + if _, err := keep.Run(map[string]any{"key": "pr.43", "until": "2000-01-01T00:00:00Z", "why": "late"}); err == nil || !p.kept["pr.43"].IsZero() { + t.Fatalf("a keep until a time past was taken: %v", err) + } + if len(a.done) != 2 || a.done[0].Verb != "mesh-forge-bridge_state_prune_now" || !strings.Contains(a.done[1].What, "pr.42") || a.done[1].Why != "the proof reads it" { + t.Fatalf("the hand acts are %+v", a.done) + } + a.err = errors.New("the hand-act log refused") + if _, err := now.Run(map[string]any{"why": "again"}); err == nil || p.pruned != 1 { + t.Fatalf("a prune not recorded ran: %v", err) + } +} diff --git a/go/stateobjects/ref_test.go b/go/stateobjects/ref_test.go index b8f29382..181f5e3f 100644 --- a/go/stateobjects/ref_test.go +++ b/go/stateobjects/ref_test.go @@ -40,4 +40,5 @@ func TestABodyIsAReferenceOnlyWhenItIsExactlyOne(t *testing.T) { t.Run("ValuesOfAnySizeThroughTheRuntime", subValuesOfAnySizeThroughTheRuntime) t.Run("AWriteLostLeavesNoObjectBehind", subAWriteLostLeavesNoObjectBehind) t.Run("AValueLargerThanTheStoreTakesIsRefused", subAValueLargerThanTheStoreTakesIsRefused) + t.Run("ThePruneToolsPruneAndKeepAsHandActs", subThePruneToolsPruneAndKeepAsHandActs) }