Files
mesh-sdk/go/prune.go
T
jschoubben 484e2bba3c S11 prune helper and tools: a kept store is pruned toward its cap at its own writes
A process judges a running sum of what its writes put against the figures
read at the bind and after each prune; a bundle judges the fill the
runtime's put answer carries. Past prune-mark (70 %) the oldest keys by
revision go down to prune-target (50 %), in one journal line; a key kept
under _keep/<key> stays, and no reader counts the keep. The generic tools
<module>_state_prune_now and <module>_state_keep act through the hand-act log.

Builds: design 47 H2 §812, H-A-380, H-A-386
2026-11-22 00:17:36 +01:00

374 lines
12 KiB
Go

package stdio
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/buscalls"
)
// A kept store's owner prunes it toward its cap, judged at its own writes (design 47 H2 §812; H-A-380, H-A-386): when
// the fill passes prune-mark of the cap, the oldest keys by their current revision are removed until it is down to
// prune-target, one journal line per prune, with no extra read, no poll and no timer. A key kept by a person
// (`<module>_state_keep`) is never pruned before its until: the keep lives in the pruned state itself under KeepPrefix.
// The prune's settings and their defaults: prune-mark below the bus module's raise mark of 75 %.
const (
PruneMarkSetting = "prune-mark"
PruneTargetSetting = "prune-target"
PruneMarkDefault = 70
PruneTargetDefault = 50
pruneSettingPercent = "a percentage of the cap between 1 and 100"
)
// KeepPrefix is the reserved prefix a keep lives under in the state it keeps a key of (H-A-386): `_keep/<key>`, with
// its until. A keep is never pruned and never counted as an entry by the readers (Keys, Page, Watch).
const KeepPrefix = "_keep/"
// KeepKey is the key the keep of key lives under.
func KeepKey(key string) string { return KeepPrefix + key }
// IsKeep is whether key is a keep rather than an entry of its state.
func IsKeep(key string) bool { return strings.HasPrefix(key, KeepPrefix) }
// Keep is a keep's value: the key it keeps is never pruned before Until.
type Keep struct {
Until time.Time `json:"until"`
}
// SettingPercent is the config value key read as a percentage ("70" or "70%"): def when none is composed or it is
// empty, an error naming the key for a value that is no whole number from 1 to 100.
func SettingPercent(key string, def int) (int, error) {
v, ok := Setting(key)
if v = strings.TrimSpace(v); !ok || v == "" {
return def, nil
}
n, err := strconv.Atoi(strings.TrimSpace(strings.TrimSuffix(v, "%")))
if err != nil || n < 1 || n > 100 {
return 0, fmt.Errorf("the setting %s (%s) holds %q, which is not %s", key, SettingWord(key), v, pruneSettingPercent)
}
return n, nil
}
// Prune is where a prune starts and where it ends, in percent of the cap.
type Prune struct {
Mark, Target int
}
// PruneSettings is the module's prune-mark and prune-target, read through the settings reader with their defaults; a
// target not below the mark is refused, naming both.
func PruneSettings() (Prune, error) {
mark, err := SettingPercent(PruneMarkSetting, PruneMarkDefault)
if err != nil {
return Prune{}, err
}
target, err := SettingPercent(PruneTargetSetting, PruneTargetDefault)
if err != nil {
return Prune{}, err
}
if target >= mark {
return Prune{}, fmt.Errorf("the setting %s (%d %%) is not below %s (%d %%): a prune would never end below its mark",
PruneTargetSetting, target, PruneMarkSetting, mark)
}
return Prune{Mark: mark, Target: target}, nil
}
// Passed is whether bytes of a cap of maxBytes passes the mark; an entry with no cap never does.
func (p Prune) Passed(bytes, maxBytes uint64) bool {
return maxBytes > 0 && bytes*100 > uint64(p.Mark)*maxBytes
}
// PruneEntry is one current value of a pruned state as the prune reads it: its key, its current revision and the
// bytes it holds on the bus.
type PruneEntry struct {
Key string
Revision uint64
Bytes uint64
Value []byte
}
// Choose is the keys a prune removes, oldest first by their current revision, until bytes is down to the target of
// maxBytes, and how many it skipped for a keep whose until is still to come. A keep itself is never chosen; a keep that
// cannot be read keeps its key (a prune never removes what a person may have kept).
func (p Prune) Choose(entries []PruneEntry, now time.Time, bytes, maxBytes uint64) (drop []string, kept int) {
keeps := map[string]bool{}
for _, e := range entries {
if !IsKeep(e.Key) {
continue
}
var k Keep
if err := json.Unmarshal(e.Value, &k); err != nil || k.Until.After(now) {
keeps[strings.TrimPrefix(e.Key, KeepPrefix)] = true
}
}
sorted := append([]PruneEntry(nil), entries...)
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Revision < sorted[j].Revision })
target := uint64(p.Target) * maxBytes / 100
for _, e := range sorted {
if bytes <= target {
break
}
if IsKeep(e.Key) {
continue
}
if keeps[e.Key] {
kept++
continue
}
drop = append(drop, e.Key)
bytes -= min(bytes, e.Bytes)
}
return drop, kept
}
// Pruned is what one prune did: the state, how many keys it removed and skipped for a keep, and its fill before and
// after against the cap.
type Pruned struct {
State string `json:"state"`
Removed int `json:"removed"`
Kept int `json:"kept"`
From uint64 `json:"from-bytes"`
To uint64 `json:"to-bytes"`
MaxBytes uint64 `json:"max-bytes"`
Mark int `json:"mark"`
Target int `json:"target"`
}
// Line is the prune's one journal line.
func (r Pruned) Line() string {
return fmt.Sprintf("pruned the state %s: removed %d keys oldest first (%d kept by a keep), from %d to %d bytes of its cap of %d (mark %d %%, target %d %%)",
r.State, r.Removed, r.Kept, r.From, r.To, r.MaxBytes, r.Mark, r.Target)
}
// Prunable is a state its owner prunes: the generic tools `<module>_state_prune_now` and `<module>_state_keep` act on it.
type Prunable interface {
// PruneNow prunes the state to its target now, whatever its fill.
PruneNow() (Pruned, error)
// Keep keeps key from the prune until the time given.
Keep(key string, until time.Time) error
}
// Pruner is the prune helper of one state, the same for a process and a bundle: Entries reads the state's current
// values (keeps included) once per prune, Remove removes one key, Figures reads the entry's bytes and cap. A process
// gives Wrote the bytes each write puts and Figures is read at Bind and after each prune; a bundle gives Filled the
// fill its write's answer carries.
type Pruner struct {
State string
Prune Prune
Entries func() ([]PruneEntry, error)
Remove func(key string) error
Figures func() (bytes, maxBytes uint64, err error)
// Say writes the prune's journal line.
Say func(line string)
Now func() time.Time
mu sync.Mutex
bytes, limit uint64
}
// Bind reads the entry's figures once, the base of the running sum Wrote keeps.
func (p *Pruner) Bind() error {
bytes, limit, err := p.Figures()
if err != nil {
return fmt.Errorf("the state %s's fill could not be read for its prune: %w", p.State, err)
}
p.mu.Lock()
p.bytes, p.limit = bytes, limit
p.mu.Unlock()
return nil
}
// Wrote adds the bytes a write put to the running sum and prunes when it passes the mark: the answer is the prune it
// ran, or nil.
func (p *Pruner) Wrote(n uint64) (*Pruned, error) {
p.mu.Lock()
defer p.mu.Unlock()
p.bytes += n
if !p.Prune.Passed(p.bytes, p.limit) {
return nil, nil
}
r, err := p.pruneLocked(p.bytes, p.limit, true)
return &r, err
}
// Filled judges the fill a write's answer carries and prunes when it passes the mark: the answer is the prune it ran,
// or nil.
func (p *Pruner) Filled(f buscalls.Fill) (*Pruned, error) {
p.mu.Lock()
defer p.mu.Unlock()
p.bytes, p.limit = f.Bytes, f.MaxBytes
if !p.Prune.Passed(f.Bytes, f.MaxBytes) {
return nil, nil
}
r, err := p.pruneLocked(f.Bytes, f.MaxBytes, false)
return &r, err
}
// PruneNow reads the figures and prunes to the target now.
func (p *Pruner) PruneNow() (Pruned, error) {
bytes, limit, err := p.Figures()
if err != nil {
return Pruned{State: p.State}, fmt.Errorf("the state %s's fill could not be read for its prune: %w", p.State, err)
}
p.mu.Lock()
defer p.mu.Unlock()
if limit == 0 {
return Pruned{State: p.State}, fmt.Errorf("the state %s names no cap, so it has no target to prune to", p.State)
}
return p.pruneLocked(bytes, limit, true)
}
// pruneLocked removes the oldest keys from bytes down to the target of limit; reread reads the figures after it, the
// new base of the running sum.
func (p *Pruner) pruneLocked(bytes, limit uint64, reread bool) (Pruned, error) {
r := Pruned{State: p.State, From: bytes, MaxBytes: limit, Mark: p.Prune.Mark, Target: p.Prune.Target}
entries, err := p.Entries()
if err != nil {
return r, fmt.Errorf("the state %s could not be read for its prune: %w", p.State, err)
}
// The running sum is an upper bound (an overwrite's past value is the bucket's history to bound), so what the
// current values hold is the start when it is lower.
var held uint64
for _, e := range entries {
held += e.Bytes
}
start := min(bytes, held)
now := time.Now
if p.Now != nil {
now = p.Now
}
drop, kept := p.Prune.Choose(entries, now(), start, limit)
r.Kept, r.To = kept, start
sizes := map[string]uint64{}
for _, e := range entries {
sizes[e.Key] = e.Bytes
}
var errs []error
for _, key := range drop {
if err := p.Remove(key); err != nil {
errs = append(errs, fmt.Errorf("%s: %w", key, err))
continue
}
r.Removed++
r.To -= min(r.To, sizes[key])
}
if reread {
if b, l, err := p.Figures(); err == nil {
r.To, r.MaxBytes = b, l
} else {
errs = append(errs, fmt.Errorf("the fill after the prune: %w", err))
}
}
p.bytes, p.limit = r.To, r.MaxBytes
if p.Say != nil {
p.Say(r.Line())
}
if len(errs) > 0 {
return r, fmt.Errorf("the prune of the state %s: %w", p.State, errors.Join(errs...))
}
return r, nil
}
// PutOverhead is what a write puts on the bus beside its key and its value, an upper bound: the bucket's subject
// prefix, the message's headers and its record.
const PutOverhead = 96
// PutBytes is what a write of value under key puts on the bus, as the prune helper counts it.
func PutBytes(key string, value []byte) uint64 { return uint64(len(key) + len(value) + PutOverhead) }
// PrunedState is a state a bundle prunes at its writes (§812): Put judges the fill the runtime's put answer carries
// (DMC-14, H-A-386) and prunes when it passes the mark. Its other calls are the state's own.
type PrunedState struct {
KeptState
pruner *Pruner
unsaid sync.Once
}
// Pruned is the state pruned by p at its writes.
func (s KeptState) Pruned(p Prune) *PrunedState {
ps := &PrunedState{KeptState: s}
ps.pruner = &Pruner{State: s.name, Prune: p, Say: func(line string) { sayf("%s", line) },
Remove: s.Delete,
Figures: func() (uint64, uint64, error) {
i, err := s.Info()
if err != nil {
return 0, 0, err
}
return i.Bytes, uint64(max(i.MaxBytes, 0)), nil
},
Entries: func() ([]PruneEntry, error) {
var out []PruneEntry
cursor := ""
for {
page, err := s.page(cursor, 768<<10, stateTimeout/2)
if err != nil {
return nil, err
}
for _, e := range page.Entries {
out = append(out, PruneEntry{Key: e.Key, Revision: e.Revision, Bytes: PutBytes(e.Key, e.Value), Value: e.Value})
}
if page.Next == "" || page.Next == cursor {
return out, nil
}
cursor = page.Next
}
}}
return ps
}
// Put writes a key whole and returns the revision written; the fill its answer carries is judged, and past the mark
// the state is pruned to its target before Put returns. A runtime whose answer carries no fill is said once, as a
// state that is never pruned.
func (s *PrunedState) Put(key string, value any) (uint64, error) {
raw, err := s.ask("put", map[string]any{"key": key, "value": value})
if err != nil {
return 0, err
}
var put struct {
Revision uint64 `json:"revision"`
Fill *buscalls.Fill `json:"fill"`
}
if err := json.Unmarshal(raw, &put); err != nil {
return 0, err
}
if put.Fill == nil {
s.unsaid.Do(func() {
sayf("the state %s is never pruned: the runtime's put answer carries no fill (design 47 H2 §812)", s.name)
})
return put.Revision, nil
}
if _, err := s.pruner.Filled(*put.Fill); err != nil {
return put.Revision, err
}
return put.Revision, nil
}
// PruneNow prunes the state to its target now.
func (s *PrunedState) PruneNow() (Pruned, error) { return s.pruner.PruneNow() }
// Keep keeps key from the prune until until, written under KeepKey.
func (s *PrunedState) Keep(key string, until time.Time) error {
if err := KeepProblem(key, until); err != nil {
return err
}
_, err := s.Put(KeepKey(key), Keep{Until: until.UTC()})
return err
}
// KeepProblem is why key cannot be kept until until, or nil: a process and a bundle check a keep alike.
func KeepProblem(key string, until time.Time) error {
switch {
case key == "" || IsKeep(key):
return fmt.Errorf("%q is not a key a keep keeps", key)
case !until.After(time.Now()):
return fmt.Errorf("a keep until %s keeps nothing: until is a time still to come", until.Format(time.RFC3339))
}
return nil
}