package stdio import ( "encoding/json" "errors" "fmt" "sort" "strconv" "strings" "sync" "time" "git.novox.be/novox/mesh-sdk/go/buscalls" ) // A kept store's owner prunes it toward its cap, judged at its own writes (design 47 H2 §812; H-A-380, H-A-386): when // the fill passes prune-mark of the cap, the oldest keys by their current revision are removed until it is down to // prune-target, one journal line per prune, with no extra read, no poll and no timer. A key kept by a person // (`_state_keep`) is never pruned before its until: the keep lives in the pruned state itself under KeepPrefix. // The prune's settings and their defaults: prune-mark below the bus module's raise mark of 75 %. const ( PruneMarkSetting = "prune-mark" PruneTargetSetting = "prune-target" PruneMarkDefault = 70 PruneTargetDefault = 50 pruneSettingPercent = "a percentage of the cap between 1 and 100" ) // KeepPrefix is the reserved prefix a keep lives under in the state it keeps a key of (H-A-386): `_keep/`, with // its until. A keep is never pruned and never counted as an entry by the readers (Keys, Page, Watch). const KeepPrefix = "_keep/" // KeepKey is the key the keep of key lives under. func KeepKey(key string) string { return KeepPrefix + key } // IsKeep is whether key is a keep rather than an entry of its state. func IsKeep(key string) bool { return strings.HasPrefix(key, KeepPrefix) } // Keep is a keep's value: the key it keeps is never pruned before Until. type Keep struct { Until time.Time `json:"until"` } // SettingPercent is the config value key read as a percentage ("70" or "70%"): def when none is composed or it is // empty, an error naming the key for a value that is no whole number from 1 to 100. func SettingPercent(key string, def int) (int, error) { v, ok := Setting(key) if v = strings.TrimSpace(v); !ok || v == "" { return def, nil } n, err := strconv.Atoi(strings.TrimSpace(strings.TrimSuffix(v, "%"))) if err != nil || n < 1 || n > 100 { return 0, fmt.Errorf("the setting %s (%s) holds %q, which is not %s", key, SettingWord(key), v, pruneSettingPercent) } return n, nil } // Prune is where a prune starts and where it ends, in percent of the cap. type Prune struct { Mark, Target int } // PruneSettings is the module's prune-mark and prune-target, read through the settings reader with their defaults; a // target not below the mark is refused, naming both. func PruneSettings() (Prune, error) { mark, err := SettingPercent(PruneMarkSetting, PruneMarkDefault) if err != nil { return Prune{}, err } target, err := SettingPercent(PruneTargetSetting, PruneTargetDefault) if err != nil { return Prune{}, err } if target >= mark { return Prune{}, fmt.Errorf("the setting %s (%d %%) is not below %s (%d %%): a prune would never end below its mark", PruneTargetSetting, target, PruneMarkSetting, mark) } return Prune{Mark: mark, Target: target}, nil } // Passed is whether bytes of a cap of maxBytes passes the mark; an entry with no cap never does. func (p Prune) Passed(bytes, maxBytes uint64) bool { return maxBytes > 0 && bytes*100 > uint64(p.Mark)*maxBytes } // PruneEntry is one current value of a pruned state as the prune reads it: its key, its current revision and the // bytes it holds on the bus. type PruneEntry struct { Key string Revision uint64 Bytes uint64 Value []byte } // Choose is the keys a prune removes, oldest first by their current revision, until bytes is down to the target of // maxBytes, and how many it skipped for a keep whose until is still to come. A keep itself is never chosen; a keep that // cannot be read keeps its key (a prune never removes what a person may have kept). func (p Prune) Choose(entries []PruneEntry, now time.Time, bytes, maxBytes uint64) (drop []string, kept int) { keeps := map[string]bool{} for _, e := range entries { if !IsKeep(e.Key) { continue } var k Keep if err := json.Unmarshal(e.Value, &k); err != nil || k.Until.After(now) { keeps[strings.TrimPrefix(e.Key, KeepPrefix)] = true } } sorted := append([]PruneEntry(nil), entries...) sort.Slice(sorted, func(i, j int) bool { return sorted[i].Revision < sorted[j].Revision }) target := uint64(p.Target) * maxBytes / 100 for _, e := range sorted { if bytes <= target { break } if IsKeep(e.Key) { continue } if keeps[e.Key] { kept++ continue } drop = append(drop, e.Key) bytes -= min(bytes, e.Bytes) } return drop, kept } // Pruned is what one prune did: the state, how many keys it removed and skipped for a keep, and its fill before and // after against the cap. type Pruned struct { State string `json:"state"` Removed int `json:"removed"` Kept int `json:"kept"` From uint64 `json:"from-bytes"` To uint64 `json:"to-bytes"` MaxBytes uint64 `json:"max-bytes"` Mark int `json:"mark"` Target int `json:"target"` } // Line is the prune's one journal line. func (r Pruned) Line() string { return fmt.Sprintf("pruned the state %s: removed %d keys oldest first (%d kept by a keep), from %d to %d bytes of its cap of %d (mark %d %%, target %d %%)", r.State, r.Removed, r.Kept, r.From, r.To, r.MaxBytes, r.Mark, r.Target) } // Prunable is a state its owner prunes: the generic tools `_state_prune_now` and `_state_keep` act on it. type Prunable interface { // PruneNow prunes the state to its target now, whatever its fill. PruneNow() (Pruned, error) // Keep keeps key from the prune until the time given. Keep(key string, until time.Time) error } // Pruner is the prune helper of one state, the same for a process and a bundle: Entries reads the state's current // values (keeps included) once per prune, Remove removes one key, Figures reads the entry's bytes and cap. A process // gives Wrote the bytes each write puts and Figures is read at Bind and after each prune; a bundle gives Filled the // fill its write's answer carries. type Pruner struct { State string Prune Prune Entries func() ([]PruneEntry, error) Remove func(key string) error Figures func() (bytes, maxBytes uint64, err error) // Say writes the prune's journal line. Say func(line string) Now func() time.Time mu sync.Mutex bytes, limit uint64 } // Bind reads the entry's figures once, the base of the running sum Wrote keeps. func (p *Pruner) Bind() error { bytes, limit, err := p.Figures() if err != nil { return fmt.Errorf("the state %s's fill could not be read for its prune: %w", p.State, err) } p.mu.Lock() p.bytes, p.limit = bytes, limit p.mu.Unlock() return nil } // Wrote adds the bytes a write put to the running sum and prunes when it passes the mark: the answer is the prune it // ran, or nil. func (p *Pruner) Wrote(n uint64) (*Pruned, error) { p.mu.Lock() defer p.mu.Unlock() p.bytes += n if !p.Prune.Passed(p.bytes, p.limit) { return nil, nil } r, err := p.pruneLocked(p.bytes, p.limit, true) return &r, err } // Filled judges the fill a write's answer carries and prunes when it passes the mark: the answer is the prune it ran, // or nil. func (p *Pruner) Filled(f buscalls.Fill) (*Pruned, error) { p.mu.Lock() defer p.mu.Unlock() p.bytes, p.limit = f.Bytes, f.MaxBytes if !p.Prune.Passed(f.Bytes, f.MaxBytes) { return nil, nil } r, err := p.pruneLocked(f.Bytes, f.MaxBytes, false) return &r, err } // PruneNow reads the figures and prunes to the target now. func (p *Pruner) PruneNow() (Pruned, error) { bytes, limit, err := p.Figures() if err != nil { return Pruned{State: p.State}, fmt.Errorf("the state %s's fill could not be read for its prune: %w", p.State, err) } p.mu.Lock() defer p.mu.Unlock() if limit == 0 { return Pruned{State: p.State}, fmt.Errorf("the state %s names no cap, so it has no target to prune to", p.State) } return p.pruneLocked(bytes, limit, true) } // pruneLocked removes the oldest keys from bytes down to the target of limit; reread reads the figures after it, the // new base of the running sum. func (p *Pruner) pruneLocked(bytes, limit uint64, reread bool) (Pruned, error) { r := Pruned{State: p.State, From: bytes, MaxBytes: limit, Mark: p.Prune.Mark, Target: p.Prune.Target} entries, err := p.Entries() if err != nil { return r, fmt.Errorf("the state %s could not be read for its prune: %w", p.State, err) } // The running sum is an upper bound (an overwrite's past value is the bucket's history to bound), so what the // current values hold is the start when it is lower. var held uint64 for _, e := range entries { held += e.Bytes } start := min(bytes, held) now := time.Now if p.Now != nil { now = p.Now } drop, kept := p.Prune.Choose(entries, now(), start, limit) r.Kept, r.To = kept, start sizes := map[string]uint64{} for _, e := range entries { sizes[e.Key] = e.Bytes } var errs []error for _, key := range drop { if err := p.Remove(key); err != nil { errs = append(errs, fmt.Errorf("%s: %w", key, err)) continue } r.Removed++ r.To -= min(r.To, sizes[key]) } if reread { if b, l, err := p.Figures(); err == nil { r.To, r.MaxBytes = b, l } else { errs = append(errs, fmt.Errorf("the fill after the prune: %w", err)) } } p.bytes, p.limit = r.To, r.MaxBytes if p.Say != nil { p.Say(r.Line()) } if len(errs) > 0 { return r, fmt.Errorf("the prune of the state %s: %w", p.State, errors.Join(errs...)) } return r, nil } // PutOverhead is what a write puts on the bus beside its key and its value, an upper bound: the bucket's subject // prefix, the message's headers and its record. const PutOverhead = 96 // PutBytes is what a write of value under key puts on the bus, as the prune helper counts it. func PutBytes(key string, value []byte) uint64 { return uint64(len(key) + len(value) + PutOverhead) } // PrunedState is a state a bundle prunes at its writes (§812): Put judges the fill the runtime's put answer carries // (DMC-14, H-A-386) and prunes when it passes the mark. Its other calls are the state's own. type PrunedState struct { KeptState pruner *Pruner unsaid sync.Once } // Pruned is the state pruned by p at its writes. func (s KeptState) Pruned(p Prune) *PrunedState { ps := &PrunedState{KeptState: s} ps.pruner = &Pruner{State: s.name, Prune: p, Say: func(line string) { sayf("%s", line) }, Remove: s.Delete, Figures: func() (uint64, uint64, error) { i, err := s.Info() if err != nil { return 0, 0, err } return i.Bytes, uint64(max(i.MaxBytes, 0)), nil }, Entries: func() ([]PruneEntry, error) { var out []PruneEntry cursor := "" for { page, err := s.page(cursor, 768<<10, stateTimeout/2) if err != nil { return nil, err } for _, e := range page.Entries { out = append(out, PruneEntry{Key: e.Key, Revision: e.Revision, Bytes: PutBytes(e.Key, e.Value), Value: e.Value}) } if page.Next == "" || page.Next == cursor { return out, nil } cursor = page.Next } }} return ps } // Put writes a key whole and returns the revision written; the fill its answer carries is judged, and past the mark // the state is pruned to its target before Put returns. A runtime whose answer carries no fill is said once, as a // state that is never pruned. func (s *PrunedState) Put(key string, value any) (uint64, error) { raw, err := s.ask("put", map[string]any{"key": key, "value": value}) if err != nil { return 0, err } var put struct { Revision uint64 `json:"revision"` Fill *buscalls.Fill `json:"fill"` } if err := json.Unmarshal(raw, &put); err != nil { return 0, err } if put.Fill == nil { s.unsaid.Do(func() { sayf("the state %s is never pruned: the runtime's put answer carries no fill (design 47 H2 §812)", s.name) }) return put.Revision, nil } if _, err := s.pruner.Filled(*put.Fill); err != nil { return put.Revision, err } return put.Revision, nil } // PruneNow prunes the state to its target now. func (s *PrunedState) PruneNow() (Pruned, error) { return s.pruner.PruneNow() } // Keep keeps key from the prune until until, written under KeepKey. func (s *PrunedState) Keep(key string, until time.Time) error { if err := KeepProblem(key, until); err != nil { return err } _, err := s.Put(KeepKey(key), Keep{Until: until.UTC()}) return err } // KeepProblem is why key cannot be kept until until, or nil: a process and a bundle check a keep alike. func KeepProblem(key string, until time.Time) error { switch { case key == "" || IsKeep(key): return fmt.Errorf("%q is not a key a keep keeps", key) case !until.After(time.Now()): return fmt.Errorf("a keep until %s keeps nothing: until is a time still to come", until.Format(time.RFC3339)) } return nil }