Files
mesh-delivery/modules/mesh-delivery-pipeline/internal/pipeline/aged.go
T
jschoubben d68e6b877d RN16: the runner ages its own kept data by its own deletes, in the step log's order
Three step-log sequences in the state aged, one per retention (7, 30, 90 days); at each step's end up to 20 entries per
sequence past their retention delete the output and the pipeline (7 days), a kept result not read for 30 days (30 days) and a
passed pipeline (90 days), never a pipeline that is its branch's newest; each delete is a journal line. The step log is
written to and read from the log's stream (design 47 G §470); a log entry names the kept results it wrote or carried and a
kept result its last read.

Builds: design 47 G §330, §410, §470
Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 05:56:54 +01:00

178 lines
5.4 KiB
Go

package pipeline
import (
"fmt"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// Retention is one of the three ages the runner's own kept data are deleted at (design 47 G §330, *The runner's kept data*): its
// name is the key of the state `aged` that holds the last step-log sequence aged at it.
type Retention struct {
Name string
Age time.Duration
}
// The retentions: a step's output and a pipeline not kept to 90 days at 7 days, a kept result not read for 30 days, a passed
// pipeline at 90 days.
var (
Retention7 = Retention{"7d", 7 * 24 * time.Hour}
Retention30 = Retention{"30d", 30 * 24 * time.Hour}
Retention90 = Retention{"90d", 90 * 24 * time.Hour}
)
// AgedPerStep is the most entries one step's end ages at one retention (§330).
const AgedPerStep = 20
// LoggedEntry is one entry of the step log with its sequence.
type LoggedEntry struct {
Sequence uint64
Entry seat.LogEntry
}
// Ager ages the runner's own kept data by its own deletes, in the step log's order (design 47 G §330): at each step's end it
// reads the step log onward from each retention's sequence, at most AgedPerStep entries, while the entry is past the
// retention, deletes what the retention says and moves the sequence; nothing is scanned whole, nothing waits on a timer.
type Ager struct {
// Log reads the step log onward from a sequence, oldest first, at most limit entries.
Log func(from uint64, limit int) ([]LoggedEntry, error)
// Aged is the state `aged`: the last sequence aged at a retention, 0 when none.
Aged interface {
Get(name string) (uint64, error)
Put(name string, sequence uint64) error
}
Pipelines interface {
Updater
Delete(key string) error
}
// Outputs deletes an object of step-outputs, Kept reads and deletes a key of the state kept.
Outputs func(name string) error
Kept interface {
Get(key string) (seat.KeptResult, bool, error)
Delete(key string) error
}
Now func() time.Time
// Logf is the runner's journal: each delete writes a line naming the key or object and its retention.
Logf func(string, ...any)
}
func (a *Ager) logf(format string, args ...any) {
if a.Logf != nil {
a.Logf(format, args...)
}
}
// Age is the ageing of one step's end; an error of one retention is said and the others go on.
func (a *Ager) Age() {
if a == nil || a.Log == nil {
return
}
for _, r := range []Retention{Retention7, Retention30, Retention90} {
if err := a.ageAt(r); err != nil {
a.logf("aged: the data past %s are not aged: %v", r.Name, err)
}
}
}
func (a *Ager) ageAt(r Retention) error {
now := time.Now().UTC()
if a.Now != nil {
now = a.Now().UTC()
}
last, err := a.Aged.Get(r.Name)
if err != nil {
return err
}
entries, err := a.Log(last+1, AgedPerStep)
if err != nil {
return err
}
moved := last
for _, e := range entries {
if now.Sub(e.Entry.At) < r.Age {
break
}
a.ageEntry(r, e.Entry, now)
moved = e.Sequence
}
if moved == last {
return nil
}
return a.Aged.Put(r.Name, moved)
}
func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) {
switch r.Name {
case Retention7.Name:
if e.Output != "" && a.Outputs != nil {
if err := a.Outputs(e.Output); err != nil {
a.logf("aged: the output %s is not deleted: %v", e.Output, err)
} else {
a.logf("aged: deleted the output %s (%s)", e.Output, r.Name)
}
}
a.deletePipeline(r, e.Pipeline, true)
case Retention30.Name:
for _, key := range e.Kept {
k, ok, err := a.Kept.Get(key)
if err != nil || !ok || k.LastRead == nil || now.Sub(*k.LastRead) < r.Age {
if err != nil {
a.logf("aged: the kept result %s is not read: %v", key, err)
}
continue
}
if err := a.Kept.Delete(key); err != nil {
a.logf("aged: the kept result %s is not deleted: %v", key, err)
continue
}
a.logf("aged: deleted the kept result %s (%s after its last read)", key, r.Name)
}
default:
a.deletePipeline(r, e.Pipeline, false)
}
}
// deletePipeline deletes the pipeline key names unless it is its branch's newest for its module, and at 7 days unless it ended
// passed on a watched branch (a run on request's is no watched branch's).
func (a *Ager) deletePipeline(r Retention, key string, keepPassed bool) {
p, _, err := a.Pipelines.Get(key)
if err != nil {
return // gone already, or never kept
}
if p.Outcome == seat.PipelineOutcomeRunning || (keepPassed && p.Outcome == seat.PipelineOutcomePassed && p.Run == "") {
return
}
if !keepPassed && (p.Outcome != seat.PipelineOutcomePassed || p.Run != "") {
return // a pipeline not passed went at 7 days; a run's too
}
newest, err := a.newest(p)
if err != nil {
a.logf("aged: the pipeline %s is not judged, so it is kept: %v", key, err)
return
}
if newest {
return
}
if err := a.Pipelines.Delete(key); err != nil {
a.logf("aged: the pipeline %s is not deleted: %v", key, err)
return
}
a.logf("aged: deleted the pipeline %s (%s)", key, r.Name)
}
// newest says whether p is its branch's newest pipeline for its module.
func (a *Ager) newest(p seat.Pipeline) (bool, error) {
pkeys, err := a.Pipelines.Keys(keys.EscapeName(p.Branch) + ".*." + p.Module)
if err != nil {
return false, fmt.Errorf("the pipelines of %s on %s are not listed: %w", p.Module, p.Branch, err)
}
for _, k := range pkeys {
if q, _, err := a.Pipelines.Get(k); err == nil && q.Source > p.Source {
return false, nil
}
}
return true, nil
}