Three step-log sequences in the state aged, one per retention (7, 30, 90 days); at each step's end up to 20 entries per sequence past their retention delete the output and the pipeline (7 days), a kept result not read for 30 days (30 days) and a passed pipeline (90 days), never a pipeline that is its branch's newest; each delete is a journal line. The step log is written to and read from the log's stream (design 47 G §470); a log entry names the kept results it wrote or carried and a kept result its last read. Builds: design 47 G §330, §410, §470 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
178 lines
5.4 KiB
Go
178 lines
5.4 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
)
|
|
|
|
// Retention is one of the three ages the runner's own kept data are deleted at (design 47 G §330, *The runner's kept data*): its
|
|
// name is the key of the state `aged` that holds the last step-log sequence aged at it.
|
|
type Retention struct {
|
|
Name string
|
|
Age time.Duration
|
|
}
|
|
|
|
// The retentions: a step's output and a pipeline not kept to 90 days at 7 days, a kept result not read for 30 days, a passed
|
|
// pipeline at 90 days.
|
|
var (
|
|
Retention7 = Retention{"7d", 7 * 24 * time.Hour}
|
|
Retention30 = Retention{"30d", 30 * 24 * time.Hour}
|
|
Retention90 = Retention{"90d", 90 * 24 * time.Hour}
|
|
)
|
|
|
|
// AgedPerStep is the most entries one step's end ages at one retention (§330).
|
|
const AgedPerStep = 20
|
|
|
|
// LoggedEntry is one entry of the step log with its sequence.
|
|
type LoggedEntry struct {
|
|
Sequence uint64
|
|
Entry seat.LogEntry
|
|
}
|
|
|
|
// Ager ages the runner's own kept data by its own deletes, in the step log's order (design 47 G §330): at each step's end it
|
|
// reads the step log onward from each retention's sequence, at most AgedPerStep entries, while the entry is past the
|
|
// retention, deletes what the retention says and moves the sequence; nothing is scanned whole, nothing waits on a timer.
|
|
type Ager struct {
|
|
// Log reads the step log onward from a sequence, oldest first, at most limit entries.
|
|
Log func(from uint64, limit int) ([]LoggedEntry, error)
|
|
// Aged is the state `aged`: the last sequence aged at a retention, 0 when none.
|
|
Aged interface {
|
|
Get(name string) (uint64, error)
|
|
Put(name string, sequence uint64) error
|
|
}
|
|
Pipelines interface {
|
|
Updater
|
|
Delete(key string) error
|
|
}
|
|
// Outputs deletes an object of step-outputs, Kept reads and deletes a key of the state kept.
|
|
Outputs func(name string) error
|
|
Kept interface {
|
|
Get(key string) (seat.KeptResult, bool, error)
|
|
Delete(key string) error
|
|
}
|
|
Now func() time.Time
|
|
// Logf is the runner's journal: each delete writes a line naming the key or object and its retention.
|
|
Logf func(string, ...any)
|
|
}
|
|
|
|
func (a *Ager) logf(format string, args ...any) {
|
|
if a.Logf != nil {
|
|
a.Logf(format, args...)
|
|
}
|
|
}
|
|
|
|
// Age is the ageing of one step's end; an error of one retention is said and the others go on.
|
|
func (a *Ager) Age() {
|
|
if a == nil || a.Log == nil {
|
|
return
|
|
}
|
|
for _, r := range []Retention{Retention7, Retention30, Retention90} {
|
|
if err := a.ageAt(r); err != nil {
|
|
a.logf("aged: the data past %s are not aged: %v", r.Name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (a *Ager) ageAt(r Retention) error {
|
|
now := time.Now().UTC()
|
|
if a.Now != nil {
|
|
now = a.Now().UTC()
|
|
}
|
|
last, err := a.Aged.Get(r.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
entries, err := a.Log(last+1, AgedPerStep)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
moved := last
|
|
for _, e := range entries {
|
|
if now.Sub(e.Entry.At) < r.Age {
|
|
break
|
|
}
|
|
a.ageEntry(r, e.Entry, now)
|
|
moved = e.Sequence
|
|
}
|
|
if moved == last {
|
|
return nil
|
|
}
|
|
return a.Aged.Put(r.Name, moved)
|
|
}
|
|
|
|
func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) {
|
|
switch r.Name {
|
|
case Retention7.Name:
|
|
if e.Output != "" && a.Outputs != nil {
|
|
if err := a.Outputs(e.Output); err != nil {
|
|
a.logf("aged: the output %s is not deleted: %v", e.Output, err)
|
|
} else {
|
|
a.logf("aged: deleted the output %s (%s)", e.Output, r.Name)
|
|
}
|
|
}
|
|
a.deletePipeline(r, e.Pipeline, true)
|
|
case Retention30.Name:
|
|
for _, key := range e.Kept {
|
|
k, ok, err := a.Kept.Get(key)
|
|
if err != nil || !ok || k.LastRead == nil || now.Sub(*k.LastRead) < r.Age {
|
|
if err != nil {
|
|
a.logf("aged: the kept result %s is not read: %v", key, err)
|
|
}
|
|
continue
|
|
}
|
|
if err := a.Kept.Delete(key); err != nil {
|
|
a.logf("aged: the kept result %s is not deleted: %v", key, err)
|
|
continue
|
|
}
|
|
a.logf("aged: deleted the kept result %s (%s after its last read)", key, r.Name)
|
|
}
|
|
default:
|
|
a.deletePipeline(r, e.Pipeline, false)
|
|
}
|
|
}
|
|
|
|
// deletePipeline deletes the pipeline key names unless it is its branch's newest for its module, and at 7 days unless it ended
|
|
// passed on a watched branch (a run on request's is no watched branch's).
|
|
func (a *Ager) deletePipeline(r Retention, key string, keepPassed bool) {
|
|
p, _, err := a.Pipelines.Get(key)
|
|
if err != nil {
|
|
return // gone already, or never kept
|
|
}
|
|
if p.Outcome == seat.PipelineOutcomeRunning || (keepPassed && p.Outcome == seat.PipelineOutcomePassed && p.Run == "") {
|
|
return
|
|
}
|
|
if !keepPassed && (p.Outcome != seat.PipelineOutcomePassed || p.Run != "") {
|
|
return // a pipeline not passed went at 7 days; a run's too
|
|
}
|
|
newest, err := a.newest(p)
|
|
if err != nil {
|
|
a.logf("aged: the pipeline %s is not judged, so it is kept: %v", key, err)
|
|
return
|
|
}
|
|
if newest {
|
|
return
|
|
}
|
|
if err := a.Pipelines.Delete(key); err != nil {
|
|
a.logf("aged: the pipeline %s is not deleted: %v", key, err)
|
|
return
|
|
}
|
|
a.logf("aged: deleted the pipeline %s (%s)", key, r.Name)
|
|
}
|
|
|
|
// newest says whether p is its branch's newest pipeline for its module.
|
|
func (a *Ager) newest(p seat.Pipeline) (bool, error) {
|
|
pkeys, err := a.Pipelines.Keys(keys.EscapeName(p.Branch) + ".*." + p.Module)
|
|
if err != nil {
|
|
return false, fmt.Errorf("the pipelines of %s on %s are not listed: %w", p.Module, p.Branch, err)
|
|
}
|
|
for _, k := range pkeys {
|
|
if q, _, err := a.Pipelines.Get(k); err == nil && q.Source > p.Source {
|
|
return false, nil
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|