RN16: the runner ages its own kept data by its own deletes, in the step log's order

Three step-log sequences in the state aged, one per retention (7, 30, 90 days); at each step's end up to 20 entries per
sequence past their retention delete the output and the pipeline (7 days), a kept result not read for 30 days (30 days) and a
passed pipeline (90 days), never a pipeline that is its branch's newest; each delete is a journal line. The step log is
written to and read from the log's stream (design 47 G §470); a log entry names the kept results it wrote or carried and a
kept result its last read.

Builds: design 47 G §330, §410, §470
Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
2026-11-12 05:56:54 +01:00
parent 267bd0a9cb
commit d68e6b877d
11 changed files with 519 additions and 9 deletions
@@ -351,6 +351,8 @@ const (
type KeptResult struct {
// sha256:<hex> of what the package's tests read
Digest string `json:"digest"`
// when a carry last read it: the runner ages it 30 days after (design 47 G section 330)
LastRead *time.Time `json:"last-read,omitempty"`
// unit or integration
Level string `json:"level"`
Package string `json:"package"`
@@ -363,7 +365,9 @@ type KeptResult struct {
type LogEntry struct {
Artifact string `json:"artifact,omitempty"`
At time.Time `json:"at"`
Outcome string `json:"outcome,omitempty"`
// the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470)
Kept []string `json:"kept,omitempty"`
Outcome string `json:"outcome,omitempty"`
// the output's name in step-outputs
Output string `json:"output,omitempty"`
Pipeline string `json:"pipeline"`
@@ -712,6 +712,11 @@
"pipeline": {
"type": "string",
"description": "the pipeline whose step kept it"
},
"last-read": {
"type": "string",
"format": "date-time",
"description": "when a carry last read it: the runner ages it 30 days after (design 47 G section 330)"
}
},
"description": "a package's passing result the runner keeps, by the digest of what its tests read (ADR 0666 section 8)",
@@ -1517,6 +1522,13 @@
"at": {
"type": "string",
"format": "date-time"
},
"kept": {
"type": "array",
"items": {
"type": "string"
},
"description": "the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470)"
}
},
"description": "one entry of the step log",
@@ -148,6 +148,8 @@ export interface GroupEnded {
export interface KeptResult {
/** sha256:<hex> of what the package's tests read */
"digest": string;
/** when a carry last read it: the runner ages it 30 days after (design 47 G section 330) */
"last-read"?: string;
/** unit or integration */
"level": string;
"package": string;
@@ -160,6 +162,8 @@ export interface KeptResult {
export interface LogEntry {
"artifact"?: string;
"at": string;
/** the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470) */
"kept"?: string[];
"outcome"?: "running" | "waiting" | "passed" | "failed" | "nothing to run" | "bypassed" | "carried" | "could not run" | "waiting for a build node" | "did not build" | "accepted" | "declined" | "superseded" | "cancelled";
/** the output's name in step-outputs */
"output"?: string;
@@ -0,0 +1,148 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
// stepLog is the runner's step log, its old `logs` section (design 47 G §470): entries appended on the log's subject, `<pipeline>`
// as the key, and read onward by sequence from the stream the bus makes for the log.
type stepLog struct {
ctx context.Context
js jetstream.JetStream
module string
}
func (l stepLog) stream() string { return buscompose.LogStreamName(l.module, seat.StateStepLog) }
// Append appends one entry; the entry's identity is its pipeline, artifact, step, what and time, so the same entry appended twice
// is kept once.
func (l stepLog) Append(e seat.LogEntry) error {
raw, err := json.Marshal(e)
if err != nil {
return err
}
msg := nats.NewMsg(buscompose.LogSubject(l.module, seat.StateStepLog) + "." + seat.StepLogByPipelineKey(keys.EscapeName(e.Pipeline)))
msg.Data = raw
ctx, cancel := context.WithTimeout(l.ctx, 10*time.Second)
defer cancel()
_, err = l.js.PublishMsg(ctx, msg, jetstream.WithMsgID(fmt.Sprintf("%s|%s|%s|%s|%s", e.Pipeline, e.Artifact, e.Step, e.What, e.At.UTC().Format(time.RFC3339Nano))))
return err
}
// Read reads the entries from sequence from on, oldest first, at most limit of them.
func (l stepLog) Read(from uint64, limit int) ([]pipeline.LoggedEntry, error) {
if from == 0 {
from = 1
}
st, err := l.js.Stream(l.ctx, l.stream())
if err != nil {
return nil, fmt.Errorf("the step log %s is not open: %w", l.stream(), err)
}
info, err := st.Info(l.ctx)
if err != nil {
return nil, err
}
if info.State.Msgs == 0 || from > info.State.LastSeq {
return nil, nil
}
cons, err := st.OrderedConsumer(l.ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverByStartSequencePolicy, OptStartSeq: from})
if err != nil {
return nil, err
}
batch, err := cons.Fetch(limit, jetstream.FetchMaxWait(2*time.Second))
if err != nil {
return nil, err
}
var out []pipeline.LoggedEntry
for m := range batch.Messages() {
meta, err := m.Metadata()
if err != nil {
return out, err
}
var e seat.LogEntry
if err := json.Unmarshal(m.Data(), &e); err != nil {
continue
}
out = append(out, pipeline.LoggedEntry{Sequence: meta.Sequence.Stream, Entry: e})
}
return out, batch.Error()
}
// agedState is the state `aged`: one key per retention holding the last step-log sequence aged at it.
type agedState struct {
ctx context.Context
kv jetstream.KeyValue
}
func (a agedState) Get(name string) (uint64, error) {
e, err := a.kv.Get(a.ctx, name)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return 0, nil
}
if err != nil {
return 0, err
}
return strconv.ParseUint(string(e.Value()), 10, 64)
}
func (a agedState) Put(name string, sequence uint64) error {
_, err := a.kv.Put(a.ctx, name, []byte(strconv.FormatUint(sequence, 10)))
return err
}
// keptStore is the state kept as the ageing reads and deletes it.
type keptStore struct {
ctx context.Context
kv jetstream.KeyValue
}
func (k keptStore) Get(key string) (seat.KeptResult, bool, error) {
e, err := k.kv.Get(k.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return seat.KeptResult{}, false, nil
}
var r seat.KeptResult
if err != nil {
return r, false, err
}
return r, true, json.Unmarshal(e.Value(), &r)
}
func (k keptStore) Delete(key string) error { return k.kv.Delete(k.ctx, key) }
// newAger is the ageing over the runner's own stores.
func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager {
return &pipeline.Ager{Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Read, Aged: agedState{ctx: ctx, kv: aged},
Pipelines: pipelineDeleter{pipelineStore{ctx: ctx, kv: pipelines}}, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf,
Outputs: func(name string) error {
o, err := js.ObjectStore(ctx, catalogue.ObjectsBucket(cred.Module, seat.StateStepOutputs))
if err != nil {
return err
}
if err := o.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
return err
}
return nil
}}
}
// pipelineDeleter is the pipelines store with the delete only its owner makes (§330).
type pipelineDeleter struct{ pipelineStore }
func (d pipelineDeleter) Delete(key string) error { return d.kv.Delete(d.ctx, key) }
@@ -53,6 +53,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// The last step-log sequence aged at each retention (§330).
aged, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "aged"), "age its own kept data")
if err != nil {
return err
}
// The bound of each ask a build seat's holder must take: a key with its own time to live, its expiry heard (§535, A-77).
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound an ask no holder takes")
if err != nil {
@@ -100,6 +105,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf)
hearer.Asker = busAsker{conn: conn, cred: cred}
hearer.Ager = newAger(ctx, js, cred, pipelines, kept, aged, logger.Printf)
hearer.Where = routerRecordOf(cred.Module, calls.Call, calls.Ask)
hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf}
// Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked
@@ -19,6 +19,7 @@ import (
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/testbus"
@@ -56,7 +57,7 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
defer cancel()
const module = "mesh-delivery-pipeline"
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers"} {
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
t.Fatal(err)
}
@@ -64,6 +65,10 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_bounds", History: 1, LimitMarkerTTL: time.Minute}); err != nil {
t.Fatal(err)
}
// The step log, the runner's old logs section (§470): the stream the bus makes for it.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
t.Fatal(err)
}
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
if err != nil {
t.Fatal(err)
@@ -161,6 +166,12 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && p.Outcome == seat.PipelineOutcomePassed })
// §470, §330: the step log is the log's stream, read onward by sequence the way the ageing reads it.
logged, err := stepLog{ctx: ctx, js: js, module: module}.Read(1, 100)
if err != nil || len(logged) == 0 || logged[0].Entry.Pipeline != key {
t.Errorf("the step log holds %+v (%v), want the entries of %s", logged, err, key)
}
// RN25 (§76, §468; A-166): the durable consumer's age is in the state consumers, none waiting once every message heard
// is acknowledged, and runner_health reads it healthy.
consumers, err := js.KeyValue(ctx, module+"_consumers")
@@ -77,9 +77,7 @@ func (s pipelineStore) Keys(filter string) ([]string, error) {
func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept jetstream.KeyValue,
say func(string, any) error, logf func(string, ...any)) *pipeline.Hearer {
return &pipeline.Hearer{Store: pipelineStore{ctx: ctx, kv: pipelines}, Say: say, Logf: logf, Name: cred.Module,
Log: func(e seat.LogEntry) error {
return say(cred.EventSubject("step-log."+keys.EscapeName(e.Pipeline)), e)
},
Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Append,
Keep: func(key string, k seat.KeptResult) error {
raw, err := json.Marshal(k)
if err == nil {
@@ -0,0 +1,177 @@
package pipeline
import (
"fmt"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// Retention is one of the three ages the runner's own kept data are deleted at (design 47 G §330, *The runner's kept data*): its
// name is the key of the state `aged` that holds the last step-log sequence aged at it.
type Retention struct {
Name string
Age time.Duration
}
// The retentions: a step's output and a pipeline not kept to 90 days at 7 days, a kept result not read for 30 days, a passed
// pipeline at 90 days.
var (
Retention7 = Retention{"7d", 7 * 24 * time.Hour}
Retention30 = Retention{"30d", 30 * 24 * time.Hour}
Retention90 = Retention{"90d", 90 * 24 * time.Hour}
)
// AgedPerStep is the most entries one step's end ages at one retention (§330).
const AgedPerStep = 20
// LoggedEntry is one entry of the step log with its sequence.
type LoggedEntry struct {
Sequence uint64
Entry seat.LogEntry
}
// Ager ages the runner's own kept data by its own deletes, in the step log's order (design 47 G §330): at each step's end it
// reads the step log onward from each retention's sequence, at most AgedPerStep entries, while the entry is past the
// retention, deletes what the retention says and moves the sequence; nothing is scanned whole, nothing waits on a timer.
type Ager struct {
// Log reads the step log onward from a sequence, oldest first, at most limit entries.
Log func(from uint64, limit int) ([]LoggedEntry, error)
// Aged is the state `aged`: the last sequence aged at a retention, 0 when none.
Aged interface {
Get(name string) (uint64, error)
Put(name string, sequence uint64) error
}
Pipelines interface {
Updater
Delete(key string) error
}
// Outputs deletes an object of step-outputs, Kept reads and deletes a key of the state kept.
Outputs func(name string) error
Kept interface {
Get(key string) (seat.KeptResult, bool, error)
Delete(key string) error
}
Now func() time.Time
// Logf is the runner's journal: each delete writes a line naming the key or object and its retention.
Logf func(string, ...any)
}
func (a *Ager) logf(format string, args ...any) {
if a.Logf != nil {
a.Logf(format, args...)
}
}
// Age is the ageing of one step's end; an error of one retention is said and the others go on.
func (a *Ager) Age() {
if a == nil || a.Log == nil {
return
}
for _, r := range []Retention{Retention7, Retention30, Retention90} {
if err := a.ageAt(r); err != nil {
a.logf("aged: the data past %s are not aged: %v", r.Name, err)
}
}
}
func (a *Ager) ageAt(r Retention) error {
now := time.Now().UTC()
if a.Now != nil {
now = a.Now().UTC()
}
last, err := a.Aged.Get(r.Name)
if err != nil {
return err
}
entries, err := a.Log(last+1, AgedPerStep)
if err != nil {
return err
}
moved := last
for _, e := range entries {
if now.Sub(e.Entry.At) < r.Age {
break
}
a.ageEntry(r, e.Entry, now)
moved = e.Sequence
}
if moved == last {
return nil
}
return a.Aged.Put(r.Name, moved)
}
func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) {
switch r.Name {
case Retention7.Name:
if e.Output != "" && a.Outputs != nil {
if err := a.Outputs(e.Output); err != nil {
a.logf("aged: the output %s is not deleted: %v", e.Output, err)
} else {
a.logf("aged: deleted the output %s (%s)", e.Output, r.Name)
}
}
a.deletePipeline(r, e.Pipeline, true)
case Retention30.Name:
for _, key := range e.Kept {
k, ok, err := a.Kept.Get(key)
if err != nil || !ok || k.LastRead == nil || now.Sub(*k.LastRead) < r.Age {
if err != nil {
a.logf("aged: the kept result %s is not read: %v", key, err)
}
continue
}
if err := a.Kept.Delete(key); err != nil {
a.logf("aged: the kept result %s is not deleted: %v", key, err)
continue
}
a.logf("aged: deleted the kept result %s (%s after its last read)", key, r.Name)
}
default:
a.deletePipeline(r, e.Pipeline, false)
}
}
// deletePipeline deletes the pipeline key names unless it is its branch's newest for its module, and at 7 days unless it ended
// passed on a watched branch (a run on request's is no watched branch's).
func (a *Ager) deletePipeline(r Retention, key string, keepPassed bool) {
p, _, err := a.Pipelines.Get(key)
if err != nil {
return // gone already, or never kept
}
if p.Outcome == seat.PipelineOutcomeRunning || (keepPassed && p.Outcome == seat.PipelineOutcomePassed && p.Run == "") {
return
}
if !keepPassed && (p.Outcome != seat.PipelineOutcomePassed || p.Run != "") {
return // a pipeline not passed went at 7 days; a run's too
}
newest, err := a.newest(p)
if err != nil {
a.logf("aged: the pipeline %s is not judged, so it is kept: %v", key, err)
return
}
if newest {
return
}
if err := a.Pipelines.Delete(key); err != nil {
a.logf("aged: the pipeline %s is not deleted: %v", key, err)
return
}
a.logf("aged: deleted the pipeline %s (%s)", key, r.Name)
}
// newest says whether p is its branch's newest pipeline for its module.
func (a *Ager) newest(p seat.Pipeline) (bool, error) {
pkeys, err := a.Pipelines.Keys(keys.EscapeName(p.Branch) + ".*." + p.Module)
if err != nil {
return false, fmt.Errorf("the pipelines of %s on %s are not listed: %w", p.Module, p.Branch, err)
}
for _, k := range pkeys {
if q, _, err := a.Pipelines.Get(k); err == nil && q.Source > p.Source {
return false, nil
}
}
return true, nil
}
@@ -0,0 +1,144 @@
package pipeline
import (
"fmt"
"strconv"
"strings"
"testing"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
type agedState map[string]uint64
func (a agedState) Get(name string) (uint64, error) { return a[name], nil }
func (a agedState) Put(name string, s uint64) error { a[name] = s; return nil }
// agedPipelines is memStore that deletes.
type agedPipelines struct {
memStore
deleted []string
}
func (a *agedPipelines) Delete(key string) error {
delete(a.memStore, key)
a.deleted = append(a.deleted, key)
return nil
}
type keptStore struct {
m map[string]seat.KeptResult
deleted []string
}
func (k *keptStore) Get(key string) (seat.KeptResult, bool, error) {
v, ok := k.m[key]
return v, ok, nil
}
func (k *keptStore) Delete(key string) error {
delete(k.m, key)
k.deleted = append(k.deleted, key)
return nil
}
type agedFixture struct {
ager *Ager
log []LoggedEntry
aged agedState
pipes *agedPipelines
kept *keptStore
outputs []string
journal []string
readFrom []uint64
}
var agedNow = time.Date(2026, 11, 12, 12, 0, 0, 0, time.UTC)
func newAgedFixture() *agedFixture {
f := &agedFixture{aged: agedState{}, pipes: &agedPipelines{memStore: memStore{}}, kept: &keptStore{m: map[string]seat.KeptResult{}}}
f.ager = &Ager{Aged: f.aged, Pipelines: f.pipes, Kept: f.kept, Now: func() time.Time { return agedNow },
Outputs: func(n string) error { f.outputs = append(f.outputs, n); return nil },
Logf: func(format string, a ...any) { f.journal = append(f.journal, sprintf(format, a...)) },
Log: func(from uint64, limit int) ([]LoggedEntry, error) {
f.readFrom = append(f.readFrom, from)
var out []LoggedEntry
for _, e := range f.log {
if e.Sequence >= from && len(out) < limit {
out = append(out, e)
}
}
return out, nil
}}
return f
}
func sprintf(format string, a ...any) string { return fmt.Sprintf(format, a...) }
func (f *agedFixture) pipeline(branch string, source int64, outcome string, run string) string {
key := branch + "." + strconv.FormatInt(source, 10) + ".app"
f.pipes.memStore[key] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{Pipeline: key, Branch: branch, Source: source, Module: "app", Outcome: outcome, Run: run}, 1}
return key
}
func days(n int) time.Time { return agedNow.Add(-time.Duration(n) * 24 * time.Hour) }
// The runner ages its own kept data in the step log's order, by the retentions of the design: at 7 days an output and the
// pipeline that did not pass or is not its branch's newest, at 30 days a kept result not read for 30 days, at 90 days a passed
// pipeline that is not its branch's newest; each retention keeps its own sequence in the state aged (design 47 G §330).
func TestTheRunnerAgesItsOwnKeptDataByTheStepLogsOrder(t *testing.T) {
f := newAgedFixture()
failed := f.pipeline("main", 1, seat.PipelineOutcomeFailed, "")
passed := f.pipeline("main", 2, seat.PipelineOutcomePassed, "")
old := f.pipeline("main", 3, seat.PipelineOutcomePassed, "")
newest := f.pipeline("main", 9, seat.PipelineOutcomeFailed, "")
stale, fresh := days(40), days(2)
f.kept.m["unit.sha256:a"] = seat.KeptResult{Digest: "sha256:a", Level: "unit", LastRead: &stale}
f.kept.m["unit.sha256:b"] = seat.KeptResult{Digest: "sha256:b", Level: "unit", LastRead: &fresh}
f.log = []LoggedEntry{
{1, seat.LogEntry{Pipeline: old, Output: "out-1", At: days(100)}},
{2, seat.LogEntry{Pipeline: failed, Output: "out-2", Kept: []string{"unit.sha256:a", "unit.sha256:b"}, At: days(35)}},
{3, seat.LogEntry{Pipeline: passed, Output: "out-3", At: days(10)}},
{4, seat.LogEntry{Pipeline: newest, Output: "out-4", At: days(9)}},
{5, seat.LogEntry{Pipeline: newest, Output: "out-5", At: days(1)}},
}
f.ager.Age()
if got := strings.Join(f.outputs, ","); got != "out-1,out-2,out-3,out-4" {
t.Errorf("outputs deleted past 7 days: %s", got)
}
if got := strings.Join(f.pipes.deleted, ","); got != failed+","+old {
t.Errorf("pipelines deleted: %s, want %s (the failed one at 7 days, the old passed one at 90; the passed at 7 stays; the newest stays)", got, failed+","+old)
}
if got := strings.Join(f.kept.deleted, ","); got != "unit.sha256:a" {
t.Errorf("kept results deleted: %s", got)
}
if f.aged["7d"] != 4 || f.aged["30d"] != 2 || f.aged["90d"] != 1 {
t.Errorf("sequences aged: %v, want 4, 2 and 1 (the entries past each retention)", f.aged)
}
for _, want := range []string{"deleted the output out-1 (7d)", "deleted the pipeline " + failed + " (7d)", "deleted the kept result unit.sha256:a (30d", "deleted the pipeline " + old + " (90d)"} {
if !strings.Contains(strings.Join(f.journal, "\n"), want) {
t.Errorf("the journal names no %q:\n%s", want, strings.Join(f.journal, "\n"))
}
}
// A second step's end reads onward from the moved sequences and deletes nothing again.
f.readFrom, f.outputs = nil, nil
f.ager.Age()
if len(f.outputs) != 0 || f.readFrom[0] != 5 {
t.Errorf("the second ageing read from %v and deleted %v; want it to read from 5", f.readFrom, f.outputs)
}
}
// One step's end ages at most 20 entries per retention.
func TestOneStepEndAgesAtMostTwentyEntries(t *testing.T) {
f := newAgedFixture()
for i := uint64(1); i <= 25; i++ {
f.log = append(f.log, LoggedEntry{i, seat.LogEntry{Pipeline: "x", Output: "o" + strconv.FormatUint(i, 10), At: days(50)}})
}
f.ager.Age()
if len(f.outputs) != AgedPerStep || f.aged["7d"] != 20 {
t.Fatalf("%d outputs deleted, sequence %d; want 20 and 20", len(f.outputs), f.aged["7d"])
}
}
@@ -85,6 +85,7 @@ func (h *Hearer) endNotTaken(key, ask string, bound time.Duration) error {
}
h.logEnd(key, *s, now)
h.Failures.StepEnded(h.Store, p, *s)
h.Ager.Age()
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err)
}
@@ -109,6 +109,8 @@ type Hearer struct {
Now func() time.Time
// Failures says a failed head as the finding pipeline-failed (§33); nil says none.
Failures *Failures
// Ager ages the runner's own kept data at each step's end (§330); nil ages none.
Ager *Ager
}
func (h *Hearer) now() time.Time {
@@ -176,9 +178,11 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
} else if err != nil {
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
}
var keptKeys []string
if h.Keep != nil {
for _, k := range ev.Kept {
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline}
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now}
keptKeys = append(keptKeys, seat.KeptByLevelDigestKey(keys.EscapeName(k.Level), keys.EscapeName(k.Digest)))
if k.Toolchain != nil {
kept.Toolchain = &seat.ToolchainUsed{Language: k.Toolchain.Language, Recipe: k.Toolchain.Recipe, Version: k.Toolchain.Version}
}
@@ -190,8 +194,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
}
}
}
h.logEnd(ev.Pipeline, *s, now)
h.logEnd(ev.Pipeline, *s, now, keptKeys...)
h.Failures.StepEnded(h.Store, p, *s)
h.Ager.Age()
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
}
@@ -214,12 +219,12 @@ func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
return nil
}
func (h *Hearer) logEnd(key string, s seat.StepResult, at time.Time) {
func (h *Hearer) logEnd(key string, s seat.StepResult, at time.Time, kept ...string) {
if h.Log == nil {
return
}
if err := h.Log(seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
Output: s.Output, At: at}); err != nil {
Output: s.Output, Kept: kept, At: at}); err != nil {
h.logf("the step log misses the end of %s of %s: %v", s.Step, key, err)
}
}