Design 47 H1: a mover's unlogged entries live under its mover key, each entry names its mover, and list answers records with counts (H-A-189, H-A-190, H-A-191)

The record still kept every ended step's unlogged step-log entry, so a step
log outage across a large pipeline grew the record as its steps had. A
mover key now holds its steps and its unlogged entries, written in the same
compare-and-set; the record keeps neither. An entry of the step log names
its mover's module, set where an entry is made, so two movers' approvals
ending at one moment are filed under their own keys and told apart when
read back; an older entry naming none goes under the one mover whose step it
matches, else stays in the record. list answers each pipeline's record with
its movers and steps counted and its steps empty; show keeps the steps. The
seat's definition changes: its Go and TypeScript contracts are generated
again.

Builds: design 47 H H-A-187, H-A-189, H-A-190, H-A-191
This commit is contained in:
2026-11-20 21:22:09 +01:00
parent 4e585f02b1
commit 3183e0e056
9 changed files with 181 additions and 35 deletions
@@ -565,8 +565,10 @@ type LogEntry struct {
Artifact string `json:"artifact,omitempty"`
At time.Time `json:"at"`
// the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470)
Kept []string `json:"kept,omitempty"`
Outcome string `json:"outcome,omitempty"`
Kept []string `json:"kept,omitempty"`
// the module of the mover whose step the entry is of (H-A-190)
Module string `json:"module,omitempty"`
Outcome string `json:"outcome,omitempty"`
// the output's name in step-outputs
Output string `json:"output,omitempty"`
Pipeline string `json:"pipeline"`
@@ -663,6 +665,8 @@ type Pipeline struct {
// how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325)
EndCount int64 `json:"end-count,omitempty"`
Ended *time.Time `json:"ended,omitempty"`
// how many movers the pipeline has, set by list alone (H-A-191)
MoverCount int64 `json:"mover-count,omitempty"`
// the modules the pipeline moves, its take-in's entries (design 47 H §746)
Movers []Mover `json:"movers"`
Outcome string `json:"outcome"`
@@ -679,11 +683,13 @@ type Pipeline struct {
// the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)
Source int64 `json:"source"`
Started *time.Time `json:"started,omitempty"`
// how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191)
StepCount int64 `json:"step-count,omitempty"`
// every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187)
Steps []StepResult `json:"steps"`
// the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107)
Superseded []string `json:"superseded,omitempty"`
// the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606)
// the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189)
Unlogged []LogEntry `json:"unlogged,omitempty"`
// the layout version
V int64 `json:"v,omitempty"`
@@ -1359,7 +1365,12 @@ type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline
type PipelinesByBranchRunRunValue = Pipeline
// PipelinesByStepsPipelineMoverValue is one value of the state pipelines under the key steps.<pipeline>.<mover>.
type PipelinesByStepsPipelineMoverValue = []StepResult
type PipelinesByStepsPipelineMoverValue struct {
// the mover's steps
Steps []StepResult `json:"steps"`
// the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189)
Unlogged []LogEntry `json:"unlogged,omitempty"`
}
// QueueByQueueNextValue is one value of the state queue under the key queue.next.
type QueueByQueueNextValue = int64
@@ -137,7 +137,7 @@
},
{
"name": "list",
"description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill. Reads only.",
"description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill: each pipeline's record with its movers and steps counted, never its steps (show gives those). Reads only.",
"input": {
"type": "object",
"properties": {
@@ -801,12 +801,28 @@
{
"key": "steps.<pipeline>.<mover>",
"value": {
"type": "array",
"items": {
"$ref": "#/defs/StepResult"
}
"type": "object",
"properties": {
"steps": {
"type": "array",
"items": {
"$ref": "#/defs/StepResult"
},
"description": "the mover's steps"
},
"unlogged": {
"type": "array",
"items": {
"$ref": "#/defs/LogEntry"
},
"description": "the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189)"
}
},
"required": [
"steps"
]
},
"description": "one mover's steps in a pipeline, its key the pipeline's and the mover's module escaped as a key's name, written by compare-and-set as the pipeline's record is; the record's own steps stay empty, so a record never grows with the catalog (H-A-187)"
"description": "one mover's steps and unlogged step-log entries in a pipeline, its key the pipeline's and the mover's module escaped as a key's name, written by compare-and-set as the pipeline's record is; the record's own steps stay empty, so a record never grows with the catalog (H-A-187)"
}
]
},
@@ -1267,7 +1283,15 @@
"items": {
"$ref": "#/defs/LogEntry"
},
"description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606)"
"description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189)"
},
"mover-count": {
"type": "integer",
"description": "how many movers the pipeline has, set by list alone (H-A-191)"
},
"step-count": {
"type": "integer",
"description": "how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191)"
},
"published": {
"$ref": "#/defs/Published",
@@ -1627,6 +1651,10 @@
"pipeline": {
"type": "string"
},
"module": {
"type": "string",
"description": "the module of the mover whose step the entry is of (H-A-190)"
},
"step": {
"type": "string",
"enum": [
@@ -192,6 +192,8 @@ export interface LogEntry {
"at": string;
/** the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470) */
"kept"?: string[];
/** the module of the mover whose step the entry is of (H-A-190) */
"module"?: string;
"outcome"?: "running" | "waiting" | "passed" | "failed" | "nothing to run" | "bypassed" | "carried" | "could not run" | "waiting for a build node" | "did not build" | "accepted" | "declined" | "superseded" | "cancelled";
/** the output's name in step-outputs */
"output"?: string;
@@ -254,6 +256,8 @@ export interface Pipeline {
/** how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325) */
"end-count"?: number;
"ended"?: string;
/** how many movers the pipeline has, set by list alone (H-A-191) */
"mover-count"?: number;
/** the modules the pipeline moves, its take-in's entries (design 47 H §746) */
"movers": Mover[];
"outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled";
@@ -270,11 +274,13 @@ export interface Pipeline {
/** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */
"source": number;
"started"?: string;
/** how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191) */
"step-count"?: number;
/** every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187) */
"steps": StepResult[];
/** the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107) */
"superseded"?: string[];
/** the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606) */
/** the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189) */
"unlogged"?: LogEntry[];
/** the layout version */
"v"?: number;
@@ -804,7 +810,12 @@ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline;
export type PipelinesByBranchRunRunValue = Pipeline;
/** is one value of the state pipelines under the key steps.<pipeline>.<mover>. */
export type PipelinesByStepsPipelineMoverValue = StepResult[];
export interface PipelinesByStepsPipelineMoverValue {
/** the mover's steps */
"steps": StepResult[];
/** the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189) */
"unlogged"?: LogEntry[];
}
/** is one value of the state queue under the key queue.next. */
export type QueueByQueueNextValue = number;
@@ -118,7 +118,9 @@ func (s *MoverStore) moverKeys(key string) (map[string][]byte, map[string]uint64
return vals, revs, nil
}
// composeSteps sets p's steps from its mover keys, in the order of p's movers, a key no mover names last in key order.
// composeSteps sets p's steps from its mover keys, in the order of p's movers, a key no mover names last in key order, and
// its unlogged step-log entries: the record's own (an entry naming no mover, written before H-A-190) and each mover's, in
// the order they ended (H-A-189).
func composeSteps(p *seat.Pipeline, key string, vals map[string][]byte) error {
var names []string
for k := range vals {
@@ -137,40 +139,81 @@ func composeSteps(p *seat.Pipeline, key string, vals map[string][]byte) error {
}
}
steps := []seat.StepResult{}
unlogged := append([]seat.LogEntry(nil), p.Unlogged...)
for _, k := range order {
v, err := seat.DecodePipelinesByStepsPipelineMoverValue(vals[k])
if err != nil {
return fmt.Errorf("%s holds no mover's steps: %w", k, err)
}
steps = append(steps, v...)
steps = append(steps, v.Steps...)
unlogged = append(unlogged, v.Unlogged...)
}
p.Steps = steps
sort.SliceStable(unlogged, func(i, j int) bool { return unlogged[i].At.Before(unlogged[j].At) })
p.Steps, p.Unlogged = steps, unlogged
return nil
}
// split is p's record, its own steps empty, and each mover's steps by its mover key.
// split is p's record, its own steps and unlogged entries empty, and each mover's steps and unlogged step-log entries by
// its mover key (H-A-187, H-A-189). An entry naming no mover (one written before H-A-190) goes under the mover whose step
// it alone matches, else it stays in the record.
func split(key string, p seat.Pipeline) ([]byte, map[string][]byte, error) {
by := map[string][]seat.StepResult{}
by := map[string]*seat.PipelinesByStepsPipelineMoverValue{}
at := func(module string) *seat.PipelinesByStepsPipelineMoverValue {
k := StepsKey(key, module)
if by[k] == nil {
by[k] = &seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{}}
}
return by[k]
}
for _, st := range p.Steps {
if st.Module == "" {
return nil, nil, fmt.Errorf("a step %s of %s names no mover, so it has no mover key", st.Step, key)
}
k := StepsKey(key, st.Module)
by[k] = append(by[k], st)
v := at(st.Module)
v.Steps = append(v.Steps, st)
}
var kept []seat.LogEntry
for _, e := range p.Unlogged {
module := e.Module
if module == "" {
module = moverOfEntry(p, e)
}
if module == "" {
kept = append(kept, e)
continue
}
v := at(module)
v.Unlogged = append(v.Unlogged, e)
}
movers := map[string][]byte{}
for k, steps := range by {
raw, err := json.Marshal(steps)
for k, v := range by {
raw, err := json.Marshal(v)
if err != nil {
return nil, nil, err
}
movers[k] = raw
}
p.Steps = []seat.StepResult{}
p.Steps, p.Unlogged = []seat.StepResult{}, kept
record, err := json.Marshal(p)
return record, movers, err
}
// moverOfEntry is the mover of the one step an entry naming no mover matches by step and artifact, empty when none or
// more than one does.
func moverOfEntry(p seat.Pipeline, e seat.LogEntry) string {
found := ""
for _, st := range p.Steps {
if st.Step != e.Step || st.Artifact != e.Artifact {
continue
}
if found != "" && found != st.Module {
return ""
}
found = st.Module
}
return found
}
// Get is pipeline key with every mover's steps and the record's revision. A record that still carries its steps inline (one
// written before H-A-187) is read whole, said once in the journal, and split at its next write.
func (s *MoverStore) Get(key string) (seat.Pipeline, uint64, error) {
@@ -6,6 +6,7 @@ import (
"fmt"
"strings"
"testing"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
@@ -192,6 +193,48 @@ func TestMoverKeys(t *testing.T) {
return fmt.Sprintf("record under 64 KiB %t, a mover key under 16 KiB %t, %d steps read", len(m.vals[key]) < 64<<10,
largest < 16<<10, len(q.Steps))
}, "record under 64 KiB true, a mover key under 16 KiB true, 1032 steps read"},
{"a mover's unlogged entries live under its mover key, each filed by the mover it names (H-A-189, H-A-190)", func(t *testing.T) string {
m, s, _ := created(t)
p, rev, _ := s.Get(key)
at := time.Date(2026, 11, 20, 20, 0, 0, 0, time.UTC)
// Two movers' approvals end alike at one moment; each entry names its mover.
p.Unlogged = []seat.LogEntry{{Pipeline: key, Module: "beta", Step: seat.StepResultStepApproval, What: seat.LogEntryWhatEnded, At: at},
{Pipeline: key, Module: "alpha", Step: seat.StepResultStepApproval, What: seat.LogEntryWhatEnded, At: at.Add(time.Second)}}
if _, err := s.Update(key, p, rev); err != nil {
return err.Error()
}
var rec seat.Pipeline
_ = json.Unmarshal(m.vals[key], &rec)
beta, _ := seat.DecodePipelinesByStepsPipelineMoverValue(m.vals["steps.main.7.beta"])
q, _, _ := s.Get(key)
var got []string
for _, e := range q.Unlogged {
got = append(got, e.Module)
}
return fmt.Sprintf("record %d unlogged, beta's key %d (%s), read %s", len(rec.Unlogged), len(beta.Unlogged),
beta.Unlogged[0].Module, strings.Join(got, " "))
}, "record 0 unlogged, beta's key 1 (beta), read beta alpha"},
{"a 129-mover pipeline with 1,000 unlogged entries keeps its record small (H-A-189)", func(t *testing.T) string {
m, s, _ := created(t)
p, rev, _ := s.Get(key)
p.Movers, p.Steps, p.Unlogged = nil, nil, nil
at := time.Date(2026, 11, 20, 20, 0, 0, 0, time.UTC)
for i := range 129 {
mod := fmt.Sprintf("module-%03d", i)
p.Movers = append(p.Movers, seat.Mover{Module: mod, Commit: strings.Repeat("c", 40), Fingerprint: "fp-" + mod})
p.Steps = append(p.Steps, seat.StepResult{Module: mod, Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeNotAsked})
}
for i := range 1000 {
mod := fmt.Sprintf("module-%03d", i%129)
p.Unlogged = append(p.Unlogged, seat.LogEntry{Pipeline: key, Module: mod, Step: seat.StepResultStepBuild, Artifact: mod + ":program",
What: seat.LogEntryWhatEnded, Outcome: seat.StepResultOutcomeNotAsked, Output: strings.Repeat("o", 200), At: at.Add(time.Duration(i) * time.Millisecond)})
}
if _, err := s.Update(key, p, rev); err != nil {
return err.Error()
}
q, _, _ := s.Get(key)
return fmt.Sprintf("record under 64 KiB %t, %d unlogged read", len(m.vals[key]) < 64<<10, len(q.Unlogged))
}, "record under 64 KiB true, 1000 unlogged read"},
{"mover keys are told from records of every key form", func(t *testing.T) string {
var out []string
for _, k := range []string{"steps.main.7.alpha", "steps.main.run.ab12.alpha", "steps.main.7.published.p.v.alpha",
@@ -70,7 +70,7 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
}
// List is the pipelines and runs of a module, a branch, a commit or a run, newest started first, at most Limit when it
// names one (§83).
// names one (§83): each one's record with its movers and steps counted, its steps empty (H-A-191).
func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) {
all, err := r.pipelines()
if err != nil {
@@ -80,6 +80,9 @@ func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) {
for _, p := range all {
if (in.Module == "" || Moves(p, in.Module)) && (in.Branch == "" || p.Branch == in.Branch) &&
(in.Commit == "" || movesCommit(p, in.Commit)) && (in.Run == "" || p.Run == in.Run) {
// Each pipeline's record with its movers and steps counted, never its steps: show gives those (H-A-191).
p.MoverCount, p.StepCount = int64(len(p.Movers)), int64(len(p.Steps))
p.Steps, p.Unlogged = []seat.StepResult{}, nil
out = append(out, p)
}
}
@@ -53,11 +53,12 @@ func TestReads(t *testing.T) {
// A pipeline kept as H-A-187 keeps it: its record's steps empty, each mover's under steps.<pipeline>.<mover>.
put(t, pipes, "next.8", seat.Pipeline{Pipeline: "next.8", Branch: "next", Source: 8, Outcome: seat.PipelineOutcomeRunning,
Started: &t0, Steps: []seat.StepResult{}, Movers: []seat.Mover{{Module: "delta", Commit: "n8"}, {Module: "epsilon", Commit: "n8"}}})
put(t, pipes, "steps.next.8.epsilon", []seat.StepResult{{Module: "epsilon", Step: seat.StepResultStepBuild, Status: "required",
Outcome: seat.StepResultOutcomePassed, Output: "next.8.build.epsilon:program"}})
put(t, pipes, "steps.next.8.delta", []seat.StepResult{{Module: "delta", Step: seat.StepResultStepBuild, Status: "required",
Outcome: seat.StepResultOutcomePassed}, {Module: "delta", Step: seat.StepResultStepApproval, Status: "required",
Outcome: seat.StepResultOutcomeWaiting, On: seat.WaitOnPerson, Started: &t2}})
put(t, pipes, "steps.next.8.epsilon", seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{{Module: "epsilon",
Step: seat.StepResultStepBuild, Status: "required", Outcome: seat.StepResultOutcomePassed, Output: "next.8.build.epsilon:program"}}})
put(t, pipes, "steps.next.8.delta", seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{{Module: "delta",
Step: seat.StepResultStepBuild, Status: "required", Outcome: seat.StepResultOutcomePassed}, {Module: "delta",
Step: seat.StepResultStepApproval, Status: "required", Outcome: seat.StepResultOutcomeWaiting, On: seat.WaitOnPerson, Started: &t2}},
Unlogged: []seat.LogEntry{{Pipeline: "next.8", Module: "delta", Step: seat.StepResultStepBuild, What: seat.LogEntryWhatEnded, At: t1}}})
put(t, kept, "unit.sha256-aa", seat.KeptResult{Level: "unit", Digest: "sha256-aa", Package: "alpha/x"})
put(t, kept, "integration.sha256-bb", seat.KeptResult{Level: "integration", Digest: "sha256-bb", Package: "alpha/y"})
r := Reads{Pipelines: pipes, Kept: kept}
@@ -102,10 +103,15 @@ func TestReads(t *testing.T) {
_, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "alpha"})
return "", err
}, "error: the pipeline next.8 does not move alpha"},
{"list names a pipeline, never a mover key", func() (string, error) {
{"list names a pipeline, never a mover key, its record with its movers and steps counted and no step (H-A-191)", func() (string, error) {
a, err := r.List(seat.ListInput{Branch: "next"})
return names(a.Pipelines), err
}, "next.8"},
var s []string
for _, p := range a.Pipelines {
s = append(s, fmt.Sprintf("%s %d movers %d steps, %d steps and %d unlogged answered", p.Pipeline, p.MoverCount, p.StepCount,
len(p.Steps), len(p.Unlogged)))
}
return strings.Join(s, "; "), err
}, "next.8 2 movers 3 steps, 0 steps and 0 unlogged answered"},
{"list by module, newest first", func() (string, error) {
a, err := r.List(seat.ListInput{Module: "alpha"})
return names(a.Pipelines), err
@@ -441,7 +441,7 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
// endEntry is the step log's entry of a step's end, with the keys of the kept results it wrote or carried.
func endEntry(key string, s seat.StepResult, at time.Time, kept ...string) seat.LogEntry {
return seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
return seat.LogEntry{Pipeline: key, Module: s.Module, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
Output: s.Output, Kept: kept, At: at}
}
@@ -7,9 +7,10 @@ import (
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// sameEntry says whether two step-log entries are one: the same pipeline, step, artifact, end or start and time.
// sameEntry says whether two step-log entries are one: the same pipeline, mover (H-A-190), step, artifact, end or start and
// time.
func sameEntry(a, b seat.LogEntry) bool {
return a.Pipeline == b.Pipeline && a.Step == b.Step && a.Artifact == b.Artifact && a.What == b.What && a.At.Equal(b.At)
return a.Pipeline == b.Pipeline && a.Module == b.Module && a.Step == b.Step && a.Artifact == b.Artifact && a.What == b.What && a.At.Equal(b.At)
}
// appendUnlogged appends the unlogged entries of pipeline key to the step log in their order and clears each appended one in a