Design 47 H1: the runner keeps each mover's steps under its own key, so a record never grows with the catalog (H-A-187)

The runner's state pipelines held a pipeline's every step in its record, and
a record of 112 or 129 movers exceeded the bus's 1 MiB message limit (ADR
0486), so main.26 and main.30 placed no step. One MoverStore per process now
reads and writes the state: the record keeps the movers, tiers, call, source
and outcome with its own steps empty, each mover's steps are written under
steps.<pipeline>.<mover> by compare-and-set as the record is, and nothing is
written when the record or a mover key changed since the read. Every read
and judgement of steps goes through it, so the placing, the step ends, the
end judgement, the hand acts and the ageing see one composed pipeline; show,
list, stands and why-carried compose from the mover keys, and show naming a
mover answers its steps alone. A record still carrying its steps inline is
read whole, said once in the journal, and split at its next write, its
mover keys written before the record.

Builds: design 47 H H-A-187
This commit is contained in:
2026-11-20 21:11:59 +01:00
parent f2d3172eac
commit 4e585f02b1
11 changed files with 755 additions and 82 deletions
@@ -19,7 +19,7 @@ import (
// §549): runner_unlogged lists them, runner_unlogged_append appends them by hand, written in the hand-act log; it is of class
// act and takes §391's default callers.
func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.HandActs, logf func(string, ...any)) []stdio.Tool {
h := &pipeline.Hearer{Store: stdioPipelines{pipelines}, Logf: logf,
h := &pipeline.Hearer{Store: &pipeline.MoverStore{Raw: stdioPipelines{pipelines}, Logf: logf}, Logf: logf,
Log: func(e seat.LogEntry) error {
_, err := log.Append(keys.EscapeName(e.Pipeline), e)
return err
@@ -67,26 +67,36 @@ func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.Hand
}
}
// stdioPipelines is the state pipelines as the runtime serves it, for the by-hand append.
// stdioPipelines is the state pipelines as bytes, as the runtime serves it, for the by-hand append: read and written through
// a pipeline.MoverStore, which keeps each mover's steps under its own key (H-A-187).
type stdioPipelines struct{ s stdio.KeptState }
func (s stdioPipelines) Get(key string) (seat.Pipeline, uint64, error) {
func (s stdioPipelines) Get(key string) ([]byte, uint64, error) {
e, err := s.s.Get(key)
if err != nil || e == nil {
return seat.Pipeline{}, 0, errors.Join(err, errors.New("no such pipeline"))
return nil, 0, err
}
var p seat.Pipeline
return p, e.Revision, json.Unmarshal(e.Value, &p)
return e.Value, e.Revision, nil
}
func (s stdioPipelines) Update(key string, p seat.Pipeline, rev uint64) (uint64, error) {
r, err := s.s.Update(key, p, rev)
func (s stdioPipelines) Create(key string, value []byte) (uint64, error) {
r, err := s.s.Create(key, json.RawMessage(value))
if err != nil && strings.Contains(err.Error(), "exists") {
return 0, pipeline.ErrExists
}
return r, err
}
func (s stdioPipelines) Update(key string, value []byte, rev uint64) (uint64, error) {
r, err := s.s.Update(key, json.RawMessage(value), rev)
if err != nil && (strings.Contains(err.Error(), "wrong last sequence") || strings.Contains(err.Error(), "revision")) {
return 0, pipeline.ErrChanged
}
return r, err
}
func (s stdioPipelines) Delete(key string) error { return s.s.Delete(key) }
func (s stdioPipelines) Keys(filter string) ([]string, error) {
all, err := s.s.Keys()
var out []string
@@ -136,9 +136,9 @@ func (k keptStore) Get(key string) (seat.KeptResult, bool, error) {
func (k keptStore) Delete(key string) error { return k.kv.Delete(k.ctx, key) }
// newAger is the ageing over the runner's own stores.
func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager {
func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines *pipeline.MoverStore, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager {
return &pipeline.Ager{Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Read, Aged: agedState{ctx: ctx, kv: aged},
Pipelines: pipelineDeleter{pipelineStore{ctx: ctx, kv: pipelines}}, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf,
Pipelines: pipelines, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf,
Outputs: func(name string) error {
o, err := js.ObjectStore(ctx, catalogue.ObjectsBucket(cred.Module, seat.StateStepOutputs))
if err != nil {
@@ -151,11 +151,6 @@ func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account,
}}
}
// pipelineDeleter is the pipelines store with the delete only its owner makes (§330).
type pipelineDeleter struct{ pipelineStore }
func (d pipelineDeleter) Delete(key string) error { return d.kv.Delete(d.ctx, key) }
// storeFills reads the fill of the runner's stores against their caps: pipelines and kept (their key-value streams), step-outputs
// (its object store's) and step-log (its log's), as the bus's own streams say it (design 47 G §366, §486).
func storeFills(ctx context.Context, js jetstream.JetStream, cred process.Account, now func() time.Time) func() ([]seat.Fill, error) {
@@ -128,9 +128,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
return err
}
hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf)
store := newPipelineStore(ctx, pipelines, logger.Printf)
hearer := newHearer(ctx, js, cred, store, kept, say, logger.Printf)
hearer.Asker = busAsker{conn: conn, cred: cred}
hearer.Ager = newAger(ctx, js, cred, pipelines, kept, aged, logger.Printf)
hearer.Ager = newAger(ctx, js, cred, store, kept, aged, logger.Printf)
// The stores' fill is said before a store is full, and a full one refuses a start (§366).
fills := &pipeline.Fills{Read: storeFills(ctx, js, cred, time.Now), Logf: logger.Printf,
Say: func(event string, f seat.Fill) error { return say("mesh.seat."+seat.Seat+".event."+event, f) }}
@@ -147,7 +148,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf}
// Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked
// for its step.
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say,
starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say,
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
@@ -170,10 +171,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return
}
// A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327).
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err)
}
n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half)
n, err := starter.Resume(store, branch, half)
if err != nil {
logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err)
return
@@ -184,19 +185,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200).
advance := func(branch string) {
rv.Holders.refresh()
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err)
}
}
advancer.Store(&advance)
// A registration-changed places the steps of its module's pipelines started before it, from it (A-327).
register := func(reg catalog.Registration) {
if n, err := starter.Registered(pipelineStore{ctx: ctx, kv: pipelines}, reg, rv.Catalog.View); err != nil || n > 0 {
if n, err := starter.Registered(store, reg, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the pipelines of %s started before its registration: %d placed (%v)", reg.Module, n, err)
}
}
registrar.Store(&register)
starter.Store = pipelineStore{ctx: ctx, kv: pipelines}
starter.Store = store
// The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in
// the state queue, placed at the start from it and on every end.
atOnce, waitingCap, err := queueSettings(getenv)
@@ -401,6 +401,8 @@ func pipelineKeys(t *testing.T, ctx context.Context, kv jetstream.KeyValue) []st
if err != nil {
t.Fatal(err)
}
// A mover's steps are a key of their own (H-A-187), placed with its pipeline: no pipeline of their own.
ks = slices.DeleteFunc(ks, pipeline.IsStepsKey)
slices.Sort(ks)
return ks
}
@@ -7,6 +7,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -221,6 +222,34 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && p.Outcome == seat.PipelineOutcomePassed })
// H-A-187 over the bus: show composes the pipeline from its record and mover keys, naming a mover its steps alone, and a
// record written before it, its steps inline, is read whole and split at its next write.
shown, err := pipeline.Reads{Pipelines: kvKept{ctx, pipelines}}.Show(seat.ShowInput{Pipeline: key, Mover: "app"})
if err != nil || len(shown.Pipeline.Steps) != len(p.Steps) || shown.Pipeline.Steps[0].Module != "app" {
t.Fatalf("show of %s naming app = %d steps (%v), want its %d", key, len(shown.Pipeline.Steps), err, len(p.Steps))
}
inline := p
inline.Pipeline = "main.2"
inlineRaw, _ := json.Marshal(inline)
if _, err := pipelines.Put(ctx, "main.2", inlineRaw); err != nil {
t.Fatal(err)
}
store := newPipelineStore(ctx, pipelines, func(f string, a ...any) { fmt.Fprintf(logs, f+"\n", a...) })
old, oldRev, err := store.Get("main.2")
if err != nil || len(old.Steps) != len(p.Steps) {
t.Fatalf("the inline record read %d steps (%v), want %d", len(old.Steps), err, len(p.Steps))
}
if _, err := store.Update("main.2", old, oldRev); err != nil {
t.Fatal(err)
}
var split seat.Pipeline
if !readPipeline(ctx, pipelines, "main.2", &split) || len(split.Steps) != len(p.Steps) {
t.Fatalf("the record split reads %d steps, want %d", len(split.Steps), len(p.Steps))
}
if _, err := pipelines.Get(ctx, pipeline.StepsKey("main.2", "app")); err != nil || !strings.Contains(logs.String(), "split into 1 mover keys") {
t.Fatalf("the mover key of main.2's app: %v; the journal says the split: %t", err, strings.Contains(logs.String(), "split into 1 mover keys"))
}
// §470, §330: the step log is the log's stream, read onward by sequence the way the ageing reads it.
logged, err := stepLog{ctx: ctx, js: js, module: module}.Read(1, 100)
if err != nil || len(logged) == 0 || logged[0].Entry.Pipeline != key {
@@ -274,13 +303,20 @@ func stepOf(p seat.Pipeline, step string) seat.StepResult {
return seat.StepResult{}
}
// readPipeline reads pipeline key as the runner does, its steps composed from its mover keys, and holds its record to
// H-A-187: a record the runner wrote carries no steps of its own, so it never grows with the catalog.
func readPipeline(ctx context.Context, kv jetstream.KeyValue, key string, p *seat.Pipeline) bool {
e, err := kv.Get(ctx, key)
if err != nil {
return false
}
*p = seat.Pipeline{}
return json.Unmarshal(e.Value(), p) == nil
var record seat.Pipeline
if json.Unmarshal(e.Value(), &record) != nil || len(record.Steps) > 0 {
panic(fmt.Sprintf("the record of %s carries %d steps of its own; each mover's steps are its own key (H-A-187)", key, len(record.Steps)))
}
got, _, err := newPipelineStore(ctx, kv, nil).Get(key)
*p = got
return err == nil
}
// waitFor asks cond until it holds or the test's context ends, the module's journal in the failure.
@@ -1,9 +1,7 @@
package main
import (
"context"
"encoding/json"
"errors"
"github.com/nats-io/nats.go/jetstream"
@@ -13,20 +11,6 @@ import (
"mesh-delivery-pipeline/internal/pipeline"
)
// createdKV is the state pipelines written by compare-and-set on a key's absence (§54).
type createdKV struct {
ctx context.Context
kv jetstream.KeyValue
}
func (c createdKV) Create(key string, value []byte) error {
_, err := c.kv.Create(c.ctx, key, value)
if errors.Is(err, jetstream.ErrKeyExists) {
return pipeline.ErrExists
}
return err
}
// hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one
// pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is
// written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard
@@ -19,64 +19,74 @@ import (
"mesh-delivery-pipeline/internal/pipeline"
)
// pipelineStore is the state pipelines as the hearer reads and writes it: a write by compare-and-set on the revision read (§27).
type pipelineStore struct {
// kvState is the state pipelines as bytes on the bus, by compare-and-set (§27); the runner reads and writes pipelines
// through one pipeline.MoverStore over it, which keeps each mover's steps under its own key (H-A-187).
type kvState struct {
ctx context.Context
kv jetstream.KeyValue
}
func (s pipelineStore) Get(key string) (seat.Pipeline, uint64, error) {
e, err := s.kv.Get(s.ctx, key)
if err != nil {
return seat.Pipeline{}, 0, err
}
var p seat.Pipeline
if err := json.Unmarshal(e.Value(), &p); err != nil {
return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
}
return p, e.Revision(), nil
// newPipelineStore is the process's one store of the state pipelines.
func newPipelineStore(ctx context.Context, kv jetstream.KeyValue, logf func(string, ...any)) *pipeline.MoverStore {
return &pipeline.MoverStore{Raw: kvState{ctx: ctx, kv: kv}, Logf: logf}
}
func (s pipelineStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) {
raw, err := json.Marshal(p)
if err != nil {
return 0, err
func (s kvState) Get(key string) ([]byte, uint64, error) {
e, err := s.kv.Get(s.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, nil
}
rev, err := s.kv.Update(s.ctx, key, raw, revision)
if err != nil {
return nil, 0, err
}
return e.Value(), e.Revision(), nil
}
func (s kvState) Create(key string, value []byte) (uint64, error) {
rev, err := s.kv.Create(s.ctx, key, value)
if errors.Is(err, jetstream.ErrKeyExists) {
return 0, pipeline.ErrExists
}
return rev, err
}
func (s kvState) Update(key string, value []byte, revision uint64) (uint64, error) {
rev, err := s.kv.Update(s.ctx, key, value, revision)
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
return 0, pipeline.ErrChanged
}
return rev, err
}
func (s pipelineStore) Keys(filter string) ([]string, error) {
all, err := s.kv.Keys(s.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
func (s kvState) Delete(key string) error {
if err := s.kv.Delete(s.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
}
// Keys are the keys matching filter, `*` one key part, as the bus filters a key-value's subjects.
func (s kvState) Keys(filter string) ([]string, error) {
l, err := s.kv.ListKeysFiltered(s.ctx, filter)
if err != nil {
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return nil, err
}
// The lister ends its own watch once the current keys are listed.
var out []string
want := strings.Split(filter, ".")
for _, k := range all {
got := strings.Split(k, ".")
if len(got) != len(want) {
continue
}
ok := true
for i := range want {
ok = ok && (want[i] == "*" || want[i] == got[i])
}
if ok {
out = append(out, k)
}
for k := range l.Keys() {
out = append(out, k)
}
return out, err
return out, nil
}
// newHearer is the hearer over the runner's own states: pipelines, kept, the step log on its stream and step-outputs on its
// objects (§27). Each store is opened where it is used, so one the grants part has not made yet fails that write loudly.
func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept jetstream.KeyValue,
func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines pipeline.Updater, kept jetstream.KeyValue,
say func(string, any) error, logf func(string, ...any)) *pipeline.Hearer {
return &pipeline.Hearer{Store: pipelineStore{ctx: ctx, kv: pipelines}, Say: say, Logf: logf, Name: cred.Module,
return &pipeline.Hearer{Store: pipelines, Say: say, Logf: logf, Name: cred.Module,
Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Append,
Keep: func(key string, k seat.KeptResult) error {
raw, err := json.Marshal(k)
@@ -0,0 +1,353 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"sort"
"strings"
"sync"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// A pipeline's steps live one key per mover (design 47 H-A-187): the state pipelines keeps the pipeline's record (its
// movers, tiers, call, source and outcome) under the pipeline's key with its own steps empty, and each mover's steps under
// `steps.<pipeline>.<mover>`, written by compare-and-set as the record is, so a record never grows with the catalog and no
// value nears the bus's message limit (ADR 0486). Every read of a pipeline composes its steps from the mover keys it names.
// StepsKey is the key of one mover's steps in the state pipelines, `steps.<pipeline>.<mover>`, the mover's module escaped
// as a key's name is (H-A-187).
func StepsKey(pipeline, mover string) string {
return seat.PipelinesByStepsPipelineMoverKey(pipeline, keys.EscapeName(mover))
}
// stepsPrefix is the start of every mover key of one pipeline.
func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") }
// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a
// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`) or a publish's (7 parts) and one mover.
func IsStepsKey(key string) bool {
parts := strings.Split(key, ".")
if parts[0] != "steps" {
return false
}
switch len(parts) {
case 4, 7:
return true
case 5:
return parts[2] == runToken
}
return false
}
// RawState is the state pipelines as bytes: a key's value and revision (nil, 0 and no error when it has none), a create by
// compare-and-set on absence (ErrExists), an update on the revision read (ErrChanged), a delete, and the keys matching a
// filter, `*` one key part.
type RawState interface {
Get(key string) ([]byte, uint64, error)
Create(key string, value []byte) (uint64, error)
Update(key string, value []byte, revision uint64) (uint64, error)
Delete(key string) error
Keys(filter string) ([]string, error)
}
// ErrNoPipeline is a read of a pipeline key the state pipelines does not hold.
var ErrNoPipeline = errors.New("no such pipeline")
// MoverStore is the state pipelines as the runner reads and writes it (§27, H-A-187): Get composes a pipeline from its
// record and its mover keys, Update writes the record and each mover key that changed, each by compare-and-set on the
// revision read, and nothing when any of them was written since. One MoverStore serves a process: a write and a read of one
// pipeline take its lock, so the process never reads a record and mover keys of two writes.
type MoverStore struct {
Raw RawState
Logf func(string, ...any)
mu sync.Mutex
locks map[string]*sync.Mutex
read map[string]readAt
inline map[string]bool
}
// readAt is what the last read of a pipeline found: its record's revision and each mover key's revision.
type readAt struct {
rev uint64
movers map[string]uint64
}
func (s *MoverStore) lock(key string) func() {
s.mu.Lock()
if s.locks == nil {
s.locks, s.read, s.inline = map[string]*sync.Mutex{}, map[string]readAt{}, map[string]bool{}
}
l, ok := s.locks[key]
if !ok {
l = &sync.Mutex{}
s.locks[key] = l
}
s.mu.Unlock()
l.Lock()
return l.Unlock
}
func (s *MoverStore) logf(f string, a ...any) {
if s.Logf != nil {
s.Logf(f, a...)
}
}
// moverKeys are the mover keys of pipeline key with their values and revisions.
func (s *MoverStore) moverKeys(key string) (map[string][]byte, map[string]uint64, error) {
ks, err := s.Raw.Keys(stepsPrefix(key) + "*")
if err != nil {
return nil, nil, fmt.Errorf("the mover keys of %s are not listed: %w", key, err)
}
vals, revs := map[string][]byte{}, map[string]uint64{}
for _, k := range ks {
v, rev, err := s.Raw.Get(k)
if err != nil {
return nil, nil, fmt.Errorf("%s is not read: %w", k, err)
}
if v != nil {
vals[k], revs[k] = v, rev
}
}
return vals, revs, nil
}
// composeSteps sets p's steps from its mover keys, in the order of p's movers, a key no mover names last in key order.
func composeSteps(p *seat.Pipeline, key string, vals map[string][]byte) error {
var names []string
for k := range vals {
names = append(names, k)
}
sort.Strings(names)
order := make([]string, 0, len(names))
for _, m := range p.Movers {
if k := StepsKey(key, m.Module); vals[k] != nil && !slices.Contains(order, k) {
order = append(order, k)
}
}
for _, k := range names {
if !slices.Contains(order, k) {
order = append(order, k)
}
}
steps := []seat.StepResult{}
for _, k := range order {
v, err := seat.DecodePipelinesByStepsPipelineMoverValue(vals[k])
if err != nil {
return fmt.Errorf("%s holds no mover's steps: %w", k, err)
}
steps = append(steps, v...)
}
p.Steps = steps
return nil
}
// split is p's record, its own steps empty, and each mover's steps by its mover key.
func split(key string, p seat.Pipeline) ([]byte, map[string][]byte, error) {
by := map[string][]seat.StepResult{}
for _, st := range p.Steps {
if st.Module == "" {
return nil, nil, fmt.Errorf("a step %s of %s names no mover, so it has no mover key", st.Step, key)
}
k := StepsKey(key, st.Module)
by[k] = append(by[k], st)
}
movers := map[string][]byte{}
for k, steps := range by {
raw, err := json.Marshal(steps)
if err != nil {
return nil, nil, err
}
movers[k] = raw
}
p.Steps = []seat.StepResult{}
record, err := json.Marshal(p)
return record, movers, err
}
// Get is pipeline key with every mover's steps and the record's revision. A record that still carries its steps inline (one
// written before H-A-187) is read whole, said once in the journal, and split at its next write.
func (s *MoverStore) Get(key string) (seat.Pipeline, uint64, error) {
defer s.lock(key)()
raw, rev, err := s.Raw.Get(key)
if err != nil {
return seat.Pipeline{}, 0, err
}
if raw == nil {
return seat.Pipeline{}, 0, fmt.Errorf("%w: %s", ErrNoPipeline, key)
}
var p seat.Pipeline
if err := json.Unmarshal(raw, &p); err != nil {
return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
}
vals, revs, err := s.moverKeys(key)
if err != nil {
return seat.Pipeline{}, 0, err
}
s.read[key] = readAt{rev: rev, movers: revs}
if len(p.Steps) > 0 {
if !s.inline[key] {
s.inline[key] = true
s.logf("the pipeline %s carries its %d steps inline in its record: they are read whole and split into one key per mover at its next write (H-A-187)", key, len(p.Steps))
}
return p, rev, nil
}
if p.Steps == nil {
p.Steps = []seat.StepResult{}
}
return p, rev, composeSteps(&p, key, vals)
}
// Update writes p as pipeline key's record and its mover keys, each by compare-and-set: nothing is written, ErrChanged,
// when the record or a mover key was written since the read revision gave. A record carrying its steps inline has its
// mover keys written before the record, so its steps are never lost between the two.
func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) {
defer s.lock(key)()
record, want, err := split(key, p)
if err != nil {
return 0, err
}
raw, cur, err := s.Raw.Get(key)
if err != nil {
return 0, err
}
if raw == nil {
return 0, fmt.Errorf("%w: %s", ErrNoPipeline, key)
}
if cur != revision {
return 0, ErrChanged
}
vals, revs, err := s.moverKeys(key)
if err != nil {
return 0, err
}
// A mover key written since the read is another write between: the read named its revision.
if seen, ok := s.read[key]; ok && seen.rev == revision {
for k, r := range revs {
if seen.movers[k] != r {
return 0, ErrChanged
}
}
for k := range seen.movers {
if _, ok := revs[k]; !ok {
return 0, ErrChanged
}
}
}
var was seat.Pipeline
inline := json.Unmarshal(raw, &was) == nil && len(was.Steps) > 0
var after map[string]uint64
if inline {
if after, err = s.writeMovers(key, want, vals, revs); err != nil {
return 0, err
}
}
rev, err := s.Raw.Update(key, record, revision)
if err != nil {
return 0, err
}
if !inline {
if after, err = s.writeMovers(key, want, vals, revs); err != nil {
return 0, err
}
} else {
s.logf("the pipeline %s's steps are split into %d mover keys and its record no longer carries them (H-A-187)", key, len(want))
delete(s.inline, key)
}
s.read[key] = readAt{rev: rev, movers: after}
return rev, nil
}
// writeMovers writes each mover key whose steps changed and deletes each that p no longer names, by compare-and-set on the
// revisions read, and answers every mover key's revision after it.
func (s *MoverStore) writeMovers(key string, want, vals map[string][]byte, revs map[string]uint64) (map[string]uint64, error) {
after := map[string]uint64{}
for _, k := range slices.Sorted(maps.Keys(want)) {
v := want[k]
old, ok := vals[k]
var rev uint64
var err error
switch {
case ok && string(old) == string(v):
after[k] = revs[k]
continue
case ok:
rev, err = s.Raw.Update(k, v, revs[k])
default:
rev, err = s.Raw.Create(k, v)
}
if err != nil {
return nil, fmt.Errorf("the steps of %s are not written: %w", k, err)
}
after[k] = rev
}
for k := range vals {
if _, ok := want[k]; !ok {
if err := s.Raw.Delete(k); err != nil {
return nil, fmt.Errorf("the steps of %s are not deleted: %w", k, err)
}
}
}
return after, nil
}
// Create writes a new pipeline, its record by compare-and-set on its absence (ErrExists: the second start writes nothing,
// §54), then its mover keys; a mover key an earlier pipeline of the key left is written over, said in the journal.
func (s *MoverStore) Create(key string, value []byte) error {
var p seat.Pipeline
if err := json.Unmarshal(value, &p); err != nil {
return fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
}
defer s.lock(key)()
record, want, err := split(key, p)
if err != nil {
return err
}
rev, err := s.Raw.Create(key, record)
if err != nil {
return err
}
vals, revs, err := s.moverKeys(key)
if err != nil {
return err
}
for k := range vals {
s.logf("%s was left by an earlier pipeline of the key: it is written over (H-A-187)", k)
}
after, err := s.writeMovers(key, want, vals, revs)
if err != nil {
return err
}
s.read[key] = readAt{rev: rev, movers: after}
return nil
}
// Delete removes pipeline key's mover keys, then its record (§330): a delete stopped between leaves the record, which the
// next ageing deletes again.
func (s *MoverStore) Delete(key string) error {
unlock := s.lock(key)
vals, _, err := s.moverKeys(key)
if err == nil {
for k := range vals {
if err = s.Raw.Delete(k); err != nil {
break
}
}
}
if err == nil {
err = s.Raw.Delete(key)
}
delete(s.read, key)
delete(s.inline, key)
unlock()
return err
}
// Keys are the keys of the state pipelines matching filter, `*` one key part.
func (s *MoverStore) Keys(filter string) ([]string, error) { return s.Raw.Keys(filter) }
@@ -0,0 +1,211 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// memRaw is the state pipelines as bytes, one revision sequence for the bucket as a key-value's stream has.
type memRaw struct {
vals map[string][]byte
revs map[string]uint64
seq uint64
}
func newMemRaw() *memRaw { return &memRaw{vals: map[string][]byte{}, revs: map[string]uint64{}} }
func (m *memRaw) Get(key string) ([]byte, uint64, error) {
v, ok := m.vals[key]
if !ok {
return nil, 0, nil
}
return append([]byte(nil), v...), m.revs[key], nil
}
func (m *memRaw) put(key string, value []byte) uint64 {
m.seq++
m.vals[key], m.revs[key] = append([]byte(nil), value...), m.seq
return m.seq
}
func (m *memRaw) Create(key string, value []byte) (uint64, error) {
if _, ok := m.vals[key]; ok {
return 0, ErrExists
}
return m.put(key, value), nil
}
func (m *memRaw) Update(key string, value []byte, rev uint64) (uint64, error) {
if m.revs[key] != rev {
return 0, ErrChanged
}
return m.put(key, value), nil
}
func (m *memRaw) Delete(key string) error {
delete(m.vals, key)
delete(m.revs, key)
return nil
}
func (m *memRaw) Keys(filter string) ([]string, error) {
var out []string
for k := range m.vals {
if ok, _ := matchKey(filter, k); ok {
out = append(out, k)
}
}
return out, nil
}
// recordSteps is the steps the raw record of key carries itself.
func recordSteps(t *testing.T, m *memRaw, key string) int {
t.Helper()
var p seat.Pipeline
if err := json.Unmarshal(m.vals[key], &p); err != nil {
t.Fatal(err)
}
return len(p.Steps)
}
// TestMoverKeys holds the state pipelines' shape to H-A-187: a pipeline's record keeps its movers and outcome, each mover's
// steps their own key written by compare-and-set as the record is, a read composing them, a record carrying its steps inline
// read whole and split at its next write, one row per case.
func TestMoverKeys(t *testing.T) {
const key = "main.7"
two := func() seat.Pipeline {
return seat.Pipeline{V: 1, Pipeline: key, Branch: "main", Source: 7, Outcome: seat.PipelineOutcomeRunning,
Movers: []seat.Mover{{Module: "beta"}, {Module: "alpha"}},
Steps: []seat.StepResult{{Module: "alpha", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning},
{Module: "beta", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning},
{Module: "beta", Step: seat.StepResultStepApproval, Outcome: seat.StepResultOutcomeNotAsked}}}
}
created := func(t *testing.T) (*memRaw, *MoverStore, *[]string) {
m, logs := newMemRaw(), &[]string{}
s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }}
raw, _ := json.Marshal(two())
if err := s.Create(key, raw); err != nil {
t.Fatal(err)
}
return m, s, logs
}
steps := func(p seat.Pipeline) string {
var out []string
for _, st := range p.Steps {
out = append(out, st.Module+" "+st.Step+" "+st.Outcome)
}
return strings.Join(out, "; ")
}
rows := []struct {
name string
run func(t *testing.T) string
want string
}{
{"a write keeps the record's steps empty and each mover's under its own key; a read composes them in the movers' order", func(t *testing.T) string {
m, s, _ := created(t)
p, _, err := s.Get(key)
return fmt.Sprintf("record %d, alpha %t, beta %t: %s %v", recordSteps(t, m, key), m.vals["steps.main.7.alpha"] != nil,
m.vals["steps.main.7.beta"] != nil, steps(p), err)
}, "record 0, alpha true, beta true: beta build running; beta approval not asked; alpha build running <nil>"},
{"an update writes the mover whose steps changed and leaves the other's key as it was", func(t *testing.T) string {
m, s, _ := created(t)
p, rev, _ := s.Get(key)
beta := m.revs["steps.main.7.beta"]
p.Steps[2].Outcome = seat.StepResultOutcomePassed
_, err := s.Update(key, p, rev)
q, _, _ := s.Get(key)
return fmt.Sprintf("beta kept %t: %s %v", m.revs["steps.main.7.beta"] == beta, steps(q), err)
}, "beta kept true: beta build running; beta approval not asked; alpha build passed <nil>"},
{"a mover key written since the read refuses the update, nothing written", func(t *testing.T) string {
m, s, _ := created(t)
p, rev, _ := s.Get(key)
m.put("steps.main.7.alpha", m.vals["steps.main.7.alpha"])
p.Outcome = seat.PipelineOutcomePassed
_, err := s.Update(key, p, rev)
return fmt.Sprintf("%t, record at %t", errors.Is(err, ErrChanged), m.revs[key] == rev)
}, "true, record at true"},
{"a record written since the read refuses the update", func(t *testing.T) string {
_, s, _ := created(t)
p, rev, _ := s.Get(key)
if _, err := s.Update(key, p, rev); err != nil {
return err.Error()
}
_, err := s.Update(key, p, rev)
return fmt.Sprint(errors.Is(err, ErrChanged))
}, "true"},
{"a record carrying its steps inline is read whole, said once, and split at its next write", func(t *testing.T) string {
m, logs := newMemRaw(), &[]string{}
s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }}
raw, _ := json.Marshal(two())
m.put(key, raw)
p, rev, _ := s.Get(key)
_, _, _ = s.Get(key)
read := fmt.Sprintf("read %d steps, %d lines", len(p.Steps), len(*logs))
_, err := s.Update(key, p, rev)
q, _, _ := s.Get(key)
return fmt.Sprintf("%s; written: record %d, %s, split said %t %v", read, recordSteps(t, m, key), steps(q),
strings.Contains(strings.Join(*logs, "\n"), "split into 2 mover keys"), err)
}, "read 3 steps, 1 lines; written: record 0, beta build running; beta approval not asked; alpha build running, split said true <nil>"},
{"a second create writes nothing", func(t *testing.T) string {
m, s, _ := created(t)
p := two()
p.Steps[0].Outcome = seat.StepResultOutcomeFailed
raw, _ := json.Marshal(p)
err := s.Create(key, raw)
q, _, _ := s.Get(key)
return fmt.Sprintf("%t %d: %s", errors.Is(err, ErrExists), len(m.vals), steps(q))
}, "true 3: beta build running; beta approval not asked; alpha build running"},
{"a delete removes the mover keys and the record", func(t *testing.T) string {
m, s, _ := created(t)
err := s.Delete(key)
return fmt.Sprintf("%d %v", len(m.vals), err)
}, "0 <nil>"},
{"a record of 129 movers never grows with their steps (main.26)", func(t *testing.T) string {
m, s, _ := created(t)
p, rev, _ := s.Get(key)
p.Movers, p.Steps = nil, nil
for i := range 129 {
mod := fmt.Sprintf("module-%03d", i)
p.Movers = append(p.Movers, seat.Mover{Module: mod, Commit: strings.Repeat("c", 40), Fingerprint: "fp-" + mod})
for _, step := range []string{seat.StepResultStepBuild, seat.StepResultStepLint, seat.StepResultStepUnit,
seat.StepResultStepIntegration, seat.StepResultStepValidate, seat.StepResultStepComposition,
seat.StepResultStepReview, seat.StepResultStepApproval} {
p.Steps = append(p.Steps, seat.StepResult{Module: mod, Step: step, Artifact: mod + ":program",
Outcome: seat.StepResultOutcomeFailed, Why: strings.Repeat("why ", 250)})
}
}
if _, err := s.Update(key, p, rev); err != nil {
return err.Error()
}
largest := 0
for k, v := range m.vals {
if IsStepsKey(k) && len(v) > largest {
largest = len(v)
}
}
q, _, _ := s.Get(key)
return fmt.Sprintf("record under 64 KiB %t, a mover key under 16 KiB %t, %d steps read", len(m.vals[key]) < 64<<10,
largest < 16<<10, len(q.Steps))
}, "record under 64 KiB true, a mover key under 16 KiB true, 1032 steps read"},
{"mover keys are told from records of every key form", func(t *testing.T) string {
var out []string
for _, k := range []string{"steps.main.7.alpha", "steps.main.run.ab12.alpha", "steps.main.7.published.p.v.alpha",
"main.7", "steps.7", "steps.run.ab12", "steps.7.published.p.v"} {
out = append(out, fmt.Sprint(IsStepsKey(k)))
}
return strings.Join(out, " ")
}, "true true true false false false false"},
}
for _, r := range rows {
t.Run(r.name, func(t *testing.T) {
if got := r.run(t); got != r.want {
t.Errorf("got %s\nwant %s", got, r.want)
}
})
}
}
@@ -39,7 +39,8 @@ func (r Reads) fills() ([]seat.Fill, error) {
return f, nil
}
// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83).
// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83), or, naming a mover, the
// pipeline's record with that mover's steps alone (H-A-187).
func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
p, ok, err := r.pipeline(in.Pipeline)
if err != nil {
@@ -52,6 +53,13 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
if err != nil {
return seat.ShowAnswer{}, err
}
// Named, a mover is answered by its steps alone (H-A-187): one mover's of a pipeline of a hundred movers.
if m := strings.TrimSpace(in.Mover); m != "" {
if !Moves(p, m) {
return seat.ShowAnswer{}, fmt.Errorf("the pipeline %s does not move %s", in.Pipeline, m)
}
p.Steps = stepsOf(p, m)
}
a := seat.ShowAnswer{Pipeline: p, Fill: fills}
for _, s := range p.Steps {
if s.Output != "" {
@@ -178,6 +186,16 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) {
if key == "" {
return seat.Pipeline{}, false, fmt.Errorf("name the pipeline's key, <branch>.<revision>.<module>")
}
all, err := r.Pipelines.Keys()
if err != nil {
return seat.Pipeline{}, false, fmt.Errorf("%s cannot be read: %w", seat.StatePipelines, err)
}
return r.pipelineOf(key, all)
}
// pipelineOf is pipeline key, its steps composed from the mover keys among all that it names (H-A-187); a record that
// still carries its steps inline is read whole.
func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) {
raw, err := r.Pipelines.Value(key)
if err != nil {
return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, key, err)
@@ -192,7 +210,24 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) {
if p.Pipeline == "" {
p.Pipeline = key
}
return p, true, nil
if len(p.Steps) > 0 {
return p, true, nil
}
prefix := stepsPrefix(key)
vals := map[string][]byte{}
for _, k := range all {
if !strings.HasPrefix(k, prefix) || strings.Contains(k[len(prefix):], ".") {
continue
}
v, err := r.Pipelines.Value(k)
if err != nil {
return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, k, err)
}
if v != nil {
vals[k] = v
}
}
return p, true, composeSteps(&p, key, vals)
}
func (r Reads) pipelines() ([]seat.Pipeline, error) {
@@ -203,7 +238,10 @@ func (r Reads) pipelines() ([]seat.Pipeline, error) {
sort.Strings(keys)
var out []seat.Pipeline
for _, k := range keys {
p, ok, err := r.pipeline(k)
if IsStepsKey(k) {
continue
}
p, ok, err := r.pipelineOf(k, keys)
if err != nil {
return nil, err
}
@@ -3,6 +3,7 @@ package pipeline
import (
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
@@ -49,6 +50,14 @@ func TestReads(t *testing.T) {
Outcome: seat.PipelineOutcomePassed, Started: &t2, Movers: []seat.Mover{{Module: "alpha", Commit: "b3"}}})
put(t, pipes, "main.7.gamma", seat.Pipeline{Pipeline: "main.7.gamma", Branch: "main",
Outcome: seat.PipelineOutcomeRunning, Started: &t1, Movers: []seat.Mover{{Module: "gamma", Commit: "c7"}}})
// A pipeline kept as H-A-187 keeps it: its record's steps empty, each mover's under steps.<pipeline>.<mover>.
put(t, pipes, "next.8", seat.Pipeline{Pipeline: "next.8", Branch: "next", Source: 8, Outcome: seat.PipelineOutcomeRunning,
Started: &t0, Steps: []seat.StepResult{}, Movers: []seat.Mover{{Module: "delta", Commit: "n8"}, {Module: "epsilon", Commit: "n8"}}})
put(t, pipes, "steps.next.8.epsilon", []seat.StepResult{{Module: "epsilon", Step: seat.StepResultStepBuild, Status: "required",
Outcome: seat.StepResultOutcomePassed, Output: "next.8.build.epsilon:program"}})
put(t, pipes, "steps.next.8.delta", []seat.StepResult{{Module: "delta", Step: seat.StepResultStepBuild, Status: "required",
Outcome: seat.StepResultOutcomePassed}, {Module: "delta", Step: seat.StepResultStepApproval, Status: "required",
Outcome: seat.StepResultOutcomeWaiting, On: seat.WaitOnPerson, Started: &t2}})
put(t, kept, "unit.sha256-aa", seat.KeptResult{Level: "unit", Digest: "sha256-aa", Package: "alpha/x"})
put(t, kept, "integration.sha256-bb", seat.KeptResult{Level: "integration", Digest: "sha256-bb", Package: "alpha/y"})
r := Reads{Pipelines: pipes, Kept: kept}
@@ -73,6 +82,30 @@ func TestReads(t *testing.T) {
_, err := r.Show(seat.ShowInput{Pipeline: "main.9.alpha"})
return "", err
}, `error: no pipeline "main.9.alpha" in pipelines`},
{"show composes a pipeline's steps from its mover keys, in its movers' order (H-A-187)", func() (string, error) {
a, err := r.Show(seat.ShowInput{Pipeline: "next.8"})
var s []string
for _, st := range a.Pipeline.Steps {
s = append(s, st.Module+" "+st.Step)
}
return strings.Join(s, ", ") + " | " + strings.Join(a.Outputs, ","), err
}, "delta build, delta approval, epsilon build | next.8.build.epsilon:program"},
{"show naming a mover answers its steps alone", func() (string, error) {
a, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "epsilon"})
var s []string
for _, st := range a.Pipeline.Steps {
s = append(s, st.Module+" "+st.Step)
}
return fmt.Sprintf("%s, %d movers", strings.Join(s, ", "), len(a.Pipeline.Movers)), err
}, "epsilon build, 2 movers"},
{"show refuses a mover the pipeline does not move", func() (string, error) {
_, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "alpha"})
return "", err
}, "error: the pipeline next.8 does not move alpha"},
{"list names a pipeline, never a mover key", func() (string, error) {
a, err := r.List(seat.ListInput{Branch: "next"})
return names(a.Pipelines), err
}, "next.8"},
{"list by module, newest first", func() (string, error) {
a, err := r.List(seat.ListInput{Module: "alpha"})
return names(a.Pipelines), err
@@ -92,7 +125,7 @@ func TestReads(t *testing.T) {
s = append(s, w.Pipeline+" "+w.Step+" "+w.On+" "+w.Since.Format("15:04"))
}
return strings.Join(s, ";"), err
}, "main.7.alpha approval person 10:01"},
}, "main.7.alpha approval person 10:01;next.8 approval person 10:02"},
{"why-carried names where a carried step came from", func() (string, error) {
a, err := r.WhyCarried(seat.WhyCarriedInput{Pipeline: "main.7.alpha", Step: "unit", Artifact: "alpha:program"})
return a.CarriedFrom, err