R6: a kept mover waiting on its kept revision says so; the record's kept-revisions are read past, not dropped

The runner seat's Mover gains unplaced (kept-revision-unread) and revision, the seat's form 3.

Builds: design 47 H2 §787, §780 (fate: the drop of kept-revisions before the decode goes)
This commit is contained in:
2026-11-22 01:16:51 +01:00
parent 427885013e
commit 8d43ce0cb9
9 changed files with 153 additions and 39 deletions
@@ -14,13 +14,13 @@ const Seat = "mesh-pipeline-runner"
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
// a reader of a value of another form says "form N, this reader knows M".
const Form = 2
const Form = 3
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
// whose fields changed while its forms did not (design 47 H2 §780).
const (
generatedForms = "2"
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=75717fd670a7 LogEntry=b5c78894cc5b Member=8a103baec01f Mover=5659e54b6dd0 MoverEnded=4b35095f6829 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=5beff763f531 StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepOutputsByPipelineStepArtifactValue=a2f06147f4b6 StepResult=94c2bea6ddcd StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
generatedForms = "3"
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=75717fd670a7 LogEntry=b5c78894cc5b Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=5beff763f531 StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepOutputsByPipelineStepArtifactValue=a2f06147f4b6 StepResult=94c2bea6ddcd StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
)
const (
@@ -637,16 +637,25 @@ type Mover struct {
// yes when the module is over its test budget (design 47 G section 419)
OverBudget bool `json:"over-budget,omitempty"`
// the artifact references, <module>:<artifact>, of the mover's dependencies whose package-published (and, for a kind-3 edge, whose build step's passed end) was heard at the pipeline's branch and source, written with the pipeline by compare-and-set; absent when none (H-A-84)
Published []string `json:"published,omitempty"`
Repository string `json:"repository"`
Tests *TestCounts `json:"tests,omitempty"`
Published []string `json:"published,omitempty"`
Repository string `json:"repository"`
// the catalog revision a kept-revision-unread mover waits for (design 47 H2 §787)
Revision int64 `json:"revision,omitempty"`
Tests *TestCounts `json:"tests,omitempty"`
// the mover's tier in the pipeline's delivery order, from 1 (ADR 0666 section 14; design 47 H §746)
Tier int64 `json:"tier"`
// why the mover has no step placed yet, where the runner knows it: kept-revision-unread for a kept mover whose kept revision the runner's view does not hold yet (design 47 H2 §787); absent once placed
Unplaced string `json:"unplaced,omitempty"`
// the version released for the mover in this pipeline (version-released), absent until one is (design 47 H-A-52)
Version string `json:"version,omitempty"`
Widens []Widening `json:"widens,omitempty"`
}
// The values of Mover's unplaced.
const (
MoverUnplacedKeptRevisionUnread = "kept-revision-unread"
)
// MoverEnded is one mover at its pipeline's end (design 47 H §746).
type MoverEnded struct {
// the commit the mover's entry was last changed at, from the pipeline's value (design 47 H-A-119)
@@ -2149,6 +2149,17 @@
"tests": {
"$ref": "#/defs/TestCounts"
},
"unplaced": {
"type": "string",
"enum": [
"kept-revision-unread"
],
"description": "why the mover has no step placed yet, where the runner knows it: kept-revision-unread for a kept mover whose kept revision the runner's view does not hold yet (design 47 H2 §787); absent once placed"
},
"revision": {
"type": "integer",
"description": "the catalog revision a kept-revision-unread mover waits for (design 47 H2 §787)"
},
"over-budget": {
"type": "boolean",
"description": "yes when the module is over its test budget (design 47 G section 419)"
@@ -2222,5 +2233,5 @@
]
}
},
"form": 2
"form": 3
}
@@ -238,9 +238,13 @@ export interface Mover {
/** the artifact references, <module>:<artifact>, of the mover's dependencies whose package-published (and, for a kind-3 edge, whose build step's passed end) was heard at the pipeline's branch and source, written with the pipeline by compare-and-set; absent when none (H-A-84) */
"published"?: string[];
"repository": string;
/** the catalog revision a kept-revision-unread mover waits for (design 47 H2 §787) */
"revision"?: number;
"tests"?: TestCounts;
/** the mover's tier in the pipeline's delivery order, from 1 (ADR 0666 section 14; design 47 H §746) */
"tier": number;
/** why the mover has no step placed yet, where the runner knows it: kept-revision-unread for a kept mover whose kept revision the runner's view does not hold yet (design 47 H2 §787); absent once placed */
"unplaced"?: "kept-revision-unread";
/** the version released for the mover in this pipeline (version-released), absent until one is (design 47 H-A-52) */
"version"?: string;
"widens"?: Widening[];
@@ -315,7 +315,7 @@ func (s *MoverStore) KeptRevisions(key string) (map[string]int64, error) {
}
// keptRevisionsField is the pipeline record's property kept-revisions, beside its movers (H-A-285): written and read by the
// runner alone, never part of the seat's Go form of a pipeline, which drops it on every decode.
// runner alone, never part of the seat's Go form of a pipeline, whose tolerant decode reads past it (design 47 H2 §780).
const keptRevisionsField = "kept-revisions"
// keptOf is the kept-revisions a record carries; none when it carries none.
@@ -335,20 +335,6 @@ func keptOf(raw []byte) (map[string]int64, error) {
return kept, nil
}
// withoutKept is record without its kept-revisions, the runner's own property and never part of the seat's Go form, so the
// seat's strict decode of the record the runner's reads make refuses no record carrying it (H-A-285, H-A-309).
func withoutKept(record []byte) ([]byte, error) {
var fields map[string]json.RawMessage
if err := json.Unmarshal(record, &fields); err != nil {
return nil, fmt.Errorf("the record is no object: %w", err)
}
if _, ok := fields[keptRevisionsField]; !ok {
return record, nil
}
delete(fields, keptRevisionsField)
return json.Marshal(fields)
}
// withKept is record with kept as its kept-revisions.
func withKept(record []byte, kept map[string]int64) ([]byte, error) {
var fields map[string]json.RawMessage
@@ -408,17 +408,18 @@ func keptOfStore(store any, key string) (map[string]int64, error) {
// keptEntries are the entries of branch's catalog revision rev, the one a same-source re-pin kept a mover at, read by its
// number (H-A-286): a kept mover is placed and pin-checked from them, never from the branch's held view; false while they
// cannot be read, and the mover waits.
func (s *Starter) keptEntries(branch string, rev int64) (map[string]catalog.CatalogEntry, bool) {
func (s *Starter) keptEntries(branch string, rev int64) (map[string]catalog.CatalogEntry, error) {
if s.Revision == nil {
s.logf("the runner reads no catalog revision, so a mover kept at revision %d of %s waits", rev, branch)
return nil, false
return nil, fmt.Errorf("the runner reads no catalog revision")
}
half, err := s.Revision(branch, rev)
if err != nil || half.Revision != rev {
s.logf("revision %d of %s, which a mover is kept at, is not read (%v): the mover waits for it (H-A-286)", rev, branch, err)
return nil, false
if err != nil {
return nil, err
}
return half.Entries, true
if half.Revision != rev {
return nil, fmt.Errorf("the view holds revision %d", half.Revision)
}
return half.Entries, nil
}
func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
@@ -320,10 +320,6 @@ func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error)
if raw == nil {
return seat.Pipeline{}, false, nil
}
raw, err = withoutKept(raw)
if err != nil {
return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err)
}
p, _, err := seat.DecodePipelinesByBranchRevisionValue(raw)
if err != nil {
return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err)
@@ -1,6 +1,8 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
@@ -189,3 +191,74 @@ func TestAKeptMoverIsPlacedLateAndResumedAtItsKeptRevision(t *testing.T) {
t.Fatalf("Resume asked %d (%v), ids %v: want the kept build %s again", n, err, asks.ids, build.Ask)
}
}
// design 47 H2 §787: a kept mover whose kept revision the runner's view does not hold yet carries unplaced
// kept-revision-unread with that revision, said once in the journal and read with show; once the revision is read it is
// placed and the mark goes.
func TestAKeptMoverWaitingOnItsRevisionSaysSo(t *testing.T) {
reg, entries, _, read := repinFixture()
unread := true
revision := func(branch string, rev int64) (Half, error) {
if unread && rev == 11 {
return Half{}, errors.New("revision 11 is not held yet")
}
return read(branch, rev)
}
store := &MoverStore{Raw: newMemRaw()}
var lines []string
registered := false
s := &Starter{Pipelines: store, Store: store, Steps: &stepAsks{}, Name: "the-runner", Now: ticking(), Revision: revision,
Say: func(string, any) error { return nil }, Logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) },
Registration: func(m string) (catalog.Registration, bool) { return reg, registered && m == "app" }}
key := Key("main", 10)
if ok, err := s.Start(Begin{Branch: "main", Source: 41, Revision: 10, Entries: entries,
Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}}}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
if n, err := s.Revised(revisionEvent(11), read); n != 0 || err != nil {
t.Fatalf("the re-pin started %d (%v)", n, err)
}
registered = true
lagging := func(string) (Half, bool) { return Half{Revision: 10, Source: 41, Entries: entries}, true }
for range 2 {
if n, err := s.Registered(store, reg, lagging); n != 0 || err != nil {
t.Fatalf("placed in %d pipelines (%v) while the kept revision is unread", n, err)
}
}
said := 0
for _, l := range lines {
if strings.Contains(l, "app waits for its kept revision 11 to be read") {
said++
}
}
shown, err := Reads{Pipelines: rawKept{store.Raw.(*memRaw)}, Kept: memKept{}}.Show(seat.ShowInput{Pipeline: key})
if err != nil || said != 1 || shown.Pipeline.Movers[0].Unplaced != seat.MoverUnplacedKeptRevisionUnread || shown.Pipeline.Movers[0].Revision != 11 {
t.Fatalf("said %d times, show %+v (%v)", said, shown.Pipeline.Movers, err)
}
if why := s.Unplaced(shown.Pipeline, "app", lagging); why != "app waits for its kept revision 11 to be read" {
t.Fatalf("the unplaced line %q", why)
}
unread = false
if n, err := s.Registered(store, reg, lagging); n != 1 || err != nil {
t.Fatalf("once read: placed in %d pipelines (%v)", n, err)
}
if p, _, _ := store.Get(key); p.Movers[0].Unplaced != "" || p.Movers[0].Revision != 0 {
t.Fatalf("the mark stays once placed: %+v", p.Movers[0])
}
}
// rawKept reads a memRaw as the reads do.
type rawKept struct{ r *memRaw }
func (k rawKept) Keys() ([]string, error) {
var out []string
for key := range k.r.vals {
out = append(out, key)
}
return out, nil
}
func (k rawKept) Value(key string) (json.RawMessage, error) {
v, _, err := k.r.Get(key)
return v, err
}
@@ -427,7 +427,7 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
s.logf("the kept-revisions of %s are not read, so its movers are placed as movers not kept: %v", p.Pipeline, err)
}
all := entries
for _, m := range p.Movers {
for i, m := range p.Movers {
// A mover that left for a newer pipeline is placed there, never here (H-A-223, H-A-361).
if placed(*p, m.Module) || slices.Contains(p.Superseded, m.Module) {
continue
@@ -440,10 +440,17 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
// A mover a same-source re-pin kept is placed from the revision it is kept at, its ids its own (H-A-286).
entries := all
if rev := kept[m.Module]; rev > 0 {
if entries, ok = s.keptEntries(p.Branch, rev); !ok {
var err error
if entries, err = s.keptEntries(p.Branch, rev); err != nil {
// It waits for its kept revision to be read, said once and carried on the mover (design 47 H2 §787).
if m.Unplaced != seat.MoverUnplacedKeptRevisionUnread || m.Revision != rev {
s.logf("%s: %s waits for its kept revision %d to be read (%v)", p.Pipeline, m.Module, rev, err)
p.Movers[i].Unplaced, p.Movers[i].Revision = seat.MoverUnplacedKeptRevisionUnread, rev
}
continue
}
}
p.Movers[i].Unplaced, p.Movers[i].Revision = "", 0
// A mover's steps wait for the publish of each mover it builds on (§746, H-A-83 (2)): placed once each is heard,
// ended not asked when one's build ended without passing.
due, waiting, failed := ready(*p, m.Module, entries)
@@ -728,7 +735,9 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
// A kept mover's steps are found by the ids its kept revision gives, from that revision's entries (H-A-286).
entries, rev := half.Entries, kept[m.Module]
if rev > 0 {
if entries, ok = s.keptEntries(p.Branch, rev); !ok {
var err error
if entries, err = s.keptEntries(p.Branch, rev); err != nil {
s.logf("%s: revision %d of %s, which %s is kept at, is not read (%v): its steps are asked again once it is", key, rev, p.Branch, m.Module, err)
continue
}
}
@@ -761,6 +770,10 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
// do not name it (A-327), else "<module> waits on the publish of <package>, …", the packages it awaits not yet published
// as the catalog half of p's branch holds its edges (§746).
func (s *Starter) Unplaced(p seat.Pipeline, module string, view func(branch string) (Half, bool)) string {
// A kept mover waiting for its kept revision says so (design 47 H2 §787).
if m, ok := MoverOf(p, module); ok && m.Unplaced == seat.MoverUnplacedKeptRevisionUnread {
return fmt.Sprintf("%s waits for its kept revision %d to be read", module, m.Revision)
}
if s.Registration == nil {
return module + " has no registration"
}
@@ -883,7 +896,7 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo
s.logf("the catalog half of %s holds revision %d: %s waits for revision %d, which its take-in made, to place its steps (§200)", p.Branch, half.Revision, key, p.Revision)
return p, nil, nil, nil, 0, nil
}
before := len(p.Steps)
before, marks := len(p.Steps), unplacedMarks(p)
asks, unregistered, ended, notAsked := s.placeMovers(&p, reg, half.Entries, now)
// Until its registration arrives, the pipeline says it waits on it (A-327): a mover with no registration has no step,
// and it holds the pipeline's end only while every placed mover's required steps passed (H-A-163, H-A-171).
@@ -892,6 +905,15 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo
s.registrationMissing(m, key)
}
if len(p.Steps) == before {
// A mover that now waits for its kept revision, or no longer does, is written so (design 47 H2 §787).
if unplacedMarks(p) == marks {
return p, nil, nil, nil, 0, nil
}
if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return p, nil, nil, nil, 0, fmt.Errorf("the movers of %s are not written: %w", key, err)
}
return p, nil, nil, nil, 0, nil
}
if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) {
@@ -904,6 +926,17 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo
}
}
// unplacedMarks is what p's movers carry of why they have no step placed (design 47 H2 §787), one line.
func unplacedMarks(p seat.Pipeline) string {
var b strings.Builder
for _, m := range p.Movers {
if m.Unplaced != "" {
fmt.Fprintf(&b, "%s %s %d;", m.Module, m.Unplaced, m.Revision)
}
}
return b.String()
}
// recovered says whether the commit of pipeline key is named a recovery; a read that fails asks on the stream check, said, since
// a step is asked either way (a recovery's only goes ahead of the first round).
func (s *Starter) recovered(key, commit string) bool {
@@ -594,8 +594,9 @@ func (v Verbs) placedAsk(p seat.Pipeline, s seat.StepResult, id string, now time
// A kept mover's step is the one placed from the revision it is kept at, under its kept ids (H-A-286).
entries, rev := half.Entries, v.keptOf(p.Pipeline)[m.Module]
if rev > 0 {
if entries, ok = v.Starter.keptEntries(p.Branch, rev); !ok {
return nil, fmt.Errorf("revision %d of %s, which %s is kept at, is not read: the step is asked when it is", rev, p.Branch, m.Module)
var err error
if entries, err = v.Starter.keptEntries(p.Branch, rev); err != nil {
return nil, fmt.Errorf("revision %d of %s, which %s is kept at, is not read (%v): the step is asked when it is", rev, p.Branch, m.Module, err)
}
}
_, placed := place(p, m, reg, entries, now, v.Starter.Name, false, rev)