R10: a publish starts only what its source's pipeline does not move

A package-published is heard in the running pipeline of the source revision that published it,
where its dependents wait (R7); every module the catalog half holds a kind-1 edge from the
package into that this pipeline does not move (a publish again of an older build, a dependent it
does not move) is a mover of one pipeline keyed <branch>.<source>.published.<package>.<version>,
at the publishing source's catalog revision and call. The dependents come from the edges the
runner holds: the runner asks outdated no more (its invoke leaves the manifest). A publish heard
before its branch's half is held is heard again once it is; the durable consumer delivers one
heard after a stop, the key written once. Unit tests 83 to 84.

Builds: design 47 H §747; lead answers H-A-19, H-A-52

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
2026-11-20 04:56:31 +01:00
parent c3d17f6b5a
commit 8da18c37c5
7 changed files with 119 additions and 122 deletions
@@ -259,7 +259,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := hearDecided(heard, hearer, cred.Module, logger.Printf); err != nil {
return err
}
if err := hearPublished(ctx, heard, starter, calls, rv.Catalog, logger.Printf); err != nil {
if err := hearPublished(heard, starter, rv.Catalog, logger.Printf); err != nil {
return err
}
if err := hearRecovery(heard, kvRecoveries{ctx, recovered}, logger.Printf); err != nil {
@@ -4,21 +4,15 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/nats-io/nats.go/jetstream"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
// outdatedVerb is the catalog's verb a publication's dependents behind are asked by, as the manifest's invokes names it.
const outdatedVerb = "seat:" + catalog.Seat + "." + catalog.VerbOutdated
// createdKV is the state pipelines written by compare-and-set on a key's absence (§54).
type createdKV struct {
ctx context.Context
@@ -33,23 +27,11 @@ func (c createdKV) Create(key string, value []byte) error {
return err
}
// hearPublished starts the pipelines of each package-published heard (§53, §334): the dependents the catalog's outdated
// names behind, each asked as a kept call. The message is acknowledged once its starts are written, on a goroutine of its
// own, so a slow catalog holds no other event.
func hearPublished(ctx context.Context, hub handlers, starter *pipeline.Starter, calls *process.Conn, v *pipeline.Views, logf func(string, ...any)) error {
outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) {
raw, err := calls.CallKept(outdatedVerb, in)
if err != nil {
return catalog.OutdatedAnswer{}, err
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return catalog.OutdatedAnswer{}, fmt.Errorf("%s answered: %s", outdatedVerb, said)
}
var a catalog.OutdatedAnswer
err = json.Unmarshal(answer, &a)
return a, err
}
// hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one
// pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is
// written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard
// after a stop.
func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error {
return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) {
var ev builds.PackagePublished
if err := json.Unmarshal(m.Data(), &ev); err != nil {
@@ -58,7 +40,7 @@ func hearPublished(ctx context.Context, hub handlers, starter *pipeline.Starter,
return
}
go func() {
n, err := starter.Published(ev, outdated, v.View)
n, err := starter.Published(ev, v.View)
if err != nil {
logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err)
_ = m.Nak()
@@ -73,11 +73,7 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) {
if !placed(p, "mesh-sdk") || !placed(p, "builder") || placed(p, "app") || placed(p, "tool") {
t.Fatalf("placed at the start: %+v, want mesh-sdk and builder only", p.Steps)
}
outdated := func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error) {
t.Error("a publish of a mover asked outdated: it is heard in its pipeline")
return catalog.OutdatedAnswer{}, nil
}
if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, outdated, view); n != 0 || err != nil {
if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); n != 0 || err != nil {
t.Fatalf("the publish started %d (%v), want none", n, err)
}
p, _, _ = mem.Get("main.4")
@@ -4,6 +4,7 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"sort"
"strconv"
@@ -718,70 +719,79 @@ func (s *Starter) Release(store Updater, key string, view func(branch string) (H
}
// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the source
// revision that published it, and places the movers that waited for it (design 47 H §746, H-A-84). It says whether such a
// pipeline moves the package's module.
func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch string) (Half, bool)) (bool, error) {
// revision that published it, and places the movers that waited for it (design 47 H §746, H-A-84). It answers that
// source's pipeline, read, and whether one is held.
func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch string) (Half, bool)) (seat.Pipeline, bool, error) {
if s.Store == nil {
return false, nil
return seat.Pipeline{}, false, nil
}
key := Key(ev.Branch, ev.Source)
for {
p, rev, err := s.Store.Get(key)
if err != nil || p.Outcome != seat.PipelineOutcomeRunning || !Moves(p, ev.Module) {
return false, nil
if err != nil {
return seat.Pipeline{}, false, nil
}
if p.Outcome != seat.PipelineOutcomeRunning || !Moves(p, ev.Module) {
return p, true, nil
}
i := slices.IndexFunc(p.Movers, func(m seat.Mover) bool { return m.Module == ev.Module })
if slices.Contains(p.Movers[i].Published, ev.Artifact) {
return true, nil
return p, true, nil
}
p.Movers[i].Published = append(p.Movers[i].Published, ev.Artifact)
if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return true, fmt.Errorf("the pipeline %s cannot be written: %w", key, err)
return p, true, fmt.Errorf("the pipeline %s cannot be written: %w", key, err)
}
_, err = s.Release(s.Store, key, view)
return true, err
return p, true, err
}
}
// outdatedKind is the edge `outdated` names a dependent behind by for a rebuild on a publish: B published before A builds,
// a package A imports (§53).
const outdatedKind = 1
// publishedKind is the edge a dependent builds on a published package by: B published before A builds, a package A
// imports (§53).
const publishedKind = 1
// Published starts, for a package-published, one pipeline keyed `<branch>.<source>.published.<package>.<version>` whose
// movers are the dependents the catalog's `outdated` names behind for that package on that branch by kind 1 (design 47 G
// §53; H §747). outdated is the kept call of the verb; view the catalog half of the publication's branch.
func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error), view func(branch string) (Half, bool)) (int, error) {
// Published hears a package-published (design 47 H §747, H-A-52): in the running pipeline of the source revision that
// published it, a mover that builds on it waits for it there (§746) and nothing starts; every module the catalog half
// holds a kind-1 edge from the package into that this pipeline does not move (a publish again of an older build, a
// dependent it does not move) is a mover of one pipeline keyed `<branch>.<source>.published.<package>.<version>`, at the
// publishing source's catalog revision. The dependents come from the edges the runner holds, never from a comparison.
func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) (int, error) {
if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 {
return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev)
}
// A publish of a mover of the running pipeline of its source revision is heard in that pipeline: its dependents there
// build once it is (§746).
if in, err := s.heardPublished(ev, view); in || err != nil {
publishing, held, err := s.heardPublished(ev, view)
if err != nil {
return 0, err
}
a, err := outdated(catalog.OutdatedInput{Branch: ev.Branch, Dependency: ev.Artifact})
if err != nil {
return 0, fmt.Errorf("outdated of %s on %s: %w", ev.Artifact, ev.Branch, err)
half, ok := view(ev.Branch)
if !ok {
return 0, fmt.Errorf("the catalog half of %s is not held yet: the publish of %s %s is heard again once it is", ev.Branch, ev.Artifact, ev.Version)
}
half, held := view(ev.Branch)
var movers []seat.Mover
for _, o := range a.Behind {
if o.Kind != outdatedKind || o.Module == "" || Moves(seat.Pipeline{Movers: movers}, o.Module) {
for _, module := range slices.Sorted(maps.Keys(half.Entries)) {
e := half.Entries[module]
if module == ev.Module || Moves(publishing, module) || !slices.ContainsFunc(e.Edges, func(x catalog.Edge) bool {
return x.Kind == publishedKind && x.From == ev.Artifact
}) {
continue
}
e := half.Entries[o.Module]
movers = append(movers, seat.Mover{Module: o.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
movers = append(movers, seat.Mover{Module: module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
}
if len(movers) == 0 {
return 0, nil
}
revision := half.Revision
if held && publishing.Revision > 0 {
revision = publishing.Revision
}
movers = Tiers(movers, half.Entries, s.logf)
ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Revision: half.Revision, Movers: movers, Entries: half.Entries,
Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version}, unheld: !held,
started, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Call: publishing.Call, Revision: revision, Movers: movers,
Entries: half.Entries, Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version},
Key: PublishedKey(ev.Branch, ev.Source, ev.Artifact, ev.Version)})
if ok {
if started {
return 1, err
}
return 0, err
@@ -21,58 +21,20 @@ func (c created) Create(key string, value []byte) error {
return nil
}
// The starts (design 47 G §53, §54; H §746, §747): a pipeline's key written by compare-and-set on its absence, so a second
// start of the same key writes nothing; a publish starts one pipeline keyed <branch>.<source>.published.<package>.<version>
// whose movers are the dependents outdated names behind by kind 1; a branch holding a dot is escaped in the key.
// The starts (design 47 H §746, §747): a pipeline's key written by compare-and-set on its absence, so a second start of the
// same key writes nothing; a branch holding a dot is escaped in the key; a start with no source revision or no mover is
// refused.
func TestStarts(t *testing.T) {
store := created{}
var said []string
s := &Starter{Pipelines: store, Say: func(subject string, body any) error {
ev, started := body.(seat.PipelineStarted)
if !started { // only pipeline-started is counted here
return nil
}
said = append(said, subject+" "+ev.Pipeline)
if ev.Published == nil || ev.Published.Artifact != "mesh-sdk:go" || ev.Published.Version != "0.1.16" {
t.Errorf("pipeline-started of %s says the publish %+v (A-202)", ev.Pipeline, ev.Published)
}
return nil
}}
view := func(branch string) (Half, bool) {
return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{
"app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app"}}}, true
}
var asked catalog.OutdatedInput
outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) {
asked = in
return catalog.OutdatedAnswer{Behind: []catalog.Outdated{
{Module: "app", Kind: 1}, {Module: "tool", Kind: 1}, {Module: "ordered", Kind: 4}}}, nil
}
n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}, outdated, view)
if err != nil || n != 1 {
t.Fatalf("a publish started %d (%v), want one pipeline", n, err)
}
if asked.Dependency != "mesh-sdk:go" || asked.Branch != "main" {
t.Errorf("outdated asked %+v, want the package on its branch", asked)
}
key := PublishedKey("main", 41, "mesh-sdk:go", "0.1.16")
var p seat.Pipeline
if err := json.Unmarshal(store[key], &p); err != nil || len(p.Movers) != 2 || p.Revision != 9 || p.Outcome != seat.PipelineOutcomeRunning ||
p.Source != 41 || p.Published == nil || p.Published.Version != "0.1.16" || Moves(p, "ordered") {
t.Fatalf("the pipeline %s = %+v (%v)", key, p, err)
}
if app, _ := MoverOf(p, "app"); app.Commit != "c1" || app.Fingerprint != "src2:aa" {
t.Errorf("the mover app = %+v", app)
s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }}
key := Key("main", 41)
if ok, err := s.Start(Begin{Branch: "main", Source: 41, Movers: []seat.Mover{{Module: "app", Commit: "c1"}}}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
before := string(store[key])
ok, err := s.Start(Begin{Branch: "main", Source: 41, Key: key, Movers: []seat.Mover{{Module: "app", Commit: "c2"}}})
if ok || err != nil || string(store[key]) != before || len(said) != 1 {
t.Errorf("a second start wrote %v (%v); said %v", ok, err, said)
if ok, err := s.Start(Begin{Branch: "main", Source: 41, Movers: []seat.Mover{{Module: "app", Commit: "c2"}}}); ok || err != nil || string(store[key]) != before {
t.Errorf("a second start wrote %v (%v)", ok, err)
}
if !strings.HasPrefix(said[0], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.41 "+key) {
t.Errorf("pipeline-started said on %q", said[0])
}
s.Say = func(string, any) error { return nil }
if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Movers: []seat.Mover{{Module: "app"}}}); err != nil {
t.Fatal(err)
}
@@ -91,6 +53,55 @@ func TestStarts(t *testing.T) {
}
}
// R10/AC1 and AC2 (design 47 H §747, H-A-52): a package-published of a mover of its source's running pipeline whose
// dependent that pipeline moves starts nothing: the dependent waits there. A publish again of an older build starts one
// pipeline keyed <branch>.<source>.published.<package>.<version>, its movers the package's kind-1 dependents from the
// edges the runner holds, at the publishing source's revision, and says pipeline-started with the publish; an ordering
// edge starts nothing; heard twice, the second writes nothing.
func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) {
mem := memStore{}
var said []seat.PipelineStarted
s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(_ string, body any) error {
if ev, ok := body.(seat.PipelineStarted); ok {
said = append(said, ev)
}
return nil
}}
edges := []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 1}}
view := func(string) (Half, bool) {
return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{
"mesh-sdk": {Module: "mesh-sdk"},
"app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges},
"tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges},
"ordered": {Module: "ordered", Edges: []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 4}}}}}, true
}
mem[Key("main", 41)] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{Pipeline: "main.41", Branch: "main", Source: 41, Revision: 8, Outcome: seat.PipelineOutcomeRunning,
Movers: []seat.Mover{{Module: "mesh-sdk"}, {Module: "app"}, {Module: "tool"}}}, 1}
ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}
if n, err := s.Published(ev, view); n != 0 || err != nil || len(mem) != 1 {
t.Fatalf("a publish in a pipeline moving its dependents started %d (%v), %d pipelines", n, err, len(mem))
}
// A publish again of an older build, at source 30: its pipeline is gone, so every kind-1 dependent is a mover.
ev.Source, ev.Version = 30, "0.1.9"
for i := range 2 {
if n, err := s.Published(ev, view); n != 1-i || err != nil {
t.Fatalf("hearing %d started %d (%v)", i+1, n, err)
}
}
key := PublishedKey("main", 30, "mesh-sdk:go", "0.1.9")
p, _, err := mem.Get(key)
if err != nil || len(p.Movers) != 2 || !Moves(p, "app") || !Moves(p, "tool") || Moves(p, "ordered") || p.Revision != 9 ||
p.Published == nil || p.Published.Version != "0.1.9" || len(said) != 1 || said[0].Published == nil {
t.Fatalf("the pipeline %s = %+v (%v), said %+v", key, p, err, said)
}
if app, _ := MoverOf(p, "app"); app.Commit != "c1" || app.Fingerprint != "src2:aa" || app.Repository != "o/app" {
t.Errorf("the mover app = %+v", app)
}
}
// memCreated is memStore with the Create a start writes through.
type memCreated struct{ memStore }
@@ -230,22 +241,23 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
if n != 1 || err != nil || len(p.Steps) == 0 {
t.Errorf("the half of next placed %d (%v): %d steps, want the take-in's pipeline placed", n, err, len(p.Steps))
}
// A-336: so does a package published heard before its branch's half is held.
// A-336: a package published heard before its branch's half is held starts nothing, heard again once it is: its
// dependents are the half's edges (§747).
held = false
outdated := func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error) {
return catalog.OutdatedAnswer{Behind: []catalog.Outdated{{Module: "app", Kind: outdatedKind}}}, nil
}
if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "next", Source: 7, Module: "mesh-sdk", Version: "0.1.17"}, outdated, late); n != 1 || err != nil {
t.Fatalf("the publish before the half started %d (%v)", n, err)
}
if p, _, _ := store.Get(PublishedKey("next", 7, "mesh-sdk:go", "0.1.17")); len(p.Steps) != 0 {
t.Fatalf("a publish before its branch's half placed %d steps from empty entries, want none", len(p.Steps))
app := entries["app"]
app.Edges = []catalog.Edge{{From: "mesh-sdk:go", To: "app:program", Kind: 1}}
entries["app"] = app
published := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "next", Source: 7, Module: "mesh-sdk", Version: "0.1.17"}
if n, err := s.Published(published, late); n != 0 || err == nil {
t.Fatalf("the publish before the half started %d (%v), want none and an error, heard again", n, err)
}
held = true
n, err = s.Waiting(store.memStore, "next", late)
if n, err := s.Published(published, late); n != 1 || err != nil {
t.Fatalf("the publish with the half held started %d (%v)", n, err)
}
p, _, _ = store.Get(PublishedKey("next", 7, "mesh-sdk:go", "0.1.17"))
if n != 1 || err != nil || len(p.Steps) == 0 {
t.Errorf("the half of next placed %d (%v): %d steps, want the publish's pipeline placed", n, err, len(p.Steps))
if len(p.Steps) == 0 {
t.Errorf("the publish's pipeline placed %d steps, want its steps", len(p.Steps))
}
// A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running,
@@ -93,8 +93,6 @@ func TestManifest(t *testing.T) {
{"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations",
// A-186: the seat list, read by its verb.
"seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act",
// RN3 (§53, §334): a publication's dependents behind.
"seat:mesh-catalog.outdated",
// RN10 (§71, §441, A-207): the permissions a change widens, asked of the grants part.
"seat:mesh-grants.grants-for",
// RN11 (§33, A-191): the author of a failed head, read as the forge records it.
@@ -68,7 +68,6 @@
"seat:mesh-catalog.registrations",
"seat:mesh-catalog.revision",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-catalog.outdated",
"seat:git.pull",
"seat:mesh-module-assigner.assignments",
"seat:mesh-grants.grants-for",