RN3 (publish half): a pipeline starts by compare-and-set on its key, and a package-published starts one per dependent behind

A start writes the pipeline's key (watched branch, source revision, module) only when it is absent and says
pipeline-started; a second start of the same revision and module writes nothing. A package-published asks the
catalog's outdated for that package on that branch, a kept call, and starts a pipeline for each dependent behind
by kind 1, keyed by the publication's branch and source revision. The start on take-in.taken, and a gathered
group's one pipeline per branch, wait for take-in.taken's body (design 47 G §476, in review).

Builds: design 47 G §54, §103, §334, §53, §75

Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
2026-11-11 12:34:26 +01:00
parent 68c10c5664
commit 38a2492394
7 changed files with 321 additions and 4 deletions
@@ -5,6 +5,7 @@ package main
import (
"context"
"encoding/json"
"fmt"
"io"
"log"
@@ -16,6 +17,8 @@ import (
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
func main() {
@@ -41,7 +44,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
defer conn.Close()
// The state pipelines, `<module>_pipelines`, holds every pipeline by compare-and-set (§27): without it the
// runner keeps nothing, so it stops.
if _, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline"); err != nil {
pipelines, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline")
if err != nil {
return err
}
// The registrations and each watched branch's catalog half, first views then events (§21, §195, §200): until a
@@ -52,7 +56,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer calls.Close()
hub := process.NewEventHub(logger.Printf)
if _, err := startViews(ctx, js, conn, cred, hub, calls.Call, logger.Printf); err != nil {
rv, err := startViews(ctx, js, conn, cred, hub, calls.Call, logger.Printf)
if err != nil {
return err
}
// A pipeline starts by compare-and-set on its key's absence and says pipeline-started (§54, §103); a publish starts
// one per dependent behind (§53, §334).
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf,
Say: func(subject string, body any) error {
data, err := json.Marshal(body)
if err == nil {
_, err = js.Publish(ctx, subject, data)
}
return err
}}
if err := hearPublished(ctx, hub, starter, calls, rv.Catalog, logger.Printf); err != nil {
return err
}
if err := hub.StartUntilBound(ctx, js, cred, nil, nil); err != nil {
@@ -0,0 +1,71 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/nats-io/nats.go/jetstream"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-pipeline/internal/pipeline"
)
// outdatedVerb is the catalog's verb a publication's dependents behind are asked by, as the manifest's invokes names it.
const outdatedVerb = "seat:" + catalog.Seat + "." + catalog.VerbOutdated
// createdKV is the state pipelines written by compare-and-set on a key's absence (§54).
type createdKV struct {
ctx context.Context
kv jetstream.KeyValue
}
func (c createdKV) Create(key string, value []byte) error {
_, err := c.kv.Create(c.ctx, key, value)
if errors.Is(err, jetstream.ErrKeyExists) {
return pipeline.ErrExists
}
return err
}
// hearPublished starts the pipelines of each package-published heard (§53, §334): the dependents the catalog's outdated
// names behind, each asked as a kept call. The message is acknowledged once its starts are written, on a goroutine of its
// own, so a slow catalog holds no other event.
func hearPublished(ctx context.Context, hub *process.EventHub, starter *pipeline.Starter, calls *process.Conn, v *pipeline.Views, logf func(string, ...any)) error {
outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) {
raw, err := calls.CallKept(outdatedVerb, in)
if err != nil {
return catalog.OutdatedAnswer{}, err
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return catalog.OutdatedAnswer{}, fmt.Errorf("%s answered: %s", outdatedVerb, said)
}
var a catalog.OutdatedAnswer
err = json.Unmarshal(answer, &a)
return a, err
}
return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) {
var ev builds.PackagePublished
if err := json.Unmarshal(m.Data(), &ev); err != nil {
logf("%s is no package-published: %v", m.Subject(), err)
_ = m.Ack()
return
}
go func() {
n, err := starter.Published(ev, outdated, v.View)
if err != nil {
logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err)
_ = m.Nak()
return
}
logf("the publish of %s %s on %s started %d pipelines", ev.Artifact, ev.Version, ev.Branch, n)
_ = m.Ack()
}()
})
}
+2
View File
@@ -12,6 +12,8 @@ require (
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
github.com/klauspost/compress v1.20.1 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
@@ -0,0 +1,136 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"strconv"
"time"
runnerclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// ErrExists is a Create of a pipeline key that is written already: the second start, which writes nothing (§54).
var ErrExists = errors.New("the pipeline's key is written already")
// Created is the runner's state pipelines as a start writes it: a key written only when it is absent (compare-and-set on
// its absence), ErrExists when it is not.
type Created interface {
Create(key string, value []byte) error
}
// Begin is what a start names: the pipeline's key (watched branch, source revision, module; design 47 G §27, §54) and
// what the catalog half of that branch's view says of the module.
type Begin struct {
Branch string
Source int64
Module string
Commit string
Fp string
Group string
Repository string
Revision int64
}
// Key is a pipeline's key in the state pipelines, `<branch>.<source revision>.<module>`, the branch escaped as a key's
// name is (§27).
func Key(branch string, source int64, module string) string {
return seat.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), module)
}
// Starter starts pipelines (design 47 G §54, §103): it writes a pipeline's key by compare-and-set on its absence, so one
// source revision and module has one pipeline whichever event starts it first, and says pipeline-started for the one it
// wrote.
type Starter struct {
Pipelines Created
// Say says one of the seat's events on its subject.
Say func(subject string, body any) error
Now func() time.Time
Logf func(string, ...any)
}
// Start writes b's pipeline, running, and says pipeline-started; false, nil when its key was written already (the second
// start writes nothing, §54).
func (s *Starter) Start(b Begin) (bool, error) {
if b.Branch == "" || b.Module == "" || b.Source <= 0 {
return false, fmt.Errorf("a pipeline names its watched branch, source revision and module: %+v", b)
}
now := time.Now().UTC()
if s.Now != nil {
now = s.Now().UTC()
}
key := Key(b.Branch, b.Source, b.Module)
p := seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fp: b.Fp,
Group: b.Group, Repository: b.Repository, Revision: b.Revision, Outcome: seat.PipelineOutcomeRunning, Started: &now,
Steps: []seat.StepResult{}}
raw, err := json.Marshal(p)
if err != nil {
return false, err
}
if err := s.Pipelines.Create(key, raw); errors.Is(err, ErrExists) {
s.logf("%s is started already: this start writes nothing", key)
return false, nil
} else if err != nil {
return false, fmt.Errorf("the pipeline %s is not written: %w", key, err)
}
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module,
Commit: b.Commit, Fp: b.Fp, Group: b.Group, Revision: b.Revision}
commitOrGroup := b.Commit
if b.Group != "" {
commitOrGroup = b.Group
}
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, b.Module, b.Branch, commitOrGroup), ev); err != nil {
s.logf("pipeline-started of %s is not said: %v", key, err)
}
return true, nil
}
func (s *Starter) logf(format string, args ...any) {
if s.Logf != nil {
s.Logf(format, args...)
}
}
// outdatedKind is the edge `outdated` names a dependent behind by for a rebuild on a publish: B published before A builds,
// a package A imports (§53).
const outdatedKind = 1
// Published starts, for a package-published, a pipeline for each dependent the catalog's `outdated` names behind for that
// package on that branch by kind 1, keyed by the publication's watched branch and source revision and the dependent
// (design 47 G §53, §334). outdated is the kept call of the verb; view the catalog half of the publication's branch. A
// dependent the half does not hold is said and started with what the event names.
func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error), view func(branch string) (Half, bool)) (int, error) {
if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 {
return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev)
}
a, err := outdated(catalog.OutdatedInput{Branch: ev.Branch, Dependency: ev.Artifact})
if err != nil {
return 0, fmt.Errorf("outdated of %s on %s: %w", ev.Artifact, ev.Branch, err)
}
half, _ := view(ev.Branch)
started := 0
var first error
for _, o := range a.Behind {
if o.Kind != outdatedKind || o.Module == "" {
continue
}
b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision}
if e, ok := half.Entries[o.Module]; ok {
b.Commit, b.Fp, b.Repository = e.Commit, e.Fp, e.Repository
} else {
s.logf("the catalog half of %s holds no entry of %s, behind %s: its pipeline starts without its commit", ev.Branch, o.Module, ev.Artifact)
}
ok, err := s.Start(b)
if err != nil && first == nil {
first = err
}
if ok {
started++
}
}
return started, first
}
@@ -0,0 +1,87 @@
package pipeline
import (
"encoding/json"
"errors"
"strings"
"testing"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
)
type created map[string][]byte
func (c created) Create(key string, value []byte) error {
if _, ok := c[key]; ok {
return ErrExists
}
c[key] = value
return nil
}
// The starts (design 47 G §54, §103, §334): a pipeline's key written by compare-and-set on its absence, so a second start
// of the same source revision and module writes nothing; a publish starts one pipeline per dependent outdated names
// behind by kind 1, keyed by the publishing branch and revision.
func TestStarts(t *testing.T) {
store := created{}
var said []string
s := &Starter{Pipelines: store, Say: func(subject string, body any) error {
ev := body.(seat.PipelineStarted)
said = append(said, subject+" "+ev.Pipeline)
return nil
}}
view := func(branch string) (Half, bool) {
return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{
"app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app"}}}, true
}
var asked catalog.OutdatedInput
outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) {
asked = in
return catalog.OutdatedAnswer{Behind: []catalog.Outdated{
{Module: "app", Kind: 1}, {Module: "tool", Kind: 1}, {Module: "ordered", Kind: 4}}}, nil
}
n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}, outdated, view)
if err != nil || n != 2 {
t.Fatalf("a publish started %d (%v), want the two dependents behind by kind 1", n, err)
}
if asked.Dependency != "mesh-sdk:go" || asked.Branch != "main" {
t.Errorf("outdated asked %+v, want the package on its branch", asked)
}
var p seat.Pipeline
if err := json.Unmarshal(store["main.41.app"], &p); err != nil || p.Commit != "c1" || p.Fp != "src2:aa" || p.Revision != 9 ||
p.Outcome != seat.PipelineOutcomeRunning || p.Source != 41 {
t.Errorf("the pipeline main.41.app = %+v (%v)", p, err)
}
if _, ok := store["main.41.ordered"]; ok {
t.Error("an ordering edge starts no pipeline")
}
// The take-in of the same source revision and module is the same pipeline: the second start writes nothing.
before := string(store["main.41.app"])
ok, err := s.Start(Begin{Branch: "main", Source: 41, Module: "app", Commit: "c2"})
if ok || err != nil || string(store["main.41.app"]) != before || len(said) != 2 {
t.Errorf("a second start wrote %v (%v); said %v", ok, err, said)
}
if !strings.HasPrefix(said[0], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.app.main.c1 main.41.app") {
t.Errorf("pipeline-started said on %q", said[0])
}
// A branch holding a dot is escaped in the key, so the key reads back; a start naming no source revision is refused.
if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Module: "app"}); err != nil {
t.Fatal(err)
}
if b, r, m, ok := seat.ParsePipelinesByBranchRevisionModuleKey(Key("release.1", 3, "app")); !ok || r != "3" || m != "app" || b == "release" {
t.Errorf("the key of release.1 is %q", Key("release.1", 3, "app"))
}
if _, err := s.Start(Begin{Branch: "main", Module: "app"}); err == nil {
t.Error("a start with no source revision is refused")
}
s.Pipelines = failing{}
if _, err := s.Start(Begin{Branch: "main", Source: 5, Module: "app"}); err == nil {
t.Error("a write the bus refuses is an error")
}
}
type failing struct{}
func (failing) Create(string, []byte) error { return errors.New("bus away") }
@@ -66,7 +66,9 @@ func TestManifest(t *testing.T) {
"mesh-catalog.catalog-revision", "mesh-conditions.findings-asked"})},
// RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and
// asked again by hand (a hand act).
{"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act"})},
{"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act",
// RN3 (§53, §334): a publication's dependents behind.
"seat:mesh-catalog.outdated"})},
{"first-view tools", equalSets(m.Tools, []string{"pipeline_first_views", "pipeline_first_view_again"})},
// RN1/AC5 (§95, §130): the new assigner is delivered before the runner.
{"program needs the assigner's program", slices.Equal(program.Needs, []string{"mesh-delivery-assigner:program"})},
+2 -1
View File
@@ -42,7 +42,8 @@
"invokes": [
"seat:mesh-catalog.registrations",
"seat:mesh-catalog.revision",
"seat:mesh-hand-acts.hand-act"
"seat:mesh-hand-acts.hand-act",
"seat:mesh-catalog.outdated"
],
"state": [
{