RN3 (publish half): a pipeline starts by compare-and-set on its key, and a package-published starts one per dependent behind
A start writes the pipeline's key (watched branch, source revision, module) only when it is absent and says pipeline-started; a second start of the same revision and module writes nothing. A package-published asks the catalog's outdated for that package on that branch, a kept call, and starts a pipeline for each dependent behind by kind 1, keyed by the publication's branch and source revision. The start on take-in.taken, and a gathered group's one pipeline per branch, wait for take-in.taken's body (design 47 G §476, in review). Builds: design 47 G §54, §103, §334, §53, §75 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
@@ -5,6 +5,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -16,6 +17,8 @@ import (
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -41,7 +44,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
defer conn.Close()
|
||||
// The state pipelines, `<module>_pipelines`, holds every pipeline by compare-and-set (§27): without it the
|
||||
// runner keeps nothing, so it stops.
|
||||
if _, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline"); err != nil {
|
||||
pipelines, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The registrations and each watched branch's catalog half, first views then events (§21, §195, §200): until a
|
||||
@@ -52,7 +56,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
defer calls.Close()
|
||||
hub := process.NewEventHub(logger.Printf)
|
||||
if _, err := startViews(ctx, js, conn, cred, hub, calls.Call, logger.Printf); err != nil {
|
||||
rv, err := startViews(ctx, js, conn, cred, hub, calls.Call, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A pipeline starts by compare-and-set on its key's absence and says pipeline-started (§54, §103); a publish starts
|
||||
// one per dependent behind (§53, §334).
|
||||
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf,
|
||||
Say: func(subject string, body any) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err == nil {
|
||||
_, err = js.Publish(ctx, subject, data)
|
||||
}
|
||||
return err
|
||||
}}
|
||||
if err := hearPublished(ctx, hub, starter, calls, rv.Catalog, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hub.StartUntilBound(ctx, js, cred, nil, nil); err != nil {
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// outdatedVerb is the catalog's verb a publication's dependents behind are asked by, as the manifest's invokes names it.
|
||||
const outdatedVerb = "seat:" + catalog.Seat + "." + catalog.VerbOutdated
|
||||
|
||||
// createdKV is the state pipelines written by compare-and-set on a key's absence (§54).
|
||||
type createdKV struct {
|
||||
ctx context.Context
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (c createdKV) Create(key string, value []byte) error {
|
||||
_, err := c.kv.Create(c.ctx, key, value)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return pipeline.ErrExists
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// hearPublished starts the pipelines of each package-published heard (§53, §334): the dependents the catalog's outdated
|
||||
// names behind, each asked as a kept call. The message is acknowledged once its starts are written, on a goroutine of its
|
||||
// own, so a slow catalog holds no other event.
|
||||
func hearPublished(ctx context.Context, hub *process.EventHub, starter *pipeline.Starter, calls *process.Conn, v *pipeline.Views, logf func(string, ...any)) error {
|
||||
outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) {
|
||||
raw, err := calls.CallKept(outdatedVerb, in)
|
||||
if err != nil {
|
||||
return catalog.OutdatedAnswer{}, err
|
||||
}
|
||||
answer, said, ok := buscalls.Unwrap(raw)
|
||||
if !ok {
|
||||
return catalog.OutdatedAnswer{}, fmt.Errorf("%s answered: %s", outdatedVerb, said)
|
||||
}
|
||||
var a catalog.OutdatedAnswer
|
||||
err = json.Unmarshal(answer, &a)
|
||||
return a, err
|
||||
}
|
||||
return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) {
|
||||
var ev builds.PackagePublished
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil {
|
||||
logf("%s is no package-published: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
n, err := starter.Published(ev, outdated, v.View)
|
||||
if err != nil {
|
||||
logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
logf("the publish of %s %s on %s started %d pipelines", ev.Artifact, ev.Version, ev.Branch, n)
|
||||
_ = m.Ack()
|
||||
}()
|
||||
})
|
||||
}
|
||||
@@ -12,6 +12,8 @@ require (
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
runnerclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient"
|
||||
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// ErrExists is a Create of a pipeline key that is written already: the second start, which writes nothing (§54).
|
||||
var ErrExists = errors.New("the pipeline's key is written already")
|
||||
|
||||
// Created is the runner's state pipelines as a start writes it: a key written only when it is absent (compare-and-set on
|
||||
// its absence), ErrExists when it is not.
|
||||
type Created interface {
|
||||
Create(key string, value []byte) error
|
||||
}
|
||||
|
||||
// Begin is what a start names: the pipeline's key (watched branch, source revision, module; design 47 G §27, §54) and
|
||||
// what the catalog half of that branch's view says of the module.
|
||||
type Begin struct {
|
||||
Branch string
|
||||
Source int64
|
||||
Module string
|
||||
Commit string
|
||||
Fp string
|
||||
Group string
|
||||
Repository string
|
||||
Revision int64
|
||||
}
|
||||
|
||||
// Key is a pipeline's key in the state pipelines, `<branch>.<source revision>.<module>`, the branch escaped as a key's
|
||||
// name is (§27).
|
||||
func Key(branch string, source int64, module string) string {
|
||||
return seat.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), module)
|
||||
}
|
||||
|
||||
// Starter starts pipelines (design 47 G §54, §103): it writes a pipeline's key by compare-and-set on its absence, so one
|
||||
// source revision and module has one pipeline whichever event starts it first, and says pipeline-started for the one it
|
||||
// wrote.
|
||||
type Starter struct {
|
||||
Pipelines Created
|
||||
// Say says one of the seat's events on its subject.
|
||||
Say func(subject string, body any) error
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
}
|
||||
|
||||
// Start writes b's pipeline, running, and says pipeline-started; false, nil when its key was written already (the second
|
||||
// start writes nothing, §54).
|
||||
func (s *Starter) Start(b Begin) (bool, error) {
|
||||
if b.Branch == "" || b.Module == "" || b.Source <= 0 {
|
||||
return false, fmt.Errorf("a pipeline names its watched branch, source revision and module: %+v", b)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if s.Now != nil {
|
||||
now = s.Now().UTC()
|
||||
}
|
||||
key := Key(b.Branch, b.Source, b.Module)
|
||||
p := seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fp: b.Fp,
|
||||
Group: b.Group, Repository: b.Repository, Revision: b.Revision, Outcome: seat.PipelineOutcomeRunning, Started: &now,
|
||||
Steps: []seat.StepResult{}}
|
||||
raw, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := s.Pipelines.Create(key, raw); errors.Is(err, ErrExists) {
|
||||
s.logf("%s is started already: this start writes nothing", key)
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
return false, fmt.Errorf("the pipeline %s is not written: %w", key, err)
|
||||
}
|
||||
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module,
|
||||
Commit: b.Commit, Fp: b.Fp, Group: b.Group, Revision: b.Revision}
|
||||
commitOrGroup := b.Commit
|
||||
if b.Group != "" {
|
||||
commitOrGroup = b.Group
|
||||
}
|
||||
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, b.Module, b.Branch, commitOrGroup), ev); err != nil {
|
||||
s.logf("pipeline-started of %s is not said: %v", key, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Starter) logf(format string, args ...any) {
|
||||
if s.Logf != nil {
|
||||
s.Logf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// outdatedKind is the edge `outdated` names a dependent behind by for a rebuild on a publish: B published before A builds,
|
||||
// a package A imports (§53).
|
||||
const outdatedKind = 1
|
||||
|
||||
// Published starts, for a package-published, a pipeline for each dependent the catalog's `outdated` names behind for that
|
||||
// package on that branch by kind 1, keyed by the publication's watched branch and source revision and the dependent
|
||||
// (design 47 G §53, §334). outdated is the kept call of the verb; view the catalog half of the publication's branch. A
|
||||
// dependent the half does not hold is said and started with what the event names.
|
||||
func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error), view func(branch string) (Half, bool)) (int, error) {
|
||||
if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 {
|
||||
return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev)
|
||||
}
|
||||
a, err := outdated(catalog.OutdatedInput{Branch: ev.Branch, Dependency: ev.Artifact})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("outdated of %s on %s: %w", ev.Artifact, ev.Branch, err)
|
||||
}
|
||||
half, _ := view(ev.Branch)
|
||||
started := 0
|
||||
var first error
|
||||
for _, o := range a.Behind {
|
||||
if o.Kind != outdatedKind || o.Module == "" {
|
||||
continue
|
||||
}
|
||||
b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision}
|
||||
if e, ok := half.Entries[o.Module]; ok {
|
||||
b.Commit, b.Fp, b.Repository = e.Commit, e.Fp, e.Repository
|
||||
} else {
|
||||
s.logf("the catalog half of %s holds no entry of %s, behind %s: its pipeline starts without its commit", ev.Branch, o.Module, ev.Artifact)
|
||||
}
|
||||
ok, err := s.Start(b)
|
||||
if err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
if ok {
|
||||
started++
|
||||
}
|
||||
}
|
||||
return started, first
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
type created map[string][]byte
|
||||
|
||||
func (c created) Create(key string, value []byte) error {
|
||||
if _, ok := c[key]; ok {
|
||||
return ErrExists
|
||||
}
|
||||
c[key] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
// The starts (design 47 G §54, §103, §334): a pipeline's key written by compare-and-set on its absence, so a second start
|
||||
// of the same source revision and module writes nothing; a publish starts one pipeline per dependent outdated names
|
||||
// behind by kind 1, keyed by the publishing branch and revision.
|
||||
func TestStarts(t *testing.T) {
|
||||
store := created{}
|
||||
var said []string
|
||||
s := &Starter{Pipelines: store, Say: func(subject string, body any) error {
|
||||
ev := body.(seat.PipelineStarted)
|
||||
said = append(said, subject+" "+ev.Pipeline)
|
||||
return nil
|
||||
}}
|
||||
view := func(branch string) (Half, bool) {
|
||||
return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app"}}}, true
|
||||
}
|
||||
var asked catalog.OutdatedInput
|
||||
outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) {
|
||||
asked = in
|
||||
return catalog.OutdatedAnswer{Behind: []catalog.Outdated{
|
||||
{Module: "app", Kind: 1}, {Module: "tool", Kind: 1}, {Module: "ordered", Kind: 4}}}, nil
|
||||
}
|
||||
n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}, outdated, view)
|
||||
if err != nil || n != 2 {
|
||||
t.Fatalf("a publish started %d (%v), want the two dependents behind by kind 1", n, err)
|
||||
}
|
||||
if asked.Dependency != "mesh-sdk:go" || asked.Branch != "main" {
|
||||
t.Errorf("outdated asked %+v, want the package on its branch", asked)
|
||||
}
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(store["main.41.app"], &p); err != nil || p.Commit != "c1" || p.Fp != "src2:aa" || p.Revision != 9 ||
|
||||
p.Outcome != seat.PipelineOutcomeRunning || p.Source != 41 {
|
||||
t.Errorf("the pipeline main.41.app = %+v (%v)", p, err)
|
||||
}
|
||||
if _, ok := store["main.41.ordered"]; ok {
|
||||
t.Error("an ordering edge starts no pipeline")
|
||||
}
|
||||
// The take-in of the same source revision and module is the same pipeline: the second start writes nothing.
|
||||
before := string(store["main.41.app"])
|
||||
ok, err := s.Start(Begin{Branch: "main", Source: 41, Module: "app", Commit: "c2"})
|
||||
if ok || err != nil || string(store["main.41.app"]) != before || len(said) != 2 {
|
||||
t.Errorf("a second start wrote %v (%v); said %v", ok, err, said)
|
||||
}
|
||||
if !strings.HasPrefix(said[0], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.app.main.c1 main.41.app") {
|
||||
t.Errorf("pipeline-started said on %q", said[0])
|
||||
}
|
||||
// A branch holding a dot is escaped in the key, so the key reads back; a start naming no source revision is refused.
|
||||
if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Module: "app"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, r, m, ok := seat.ParsePipelinesByBranchRevisionModuleKey(Key("release.1", 3, "app")); !ok || r != "3" || m != "app" || b == "release" {
|
||||
t.Errorf("the key of release.1 is %q", Key("release.1", 3, "app"))
|
||||
}
|
||||
if _, err := s.Start(Begin{Branch: "main", Module: "app"}); err == nil {
|
||||
t.Error("a start with no source revision is refused")
|
||||
}
|
||||
s.Pipelines = failing{}
|
||||
if _, err := s.Start(Begin{Branch: "main", Source: 5, Module: "app"}); err == nil {
|
||||
t.Error("a write the bus refuses is an error")
|
||||
}
|
||||
}
|
||||
|
||||
type failing struct{}
|
||||
|
||||
func (failing) Create(string, []byte) error { return errors.New("bus away") }
|
||||
@@ -66,7 +66,9 @@ func TestManifest(t *testing.T) {
|
||||
"mesh-catalog.catalog-revision", "mesh-conditions.findings-asked"})},
|
||||
// RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and
|
||||
// asked again by hand (a hand act).
|
||||
{"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act"})},
|
||||
{"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act",
|
||||
// RN3 (§53, §334): a publication's dependents behind.
|
||||
"seat:mesh-catalog.outdated"})},
|
||||
{"first-view tools", equalSets(m.Tools, []string{"pipeline_first_views", "pipeline_first_view_again"})},
|
||||
// RN1/AC5 (§95, §130): the new assigner is delivered before the runner.
|
||||
{"program needs the assigner's program", slices.Equal(program.Needs, []string{"mesh-delivery-assigner:program"})},
|
||||
|
||||
@@ -42,7 +42,8 @@
|
||||
"invokes": [
|
||||
"seat:mesh-catalog.registrations",
|
||||
"seat:mesh-catalog.revision",
|
||||
"seat:mesh-hand-acts.hand-act"
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-catalog.outdated"
|
||||
],
|
||||
"state": [
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user