diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 9c35c107..5805b1e0 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -5,6 +5,7 @@ package main import ( "context" + "encoding/json" "fmt" "io" "log" @@ -16,6 +17,8 @@ import ( seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" ) func main() { @@ -41,7 +44,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro defer conn.Close() // The state pipelines, `_pipelines`, holds every pipeline by compare-and-set (§27): without it the // runner keeps nothing, so it stops. - if _, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline"); err != nil { + pipelines, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StatePipelines), "keep a pipeline") + if err != nil { return err } // The registrations and each watched branch's catalog half, first views then events (§21, §195, §200): until a @@ -52,7 +56,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer calls.Close() hub := process.NewEventHub(logger.Printf) - if _, err := startViews(ctx, js, conn, cred, hub, calls.Call, logger.Printf); err != nil { + rv, err := startViews(ctx, js, conn, cred, hub, calls.Call, logger.Printf) + if err != nil { + return err + } + // A pipeline starts by compare-and-set on its key's absence and says pipeline-started (§54, §103); a publish starts + // one per dependent behind (§53, §334). + starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, + Say: func(subject string, body any) error { + data, err := json.Marshal(body) + if err == nil { + _, err = js.Publish(ctx, subject, data) + } + return err + }} + if err := hearPublished(ctx, hub, starter, calls, rv.Catalog, logger.Printf); err != nil { return err } if err := hub.StartUntilBound(ctx, js, cred, nil, nil); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go new file mode 100644 index 00000000..a2341544 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/nats-io/nats.go/jetstream" + + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-pipeline/internal/pipeline" +) + +// outdatedVerb is the catalog's verb a publication's dependents behind are asked by, as the manifest's invokes names it. +const outdatedVerb = "seat:" + catalog.Seat + "." + catalog.VerbOutdated + +// createdKV is the state pipelines written by compare-and-set on a key's absence (§54). +type createdKV struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (c createdKV) Create(key string, value []byte) error { + _, err := c.kv.Create(c.ctx, key, value) + if errors.Is(err, jetstream.ErrKeyExists) { + return pipeline.ErrExists + } + return err +} + +// hearPublished starts the pipelines of each package-published heard (§53, §334): the dependents the catalog's outdated +// names behind, each asked as a kept call. The message is acknowledged once its starts are written, on a goroutine of its +// own, so a slow catalog holds no other event. +func hearPublished(ctx context.Context, hub *process.EventHub, starter *pipeline.Starter, calls *process.Conn, v *pipeline.Views, logf func(string, ...any)) error { + outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) { + raw, err := calls.CallKept(outdatedVerb, in) + if err != nil { + return catalog.OutdatedAnswer{}, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return catalog.OutdatedAnswer{}, fmt.Errorf("%s answered: %s", outdatedVerb, said) + } + var a catalog.OutdatedAnswer + err = json.Unmarshal(answer, &a) + return a, err + } + return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) { + var ev builds.PackagePublished + if err := json.Unmarshal(m.Data(), &ev); err != nil { + logf("%s is no package-published: %v", m.Subject(), err) + _ = m.Ack() + return + } + go func() { + n, err := starter.Published(ev, outdated, v.View) + if err != nil { + logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err) + _ = m.Nak() + return + } + logf("the publish of %s %s on %s started %d pipelines", ev.Artifact, ev.Version, ev.Branch, n) + _ = m.Ack() + }() + }) +} diff --git a/modules/mesh-delivery-pipeline/go.mod b/modules/mesh-delivery-pipeline/go.mod index 41926627..daa61da4 100644 --- a/modules/mesh-delivery-pipeline/go.mod +++ b/modules/mesh-delivery-pipeline/go.mod @@ -12,6 +12,8 @@ require ( git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 github.com/klauspost/compress v1.20.1 // indirect github.com/nats-io/nkeys v0.4.16 // indirect diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go new file mode 100644 index 00000000..3d693669 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -0,0 +1,136 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + runnerclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient" + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// ErrExists is a Create of a pipeline key that is written already: the second start, which writes nothing (§54). +var ErrExists = errors.New("the pipeline's key is written already") + +// Created is the runner's state pipelines as a start writes it: a key written only when it is absent (compare-and-set on +// its absence), ErrExists when it is not. +type Created interface { + Create(key string, value []byte) error +} + +// Begin is what a start names: the pipeline's key (watched branch, source revision, module; design 47 G §27, §54) and +// what the catalog half of that branch's view says of the module. +type Begin struct { + Branch string + Source int64 + Module string + Commit string + Fp string + Group string + Repository string + Revision int64 +} + +// Key is a pipeline's key in the state pipelines, `..`, the branch escaped as a key's +// name is (§27). +func Key(branch string, source int64, module string) string { + return seat.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), module) +} + +// Starter starts pipelines (design 47 G §54, §103): it writes a pipeline's key by compare-and-set on its absence, so one +// source revision and module has one pipeline whichever event starts it first, and says pipeline-started for the one it +// wrote. +type Starter struct { + Pipelines Created + // Say says one of the seat's events on its subject. + Say func(subject string, body any) error + Now func() time.Time + Logf func(string, ...any) +} + +// Start writes b's pipeline, running, and says pipeline-started; false, nil when its key was written already (the second +// start writes nothing, §54). +func (s *Starter) Start(b Begin) (bool, error) { + if b.Branch == "" || b.Module == "" || b.Source <= 0 { + return false, fmt.Errorf("a pipeline names its watched branch, source revision and module: %+v", b) + } + now := time.Now().UTC() + if s.Now != nil { + now = s.Now().UTC() + } + key := Key(b.Branch, b.Source, b.Module) + p := seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fp: b.Fp, + Group: b.Group, Repository: b.Repository, Revision: b.Revision, Outcome: seat.PipelineOutcomeRunning, Started: &now, + Steps: []seat.StepResult{}} + raw, err := json.Marshal(p) + if err != nil { + return false, err + } + if err := s.Pipelines.Create(key, raw); errors.Is(err, ErrExists) { + s.logf("%s is started already: this start writes nothing", key) + return false, nil + } else if err != nil { + return false, fmt.Errorf("the pipeline %s is not written: %w", key, err) + } + ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, + Commit: b.Commit, Fp: b.Fp, Group: b.Group, Revision: b.Revision} + commitOrGroup := b.Commit + if b.Group != "" { + commitOrGroup = b.Group + } + if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, b.Module, b.Branch, commitOrGroup), ev); err != nil { + s.logf("pipeline-started of %s is not said: %v", key, err) + } + return true, nil +} + +func (s *Starter) logf(format string, args ...any) { + if s.Logf != nil { + s.Logf(format, args...) + } +} + +// outdatedKind is the edge `outdated` names a dependent behind by for a rebuild on a publish: B published before A builds, +// a package A imports (§53). +const outdatedKind = 1 + +// Published starts, for a package-published, a pipeline for each dependent the catalog's `outdated` names behind for that +// package on that branch by kind 1, keyed by the publication's watched branch and source revision and the dependent +// (design 47 G §53, §334). outdated is the kept call of the verb; view the catalog half of the publication's branch. A +// dependent the half does not hold is said and started with what the event names. +func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error), view func(branch string) (Half, bool)) (int, error) { + if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 { + return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev) + } + a, err := outdated(catalog.OutdatedInput{Branch: ev.Branch, Dependency: ev.Artifact}) + if err != nil { + return 0, fmt.Errorf("outdated of %s on %s: %w", ev.Artifact, ev.Branch, err) + } + half, _ := view(ev.Branch) + started := 0 + var first error + for _, o := range a.Behind { + if o.Kind != outdatedKind || o.Module == "" { + continue + } + b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision} + if e, ok := half.Entries[o.Module]; ok { + b.Commit, b.Fp, b.Repository = e.Commit, e.Fp, e.Repository + } else { + s.logf("the catalog half of %s holds no entry of %s, behind %s: its pipeline starts without its commit", ev.Branch, o.Module, ev.Artifact) + } + ok, err := s.Start(b) + if err != nil && first == nil { + first = err + } + if ok { + started++ + } + } + return started, first +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go new file mode 100644 index 00000000..da18b6ba --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -0,0 +1,87 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" +) + +type created map[string][]byte + +func (c created) Create(key string, value []byte) error { + if _, ok := c[key]; ok { + return ErrExists + } + c[key] = value + return nil +} + +// The starts (design 47 G §54, §103, §334): a pipeline's key written by compare-and-set on its absence, so a second start +// of the same source revision and module writes nothing; a publish starts one pipeline per dependent outdated names +// behind by kind 1, keyed by the publishing branch and revision. +func TestStarts(t *testing.T) { + store := created{} + var said []string + s := &Starter{Pipelines: store, Say: func(subject string, body any) error { + ev := body.(seat.PipelineStarted) + said = append(said, subject+" "+ev.Pipeline) + return nil + }} + view := func(branch string) (Half, bool) { + return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{ + "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app"}}}, true + } + var asked catalog.OutdatedInput + outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) { + asked = in + return catalog.OutdatedAnswer{Behind: []catalog.Outdated{ + {Module: "app", Kind: 1}, {Module: "tool", Kind: 1}, {Module: "ordered", Kind: 4}}}, nil + } + n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}, outdated, view) + if err != nil || n != 2 { + t.Fatalf("a publish started %d (%v), want the two dependents behind by kind 1", n, err) + } + if asked.Dependency != "mesh-sdk:go" || asked.Branch != "main" { + t.Errorf("outdated asked %+v, want the package on its branch", asked) + } + var p seat.Pipeline + if err := json.Unmarshal(store["main.41.app"], &p); err != nil || p.Commit != "c1" || p.Fp != "src2:aa" || p.Revision != 9 || + p.Outcome != seat.PipelineOutcomeRunning || p.Source != 41 { + t.Errorf("the pipeline main.41.app = %+v (%v)", p, err) + } + if _, ok := store["main.41.ordered"]; ok { + t.Error("an ordering edge starts no pipeline") + } + // The take-in of the same source revision and module is the same pipeline: the second start writes nothing. + before := string(store["main.41.app"]) + ok, err := s.Start(Begin{Branch: "main", Source: 41, Module: "app", Commit: "c2"}) + if ok || err != nil || string(store["main.41.app"]) != before || len(said) != 2 { + t.Errorf("a second start wrote %v (%v); said %v", ok, err, said) + } + if !strings.HasPrefix(said[0], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.app.main.c1 main.41.app") { + t.Errorf("pipeline-started said on %q", said[0]) + } + // A branch holding a dot is escaped in the key, so the key reads back; a start naming no source revision is refused. + if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Module: "app"}); err != nil { + t.Fatal(err) + } + if b, r, m, ok := seat.ParsePipelinesByBranchRevisionModuleKey(Key("release.1", 3, "app")); !ok || r != "3" || m != "app" || b == "release" { + t.Errorf("the key of release.1 is %q", Key("release.1", 3, "app")) + } + if _, err := s.Start(Begin{Branch: "main", Module: "app"}); err == nil { + t.Error("a start with no source revision is refused") + } + s.Pipelines = failing{} + if _, err := s.Start(Begin{Branch: "main", Source: 5, Module: "app"}); err == nil { + t.Error("a write the bus refuses is an error") + } +} + +type failing struct{} + +func (failing) Create(string, []byte) error { return errors.New("bus away") } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index dbad350d..c60c345f 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -66,7 +66,9 @@ func TestManifest(t *testing.T) { "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked"})}, // RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and // asked again by hand (a hand act). - {"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act"})}, + {"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", + // RN3 (§53, §334): a publication's dependents behind. + "seat:mesh-catalog.outdated"})}, {"first-view tools", equalSets(m.Tools, []string{"pipeline_first_views", "pipeline_first_view_again"})}, // RN1/AC5 (§95, §130): the new assigner is delivered before the runner. {"program needs the assigner's program", slices.Equal(program.Needs, []string{"mesh-delivery-assigner:program"})}, diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 5c71ce13..d0055720 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -42,7 +42,8 @@ "invokes": [ "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", - "seat:mesh-hand-acts.hand-act" + "seat:mesh-hand-acts.hand-act", + "seat:mesh-catalog.outdated" ], "state": [ {