diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index deb5d530..76a234ad 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -259,7 +259,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hearDecided(heard, hearer, cred.Module, logger.Printf); err != nil { return err } - if err := hearPublished(ctx, heard, starter, calls, rv.Catalog, logger.Printf); err != nil { + if err := hearPublished(heard, starter, rv.Catalog, logger.Printf); err != nil { return err } if err := hearRecovery(heard, kvRecoveries{ctx, recovered}, logger.Printf); err != nil { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go index a2985e5f..9b63d931 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -4,21 +4,15 @@ import ( "context" "encoding/json" "errors" - "fmt" "github.com/nats-io/nats.go/jetstream" builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-sdk/go/buscalls" - "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-pipeline/internal/pipeline" ) -// outdatedVerb is the catalog's verb a publication's dependents behind are asked by, as the manifest's invokes names it. -const outdatedVerb = "seat:" + catalog.Seat + "." + catalog.VerbOutdated - // createdKV is the state pipelines written by compare-and-set on a key's absence (§54). type createdKV struct { ctx context.Context @@ -33,23 +27,11 @@ func (c createdKV) Create(key string, value []byte) error { return err } -// hearPublished starts the pipelines of each package-published heard (§53, §334): the dependents the catalog's outdated -// names behind, each asked as a kept call. The message is acknowledged once its starts are written, on a goroutine of its -// own, so a slow catalog holds no other event. -func hearPublished(ctx context.Context, hub handlers, starter *pipeline.Starter, calls *process.Conn, v *pipeline.Views, logf func(string, ...any)) error { - outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) { - raw, err := calls.CallKept(outdatedVerb, in) - if err != nil { - return catalog.OutdatedAnswer{}, err - } - answer, said, ok := buscalls.Unwrap(raw) - if !ok { - return catalog.OutdatedAnswer{}, fmt.Errorf("%s answered: %s", outdatedVerb, said) - } - var a catalog.OutdatedAnswer - err = json.Unmarshal(answer, &a) - return a, err - } +// hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one +// pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is +// written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard +// after a stop. +func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error { return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) { var ev builds.PackagePublished if err := json.Unmarshal(m.Data(), &ev); err != nil { @@ -58,7 +40,7 @@ func hearPublished(ctx context.Context, hub handlers, starter *pipeline.Starter, return } go func() { - n, err := starter.Published(ev, outdated, v.View) + n, err := starter.Published(ev, v.View) if err != nil { logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err) _ = m.Nak() diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go index 55074454..b9fd84a4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go @@ -73,11 +73,7 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if !placed(p, "mesh-sdk") || !placed(p, "builder") || placed(p, "app") || placed(p, "tool") { t.Fatalf("placed at the start: %+v, want mesh-sdk and builder only", p.Steps) } - outdated := func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error) { - t.Error("a publish of a mover asked outdated: it is heard in its pipeline") - return catalog.OutdatedAnswer{}, nil - } - if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, outdated, view); n != 0 || err != nil { + if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); n != 0 || err != nil { t.Fatalf("the publish started %d (%v), want none", n, err) } p, _, _ = mem.Get("main.4") diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 7d1aafec..d0e46489 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "slices" "sort" "strconv" @@ -718,70 +719,79 @@ func (s *Starter) Release(store Updater, key string, view func(branch string) (H } // heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the source -// revision that published it, and places the movers that waited for it (design 47 H §746, H-A-84). It says whether such a -// pipeline moves the package's module. -func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch string) (Half, bool)) (bool, error) { +// revision that published it, and places the movers that waited for it (design 47 H §746, H-A-84). It answers that +// source's pipeline, read, and whether one is held. +func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch string) (Half, bool)) (seat.Pipeline, bool, error) { if s.Store == nil { - return false, nil + return seat.Pipeline{}, false, nil } key := Key(ev.Branch, ev.Source) for { p, rev, err := s.Store.Get(key) - if err != nil || p.Outcome != seat.PipelineOutcomeRunning || !Moves(p, ev.Module) { - return false, nil + if err != nil { + return seat.Pipeline{}, false, nil + } + if p.Outcome != seat.PipelineOutcomeRunning || !Moves(p, ev.Module) { + return p, true, nil } i := slices.IndexFunc(p.Movers, func(m seat.Mover) bool { return m.Module == ev.Module }) if slices.Contains(p.Movers[i].Published, ev.Artifact) { - return true, nil + return p, true, nil } p.Movers[i].Published = append(p.Movers[i].Published, ev.Artifact) if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { continue } else if err != nil { - return true, fmt.Errorf("the pipeline %s cannot be written: %w", key, err) + return p, true, fmt.Errorf("the pipeline %s cannot be written: %w", key, err) } _, err = s.Release(s.Store, key, view) - return true, err + return p, true, err } } -// outdatedKind is the edge `outdated` names a dependent behind by for a rebuild on a publish: B published before A builds, -// a package A imports (§53). -const outdatedKind = 1 +// publishedKind is the edge a dependent builds on a published package by: B published before A builds, a package A +// imports (§53). +const publishedKind = 1 -// Published starts, for a package-published, one pipeline keyed `..published..` whose -// movers are the dependents the catalog's `outdated` names behind for that package on that branch by kind 1 (design 47 G -// §53; H §747). outdated is the kept call of the verb; view the catalog half of the publication's branch. -func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error), view func(branch string) (Half, bool)) (int, error) { +// Published hears a package-published (design 47 H §747, H-A-52): in the running pipeline of the source revision that +// published it, a mover that builds on it waits for it there (§746) and nothing starts; every module the catalog half +// holds a kind-1 edge from the package into that this pipeline does not move (a publish again of an older build, a +// dependent it does not move) is a mover of one pipeline keyed `..published..`, at the +// publishing source's catalog revision. The dependents come from the edges the runner holds, never from a comparison. +func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) (int, error) { if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 { return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev) } - // A publish of a mover of the running pipeline of its source revision is heard in that pipeline: its dependents there - // build once it is (§746). - if in, err := s.heardPublished(ev, view); in || err != nil { + publishing, held, err := s.heardPublished(ev, view) + if err != nil { return 0, err } - a, err := outdated(catalog.OutdatedInput{Branch: ev.Branch, Dependency: ev.Artifact}) - if err != nil { - return 0, fmt.Errorf("outdated of %s on %s: %w", ev.Artifact, ev.Branch, err) + half, ok := view(ev.Branch) + if !ok { + return 0, fmt.Errorf("the catalog half of %s is not held yet: the publish of %s %s is heard again once it is", ev.Branch, ev.Artifact, ev.Version) } - half, held := view(ev.Branch) var movers []seat.Mover - for _, o := range a.Behind { - if o.Kind != outdatedKind || o.Module == "" || Moves(seat.Pipeline{Movers: movers}, o.Module) { + for _, module := range slices.Sorted(maps.Keys(half.Entries)) { + e := half.Entries[module] + if module == ev.Module || Moves(publishing, module) || !slices.ContainsFunc(e.Edges, func(x catalog.Edge) bool { + return x.Kind == publishedKind && x.From == ev.Artifact + }) { continue } - e := half.Entries[o.Module] - movers = append(movers, seat.Mover{Module: o.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository}) + movers = append(movers, seat.Mover{Module: module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository}) } if len(movers) == 0 { return 0, nil } + revision := half.Revision + if held && publishing.Revision > 0 { + revision = publishing.Revision + } movers = Tiers(movers, half.Entries, s.logf) - ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Revision: half.Revision, Movers: movers, Entries: half.Entries, - Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version}, unheld: !held, + started, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Call: publishing.Call, Revision: revision, Movers: movers, + Entries: half.Entries, Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version}, Key: PublishedKey(ev.Branch, ev.Source, ev.Artifact, ev.Version)}) - if ok { + if started { return 1, err } return 0, err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 1d58f7fb..21f7ad93 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -21,58 +21,20 @@ func (c created) Create(key string, value []byte) error { return nil } -// The starts (design 47 G §53, §54; H §746, §747): a pipeline's key written by compare-and-set on its absence, so a second -// start of the same key writes nothing; a publish starts one pipeline keyed ..published.. -// whose movers are the dependents outdated names behind by kind 1; a branch holding a dot is escaped in the key. +// The starts (design 47 H §746, §747): a pipeline's key written by compare-and-set on its absence, so a second start of the +// same key writes nothing; a branch holding a dot is escaped in the key; a start with no source revision or no mover is +// refused. func TestStarts(t *testing.T) { store := created{} - var said []string - s := &Starter{Pipelines: store, Say: func(subject string, body any) error { - ev, started := body.(seat.PipelineStarted) - if !started { // only pipeline-started is counted here - return nil - } - said = append(said, subject+" "+ev.Pipeline) - if ev.Published == nil || ev.Published.Artifact != "mesh-sdk:go" || ev.Published.Version != "0.1.16" { - t.Errorf("pipeline-started of %s says the publish %+v (A-202)", ev.Pipeline, ev.Published) - } - return nil - }} - view := func(branch string) (Half, bool) { - return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{ - "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app"}}}, true - } - var asked catalog.OutdatedInput - outdated := func(in catalog.OutdatedInput) (catalog.OutdatedAnswer, error) { - asked = in - return catalog.OutdatedAnswer{Behind: []catalog.Outdated{ - {Module: "app", Kind: 1}, {Module: "tool", Kind: 1}, {Module: "ordered", Kind: 4}}}, nil - } - n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}, outdated, view) - if err != nil || n != 1 { - t.Fatalf("a publish started %d (%v), want one pipeline", n, err) - } - if asked.Dependency != "mesh-sdk:go" || asked.Branch != "main" { - t.Errorf("outdated asked %+v, want the package on its branch", asked) - } - key := PublishedKey("main", 41, "mesh-sdk:go", "0.1.16") - var p seat.Pipeline - if err := json.Unmarshal(store[key], &p); err != nil || len(p.Movers) != 2 || p.Revision != 9 || p.Outcome != seat.PipelineOutcomeRunning || - p.Source != 41 || p.Published == nil || p.Published.Version != "0.1.16" || Moves(p, "ordered") { - t.Fatalf("the pipeline %s = %+v (%v)", key, p, err) - } - if app, _ := MoverOf(p, "app"); app.Commit != "c1" || app.Fingerprint != "src2:aa" { - t.Errorf("the mover app = %+v", app) + s := &Starter{Pipelines: store, Say: func(string, any) error { return nil }} + key := Key("main", 41) + if ok, err := s.Start(Begin{Branch: "main", Source: 41, Movers: []seat.Mover{{Module: "app", Commit: "c1"}}}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) } before := string(store[key]) - ok, err := s.Start(Begin{Branch: "main", Source: 41, Key: key, Movers: []seat.Mover{{Module: "app", Commit: "c2"}}}) - if ok || err != nil || string(store[key]) != before || len(said) != 1 { - t.Errorf("a second start wrote %v (%v); said %v", ok, err, said) + if ok, err := s.Start(Begin{Branch: "main", Source: 41, Movers: []seat.Mover{{Module: "app", Commit: "c2"}}}); ok || err != nil || string(store[key]) != before { + t.Errorf("a second start wrote %v (%v)", ok, err) } - if !strings.HasPrefix(said[0], "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.41 "+key) { - t.Errorf("pipeline-started said on %q", said[0]) - } - s.Say = func(string, any) error { return nil } if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Movers: []seat.Mover{{Module: "app"}}}); err != nil { t.Fatal(err) } @@ -91,6 +53,55 @@ func TestStarts(t *testing.T) { } } +// R10/AC1 and AC2 (design 47 H §747, H-A-52): a package-published of a mover of its source's running pipeline whose +// dependent that pipeline moves starts nothing: the dependent waits there. A publish again of an older build starts one +// pipeline keyed ..published.., its movers the package's kind-1 dependents from the +// edges the runner holds, at the publishing source's revision, and says pipeline-started with the publish; an ordering +// edge starts nothing; heard twice, the second writes nothing. +func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) { + mem := memStore{} + var said []seat.PipelineStarted + s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(_ string, body any) error { + if ev, ok := body.(seat.PipelineStarted); ok { + said = append(said, ev) + } + return nil + }} + edges := []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 1}} + view := func(string) (Half, bool) { + return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{ + "mesh-sdk": {Module: "mesh-sdk"}, + "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges}, + "tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges}, + "ordered": {Module: "ordered", Edges: []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 4}}}}}, true + } + mem[Key("main", 41)] = struct { + p seat.Pipeline + rev uint64 + }{seat.Pipeline{Pipeline: "main.41", Branch: "main", Source: 41, Revision: 8, Outcome: seat.PipelineOutcomeRunning, + Movers: []seat.Mover{{Module: "mesh-sdk"}, {Module: "app"}, {Module: "tool"}}}, 1} + ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"} + if n, err := s.Published(ev, view); n != 0 || err != nil || len(mem) != 1 { + t.Fatalf("a publish in a pipeline moving its dependents started %d (%v), %d pipelines", n, err, len(mem)) + } + // A publish again of an older build, at source 30: its pipeline is gone, so every kind-1 dependent is a mover. + ev.Source, ev.Version = 30, "0.1.9" + for i := range 2 { + if n, err := s.Published(ev, view); n != 1-i || err != nil { + t.Fatalf("hearing %d started %d (%v)", i+1, n, err) + } + } + key := PublishedKey("main", 30, "mesh-sdk:go", "0.1.9") + p, _, err := mem.Get(key) + if err != nil || len(p.Movers) != 2 || !Moves(p, "app") || !Moves(p, "tool") || Moves(p, "ordered") || p.Revision != 9 || + p.Published == nil || p.Published.Version != "0.1.9" || len(said) != 1 || said[0].Published == nil { + t.Fatalf("the pipeline %s = %+v (%v), said %+v", key, p, err, said) + } + if app, _ := MoverOf(p, "app"); app.Commit != "c1" || app.Fingerprint != "src2:aa" || app.Repository != "o/app" { + t.Errorf("the mover app = %+v", app) + } +} + // memCreated is memStore with the Create a start writes through. type memCreated struct{ memStore } @@ -230,22 +241,23 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T if n != 1 || err != nil || len(p.Steps) == 0 { t.Errorf("the half of next placed %d (%v): %d steps, want the take-in's pipeline placed", n, err, len(p.Steps)) } - // A-336: so does a package published heard before its branch's half is held. + // A-336: a package published heard before its branch's half is held starts nothing, heard again once it is: its + // dependents are the half's edges (§747). held = false - outdated := func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error) { - return catalog.OutdatedAnswer{Behind: []catalog.Outdated{{Module: "app", Kind: outdatedKind}}}, nil - } - if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "next", Source: 7, Module: "mesh-sdk", Version: "0.1.17"}, outdated, late); n != 1 || err != nil { - t.Fatalf("the publish before the half started %d (%v)", n, err) - } - if p, _, _ := store.Get(PublishedKey("next", 7, "mesh-sdk:go", "0.1.17")); len(p.Steps) != 0 { - t.Fatalf("a publish before its branch's half placed %d steps from empty entries, want none", len(p.Steps)) + app := entries["app"] + app.Edges = []catalog.Edge{{From: "mesh-sdk:go", To: "app:program", Kind: 1}} + entries["app"] = app + published := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "next", Source: 7, Module: "mesh-sdk", Version: "0.1.17"} + if n, err := s.Published(published, late); n != 0 || err == nil { + t.Fatalf("the publish before the half started %d (%v), want none and an error, heard again", n, err) } held = true - n, err = s.Waiting(store.memStore, "next", late) + if n, err := s.Published(published, late); n != 1 || err != nil { + t.Fatalf("the publish with the half held started %d (%v)", n, err) + } p, _, _ = store.Get(PublishedKey("next", 7, "mesh-sdk:go", "0.1.17")) - if n != 1 || err != nil || len(p.Steps) == 0 { - t.Errorf("the half of next placed %d (%v): %d steps, want the publish's pipeline placed", n, err, len(p.Steps)) + if len(p.Steps) == 0 { + t.Errorf("the publish's pipeline placed %d steps, want its steps", len(p.Steps)) } // A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running, diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index 3fe866a6..f55cb743 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -93,8 +93,6 @@ func TestManifest(t *testing.T) { {"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations", // A-186: the seat list, read by its verb. "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", - // RN3 (§53, §334): a publication's dependents behind. - "seat:mesh-catalog.outdated", // RN10 (§71, §441, A-207): the permissions a change widens, asked of the grants part. "seat:mesh-grants.grants-for", // RN11 (§33, A-191): the author of a failed head, read as the forge records it. diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 6e1ea095..0fcd5898 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -68,7 +68,6 @@ "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", - "seat:mesh-catalog.outdated", "seat:git.pull", "seat:mesh-module-assigner.assignments", "seat:mesh-grants.grants-for",