From a926a40eed397cf8372a697ba443dc6d383a7cbd Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 22:55:36 +0100 Subject: [PATCH 01/11] Design 47 H1: every step ask fetches with the entry's pins, as the build does (H-A-203) The runner's lint, unit and integration asks carried no pins, only the build ask did, so the Go build module, which refuses a requirement of another repository with no pin, could not run twelve test steps of the contracts mover (main.45). stepAsk now puts the entry's packages into the fetch in the form the entry keeps and the builder sends, so a test runs against what the build was built against; a package the entry names with no pin yet is left out. Builds: design 47 H H-A-203, H-A-192, H-A-183 --- .../testruns_integration_test.go | 14 +++++++++++++- .../internal/pipeline/place.go | 12 ++++++++++++ .../internal/pipeline/place_test.go | 9 ++++++++- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index 95ed9670..a0aa6696 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -88,7 +88,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil { + askStream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}) + if err != nil { t.Fatal(err) } @@ -116,6 +117,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) entries := []catalog.CatalogEntry{ {Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app", + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25"}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, {Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}}, {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, @@ -331,6 +333,16 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { passBuild(p3, "app", "c1", "src2:aa") unitRun, unit := runOfStep(p3, "app:program", "unit") integRun, integ := runOfStep(p3, "app:program", "integration") + // A test ask fetches with the entry's pins, as the builder gives them the build (H-A-203). + last, err := askStream.GetLastMsgForSubject(ctx, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner") + if err != nil { + t.Fatal(err) + } + var testAsk nodebuildgo.StepAsk + if err := json.Unmarshal(last.Data, &testAsk); err != nil || testAsk.Step == nil || + testAsk.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25" { + t.Fatalf("the last test ask = %s (%v), want its fetch to carry the entry's pin", last.Data, err) + } if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 { t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index af5758d6..4110b3d1 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -234,6 +234,18 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar if paths := artifactPaths(e, a.Ref); len(paths) > 0 { fetch.Paths = paths } + // The entry's pins, the same the builder gives the build (H-A-203, H-A-183): a test runs against what the build was + // built against, and the Go build module refuses a requirement of another repository with no pin (H-A-192). A + // package the entry names with no pin yet is left out, as the builder pins it only from its publish. + for path, version := range e.Packages { + if version == "" { + continue + } + if fetch.Packages == nil { + fetch.Packages = map[string]string{} + } + fetch.Packages[path] = version + } return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now, Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: m.Commit, Fetch: fetch, Fingerprint: m.Fingerprint, Module: m.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index d54f44a8..21b3047e 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -44,7 +44,7 @@ func placeFixture() (catalog.Registration, map[string]catalog.CatalogEntry) { Artifacts: map[string]catalog.StepStatuses{"tools": {Lint: "bypass"}}, }} return reg, map[string]catalog.CatalogEntry{ - "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, + "app": {Module: "app", Path: "modules/app", Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": ""}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, {Ref: "app:web", Language: "rust"}}, Inputs: &catalog.EntryInputs{Paths: map[string][]string{"app:program": {"cmd/app"}}}}, "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, } @@ -104,6 +104,13 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { len(a.Step.Fetch.Paths) != 1 || a.Step.Fetch.Paths[0] != "cmd/app" { t.Errorf("the ask = %+v at %s", a, asks.at[1]) } + // Every step ask fetches with the entry's pins, as the builder gives them the build; a package with no pin yet is left + // out (H-A-203, H-A-192). + for _, x := range asks.asks[1:] { + if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && (len(b.Step.Fetch.Packages) != 1 || b.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25") { + t.Errorf("the %s ask of %s fetches the pins %v, want the entry's", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages) + } + } // An artifact that names no paths still fetches its module alone: the directory with no paths (principle 10, A-447). for _, x := range asks.asks[2:] { if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && b.Step.Artifact == "app:tools" && (b.Step.Fetch.Dir != "modules/app" || len(b.Step.Fetch.Paths) != 0) { From e3fed7751558298365c74fa7c899fb72037deb17 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 22:58:30 +0100 Subject: [PATCH 02/11] Design 47 H1: a Go step waits for its entry's pins as the build does (H-A-206) An entry may name a package with no version yet, and the runner cannot pin it from a publish it does not hear, so a step asked then is refused by the Go build module. The placing now asks a Go artifact's lint, unit and integration steps only when every pin of the entry has a version; until then each stands waiting, its why "waits on the publish of " as the build's wait reads, said in the journal and read by show. When the mover's build ends without passing, the held steps end as it ended; the publish re-pins the entry and moves the mover, whose pipeline runs them. A held step still counts in the judgement of a seat with no holder. Builds: design 47 H H-A-206, H-A-203, H-A-165 --- .../internal/pipeline/place.go | 48 +++++++++++++++++++ .../internal/pipeline/place_test.go | 44 +++++++++++++++-- .../internal/pipeline/start.go | 7 ++- .../internal/pipeline/steps.go | 3 ++ 4 files changed, 98 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 4110b3d1..d978de57 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -4,6 +4,8 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "maps" + "slices" "sort" "strconv" "strings" @@ -140,6 +142,13 @@ func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[ steps = append(steps, s) continue } + // A Go step is asked only once every pin of its entry has a version (H-A-206): until then it waits on the + // publish, as the build does (H-A-165), and the publish re-pins the entry and moves the mover (H-A-204). + if path := unpinnedPackage(entry); path != "" && language == sdkstep.AskGo { + s.Seat, s.Outcome, s.Started, s.Why = bs, seat.StepResultOutcomeWaiting, &now, PinWait+path + steps = append(steps, s) + continue + } s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now s.Ask = AskID(startOf(p, now), p.Pipeline, m.Module, a.Ref, step) body := stepAsk(p, m, entry, a, language, step, s.Ask, asker, now) @@ -227,6 +236,45 @@ type Asked struct { Commit string } +// PinWait opens the why of a Go step held because its entry names a package with no version yet (H-A-206), the build's +// own wait form (H-A-165); the package's path follows. +const PinWait = "waits on the publish of " + +// unpinnedPackage is the first package, in path order, the entry names with no version, empty when every one has one. +func unpinnedPackage(e catalog.CatalogEntry) string { + for _, path := range slices.Sorted(maps.Keys(e.Packages)) { + if e.Packages[path] == "" { + return path + } + } + return "" +} + +// pinHeld says whether a step waits for its entry's pins (H-A-206). +func pinHeld(s seat.StepResult) bool { + return s.Outcome == seat.StepResultOutcomeWaiting && strings.HasPrefix(s.Why, PinWait) +} + +// endPinHeld ends, as the mover's build ended without passing, each of its steps waiting for its entry's pins (H-A-206): +// with the build's outcome and why, so no step waits on a publish the build no longer waits for. It answers the steps it +// ended; a build that passed ends none, the publish re-pinning the entry and moving the mover. +func endPinHeld(p *seat.Pipeline, build seat.StepResult, now time.Time) []int { + if stepPassed(build) { + return nil + } + var ended []int + for i := range p.Steps { + st := &p.Steps[i] + if st.Module != build.Module || !pinHeld(*st) { + continue + } + st.Outcome, st.Why, st.Ended = build.Outcome, "the build ended "+build.Outcome+" while this step waited: "+st.Why+"; "+build.Why, &now + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) + ended = append(ended, i) + } + return ended +} + func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk { // A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the // artifact's named paths in it when the entry has them. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index 21b3047e..0ffc885f 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -2,6 +2,7 @@ package pipeline import ( "encoding/json" + "fmt" "strconv" "strings" "testing" @@ -44,7 +45,7 @@ func placeFixture() (catalog.Registration, map[string]catalog.CatalogEntry) { Artifacts: map[string]catalog.StepStatuses{"tools": {Lint: "bypass"}}, }} return reg, map[string]catalog.CatalogEntry{ - "app": {Module: "app", Path: "modules/app", Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": ""}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, + "app": {Module: "app", Path: "modules/app", Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25"}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, {Ref: "app:web", Language: "rust"}}, Inputs: &catalog.EntryInputs{Paths: map[string][]string{"app:program": {"cmd/app"}}}}, "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, } @@ -104,13 +105,50 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { len(a.Step.Fetch.Paths) != 1 || a.Step.Fetch.Paths[0] != "cmd/app" { t.Errorf("the ask = %+v at %s", a, asks.at[1]) } - // Every step ask fetches with the entry's pins, as the builder gives them the build; a package with no pin yet is left - // out (H-A-203, H-A-192). + // Every step ask fetches with the entry's pins, as the builder gives them the build (H-A-203, H-A-192). for _, x := range asks.asks[1:] { if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && (len(b.Step.Fetch.Packages) != 1 || b.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25") { t.Errorf("the %s ask of %s fetches the pins %v, want the entry's", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages) } } + // An entry naming a package with no version yet asks no Go step: each waits on its publish, as the build does, said in + // the journal (H-A-206); the build is still asked. + waiting := map[string]catalog.CatalogEntry{} + for k, e := range entries { + waiting[k] = e + } + held := waiting["app"] + held.Packages = map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": ""} + waiting["app"] = held + var journal []string + heldAsks := &stepAsks{} + hs := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: heldAsks, Name: "the-runner", + Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }, + Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }} + if ok, err := hs.Start(Begin{Branch: "main", Source: 5, Entries: waiting, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + for _, x := range heldAsks.asks { + if b, ok := x.(nodebuildgo.StepAsk); ok { + t.Errorf("the %s of %s is asked with a package unpinned", b.Step.Step, b.Step.Artifact) + } + } + var hp seat.Pipeline + if err := json.Unmarshal(store["main.5"], &hp); err != nil { + t.Fatal(err) + } + if st := hp.Steps[1]; st.Step != "lint" || st.Outcome != seat.StepResultOutcomeWaiting || st.Ask != "" || st.Why != "waits on the publish of git.novox.be/novox/x/go" { + t.Errorf("the held lint step = %+v; want it waiting on the publish", st) + } + // The build ending without passing past its bound ends each held step as it ended (H-A-206). + failed := hp.Steps[0] + failed.Outcome, failed.Why = seat.StepResultOutcomeFailed, "package git.novox.be/novox/x/go has no released version" + if ended := endPinHeld(&hp, failed, time.Now()); len(ended) == 0 || hp.Steps[1].Outcome != seat.StepResultOutcomeFailed || hp.Steps[1].Ended == nil { + t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", hp.Steps[1], len(ended)) + } + if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") { + t.Errorf("the journal says %q; want the held step named", journal) + } // An artifact that names no paths still fetches its module alone: the directory with no paths (principle 10, A-447). for _, x := range asks.asks[2:] { if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && b.Step.Artifact == "app:tools" && (b.Step.Fetch.Dir != "modules/app" || len(b.Step.Fetch.Paths) != 0) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index cfb8cc48..cb039452 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -189,7 +189,7 @@ func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]A } var none []string for _, st := range p.Steps { - if !open(st) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) { + if (!open(st) && !pinHeld(st)) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) { continue } if holders, read := s.Holders(st.Seat); read && len(holders) == 0 { @@ -337,6 +337,11 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog } steps, a := Place(*p, m, r, entries, now, s.Name) supersededWhy(*p, m.Module, entries, steps) + for _, st := range steps { + if pinHeld(st) { + s.logf("%s: %s's %s of %s is not asked: it %s, as its build does (H-A-206)", p.Pipeline, m.Module, st.Step, st.Artifact, st.Why) + } + } if failed != "" { why := "its dependency " + failed + " did not build" for k := range steps { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 9b39cf26..854af132 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -272,6 +272,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh // Every mover's validate reads the one run (§745, H-A-51); its end asks each mover's composition, or ends it not // asked when validate did not pass. var due []Asked + if s.Step == seat.StepResultStepBuild { + readers = append(readers, endPinHeld(&p, *s, now)...) + } if s.Step == seat.StepResultStepValidate { readers = append(readers, sameValidate(&p, i, now)...) var notAsked []int From cb26645614e1178cecd85e9ef34be3cf28a389ad Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:05:08 +0100 Subject: [PATCH 03/11] Design 47 H1: the build step's end reports the pins it was built with (H-A-207) A test step held for an entry's empty pin had nothing to run with once the build passed: the builder took the pin from the registry and kept it to itself. The seat mesh-builds' step end gains packages, and the builder sets it from the pins each entry was built with, the registry's included, so the runner asks the held steps with what the build was built against. The seat's definition changes: its Go and TypeScript contracts are generated again. Builds: design 47 H H-A-207 --- .../internal/build/step.go | 19 ++++++++++++++++--- .../internal/build/step_test.go | 4 ++++ .../go/seats/meshbuilds/contract.go | 8 +++++--- .../seats/mesh-builds/seat.json | 7 +++++++ .../ts/seats/meshbuilds/index.d.ts | 2 ++ 5 files changed, 34 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index 09687285..cfff2cac 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -41,7 +41,18 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui } var failed []string for _, e := range run.Entries { - outcome, said := b.buildEntry(ctx, e) + var pins map[string]string + outcome, said := b.buildEntry(ctx, e, &pins) + // The end reports the pins each entry was built with, so the runner asks its held test steps with them (H-A-207). + for path, version := range pins { + if version == "" { + continue + } + if end.Packages == nil { + end.Packages = map[string]string{} + } + end.Packages[path] = version + } switch outcome { case meshbuilds.StepEndedOutcomeCouldNotRun: return finish(outcome, said) @@ -55,8 +66,9 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui return finish(meshbuilds.StepEndedOutcomePassed, "") } -// buildEntry builds one entry at its fingerprint and waits for its end. -func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outcome, said string) { +// buildEntry builds one entry at its fingerprint and waits for its end; pins is set to the pins it is built with, the +// entry's own and those the registry gave a package the entry named with none (H-A-207). +func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) { entry, err := b.StepEntry(ctx, e.Fingerprint) if err != nil { return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry %s at %s cannot be read: %v", e.Module, e.Fingerprint, err) @@ -70,6 +82,7 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc return outcome, said } entry.Packages = pinned + *pins = pinned key, err := builds.BuilderBuildKey(entry.Module, entry.Fp) if err != nil { return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error() diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 6a777cf2..08bbbedd 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -189,6 +189,10 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) { !reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) { t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks) } + // The end reports the pins the build was built with, the one the publish gave included (H-A-207). + if !reflect.DeepEqual(end.Packages, map[string]string{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) { + t.Errorf("the end reports the pins %v; want the ones it was built with", end.Packages) + } } func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) { diff --git a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go index 836f14e0..4cc3285b 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go @@ -114,9 +114,11 @@ type StepEnded struct { // the node the builder ran it on, as in the subject Node string `json:"node"` // never passed for a build that built nothing - Outcome string `json:"outcome"` - Pipeline string `json:"pipeline"` - Rerun bool `json:"rerun,omitempty"` + Outcome string `json:"outcome"` + // the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207) + Packages map[string]string `json:"packages,omitempty"` + Pipeline string `json:"pipeline"` + Rerun bool `json:"rerun,omitempty"` // why it failed or could not run: the entry not built, the part not heard Said string `json:"said,omitempty"` Started *time.Time `json:"started,omitempty"` diff --git a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json index d10eec4f..fc3d717d 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json @@ -363,6 +363,13 @@ "$ref": "#/defs/BuildEntry" }, "description": "each entry the step built, by module and fingerprint" + }, + "packages": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207)" } }, "required": [ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts index d142d6d9..947f2e52 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts @@ -89,6 +89,8 @@ export interface StepEnded { "node": string; /** never passed for a build that built nothing */ "outcome": "passed" | "failed" | "nothing to run" | "could not run"; + /** the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207) */ + "packages"?: Record; "pipeline": string; "rerun"?: boolean; /** why it failed or could not run: the entry not built, the part not heard */ From 64cba95a101d8d8e3bf4501fae7d1e3496a5f64f Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:07:33 +0100 Subject: [PATCH 04/11] Design 47 H1: a held test step is asked with the pins its build passed with (H-A-207) When the build of an entry with an empty pin passed, its held lint, unit and integration steps had no pin to run with and nothing ended them, so a pipeline stood running and, one pipeline at a time, the queue stopped. The runner reads the pins the build's end names and, when the build passed, asks each held step with them, its fetch the entry's as placed; a passed build whose end names no pins ends the held steps could not run, naming why, as does an artifact the branch's half no longer names. Builds: design 47 H H-A-207, H-A-206, H-A-203 --- .../cmd/mesh-delivery-pipeline/main.go | 5 ++ .../cmd/mesh-delivery-pipeline/steps.go | 6 +- .../internal/pipeline/place.go | 82 +++++++++++++++++-- .../internal/pipeline/place_test.go | 53 +++++++++++- .../internal/pipeline/steps.go | 29 ++++++- 5 files changed, 164 insertions(+), 11 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 9488c743..8d0fdd22 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -154,6 +154,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} hearer.AskSteps = starter.AskSteps hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } + hearer.AskPinHeld = func(key, module string, pins map[string]string) { + if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, rv.Catalog.View); err != nil { + logger.Printf("the steps of %s's %s that waited for its pins are not asked: %v", key, module, err) + } + } hearer.Release = func(key string) { if _, err := starter.Release(hearer.Store, key, rv.Catalog.View); err != nil { logger.Printf("the movers of %s waiting for a build are not placed: %v", key, err) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index d6915534..c4aa18c1 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -128,7 +128,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) if bs.Shared != nil { shared = bs.Shared(m.Data()) } - _, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared) + var pins map[string]string + if bs.Pins != nil { + pins = bs.Pins(m.Data()) + } + _, err = h.OnStepEndedPinned(bs, m.Subject(), ev, shared, pins) switch { case errors.Is(err, pipeline.ErrRefused): logf("%v", err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index d978de57..5c096fe9 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -3,6 +3,7 @@ package pipeline import ( "crypto/sha256" "encoding/hex" + "errors" "fmt" "maps" "slices" @@ -255,12 +256,16 @@ func pinHeld(s seat.StepResult) bool { return s.Outcome == seat.StepResultOutcomeWaiting && strings.HasPrefix(s.Why, PinWait) } -// endPinHeld ends, as the mover's build ended without passing, each of its steps waiting for its entry's pins (H-A-206): -// with the build's outcome and why, so no step waits on a publish the build no longer waits for. It answers the steps it -// ended; a build that passed ends none, the publish re-pinning the entry and moving the mover. -func endPinHeld(p *seat.Pipeline, build seat.StepResult, now time.Time) []int { +// endPinHeld ends each of build's mover's steps waiting for its entry's pins (H-A-206, H-A-207): as the build ended when +// it did not pass, so no step waits on a publish the build no longer waits for; could not run, loud, when the build passed +// and its end names no pins to run them with. It answers the steps it ended. +func endPinHeld(p *seat.Pipeline, build seat.StepResult, pins map[string]string, now time.Time) []int { + outcome, why := build.Outcome, "the build ended "+build.Outcome+" while this step waited: %s; "+build.Why if stepPassed(build) { - return nil + if len(pins) > 0 { + return nil + } + outcome, why = seat.StepResultOutcomeCouldNotRun, "the build passed, but its end names no pins to run this step with (H-A-207); it %s" } var ended []int for i := range p.Steps { @@ -268,13 +273,78 @@ func endPinHeld(p *seat.Pipeline, build seat.StepResult, now time.Time) []int { if st.Module != build.Module || !pinHeld(*st) { continue } - st.Outcome, st.Why, st.Ended = build.Outcome, "the build ended "+build.Outcome+" while this step waited: "+st.Why+"; "+build.Why, &now + st.Outcome, st.Why, st.Ended = outcome, fmt.Sprintf(why, st.Why), &now p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) ended = append(ended, i) } return ended } +// heldOn says whether module has a step of p waiting for its entry's pins. +func heldOn(p seat.Pipeline, module string) bool { + return slices.ContainsFunc(p.Steps, func(st seat.StepResult) bool { return st.Module == module && pinHeld(st) }) +} + +// AskPinHeld asks the steps of pipeline key's mover module that waited for its entry's pins (H-A-206) with the pins its +// build passed with (H-A-207): each running under its own ask, its fetch the entry's as Place builds it with those pins. +// A step whose artifact the branch's half no longer names ends could not run, saying so. It answers how many it asked. +func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]string, view func(branch string) (Half, bool)) (int, error) { + for { + p, rev, err := store.Get(key) + if err != nil { + return 0, fmt.Errorf("the pipeline %s is not read: %w", key, err) + } + m, moves := MoverOf(p, module) + if !moves || !heldOn(p, module) { + return 0, nil + } + var entry catalog.CatalogEntry + if view != nil { + if half, ok := view(p.Branch); ok { + entry = half.Entries[module] + } + } + entry.Packages = pins + now := s.now() + var asks []Asked + var ended []int + for i := range p.Steps { + st := &p.Steps[i] + if st.Module != module || !pinHeld(*st) { + continue + } + j := slices.IndexFunc(entry.Artifacts, func(a catalog.Artifact) bool { return a.Ref == st.Artifact }) + language, err := sdkstep.AskLanguageOfSeat(st.Seat) + if j < 0 || err != nil { + st.Outcome, st.Ended = seat.StepResultOutcomeCouldNotRun, &now + st.Why = fmt.Sprintf("its build passed with its pins, but the catalog half of %s names no artifact %s of %s to ask it of (H-A-207)", p.Branch, st.Artifact, module) + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) + ended = append(ended, i) + continue + } + a := entry.Artifacts[j] + st.Outcome, st.Why, st.Started = seat.StepResultOutcomeRunning, "", &now + st.Ask = AskID(startOf(p, now), p.Pipeline, module, a.Ref, st.Step) + ask := Asked{Seat: st.Seat, ID: st.Ask, Ask: stepAsk(p, m, entry, a, language, st.Step, st.Ask, s.Name, now), Toolchain: a.Language, Commit: m.Commit} + if a.Package != nil { + ask.Package = a.Package.Name + } + asks = append(asks, ask) + } + if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return 0, fmt.Errorf("the pipeline %s is not written: %w", key, err) + } + s.logf("%s: %s's build passed with its pins: %d steps that waited for them are asked, %d could not run (H-A-207)", key, module, len(asks), len(ended)) + s.sayNotAsked(p, ended) + if s.Steps == nil || len(asks) == 0 { + return 0, nil + } + return len(asks), s.askSteps(key, asks) + } +} + func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk { // A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the // artifact's named paths in it when the entry has them. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index 0ffc885f..f152f0a0 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -3,6 +3,7 @@ package pipeline import ( "encoding/json" "fmt" + "reflect" "strconv" "strings" "testing" @@ -143,8 +144,56 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { // The build ending without passing past its bound ends each held step as it ended (H-A-206). failed := hp.Steps[0] failed.Outcome, failed.Why = seat.StepResultOutcomeFailed, "package git.novox.be/novox/x/go has no released version" - if ended := endPinHeld(&hp, failed, time.Now()); len(ended) == 0 || hp.Steps[1].Outcome != seat.StepResultOutcomeFailed || hp.Steps[1].Ended == nil { - t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", hp.Steps[1], len(ended)) + failedP := hp + failedP.Steps = append([]seat.StepResult(nil), hp.Steps...) + if ended := endPinHeld(&failedP, failed, nil, time.Now()); len(ended) == 0 || failedP.Steps[1].Outcome != seat.StepResultOutcomeFailed || failedP.Steps[1].Ended == nil { + t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", failedP.Steps[1], len(ended)) + } + // A build passed whose end names no pins ends them could not run, loud (H-A-207). + passed := hp.Steps[0] + passed.Outcome = seat.StepResultOutcomePassed + bare := hp + bare.Steps = append([]seat.StepResult(nil), hp.Steps...) + if ended := endPinHeld(&bare, passed, nil, time.Now()); len(ended) == 0 || bare.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || + !strings.Contains(bare.Steps[1].Why, "names no pins") { + t.Errorf("after a build passed naming no pins the held lint step = %+v; want could not run naming it", bare.Steps[1]) + } + // A build passed naming its pins asks the held steps with them: its end heard, the runner asks each with the pins the + // build was built with (H-A-207). + pinsBuilt := map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": "v0.1.4"} + ms := memStore{} + ms["main.5"] = struct { + p seat.Pipeline + rev uint64 + }{hp, 1} + var askedWith map[string]string + h := &Hearer{Store: ms, Say: func(string, any) error { return nil }, Name: "the-runner", + AskPinHeld: func(key, module string, pins map[string]string) { + askedWith = pins + if _, err := hs.AskPinHeld(ms, key, module, pins, func(string) (Half, bool) { return Half{Entries: waiting}, true }); err != nil { + t.Errorf("ask held: %v", err) + } + }} + buildEnd := StepEnded{ID: hp.Steps[0].Ask, Pipeline: "main.5", Step: "build", Node: "n1", Outcome: seat.StepResultOutcomePassed, + Module: "app", Commit: "c1", Fingerprint: "src2:ab"} + if _, err := h.OnStepEndedPinned(StepSeats[3], StepSeats[3].Prefix+"n1", buildEnd, nil, pinsBuilt); err != nil { + t.Fatalf("the build's end: %v", err) + } + if !reflect.DeepEqual(askedWith, pinsBuilt) { + t.Errorf("the held steps were asked with %v; want the build's pins", askedWith) + } + var askedHeld int + for _, x := range heldAsks.asks { + if b, ok := x.(nodebuildgo.StepAsk); ok { + askedHeld++ + if !reflect.DeepEqual(b.Step.Fetch.Packages, pinsBuilt) { + t.Errorf("the %s of %s fetches %v; want the build's pins", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages) + } + } + } + after, _, _ := ms.Get("main.5") + if askedHeld == 0 || after.Steps[1].Outcome == seat.StepResultOutcomeWaiting || after.Steps[1].Ask == "" { + t.Errorf("after the build passed %d held steps were asked, lint = %+v; want each asked", askedHeld, after.Steps[1]) } if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") { t.Errorf("the journal says %q; want the held step named", journal) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 854af132..5ac10c50 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -31,6 +31,8 @@ type StepSeat struct { // Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for // a seat whose ends name none. Shared func([]byte) []seat.Shared + // Pins reads the pins a build's end names it was built with (H-A-207); nil for a seat whose ends name none. + Pins func([]byte) map[string]string } // StepSeats are the three build seats, from their contracts. @@ -39,7 +41,7 @@ var StepSeats = []StepSeat{ {Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")}, {Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")}, // The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99). - {Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded}, + {Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded, Pins: buildPins}, // The seat mesh-wants answers the composition step (§22, §24). {Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared}, } @@ -77,6 +79,15 @@ func decodeWantsEnded(raw []byte) (StepEnded, error) { Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil } +// buildPins is the pins a build step's end of the seat mesh-builds names its entry was built with (H-A-207). +func buildPins(raw []byte) map[string]string { + var b builds.StepEnded + if json.Unmarshal(raw, &b) != nil { + return nil + } + return b.Packages +} + // decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one // entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is // refused (A-99). @@ -155,6 +166,9 @@ type Hearer struct { // Release places the movers of a pipeline whose awaited publishes are heard now, after a build step passed (a kind-3 // edge, §746; Starter.Release); nil places none. Release func(key string) + // AskPinHeld asks a mover's steps that waited for its entry's pins with the pins its build passed with (H-A-207; + // Starter.AskPinHeld); nil leaves them waiting. + AskPinHeld func(key, module string, pins map[string]string) // Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748; // Starter.Turn); nil takes none. Turn func(key string) @@ -197,6 +211,12 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi // of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726, // A-469m). func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) { + return h.OnStepEndedPinned(bs, subject, ev, shared, nil) +} + +// OnStepEndedPinned is OnStepEndedShared of a build's end naming the pins it was built with (H-A-207): a passed build's +// mover's steps that waited for its entry's pins are asked with them. +func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared, pins map[string]string) (seat.Pipeline, error) { node := strings.TrimPrefix(subject, bs.Prefix) if node == subject || node == "" || strings.Contains(node, ".") { return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat) @@ -272,8 +292,10 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh // Every mover's validate reads the one run (§745, H-A-51); its end asks each mover's composition, or ends it not // asked when validate did not pass. var due []Asked + askHeld := false if s.Step == seat.StepResultStepBuild { - readers = append(readers, endPinHeld(&p, *s, now)...) + readers = append(readers, endPinHeld(&p, *s, pins, now)...) + askHeld = stepPassed(*s) && heldOn(p, s.Module) } if s.Step == seat.StepResultStepValidate { readers = append(readers, sameValidate(&p, i, now)...) @@ -327,6 +349,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh if release && h.Release != nil { h.Release(ev.Pipeline) } + if askHeld && h.AskPinHeld != nil { + h.AskPinHeld(ev.Pipeline, s.Module, pins) + } if h.AskSteps != nil && len(due) > 0 { if err := h.AskSteps(ev.Pipeline, due); err != nil { h.logf("the compositions of %s are written running and not asked, asked again at the next start: %v", ev.Pipeline, err) From e29aa8aca9139128bffacf6d6305e397d00390f3 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:12:29 +0100 Subject: [PATCH 05/11] Design 47 H1: a done build key keeps the pins it was built with, and its step's end reports those (H-A-207) A build step whose key was done before reported the pins it resolved at that moment, so a publish since would have the runner ask the waiting test steps against a version the build was never built with. The done key's record keeps the versions its Go ask was built with, beside its digests, and the step's end reports the record's; a done key keeping none reports none, so the runner ends those steps could not run, loud. Builds: design 47 H H-A-207, H-A-203 --- .../mesh-delivery-builder/builds/builds.go | 5 +++++ .../internal/build/run.go | 18 ++++++++++++++- .../internal/build/step.go | 22 ++++++++++++++++--- .../internal/build/step_test.go | 22 +++++++++++++++++++ 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-builder/builds/builds.go b/modules/mesh-delivery-builder/builds/builds.go index 8981c7dc..04c3c6cd 100644 --- a/modules/mesh-delivery-builder/builds/builds.go +++ b/modules/mesh-delivery-builder/builds/builds.go @@ -242,6 +242,11 @@ type BuilderBuild struct { // Imports are, on a done key, the mesh packages the built programs import, as the build module answered them // (BuildAnswer.Imports; novox/hq ADR 0593 §2): the builder carries them onto the build's end (BuildLogEntry.Imports). Imports []string `json:"imports,omitempty"` + // Packages are, on a done key only, the mesh packages' versions its Go build was built with, by module path, the ones + // the registry gave a package the entry named with none included (H-A-207): the build step's end reports them, so the + // runner asks the steps that waited for the pins with what the build was built against. A reader that does not know + // the field reads the build as reporting none. + Packages map[string]string `json:"packages,omitempty"` // Needs are, on a done key only, the reader capabilities (`form:`) the build's reference forms need of a // node-engine, beside the ones its manifest needs (novox/hq ADR 0618 §1). A reader that does not know the field // reads the build as needing none. diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ba6f3e3c..b893e9a1 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -833,6 +833,7 @@ type lane struct { tcVersion string // and the version it ran kept string // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty outOfStep []string // the generated files its build module's compare found out of step, passed or failed + pins map[string]string // the mesh packages' versions its Go ask was built with (H-A-207) err error } @@ -982,7 +983,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module used = nil } done := builds.BuilderBuild{State: builds.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used, - Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts)} // the forms it is written in (ADR 0618 §1) + Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts), Packages: pinsOf(lanes)} // the forms it is written in (ADR 0618 §1) for _, l := range lanes { if l.kept != "" { done.KeptOn = l.node // design 47 G §671: the build node that keeps the build @@ -1144,6 +1145,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes for _, fb := range req.Builds { delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465) } + l.pins = maps.Clone(ask.Packages) // kept with the done key, so its step's end reports them (H-A-207) } if id := asked.resume(language); id != "" { ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it @@ -1326,6 +1328,20 @@ func (b *Builder) takeOutOfStep(key string) []string { // importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593 // §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that // kept one reports only what it built. +// pinsOf is the mesh packages' versions the lanes were built with (H-A-207), nil when none was pinned. +func pinsOf(lanes []*lane) map[string]string { + var out map[string]string + for _, l := range lanes { + for p, v := range l.pins { + if out == nil { + out = map[string]string{} + } + out[p] = v + } + } + return out +} + func importsOf(lanes []*lane) []string { seen := map[string]bool{} for _, l := range lanes { diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index cfff2cac..bd24c5a3 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -66,8 +66,8 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui return finish(meshbuilds.StepEndedOutcomePassed, "") } -// buildEntry builds one entry at its fingerprint and waits for its end; pins is set to the pins it is built with, the -// entry's own and those the registry gave a package the entry named with none (H-A-207). +// buildEntry builds one entry at its fingerprint and waits for its end; pins is set, when it is done, to the pins its +// done key keeps it was built with, the registry's included (H-A-207). func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) { entry, err := b.StepEntry(ctx, e.Fingerprint) if err != nil { @@ -82,7 +82,6 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins return outcome, said } entry.Packages = pinned - *pins = pinned key, err := builds.BuilderBuildKey(entry.Module, entry.Fp) if err != nil { return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error() @@ -91,6 +90,8 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins defer b.dropEnd(key, ended) if done, failedWords, found := b.keyEnd(ctx, key); found { if done { + // A key done before reports the pins it was built with, kept with it, never the ones a publish gives now. + *pins = b.keptPins(ctx, key) return b.passedWithItsVersion(ctx, entry) } return meshbuilds.StepEndedOutcomeFailed, failedWords @@ -101,6 +102,7 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins select { case log := <-ended: if log.Outcome == builds.BuildDone { + *pins = b.keptPins(ctx, key) return b.passedWithItsVersion(ctx, entry) } return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error) @@ -149,6 +151,20 @@ func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.Ca return meshbuilds.StepEndedOutcomePassed, "" } +// keptPins are the pins a done key keeps it was built with (H-A-207); none when it keeps none or cannot be read, so the +// runner ends the steps that waited for them could not run, loud. +func (b *Builder) keptPins(ctx context.Context, key string) map[string]string { + raw, _, ok, err := b.Claimer.Store.Get(ctx, key) + if err != nil || !ok { + return nil + } + v, err := builds.DecodeBuilderBuild(raw) + if err != nil || v.State != builds.BuildDone { + return nil + } + return v.Packages +} + // keyEnd reads the build key: whether it is done, and its error when it failed for good; found only for a key that has ended. func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed string, found bool) { raw, _, ok, err := b.Claimer.Store.Get(ctx, key) diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 08bbbedd..c70f1ae4 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -193,6 +193,28 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) { if !reflect.DeepEqual(end.Packages, map[string]string{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) { t.Errorf("the end reports the pins %v; want the ones it was built with", end.Packages) } + // A done key reports the pins it keeps it was built with, never the newer one a publish gives now (H-A-207). + r.b.PackagePublished(unpinnedX, "v0.1.5") + again := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}) + if again.Outcome != meshbuilds.StepEndedOutcomePassed || again.Packages[unpinnedX] != "v0.1.3" { + t.Errorf("the done key's end = %+v; want passed reporting x at v0.1.3, as built", again) + } + // A done key that keeps no pins reports none, so the runner ends the waiting steps loud. + v, err := builds.DecodeBuilderBuild(r.st.vals[r.key]) + if err != nil { + t.Fatal(err) + } + v.Packages = nil + raw, err := builds.EncodeBuilderBuild(v) + if err != nil { + t.Fatal(err) + } + r.st.vals[r.key] = raw + if bare := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); len(bare.Packages) != 0 { + t.Errorf("a done key keeping no pins reported %v; want none", bare.Packages) + } } func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) { From c418f4903846d723fc5130c420439ea9c5f97506 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:13:47 +0100 Subject: [PATCH 06/11] Design 47 H1: held test steps are asked from the pipeline's own catalog revision (H-A-205, H-A-207) The runner built the asks of the steps that waited for their pins from the branch's newest catalog half, so a newer revision that moved the module changed what a passed build's tests fetched. The entry is now read at the pipeline's own revision through the runner's revision reader, which keeps what it read; a revision that cannot be read, or whose entry is of another fingerprint than the mover's, ends the held steps could not run, naming why. Builds: design 47 H H-A-205, H-A-207, H-A-203 --- .../cmd/mesh-delivery-pipeline/main.go | 5 +-- .../internal/pipeline/place.go | 34 ++++++++++++++----- .../internal/pipeline/place_test.go | 33 ++++++++++++++++-- 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 8d0fdd22..80a28898 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -142,7 +142,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } logger.Printf("widens: each read bound %s, %d catalog revisions kept", readBound, revisionsKept) - hearer.Widens = widenerOf(calls.Call, readBound, revisionsKept, logger.Printf).Widens + widener := widenerOf(calls.Call, readBound, revisionsKept, logger.Printf) + hearer.Widens = widener.Widens hearer.LogLast = stepLog{ctx: ctx, js: js, module: cred.Module}.Last hearer.Where = routerRecordOf(cred.Module, calls.Call, calls.Ask) hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf} @@ -155,7 +156,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro hearer.AskSteps = starter.AskSteps hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } hearer.AskPinHeld = func(key, module string, pins map[string]string) { - if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, rv.Catalog.View); err != nil { + if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil { logger.Printf("the steps of %s's %s that waited for its pins are not asked: %v", key, module, err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 5c096fe9..4fcc2aee 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -280,6 +280,14 @@ func endPinHeld(p *seat.Pipeline, build seat.StepResult, pins map[string]string, return ended } +// readRevision is revision's read, an error when none is given. +func readRevision(revision func(branch string, revision int64) (Half, error), branch string, n int64) (Half, error) { + if revision == nil { + return Half{}, errors.New("the runner reads no catalog revision") + } + return revision(branch, n) +} + // heldOn says whether module has a step of p waiting for its entry's pins. func heldOn(p seat.Pipeline, module string) bool { return slices.ContainsFunc(p.Steps, func(st seat.StepResult) bool { return st.Module == module && pinHeld(st) }) @@ -287,8 +295,10 @@ func heldOn(p seat.Pipeline, module string) bool { // AskPinHeld asks the steps of pipeline key's mover module that waited for its entry's pins (H-A-206) with the pins its // build passed with (H-A-207): each running under its own ask, its fetch the entry's as Place builds it with those pins. -// A step whose artifact the branch's half no longer names ends could not run, saying so. It answers how many it asked. -func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]string, view func(branch string) (Half, bool)) (int, error) { +// The entry is read at the pipeline's own catalog revision through revision, which keeps what it read (H-A-205), never +// the branch's newest: a revision that cannot be read, or whose entry is of another fingerprint than the mover's, or that +// names no artifact of the step, ends the step could not run, saying so. It answers how many it asked. +func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]string, revision func(branch string, revision int64) (Half, error)) (int, error) { for { p, rev, err := store.Get(key) if err != nil { @@ -299,10 +309,14 @@ func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string] return 0, nil } var entry catalog.CatalogEntry - if view != nil { - if half, ok := view(p.Branch); ok { - entry = half.Entries[module] - } + unread := "" + switch half, err := readRevision(revision, p.Branch, p.Revision); { + case err != nil: + unread = fmt.Sprintf("revision %d of %s, the pipeline's, cannot be read for its entry: %v", p.Revision, p.Branch, err) + case half.Entries[module].Fp != m.Fingerprint: + unread = fmt.Sprintf("revision %d of %s holds %s at %q, not the mover's %s", p.Revision, p.Branch, module, half.Entries[module].Fp, m.Fingerprint) + default: + entry = half.Entries[module] } entry.Packages = pins now := s.now() @@ -315,9 +329,13 @@ func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string] } j := slices.IndexFunc(entry.Artifacts, func(a catalog.Artifact) bool { return a.Ref == st.Artifact }) language, err := sdkstep.AskLanguageOfSeat(st.Seat) - if j < 0 || err != nil { + if unread != "" || j < 0 || err != nil { + why := unread + if why == "" { + why = fmt.Sprintf("revision %d of %s names no artifact %s of %s to ask it of", p.Revision, p.Branch, st.Artifact, module) + } st.Outcome, st.Ended = seat.StepResultOutcomeCouldNotRun, &now - st.Why = fmt.Sprintf("its build passed with its pins, but the catalog half of %s names no artifact %s of %s to ask it of (H-A-207)", p.Branch, st.Artifact, module) + st.Why = "its build passed with its pins, but " + why + " (H-A-207, H-A-205)" p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) ended = append(ended, i) continue diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index f152f0a0..dd10438a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -120,13 +120,14 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { } held := waiting["app"] held.Packages = map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": ""} + held.Fp = "src2:ab" waiting["app"] = held var journal []string heldAsks := &stepAsks{} hs := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: heldAsks, Name: "the-runner", Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }, Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }} - if ok, err := hs.Start(Begin{Branch: "main", Source: 5, Entries: waiting, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}}); !ok || err != nil { + if ok, err := hs.Start(Begin{Branch: "main", Source: 5, Revision: 3, Entries: waiting, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}}); !ok || err != nil { t.Fatalf("start = %v, %v", ok, err) } for _, x := range heldAsks.asks { @@ -166,11 +167,23 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { p seat.Pipeline rev uint64 }{hp, 1} + // The entry is read at the pipeline's own revision (3), never the newer revision 4 that moved the module (H-A-205). + moved := held + moved.Fp, moved.Path = "src2:ac", "modules/moved" + revisions := func(branch string, n int64) (Half, error) { + switch n { + case 3: + return Half{Revision: 3, Entries: waiting}, nil + case 0, 4: + return Half{Revision: 4, Entries: map[string]catalog.CatalogEntry{"app": moved}}, nil + } + return Half{}, fmt.Errorf("no revision %d", n) + } var askedWith map[string]string h := &Hearer{Store: ms, Say: func(string, any) error { return nil }, Name: "the-runner", AskPinHeld: func(key, module string, pins map[string]string) { askedWith = pins - if _, err := hs.AskPinHeld(ms, key, module, pins, func(string) (Half, bool) { return Half{Entries: waiting}, true }); err != nil { + if _, err := hs.AskPinHeld(ms, key, module, pins, revisions); err != nil { t.Errorf("ask held: %v", err) } }} @@ -189,12 +202,28 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { if !reflect.DeepEqual(b.Step.Fetch.Packages, pinsBuilt) { t.Errorf("the %s of %s fetches %v; want the build's pins", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages) } + if b.Step.Fetch.Dir != "modules/app" { + t.Errorf("the %s of %s fetches %s; want the pipeline's revision's directory", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Dir) + } } } after, _, _ := ms.Get("main.5") if askedHeld == 0 || after.Steps[1].Outcome == seat.StepResultOutcomeWaiting || after.Steps[1].Ask == "" { t.Errorf("after the build passed %d held steps were asked, lint = %+v; want each asked", askedHeld, after.Steps[1]) } + // A mover whose fingerprint the pipeline's revision does not hold ends its held steps could not run, naming it. + other := hp + other.Pipeline, other.Steps, other.Movers = "main.6", append([]seat.StepResult(nil), hp.Steps...), []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:zz", Repository: "o/app"}} + ms["main.6"] = struct { + p seat.Pipeline + rev uint64 + }{other, 1} + if n, err := hs.AskPinHeld(ms, "main.6", "app", pinsBuilt, revisions); n != 0 || err != nil { + t.Errorf("asked %d held steps of a mover its revision does not hold (%v); want none", n, err) + } + if o, _, _ := ms.Get("main.6"); o.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(o.Steps[1].Why, "not the mover's src2:zz") { + t.Errorf("the held lint step of a moved fingerprint = %+v; want could not run naming it", o.Steps[1]) + } if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") { t.Errorf("the journal says %q; want the held step named", journal) } From 189d75ad40cc594043f6aa1ed1986e2a3174ade8 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:02:48 +0100 Subject: [PATCH 07/11] Design 47 H1 (5a): a pipeline is keyed by its catalog revision, and every step ask names revision and source One source now has several catalog revisions, so a pipeline keyed by its source could not tell them apart. A pipeline is keyed .; one of a source (a publish's, a version-released's) is found by the source it names, never by its key, so pipelines kept under their source are read as they are. The Go, Node and image step asks name the catalog revision in revision and the gate's source in a new optional source (the build-step part of the seats node-build-go, node-build-node and node-build-image, seatgen regenerated). Builds: design 47 H H-A-211, H-A-213 (4), H-A-219 --- .../go/seats/nodebuildgo/contract.go | 4 +- .../go/seats/nodebuildimage/contract.go | 4 +- .../go/seats/nodebuildnode/contract.go | 4 +- .../parts/build-step.json | 6 ++- .../seats/nodebuildgo/parts/build-step.d.ts | 4 +- .../nodebuildimage/parts/build-step.d.ts | 4 +- .../seats/nodebuildnode/parts/build-step.d.ts | 4 +- .../internal/pipeline/ended_test.go | 14 +++++- .../internal/pipeline/order_test.go | 14 +++--- .../internal/pipeline/pins_test.go | 6 +-- .../internal/pipeline/place.go | 2 +- .../internal/pipeline/place_test.go | 27 ++++++---- .../internal/pipeline/queue_test.go | 2 +- .../internal/pipeline/released.go | 11 ++-- .../internal/pipeline/start.go | 50 ++++++++++++++++--- .../internal/pipeline/start_test.go | 8 +-- .../internal/pipeline/testruns_test.go | 2 +- .../internal/pipeline/verbs.go | 4 +- .../internal/pipeline/verbs_test.go | 2 +- 19 files changed, 121 insertions(+), 51 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go index 7ed4f8fe..213116c3 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go @@ -905,8 +905,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go index d2136e78..391f8658 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go @@ -903,8 +903,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go index 47ca8f1d..bd89858a 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go @@ -903,8 +903,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/parts/build-step.json b/modules/mesh-delivery-contracts/parts/build-step.json index fcaebd62..0390aecb 100644 --- a/modules/mesh-delivery-contracts/parts/build-step.json +++ b/modules/mesh-delivery-contracts/parts/build-step.json @@ -1521,7 +1521,11 @@ }, "revision": { "type": "integer", - "description": "the source revision of the watched branch the pipeline runs for; absent for a run on request" + "description": "the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request" + }, + "source": { + "type": "integer", + "description": "the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request" }, "fingerprint": { "type": "string", diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go index abf01a7c..ee636c4a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go @@ -113,9 +113,19 @@ func TestAReleasedVersionIsTheMoversVersion(t *testing.T) { other := store[pkey] other.p.Pipeline, other.p.Branch, other.p.Movers = "beta.5", "beta", append([]seat.Mover(nil), other.p.Movers...) store["beta.5"] = other + // A pipeline is keyed by its revision (H-A-213 (4)): main.8, a later revision of source 5, is written too; main.6, of + // source 6, is not. + for key, source := range map[string]int64{"main.8": 5, "main.6": 6} { + again := store[pkey] + again.p.Pipeline, again.p.Source, again.p.Movers = key, source, append([]seat.Mover(nil), again.p.Movers...) + store[key] = again + } ev := catalog.VersionReleasedEvent{Module: "app", Version: "0.4.1", Source: 5} - if n, err := h.Released(ev, "main"); n != 1 || err != nil { - t.Fatalf("released wrote %d (%v), want app's mover of main.5", n, err) + if n, err := h.Released(ev, "main"); n != 2 || err != nil { + t.Fatalf("released wrote %d (%v), want app's mover of main.5 and main.8", n, err) + } + if m, _ := MoverOf(store["main.6"].p, "app"); m.Version != "" { + t.Errorf("main.6's mover = %+v, want no version: another source", m) } if m, _ := MoverOf(store[pkey].p, "app"); m.Version != "0.4.1" { t.Errorf("app's mover = %+v, want version 0.4.1", m) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go index 0065aabd..73d1e020 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go @@ -70,14 +70,14 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if n, err := s.TakenIn(ev, view); n != 1 || err != nil { t.Fatalf("the take-in started %d (%v)", n, err) } - p, _, _ := mem.Get("main.4") + p, _, _ := mem.Get("main.7") if !placed(p, "mesh-sdk") || !placed(p, "builder") || placed(p, "app") || placed(p, "tool") { t.Fatalf("placed at the start: %+v, want mesh-sdk and builder only", p.Steps) } if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); n != 0 || err != nil { t.Fatalf("the publish started %d (%v), want none", n, err) } - p, _, _ = mem.Get("main.4") + p, _, _ = mem.Get("main.7") if sdk, _ := MoverOf(p, "mesh-sdk"); !placed(p, "app") || placed(p, "tool") || len(mem) != 1 || !slices.Contains(sdk.Published, "mesh-sdk:go") { t.Fatalf("after mesh-sdk's publish: %d pipelines, app placed %v, tool placed %v, mesh-sdk %+v", len(mem), placed(p, "app"), placed(p, "tool"), sdk) } @@ -87,22 +87,22 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { b := stepsOf(p, "builder")[0] bs := stepSeatOf(t, builds.Seat) end := func(outcome string) StepEnded { - e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.4","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`)) + e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.7","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`)) if err != nil { t.Fatal(err) } return e } - before := mem["main.4"] + before := mem["main.7"] if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("passed")); err != nil { t.Fatal(err) } - if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning { + if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning { t.Errorf("after builder's build passed, tool's steps = %+v, want placed and asked", stepsOf(p, "tool")) } // The same pipeline, builder's build failed: tool is placed with every step not asked, each saying its own step-ended // naming the dependency (H-A-101 (2)). - mem["main.4"] = before + mem["main.7"] = before var notAsked []seat.StepEvent s.Say = func(subject string, body any) error { if ev, ok := body.(seat.StepEvent); ok && strings.Contains(subject, ".step-ended.tool.") { @@ -113,7 +113,7 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("failed")); err != nil { t.Fatal(err) } - if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked { + if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked { t.Errorf("after builder's build failed, tool's steps = %+v, want placed not asked", stepsOf(p, "tool")) } if len(notAsked) == 0 || len(notAsked) != len(slices.DeleteFunc(stepsOf(p, "tool"), func(st seat.StepResult) bool { return st.Outcome != seat.StepResultOutcomeNotAsked })) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go index 455f8fa5..bd99d2c8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go @@ -51,7 +51,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing. if n, err := s.TakenIn(taken, view); n != 1 || err != nil { t.Fatalf("the take-in started %d (%v)", n, err) } - p, _, _ := store.Get("main.947") + p, _, _ := store.Get("main.357") if p.Revision != 357 || len(p.Steps) != 0 || len(asks.asks) != 0 { t.Fatalf("a take-in of revision 357 with the view at 356 started %+v with %d asks, want revision 357 and no step", p, len(asks.asks)) } @@ -60,7 +60,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing. } viewRevision = 357 n, err := s.Waiting(store.memStore, "main", view) - p, _, _ = store.Get("main.947") + p, _, _ = store.Get("main.357") if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Revision != 357 { t.Errorf("the view reaching 357 placed %d (%v): %d steps, %d asks at revision %d", n, err, len(p.Steps), len(asks.asks), p.Revision) } @@ -87,7 +87,7 @@ func TestAViewThatReachesTheRevisionDuringTheStartsPlacesTheHeldBackPipelines(t if n, err := s.TakenIn(taken, view); n != 1 || err != nil { t.Fatalf("started %d (%v)", n, err) } - if p, _, _ := store.Get("main.947"); len(p.Steps) == 0 || len(asks.asks) == 0 { + if p, _, _ := store.Get("main.357"); len(p.Steps) == 0 || len(asks.asks) == 0 { t.Errorf("the pipeline has %d steps and %d asks after the view reached 357 during the start, want them placed", len(p.Steps), len(asks.asks)) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 4fcc2aee..a7dbf327 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -384,7 +384,7 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar } return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now, Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: m.Commit, Fetch: fetch, Fingerprint: m.Fingerprint, - Module: m.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}} + Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}} } // askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index dd10438a..7f46c658 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -102,7 +102,7 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { a := asks.asks[1].(nodebuildgo.StepAsk) if asks.at[1] != "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner" || a.Kind != "step" || a.Asker != "the-runner" || a.ID != p.Steps[1].Ask || asks.ids[1] != a.ID || a.Step.Commit != "c1" || a.Step.Fingerprint != "src2:aa" || a.Step.Pipeline != "main.4" || a.Step.Artifact != "app:program" || - a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Revision != 4 || + a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Source != 4 || len(a.Step.Fetch.Paths) != 1 || a.Step.Fetch.Paths[0] != "cmd/app" { t.Errorf("the ask = %+v at %s", a, asks.at[1]) } @@ -136,7 +136,7 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { } } var hp seat.Pipeline - if err := json.Unmarshal(store["main.5"], &hp); err != nil { + if err := json.Unmarshal(store["main.3"], &hp); err != nil { t.Fatal(err) } if st := hp.Steps[1]; st.Step != "lint" || st.Outcome != seat.StepResultOutcomeWaiting || st.Ask != "" || st.Why != "waits on the publish of git.novox.be/novox/x/go" { @@ -213,15 +213,15 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { } // A mover whose fingerprint the pipeline's revision does not hold ends its held steps could not run, naming it. other := hp - other.Pipeline, other.Steps, other.Movers = "main.6", append([]seat.StepResult(nil), hp.Steps...), []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:zz", Repository: "o/app"}} - ms["main.6"] = struct { + other.Pipeline, other.Steps, other.Movers = "main.4", append([]seat.StepResult(nil), hp.Steps...), []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:zz", Repository: "o/app"}} + ms["main.4"] = struct { p seat.Pipeline rev uint64 }{other, 1} - if n, err := hs.AskPinHeld(ms, "main.6", "app", pinsBuilt, revisions); n != 0 || err != nil { + if n, err := hs.AskPinHeld(ms, "main.4", "app", pinsBuilt, revisions); n != 0 || err != nil { t.Errorf("asked %d held steps of a mover its revision does not hold (%v); want none", n, err) } - if o, _, _ := ms.Get("main.6"); o.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(o.Steps[1].Why, "not the mover's src2:zz") { + if o, _, _ := ms.Get("main.4"); o.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(o.Steps[1].Why, "not the mover's src2:zz") { t.Errorf("the held lint step of a moved fingerprint = %+v; want could not run naming it", o.Steps[1]) } if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") { @@ -532,7 +532,8 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }} start := func(source, revision int64) seat.Pipeline { t.Helper() - if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { + // Keyed by its source, so pipelines of one revision are several (one kept under its source before H-A-213 (4)). + if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { t.Fatalf("start %d = %v, %v", source, ok, err) } p, _, _ := store.Get(Key("main", source)) @@ -551,6 +552,12 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" { t.Fatalf("the test steps asked = %q, want the program's three alone", got) } + // Every step ask names the catalog revision in revision and the gate's source in source (H-A-219). + for _, a := range asks.asks { + if b, ok := a.(nodebuildgo.StepAsk); ok && (b.Step.Revision != 9 || b.Step.Source != 4) { + t.Errorf("the %s ask of %s names revision %d and source %d, want 9 and 4", b.Step.Step, b.Step.Artifact, b.Step.Revision, b.Step.Source) + } + } for _, level := range buildSteps { asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level) if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil { @@ -751,13 +758,13 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) { } } } - p, _, _ := mem.Get("main.4") + p, _, _ := mem.Get("main.7") if validates != 1 || compositions != 0 || stepNamed(p, "", seat.StepResultStepComposition).Outcome != seat.StepResultOutcomeWaiting { t.Fatalf("placed: %d validate and %d composition asks, steps %+v; want one validate, no composition yet", validates, compositions, p.Steps) } h := &Hearer{Store: mem, Name: "the-runner", Say: func(string, any) error { return nil }, AskSteps: s.AskSteps} bs := stepSeatOf(t, wants.Seat) - raw := `{"id":"` + validateAsk + `","pipeline":"main.4","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}` + raw := `{"id":"` + validateAsk + `","pipeline":"main.7","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}` ev, err := bs.Decode([]byte(raw)) if err != nil { t.Fatal(err) @@ -766,7 +773,7 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) { if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil { t.Fatal(err) } - p, _, _ = mem.Get("main.4") + p, _, _ = mem.Get("main.7") for _, st := range p.Steps { if st.Step == seat.StepResultStepValidate && st.Outcome != outcome { t.Errorf("%s: %s's validate is %s, want every mover's read from the one run", outcome, st.Module, st.Outcome) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go index 586b656e..54249bca 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go @@ -96,7 +96,7 @@ func queueFixture() (*Starter, memStore, memQueue, *[]string, *[]seat.Wait, *ste func takeIn(t *testing.T, s *Starter, view func(string) (Half, bool)) func(source int64, modules ...string) { return func(source int64, modules ...string) { t.Helper() - ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7} + ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: source} for _, m := range modules { ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/released.go b/modules/mesh-delivery-pipeline/internal/pipeline/released.go index 76028fce..840ca598 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/released.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/released.go @@ -3,7 +3,6 @@ package pipeline import ( "errors" "fmt" - "strconv" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/keys" @@ -21,12 +20,13 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int, if branch != "" { b = keys.EscapeName(branch) } - source := strconv.FormatInt(ev.Source, 10) + // A pipeline is keyed by its revision (H-A-213 (4)), so the pipelines of a source are those naming it, of either key + // form a source's pipelines were kept under. var pkeys []string - for _, f := range []string{b + "." + source, b + "." + source + ".published.*.*"} { + for _, f := range []string{b + ".*", b + ".*.published.*.*"} { k, err := h.Store.Keys(f) if err != nil { - return 0, fmt.Errorf("the pipelines of source %s are not listed: %w", source, err) + return 0, fmt.Errorf("the pipelines of source %d are not listed: %w", ev.Source, err) } pkeys = append(pkeys, k...) } @@ -37,6 +37,9 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int, if err != nil { return wrote, fmt.Errorf("the pipeline %s is not read: %w", key, err) } + if p.Source != ev.Source || p.Run != "" { + break + } i := -1 for j, m := range p.Movers { if m.Module == ev.Module { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index cb039452..7f99aa83 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -28,7 +28,7 @@ type Created interface { Create(key string, value []byte) error } -// Begin is what a start names: the pipeline's key (watched branch and source revision, design 47 H §746), the take-in's +// Begin is what a start names: the pipeline's key (watched branch and catalog revision, design 47 H §746, H-A-213), the take-in's // call and catalog graph revision, its movers, and the catalog half of that branch. type Begin struct { Branch string @@ -41,16 +41,26 @@ type Begin struct { Entries map[string]catalog.CatalogEntry // Published is the package's publish that started the pipeline, nil for any other start (§747). Published *seat.Published - // Key is the pipeline's key when it is not . (a publish's, §747); empty for a take-in's. + // Key is the pipeline's key when it is not . (a publish's, §747); empty for a take-in's. Key string // unheld says the branch's catalog half is not held yet: the pipeline starts with no steps, placed once it is (A-336). unheld bool } -// Key is a take-in's pipeline key in the state pipelines, `.`, the branch escaped as a key's name is -// (design 47 H §746). -func Key(branch string, source int64) string { - return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10)) +// Key is a pipeline's key in the state pipelines, `.`, the catalog revision's number the indexer counts, +// the branch escaped as a key's name is (design 47 H §746, H-A-213 (4)); a take-in's number is unchanged where it equalled +// its source. A pipeline kept under its source before (`.`) has the same form and is read as it is. +func Key(branch string, revision int64) string { + return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10)) +} + +// keyRevision is the number b's pipeline is keyed by (H-A-213 (4)): its revision, or its source where the start names no +// revision (a take-in.taken of an indexer whose revision numbers were its sources, H-A-211). +func keyRevision(b Begin) int64 { + if b.Revision > 0 { + return b.Revision + } + return b.Source } // PublishedKey is the key of a pipeline a package's publish starts, `..published..`, @@ -495,7 +505,7 @@ func (s *Starter) Start(b Begin) (bool, error) { now := s.now() key := b.Key if key == "" { - key = Key(b.Branch, b.Source) + key = Key(b.Branch, keyRevision(b)) } if err := s.Fills.refuseStart(key); err != nil { return false, err @@ -899,7 +909,10 @@ func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch st if s.Store == nil { return seat.Pipeline{}, false, nil } - key := Key(ev.Branch, ev.Source) + key, ok := s.sourcePipeline(ev.Branch, ev.Source) + if !ok { + return seat.Pipeline{}, false, nil + } for { p, rev, err := s.Store.Get(key) if err != nil { @@ -923,6 +936,27 @@ func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch st } } +// sourcePipeline is the key of branch's pipeline of the lowest-numbered revision of source, the take-in's (H-A-213 (4), +// H-A-218): a pipeline is keyed by its revision, so one of a source is found by the source it names, never by its key. +func (s *Starter) sourcePipeline(branch string, source int64) (string, bool) { + pkeys, err := s.Store.Keys(keys.EscapeName(branch) + ".*") + if err != nil { + s.logf("the pipelines of %s are not listed, so source %d's is not found: %v", branch, source, err) + return "", false + } + found, low := "", int64(0) + for _, k := range pkeys { + p, _, err := s.Store.Get(k) + if err != nil || p.Branch != branch || p.Source != source || p.Run != "" { + continue + } + if r := keyRevision(Begin{Revision: p.Revision, Source: p.Source}); found == "" || r < low { + found, low = k, r + } + } + return found, found != "" +} + // publishedKind is the edge a dependent builds on a published package by: B published before A builds, a package A // imports (§53). const publishedKind = 1 diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 019127c7..9418cfca 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -153,8 +153,8 @@ func TestATakeInIsOnePipeline(t *testing.T) { t.Fatalf("written %d pipelines and said %d starts, want one each", len(store), len(said)) } var p seat.Pipeline - if err := json.Unmarshal(store["main.7"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 { - t.Fatalf("the pipeline main.7 = %+v (%v)", p, err) + if err := json.Unmarshal(store["main.12"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 { + t.Fatalf("the pipeline main.12 = %+v (%v)", p, err) } if m, _ := MoverOf(p, "mesh-sdk"); m.Commit != "c1" || m.Fingerprint != "src2:aa" || m.Repository != "o/sdk" || m.Tier != 1 { t.Errorf("the mover mesh-sdk = %+v", m) @@ -318,8 +318,8 @@ func TestTakenInStartsOnlyItsEntries(t *testing.T) { if err != nil || n != 1 { t.Fatalf("the take-in started %d (%v), want its one entry", n, err) } - if p, _, err := mem.Get("main.9"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") { - t.Errorf("main.9 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err) + if p, _, err := mem.Get("main.12"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") { + t.Errorf("main.12 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index 780009af..ba0712e4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -113,7 +113,7 @@ func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline { "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, } - if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { + if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { t.Fatalf("start = %v, %v", ok, err) } p, _, _ := f.store.Get(Key("main", source)) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 5264f76e..d86cec3b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -582,11 +582,11 @@ func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) { if !ok || half.Source <= 0 { return seat.StartAnswer{}, fmt.Errorf("the catalog half of %s holds no entry of %s at a source revision: nothing was started", in.Branch, in.Module) } - key := Key(in.Branch, half.Source) + key := Key(in.Branch, keyRevision(Begin{Revision: half.Revision, Source: half.Source})) if err := v.record(seat.VerbStart, "started the pipeline "+key+" by hand", in.Why); err != nil { return seat.StartAnswer{}, err } - // A hand start keys the newest revision's . with the module as its one mover (H-A-52). + // A hand start keys the newest revision's . with the module as its one mover (H-A-52, H-A-213 (4)). started, err := v.Starter.Start(Begin{Branch: in.Branch, Source: half.Source, Revision: half.Revision, Entries: half.Entries, Movers: []seat.Mover{{Module: in.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository, Tier: 1}}}) if err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go index d3a273a0..a2c8adae 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go @@ -287,7 +287,7 @@ func TestStartAndAskAgain(t *testing.T) { _, entries := placeFixture() v.View = func(string) (Half, bool) { return Half{Source: 7, Revision: 3, Entries: entries}, true } a, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "no pipeline for the head"}) - if err != nil || a.Pipeline != "main.7" || made["main.7"] == nil { + if err != nil || a.Pipeline != "main.3" || made["main.3"] == nil { t.Fatalf("start = %+v, %v; written %v", a, err, len(made)) } if _, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "again"}); err == nil { From 91b5e8d0866ad6d54618969bbcd66ef530dcc631 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:07:55 +0100 Subject: [PATCH 08/11] Design 47 H1 (5b): a release, take-in again or recompute revision starts one pipeline, read by its number A re-pin on a module's release, a take-in again and a recompute that moves a fingerprint each keep a catalog revision of their own. The runner hears a trunk catalog-revision of those kinds, reads that revision by its number through the catalog seat's revision verb, and starts one pipeline keyed . with the revision's movers. A package-published no longer starts per-artifact pipelines: it only places the movers waiting for it. A branch's newest pipeline for a mover is the one of the highest revision number. Builds: design 47 H H-A-212, H-A-213 (2) and (4), H-A-217 --- .../mesh-delivery-pipeline/candidate_test.go | 25 +++++ .../cmd/mesh-delivery-pipeline/main.go | 10 ++ .../cmd/mesh-delivery-pipeline/start.go | 37 +++++-- .../cmd/mesh-delivery-pipeline/views.go | 7 +- .../cmd/mesh-delivery-pipeline/widens.go | 3 +- .../internal/pipeline/aged.go | 4 +- .../internal/pipeline/aged_test.go | 20 ++++ .../internal/pipeline/order_test.go | 4 +- .../internal/pipeline/registrations.go | 4 + .../internal/pipeline/start.go | 102 +++++++++--------- .../internal/pipeline/start_test.go | 102 +++++++++--------- 11 files changed, 205 insertions(+), 113 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go index 26a92e9c..f0d87ef5 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go @@ -71,3 +71,28 @@ func TestPlaceLostRunsReadsEachCandidateNewest(t *testing.T) { t.Errorf("asked %+v, want the candidate's group alone", asked[1]) } } + +// H-A-213 (2), (4): a catalog-revision heard is handed whole to the start of a revision's pipeline; a start that fails is +// an error, so the bus delivers the event again, and a body that is no event is ended. +func TestARevisionHeardIsHandedToItsStart(t *testing.T) { + var got []catalog.CatalogRevisionEvent + fail := false + hear := revisedHear(func(ev catalog.CatalogRevisionEvent) (int, error) { + got = append(got, ev) + if fail { + return 0, errors.New("not read") + } + return 1, nil + }, func(string, ...any) {}) + data := []byte(`{"v":1,"kind":"trunk","branch":"main","revision":10,"source":41,"call-kind":"release","call":"release of mesh-sdk 0.1.17"}`) + if err := hear(t.Context(), data); err != nil || len(got) != 1 || got[0].Revision != 10 || got[0].CallKind != "release" || got[0].Source != 41 { + t.Fatalf("heard %+v (%v), want the release revision 10", got, err) + } + fail = true + if err := hear(t.Context(), data); err == nil { + t.Error("a start that failed is not an error, so the event is never heard again") + } + if err := hear(t.Context(), []byte("no event")); err != nil { + t.Errorf("a body that is no event = %v, want it ended", err) + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 80a28898..85891217 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -98,6 +98,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro var advancer atomic.Pointer[func(string)] var placer atomic.Pointer[func(catalog.CatalogRevisionEvent) error] var registrar atomic.Pointer[func(catalog.Registration)] + var reviser atomic.Pointer[func(catalog.CatalogRevisionEvent) (int, error)] rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) { if f := resumer.Load(); f != nil { (*f)(branch) @@ -115,6 +116,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if f := registrar.Load(); f != nil { (*f)(reg) } + }, func(ev catalog.CatalogRevisionEvent) (int, error) { + if f := reviser.Load(); f != nil { + return (*f)(ev) + } + return 0, fmt.Errorf("the revision %d of %s is heard before the runner can start its pipeline", ev.Revision, ev.Branch) }, logger.Printf) if err != nil { return err @@ -203,6 +209,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } registrar.Store(®ister) + // A release's, a take-in again's or a recompute's revision starts one pipeline, its revision read by its number + // (H-A-213 (2), (4)). + revise := func(ev catalog.CatalogRevisionEvent) (int, error) { return starter.Revised(ev, widener.Revision) } + reviser.Store(&revise) starter.Store = store // The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in // the state queue, placed at the start from it and on every end. diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go index 5be6db24..1f9365ce 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "github.com/nats-io/nats.go/jetstream" @@ -11,10 +12,10 @@ import ( "mesh-delivery-pipeline/internal/pipeline" ) -// hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one -// pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is -// written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard -// after a stop. +// hearPublished hears each package-published (design 47 H §746, §747): in the running pipeline of the revision that +// published it, the movers waiting for it are placed; it starts no pipeline (a release's revision does, H-A-213 (2)). The +// message is acknowledged once that is written, on a goroutine of its own, so a slow write holds no other event; the +// durable consumer delivers a publish heard after a stop. func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error { return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) { var ev builds.PackagePublished @@ -24,18 +25,38 @@ func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, l return } go func() { - n, err := starter.Published(ev, v.View) - if err != nil { - logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err) + if err := starter.Published(ev, v.View); err != nil { + logf("the publish of %s %s on %s: %v; heard again", ev.Artifact, ev.Version, ev.Branch, err) _ = m.Nak() return } - logf("the publish of %s %s on %s started %d pipelines", ev.Artifact, ev.Version, ev.Branch, n) + logf("the publish of %s %s on %s is heard", ev.Artifact, ev.Version, ev.Branch) _ = m.Ack() }() }) } +// revisedHear is the handler of catalog-revision that starts the pipeline of a revision whose call kind is not a +// take-in's (design 47 H-A-212, H-A-213 (2), H-A-217): every other revision is left to the views that hear it. An error +// has the event heard again; a second start of its key writes nothing. +func revisedHear(revised func(ev catalog.CatalogRevisionEvent) (int, error), logf func(string, ...any)) func(context.Context, []byte) error { + return func(_ context.Context, data []byte) error { + var ev catalog.CatalogRevisionEvent + if err := json.Unmarshal(data, &ev); err != nil { + logf("a catalog-revision is not read: %v", err) + return nil + } + n, err := revised(ev) + if err != nil { + return err + } + if n > 0 { + logf("the revision %d of %s (%s, %s) started %d pipeline", ev.Revision, ev.Branch, ev.CallKind, ev.Call, n) + } + return nil + } +} + // hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged // once it is written, and heard again when a write failed, so the key already written starts nothing twice. func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go index 3a620f52..440e2c3d 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -98,7 +98,7 @@ func (h *holderView) Holders(seatName string) ([]string, bool) { // after, then start hub once. func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State, call meshcatalogclient.Call, onReady, onAdvance func(branch string), candidate func(ctx context.Context, ev catalog.CatalogRevisionEvent) error, - registered func(reg catalog.Registration), logf func(string, ...any)) (*runnerViews, error) { + registered func(reg catalog.Registration), revised func(ev catalog.CatalogRevisionEvent) (int, error), logf func(string, ...any)) (*runnerViews, error) { findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf) if err != nil { return nil, err @@ -153,6 +153,11 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr if err := meshcatalogclient.WireCandidatesProcess(reader, candidate); err != nil { return nil, err } + // A trunk revision of a release, a take-in again or a recompute starts one pipeline of its own, read by its number + // (H-A-212, H-A-213 (2), H-A-217), heard on the same event's one subscription. + if err := reader.AdoptEvent(catalog.Seat, catalog.EventCatalogRevision, revisedHear(revised, logf)); err != nil { + return nil, err + } reader.Start(ctx, js, cred, status) return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings, Holders: &holderView{reader: reader, catalog: cv, logf: logf}}, nil } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go index 211738a7..12fef5af 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go @@ -166,7 +166,8 @@ func catalogRevision(call func(string, any) (json.RawMessage, error), kept *revi if h, ok := kept.get(branch, a.Revision.Revision); ok { return h, nil } - half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{}} + half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{}, + Call: a.Revision.Call, Movers: a.Revision.Movers} in.Revision = a.Revision.Revision // the pages of one revision, though a newer one is said between them } for _, e := range a.Entries { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go index fd32e2f8..60a7d333 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go @@ -231,11 +231,11 @@ func (a *Ager) newest(p seat.Pipeline) (bool, error) { if err != nil { return false, fmt.Errorf("the pipelines of %s are not listed: %w", p.Branch, err) } - // Newest for one of its movers: no later pipeline of the branch moves it. + // Newest for one of its movers: no pipeline of a later revision of the branch moves it (H-A-213 (4)). for _, m := range p.Movers { newer := false for _, k := range pkeys { - if q, _, err := a.Pipelines.Get(k); err == nil && q.Source > p.Source && Moves(q, m.Module) { + if q, _, err := a.Pipelines.Get(k); err == nil && keyRevision(Begin{Revision: q.Revision, Source: q.Source}) > keyRevision(Begin{Revision: p.Revision, Source: p.Source}) && Moves(q, m.Module) { newer = true break } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go index 2270238e..47da9a57 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go @@ -142,3 +142,23 @@ func TestOneStepEndAgesAtMostTwentyEntries(t *testing.T) { t.Fatalf("%d outputs deleted, sequence %d; want 20 and 20", len(f.outputs), f.aged["7d"]) } } + +// H-A-213 (4): a branch's newest pipeline for a mover is the one of the highest revision number, never of the highest source: +// a release revision of the same source is newer than its take-in's. +func TestTheNewestPipelineIsOfTheHighestRevision(t *testing.T) { + f := newAgedFixture() + for key, p := range map[string]seat.Pipeline{ + "main.41": {Pipeline: "main.41", Branch: "main", Source: 41, Revision: 41, Movers: []seat.Mover{{Module: "app"}}}, + "main.42": {Pipeline: "main.42", Branch: "main", Source: 41, Revision: 42, Movers: []seat.Mover{{Module: "app"}}}, + } { + f.pipes.memStore[key] = struct { + p seat.Pipeline + rev uint64 + }{p, 1} + } + for key, want := range map[string]bool{"main.41": false, "main.42": true} { + if got, err := f.ager.newest(f.pipes.memStore[key].p); got != want || err != nil { + t.Errorf("%s newest = %v (%v), want %v", key, got, err, want) + } + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go index 73d1e020..95f5536a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go @@ -74,8 +74,8 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if !placed(p, "mesh-sdk") || !placed(p, "builder") || placed(p, "app") || placed(p, "tool") { t.Fatalf("placed at the start: %+v, want mesh-sdk and builder only", p.Steps) } - if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); n != 0 || err != nil { - t.Fatalf("the publish started %d (%v), want none", n, err) + if err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); err != nil { + t.Fatalf("the publish is not heard: %v", err) } p, _, _ = mem.Get("main.7") if sdk, _ := MoverOf(p, "mesh-sdk"); !placed(p, "app") || placed(p, "tool") || len(mem) != 1 || !slices.Contains(sdk.Published, "mesh-sdk:go") { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go index 751fbf9a..1f0cc57a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go @@ -99,6 +99,10 @@ type Half struct { Revision int64 Source int64 Entries map[string]catalog.CatalogEntry + // Call and Movers are the revision's own, as the revision verb answers them (H-A-213): what made it and its movers; + // a view's half names none. + Call string + Movers []catalog.TakeInEntry } // Views is the runner's views of the catalog half, one per watched branch the registrations name (design 47 G §21, diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 7f99aa83..7d1a50ea 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "maps" "slices" "sort" "strconv" @@ -39,9 +38,7 @@ type Begin struct { Movers []seat.Mover // Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them. Entries map[string]catalog.CatalogEntry - // Published is the package's publish that started the pipeline, nil for any other start (§747). - Published *seat.Published - // Key is the pipeline's key when it is not . (a publish's, §747); empty for a take-in's. + // Key is the pipeline's key when it is not .; empty for every start the runner makes. Key string // unheld says the branch's catalog half is not held yet: the pipeline starts with no steps, placed once it is (A-336). unheld bool @@ -63,14 +60,8 @@ func keyRevision(b Begin) int64 { return b.Source } -// PublishedKey is the key of a pipeline a package's publish starts, `..published..`, -// each part escaped as a key's name is (design 47 H §747). -func PublishedKey(branch string, source int64, pkg, version string) string { - return seat.PipelinesByBranchSourcePublishedPackageVersionKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), - keys.EscapeName(pkg), keys.EscapeName(version)) -} - -// keyForms are the filters of every key form of the state pipelines: a take-in's, a run's and a publish's. +// keyForms are the filters of every key form of the state pipelines: a revision's, a run's, and a publish's kept from before +// a release revision started one pipeline (H-A-213 (2)). var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*"} // AllKeys are every pipeline's key, of each key form (§746, §747; H-A-83). @@ -292,7 +283,7 @@ func (s *Starter) sayStarted(key string, asks []Asked) { func newPipeline(b Begin, key string, now time.Time) seat.Pipeline { movers := append([]seat.Mover(nil), b.Movers...) return seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Call: b.Call, Revision: b.Revision, Movers: movers, - Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}, Published: b.Published} + Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}} } func (s *Starter) create(key string, p seat.Pipeline) error { @@ -957,49 +948,60 @@ func (s *Starter) sourcePipeline(branch string, source int64) (string, bool) { return found, found != "" } -// publishedKind is the edge a dependent builds on a published package by: B published before A builds, a package A -// imports (§53). -const publishedKind = 1 - -// Published hears a package-published (design 47 H §747, H-A-52): in the running pipeline of the source revision that -// published it, a mover that builds on it waits for it there (§746) and nothing starts; every module the catalog half -// holds a kind-1 edge from the package into that this pipeline does not move (a publish again of an older build, a -// dependent it does not move) is a mover of one pipeline keyed `..published..`, at the -// publishing source's catalog revision. The dependents come from the edges the runner holds, never from a comparison. -func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) (int, error) { +// Published hears a package-published (design 47 H §746, H-A-52): in the running pipeline of the revision that published +// it, a mover that builds on it waits for it there and is placed once it is heard. It starts no pipeline: a re-pin on a +// module's release is a catalog revision of its own, whose pipeline Revised starts (H-A-212, H-A-213 (2)). +func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) error { if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 { - return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev) + return fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev) } - publishing, held, err := s.heardPublished(ev, view) - if err != nil { - return 0, err - } - half, ok := view(ev.Branch) - if !ok { - return 0, fmt.Errorf("the catalog half of %s is not held yet: the publish of %s %s is heard again once it is", ev.Branch, ev.Artifact, ev.Version) - } - var movers []seat.Mover - for _, module := range slices.Sorted(maps.Keys(half.Entries)) { - e := half.Entries[module] - if module == ev.Module || Moves(publishing, module) || !slices.ContainsFunc(e.Edges, func(x catalog.Edge) bool { - return x.Kind == publishedKind && x.From == ev.Artifact - }) { - continue - } - movers = append(movers, seat.Mover{Module: module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository}) - } - if len(movers) == 0 { + _, _, err := s.heardPublished(ev, view) + return err +} + +// revisedKinds are the call kinds of a catalog revision whose pipeline Revised starts: every kind but a take-in's, whose +// pipeline its take-in.taken starts (H-A-213 (2), H-A-217). +var revisedKinds = []string{catalog.CatalogRevisionEventCallKindRelease, catalog.CatalogRevisionEventCallKindTakeInAgain, + catalog.CatalogRevisionEventCallKindRecompute} + +// Revised starts, on a catalog-revision of the trunk kind whose call kind is release, take-in again or recompute, ONE +// pipeline keyed . by compare-and-set on its absence (design 47 H §746, H-A-212, H-A-213 (2) and (4), +// H-A-217): the revision is read by its number through the catalog seat's revision verb, never the newest half, and its +// movers are the revision's movers, its steps placed from the revision's entries. A revision of another kind, or one with +// no mover, starts nothing; a second hearing writes nothing. +func (s *Starter) Revised(ev catalog.CatalogRevisionEvent, read func(branch string, revision int64) (Half, error)) (int, error) { + if ev.Kind != "trunk" || !slices.Contains(revisedKinds, ev.CallKind) { return 0, nil } - revision := half.Revision - if held && publishing.Revision > 0 { - revision = publishing.Revision + branch, err := keys.UnescapeName(ev.Branch) + if err != nil || branch == "" || ev.Revision <= 0 { + return 0, fmt.Errorf("a catalog-revision names its watched branch and revision: branch %q revision %d", ev.Branch, ev.Revision) + } + if read == nil { + return 0, fmt.Errorf("the revision %d of %s cannot be read: the runner holds no revision reader", ev.Revision, branch) + } + half, err := read(branch, ev.Revision) + if err != nil { + return 0, fmt.Errorf("the revision %d of %s is not read, so its pipeline is not started: %w", ev.Revision, branch, err) + } + if half.Revision != ev.Revision { + return 0, fmt.Errorf("the revision verb answered revision %d of %s for %d", half.Revision, branch, ev.Revision) + } + if len(half.Movers) == 0 { + s.logf("the revision %d of %s (%s) names no mover: it starts no pipeline", ev.Revision, branch, ev.Call) + return 0, nil + } + movers := make([]seat.Mover, 0, len(half.Movers)) + for _, e := range half.Movers { + movers = append(movers, seat.Mover{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint, Repository: e.Repository}) } movers = Tiers(movers, half.Entries, s.logf) - started, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Call: publishing.Call, Revision: revision, Movers: movers, - Entries: half.Entries, Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version}, - Key: PublishedKey(ev.Branch, ev.Source, ev.Artifact, ev.Version)}) - if started { + call := half.Call + if call == "" { + call = ev.Call + } + ok, err := s.Start(Begin{Branch: branch, Source: half.Source, Call: call, Revision: half.Revision, Movers: movers, Entries: half.Entries}) + if ok { return 1, err } return 0, err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 9418cfca..7baa680c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -11,6 +11,7 @@ import ( builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" ) type created map[string][]byte @@ -55,12 +56,11 @@ func TestStarts(t *testing.T) { } } -// R10/AC1 and AC2 (design 47 H §747, H-A-52): a package-published of a mover of its source's running pipeline whose -// dependent that pipeline moves starts nothing: the dependent waits there. A publish again of an older build starts one -// pipeline keyed ..published.., its movers the package's kind-1 dependents from the -// edges the runner holds, at the publishing source's revision, and says pipeline-started with the publish; an ordering -// edge starts nothing; heard twice, the second writes nothing. -func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) { +// H-A-212, H-A-213 (2) and (4), H-A-217: a package-published starts no pipeline, the movers waiting for it in its +// pipeline placed there; a trunk catalog-revision of a release, a take-in again or a recompute starts ONE pipeline keyed +// ., the revision read by its number, its movers the revision's, its source and call the revision's; +// heard twice the second writes nothing; a take-in's revision, a candidate's and one with no mover start nothing. +func TestARevisionOfAReleaseStartsOnePipeline(t *testing.T) { mem := memStore{} var said []seat.PipelineStarted s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(_ string, body any) error { @@ -70,37 +70,60 @@ func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) { return nil }} edges := []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 1}} - view := func(string) (Half, bool) { - return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{ - "mesh-sdk": {Module: "mesh-sdk"}, - "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges}, - "tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges}, - "ordered": {Module: "ordered", Edges: []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 4}}}}}, true + entries := map[string]catalog.CatalogEntry{ + "mesh-sdk": {Module: "mesh-sdk"}, + "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges}, + "tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges}} + view := func(string) (Half, bool) { return Half{Revision: 9, Entries: entries}, true } + ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 30, Module: "mesh-sdk", Version: "0.1.9"} + if err := s.Published(ev, view); err != nil || len(mem) != 0 { + t.Fatalf("a publish of an older build wrote %d pipelines (%v), want none", len(mem), err) } - mem[Key("main", 41)] = struct { - p seat.Pipeline - rev uint64 - }{seat.Pipeline{Pipeline: "main.41", Branch: "main", Source: 41, Revision: 8, Outcome: seat.PipelineOutcomeRunning, - Movers: []seat.Mover{{Module: "mesh-sdk"}, {Module: "app"}, {Module: "tool"}}}, 1} - ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"} - if n, err := s.Published(ev, view); n != 0 || err != nil || len(mem) != 1 { - t.Fatalf("a publish in a pipeline moving its dependents started %d (%v), %d pipelines", n, err, len(mem)) + var asked []int64 + movers := []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}, {Module: "tool", Commit: "c2", Fingerprint: "src2:bc"}} + read := func(branch string, revision int64) (Half, error) { + asked = append(asked, revision) + if branch != "release.1" { + return Half{}, fmt.Errorf("branch %q", branch) + } + r := Half{Revision: revision, Source: 41, Entries: entries, Call: "release of mesh-sdk 0.1.17"} + if revision != 12 { + r.Movers = movers + } + return r, nil } - // A publish again of an older build, at source 30: its pipeline is gone, so every kind-1 dependent is a mover. - ev.Source, ev.Version = 30, "0.1.9" + release := catalog.CatalogRevisionEvent{Kind: "trunk", Branch: keys.EscapeName("release.1"), Revision: 10, Source: 41, CallKind: "release", Call: "release of mesh-sdk 0.1.17"} for i := range 2 { - if n, err := s.Published(ev, view); n != 1-i || err != nil { + if n, err := s.Revised(release, read); n != 1-i || err != nil { t.Fatalf("hearing %d started %d (%v)", i+1, n, err) } } - key := PublishedKey("main", 30, "mesh-sdk:go", "0.1.9") - p, _, err := mem.Get(key) - if err != nil || len(p.Movers) != 2 || !Moves(p, "app") || !Moves(p, "tool") || Moves(p, "ordered") || p.Revision != 9 || - p.Published == nil || p.Published.Version != "0.1.9" || len(said) != 1 || said[0].Published == nil { - t.Fatalf("the pipeline %s = %+v (%v), said %+v", key, p, err, said) + p, _, err := mem.Get(Key("release.1", 10)) + if err != nil || len(mem) != 1 || len(p.Movers) != 2 || p.Revision != 10 || p.Source != 41 || p.Call != "release of mesh-sdk 0.1.17" || + len(said) != 1 || len(asked) != 2 || asked[0] != 10 { + t.Fatalf("the pipeline %s = %+v (%v), %d written, said %d, read %v", Key("release.1", 10), p, err, len(mem), len(said), asked) } - if app, _ := MoverOf(p, "app"); app.Commit != "c1" || app.Fingerprint != "src2:aa" || app.Repository != "o/app" { - t.Errorf("the mover app = %+v", app) + if app, _ := MoverOf(p, "app"); app.Fingerprint != "src2:ab" || app.Repository != "o/app" || app.Tier != 1 { + t.Errorf("the mover app = %+v, want the revision's", app) + } + for _, c := range []struct { + name string + ev catalog.CatalogRevisionEvent + want int + }{ + {"take-in again", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 11, CallKind: "take-in-again"}, 1}, + {"recompute with no mover", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 12, CallKind: "recompute"}, 0}, + {"recompute", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 13, CallKind: "recompute"}, 1}, + {"take-in", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 14, CallKind: "take-in"}, 0}, + {"candidate", catalog.CatalogRevisionEvent{Kind: "candidate", Branch: release.Branch, Revision: 15, CallKind: "release"}, 0}, + } { + if n, err := s.Revised(c.ev, read); n != c.want || err != nil { + t.Errorf("%s: started %d (%v), want %d", c.name, n, err, c.want) + } + } + if _, err := s.Revised(catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 16, CallKind: "release"}, + func(string, int64) (Half, error) { return Half{}, errors.New("no answer") }); err == nil { + t.Error("a revision that cannot be read is not an error, so it is never heard again") } } @@ -243,25 +266,6 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T if n != 1 || err != nil || len(p.Steps) == 0 { t.Errorf("the half of next placed %d (%v): %d steps, want the take-in's pipeline placed", n, err, len(p.Steps)) } - // A-336: a package published heard before its branch's half is held starts nothing, heard again once it is: its - // dependents are the half's edges (§747). - held = false - app := entries["app"] - app.Edges = []catalog.Edge{{From: "mesh-sdk:go", To: "app:program", Kind: 1}} - entries["app"] = app - published := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "next", Source: 7, Module: "mesh-sdk", Version: "0.1.17"} - if n, err := s.Published(published, late); n != 0 || err == nil { - t.Fatalf("the publish before the half started %d (%v), want none and an error, heard again", n, err) - } - held = true - if n, err := s.Published(published, late); n != 1 || err != nil { - t.Fatalf("the publish with the half held started %d (%v)", n, err) - } - p, _, _ = store.Get(PublishedKey("next", 7, "mesh-sdk:go", "0.1.17")) - if len(p.Steps) == 0 { - t.Errorf("the publish's pipeline placed %d steps, want its steps", len(p.Steps)) - } - // A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running, // no step placed) is placed and asked as a newly started one when the half is held; no hold is read any more. var old seat.Pipeline From 4af1ba99cfcbf2ccdf8b7afa291d9727551914f6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:12:36 +0100 Subject: [PATCH 09/11] Design 47 H1 (5a): the runner's integration tests key a pipeline by its catalog revision The queue's and the test runs' integration tests took several sources in at one catalog revision, which now are one key. Each take-in has its own revision; the test runs' test also checks that a test ask names revision and source. A replaced reader of another pipeline's run stays the unit tests' (one revision is one pipeline). Builds: design 47 H H-A-213 (4), H-A-219 --- .../queue_integration_test.go | 4 +- .../testruns_integration_test.go | 52 +++++++------------ 2 files changed, 20 insertions(+), 36 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index 6b319931..8e9fb2d3 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -101,7 +101,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { } } answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs}) - answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries}) + answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 100, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries}) answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) // The router of the operator's asks holds the seat operator-channel; its record of each ask is in its state asks. const router = "operator-router" @@ -218,7 +218,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { } take := func(source int64, movers ...string) { t.Helper() - ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: 1, Call: fmt.Sprintf("%016x", source)} + ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: source, Call: fmt.Sprintf("%016x", source)} for _, m := range movers { ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index a0aa6696..e39fabf9 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -37,7 +37,7 @@ import ( // ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step // that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a // run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed; -// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended, +// a run that keeps saying its progress past its bound is not ended, // and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the // same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its // other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and @@ -327,7 +327,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { // One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its // ask is taken (A-469b), its holder present and nothing failing it (A-466y). - const p3 = "main.3" + // A pipeline is keyed by its catalog revision (H-A-213 (4)): source 3's take-in of revision 1 is main.1. + const p3 = "main.1" takeIn("app", "c1", "src2:aa", 3, 1) placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747) passBuild(p3, "app", "c1", "src2:aa") @@ -340,8 +341,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { } var testAsk nodebuildgo.StepAsk if err := json.Unmarshal(last.Data, &testAsk); err != nil || testAsk.Step == nil || - testAsk.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25" { - t.Fatalf("the last test ask = %s (%v), want its fetch to carry the entry's pin", last.Data, err) + testAsk.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25" || testAsk.Step.Revision != 1 || testAsk.Step.Source != 3 { + t.Fatalf("the last test ask = %s (%v), want its fetch to carry the entry's pin, revision 1 and source 3 (H-A-219)", last.Data, err) } if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 { t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers) @@ -483,39 +484,20 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { // A second revision with the same digests asks its own runs, so a failed test runs again (§724). revision.Store(2) publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4}) - const p4, p5 = "main.4", "main.5" + const p4 = "main.2" takeIn("app", "c1", "src2:aa", 4, 2) placed(p4) run4, r4 := runOfStep(p4, "app:program", "integration") if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask { t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask) } - // A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads - // the run, the first is cancelled and leaves it, and the run's ask is not dropped. - takeIn("app", "c1", "src2:aa", 5, 2) - placed(p5) - waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 }) - if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" { - t.Fatalf("cancel: %s", e) - } - waitFor(t, ctx, logs, func() bool { - r, _ := runOf(run4) - return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5 - }) - if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask { - t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r) - } - mu.Lock() - dropped := append([]string(nil), drops...) - mu.Unlock() - if len(dropped) != 0 { - t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String()) - } + // A replaced reader leaving the ask to the other readers of another pipeline (§726) is the unit tests': one revision is + // one pipeline (H-A-213 (4)). // A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720). taken(r4.Ask) waitFor(t, ctx, logs, func() bool { - return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun && - step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun + return step(read(p4), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun && + step(read(p4), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun }) if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") { t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r) @@ -528,8 +510,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { // A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not // asked (§535, §647; A-470g, A-470h). - const web = "main.7" - takeIn("web", "w1", "src2:ww", 7, 2) + revision.Store(3) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 7}) + const web = "main.3" + takeIn("web", "w1", "src2:ww", 7, 3) waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 }) if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" { t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e) @@ -580,10 +564,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { t.Fatal(err) } } - revision.Store(3) - publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5}) - const p6 = "main.6" - takeIn("app", "c1", "src2:aa", 6, 3) + revision.Store(4) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 4, Kind: "trunk", Branch: "main", Source: 6}) + const p6 = "main.4" + takeIn("app", "c1", "src2:aa", 6, 4) waitFor(t, ctx, logs, func() bool { s := step(read(p6), "app:program", "unit") return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib") From 671dc5a9630ec84f0abb5bbe09390c9f9a50c1c7 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:17:55 +0100 Subject: [PATCH 10/11] Design 47 H1 (5c): the runner takes a publish into the pipeline of the revision it names One source now has several catalog revisions, so the publishing pipeline is found by the branch and revision the package-published names. A publish naming no revision, from a reservation made before the field, goes to its source's lowest-numbered revision, the take-in's. The seat mesh-builds' package-published gains its optional revision here too, identical to the builder branch's, so this branch compiles alone. Builds: design 47 H H-A-218, H-A-220 --- .../go/seats/meshbuilds/contract.go | 2 ++ .../seats/mesh-builds/seat.json | 4 ++++ .../ts/seats/meshbuilds/index.d.ts | 2 ++ .../internal/pipeline/start.go | 12 ++++++---- .../internal/pipeline/start_test.go | 24 +++++++++++++++++++ 5 files changed, 40 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go index 4cc3285b..179a6890 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go @@ -95,6 +95,8 @@ type PackagePublished struct { Digest string `json:"digest"` // the module Module string `json:"module"` + // the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field + Revision int64 `json:"revision,omitempty"` // the source revision of that branch the build was made from Source int64 `json:"source"` V int64 `json:"v"` diff --git a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json index fc3d717d..e100e48c 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json @@ -180,6 +180,10 @@ "type": "integer", "description": "the source revision of that branch the build was made from" }, + "revision": { + "type": "integer", + "description": "the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field" + }, "dependencies": { "type": "array", "items": { diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts index 947f2e52..bf8e68c7 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts @@ -69,6 +69,8 @@ export interface PackagePublished { "digest": string; /** the module */ "module": string; + /** the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field */ + "revision"?: number; /** the source revision of that branch the build was made from */ "source": number; "v": number; diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 7d1a50ea..41f92321 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -893,14 +893,18 @@ func (s *Starter) Release(store Updater, key string, view func(branch string) (H }, s.Registration, view, "the publishes its movers waited for") } -// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the source -// revision that published it, and places the movers that waited for it (design 47 H §746, H-A-84). It answers that -// source's pipeline, read, and whether one is held. +// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the catalog +// revision that published it, found by the branch and revision the publish names (H-A-218), and places the movers that +// waited for it (design 47 H §746, H-A-84). It answers that pipeline, read, and whether one is held. func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch string) (Half, bool)) (seat.Pipeline, bool, error) { if s.Store == nil { return seat.Pipeline{}, false, nil } - key, ok := s.sourcePipeline(ev.Branch, ev.Source) + key, ok := Key(ev.Branch, ev.Revision), true + if ev.Revision <= 0 { + // A publish of an older builder names no revision: its source's lowest-numbered revision, the take-in's (H-A-218). + key, ok = s.sourcePipeline(ev.Branch, ev.Source) + } if !ok { return seat.Pipeline{}, false, nil } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 7baa680c..3c5cbfe0 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -339,3 +339,27 @@ func TestATakeInOfTheOldFormIsRefused(t *testing.T) { t.Errorf("the new form = %+v (%v)", ev, err) } } + +// H-A-218: a package-published is taken into the pipeline of the branch and revision it names; one naming no revision (an +// older builder) into its source's lowest-numbered revision, the take-in's. +func TestAPublishIsTakenIntoThePipelineOfItsRevision(t *testing.T) { + mem := memStore{} + for _, r := range []int64{7, 8} { + mem[Key("main", r)] = struct { + p seat.Pipeline + rev uint64 + }{seat.Pipeline{Pipeline: Key("main", r), Branch: "main", Source: 4, Revision: r, Outcome: seat.PipelineOutcomeRunning, + Movers: []seat.Mover{{Module: "mesh-sdk"}}}, 1} + } + s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(string, any) error { return nil }} + view := func(string) (Half, bool) { return Half{Revision: 8}, true } + published := func(key string) []string { m, _ := MoverOf(mem[key].p, "mesh-sdk"); return m.Published } + ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Revision: 8, Version: "0.1.18"} + if err := s.Published(ev, view); err != nil || len(published("main.8")) != 1 || len(published("main.7")) != 0 { + t.Fatalf("a publish of revision 8 (%v): main.8 %v, main.7 %v; want main.8's mover alone", err, published("main.8"), published("main.7")) + } + ev.Revision, ev.Artifact = 0, "mesh-sdk:ts" + if err := s.Published(ev, view); err != nil || len(published("main.7")) != 1 || len(published("main.8")) != 1 { + t.Errorf("a publish naming no revision (%v): main.7 %v, main.8 %v; want main.7's, the source's lowest", err, published("main.7"), published("main.8")) + } +} From 626a6fb3586545a8d7ec0143660ce39b457cb91f Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:25:08 +0100 Subject: [PATCH 11/11] Design 47 H1: the pipeline-runner seat names a pipeline's key and its subjects by revision A pipeline is keyed by its catalog revision's number, so the seat's key form is . (the generated PipelinesByBranchRevisionKey and its siblings; the fixtures follow), and pipeline-started, pipeline-ended, wait-held and wait-ended are said on ., carrying the same number as the key; every reader takes the body. The client's SourceSubject is RevisionSubject. The published key form, under which pipelines were kept before, stays. Builds: design 47 H H-A-213 (4), H-A-219, H-A-221 --- .../meshpipelinerunnerclient/client.go | 34 +++++------ .../meshpipelinerunnerclient/client_test.go | 6 +- .../go/seats/meshpipelinerunner/contract.go | 56 +++++++++---------- .../seats/meshpipelinerunner/contract_test.go | 6 +- .../mesh-pipeline-runner/fixtures/states.json | 6 +- .../seats/mesh-pipeline-runner/seat.json | 26 ++++----- .../ts/seats/meshpipelinerunner/index.d.ts | 34 +++++------ .../ts/seats/meshpipelinerunner/index.js | 30 +++++----- .../internal/index/catalog_revisions.go | 2 +- .../internal/pipeline/queue.go | 4 +- .../internal/pipeline/read.go | 2 +- .../internal/pipeline/start.go | 12 +++- .../internal/pipeline/start_test.go | 6 +- .../internal/pipeline/steps.go | 6 +- 14 files changed, 119 insertions(+), 111 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client.go b/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client.go index eb466b0b..e1702ce1 100644 --- a/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client.go +++ b/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client.go @@ -17,8 +17,9 @@ import ( // eventPrefix starts every event subject of the seat. const eventPrefix = "mesh.seat." + seat.Seat + ".event." -// perSource are the events said per source revision, on .. (design 47 H §746, H-A-72 (3)). -var perSource = []string{seat.EventPipelineStarted, seat.EventPipelineEnded, seat.EventWaitHeld, seat.EventWaitEnded} +// perRevision are the events said per pipeline, on .., its catalog revision's number as its key +// names it (design 47 H §746, H-A-72 (3), H-A-221). +var perRevision = []string{seat.EventPipelineStarted, seat.EventPipelineEnded, seat.EventWaitHeld, seat.EventWaitEnded} // Subject is where the runner says a step's event or a run's start: mesh.seat.mesh-pipeline-runner.event.. // .., each name escaped as a key's name is, so a branch holding a dot is one token (design 47 H @@ -27,41 +28,42 @@ func Subject(event, module, branch, commit string) string { return eventPrefix + event + "." + keys.EscapeName(module) + "." + keys.EscapeName(branch) + "." + keys.EscapeName(commit) } -// SourceSubject is where the runner says a pipeline's start and end and its waits: mesh.seat.mesh-pipeline-runner. -// event..., the branch escaped as a key's name is (design 47 H §746, H-A-72 (3)). -func SourceSubject(event, branch string, source int64) string { - return eventPrefix + event + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10) +// RevisionSubject is where the runner says a pipeline's start and end and its waits: mesh.seat.mesh-pipeline-runner. +// event..., the pipeline's catalog revision number, the branch escaped as a key's name is (design +// 47 H §746, H-A-72 (3), H-A-221). +func RevisionSubject(event, branch string, revision int64) string { + return eventPrefix + event + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10) } // Of is what an event's subject names: a step's event or a run's start names the module, the branch and the commit; an -// event per source revision names the branch and the source. +// event per pipeline names the branch and the pipeline's catalog revision (H-A-221). type Of struct { Event, Module, Branch, Commit string - Source int64 + Revision int64 } -// Parse reads a subject Subject or SourceSubject built: an event of the seat's, and its names unescaped. +// Parse reads a subject Subject or RevisionSubject built: an event of the seat's, and its names unescaped. func Parse(subject string) (Of, error) { rest, ok := strings.CutPrefix(subject, eventPrefix) toks := strings.Split(rest, ".") - bySource := len(toks) > 0 && slices.Contains(perSource, toks[0]) + byRevision := len(toks) > 0 && slices.Contains(perRevision, toks[0]) want := 4 - if bySource { + if byRevision { want = 3 } if !ok || len(toks) != want || !slices.Contains(seat.Events(), toks[0]) { return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q is no event subject of the seat %s", subject, seat.Seat) } - if bySource { + if byRevision { branch, err := keys.UnescapeName(toks[1]) if err != nil || branch == "" { return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: branch: %v", subject, err) } - source, err := strconv.ParseInt(toks[2], 10, 64) - if err != nil || source < 1 { - return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: no source revision", subject) + revision, err := strconv.ParseInt(toks[2], 10, 64) + if err != nil || revision < 1 { + return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: no revision", subject) } - return Of{Event: toks[0], Branch: branch, Source: source}, nil + return Of{Event: toks[0], Branch: branch, Revision: revision}, nil } names := make([]string, 0, 3) for i, t := range toks[1:] { diff --git a/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client_test.go b/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client_test.go index 355017f2..f76ef511 100644 --- a/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client_test.go +++ b/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client_test.go @@ -17,14 +17,14 @@ func TestSubjectsAndBodies(t *testing.T) { if of, err := Parse(s); err != nil || of != (Of{Event: seat.EventStepEnded, Module: "nats", Branch: "release.1", Commit: "c1"}) { t.Errorf("parse %s: %+v, %v", s, of, err) } - ps := SourceSubject(seat.EventPipelineEnded, "release.1", 3589) + ps := RevisionSubject(seat.EventPipelineEnded, "release.1", 3589) if ps != "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.release=2e1.3589" { t.Errorf("pipeline-ended subject %s", ps) } - if of, err := Parse(ps); err != nil || of != (Of{Event: seat.EventPipelineEnded, Branch: "release.1", Source: 3589}) { + if of, err := Parse(ps); err != nil || of != (Of{Event: seat.EventPipelineEnded, Branch: "release.1", Revision: 3589}) { t.Errorf("parse %s: %+v, %v", ps, of, err) } - if of, err := Parse(SourceSubject(seat.EventWaitHeld, "main", 7)); err != nil || of.Source != 7 || of.Module != "" { + if of, err := Parse(RevisionSubject(seat.EventWaitHeld, "main", 7)); err != nil || of.Revision != 7 || of.Module != "" { t.Errorf("wait-held: %+v, %v", of, err) } for _, bad := range []string{"not.an.event.subject", diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go index 058625fb..8b5f340e 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go @@ -65,9 +65,9 @@ const ( // StreamPipelineStarted is the stream pipeline-started is kept in. const StreamPipelineStarted = "SEAT_mesh-pipeline-runner_pipeline-started" -// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started... -func SubjectPipelineStarted(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source +// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started... +func SubjectPipelineStarted(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision } // SubjectPipelineStartedAll hears every pipeline-started. @@ -76,9 +76,9 @@ const SubjectPipelineStartedAll = "mesh.seat.mesh-pipeline-runner.event.pipeline // StreamPipelineEnded is the stream pipeline-ended is kept in. const StreamPipelineEnded = "SEAT_mesh-pipeline-runner_pipeline-ended" -// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended... -func SubjectPipelineEnded(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source +// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended... +func SubjectPipelineEnded(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision } // SubjectPipelineEndedAll hears every pipeline-ended. @@ -120,9 +120,9 @@ const SubjectStepEndedAll = "mesh.seat.mesh-pipeline-runner.event.step-ended.*.* // StreamWaitHeld is the stream wait-held is kept in. const StreamWaitHeld = "SEAT_mesh-pipeline-runner_wait-held" -// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held... -func SubjectWaitHeld(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source +// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held... +func SubjectWaitHeld(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision } // SubjectWaitHeldAll hears every wait-held. @@ -131,9 +131,9 @@ const SubjectWaitHeldAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*" // StreamWaitEnded is the stream wait-ended is kept in. const StreamWaitEnded = "SEAT_mesh-pipeline-runner_wait-ended" -// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended... -func SubjectWaitEnded(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source +// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended... +func SubjectWaitEnded(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision } // SubjectWaitEndedAll hears every wait-ended. @@ -156,11 +156,11 @@ const ( StateStepOutputs = "step-outputs" ) -// PipelinesByBranchSourceKey is the key of one value of the state pipelines under the form ".". -func PipelinesByBranchSourceKey(branch, source string) string { return branch + "." + source } +// PipelinesByBranchRevisionKey is the key of one value of the state pipelines under the form ".". +func PipelinesByBranchRevisionKey(branch, revision string) string { return branch + "." + revision } -// ParsePipelinesByBranchSourceKey reads the parts of a key of the form ".", in the order of the form; the last result is false for a key of another form. -func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) { +// ParsePipelinesByBranchRevisionKey reads the parts of a key of the form ".", in the order of the form; the last result is false for a key of another form. +func ParsePipelinesByBranchRevisionKey(key string) (string, string, bool) { rest := key v0, after0, found0 := strings.Cut(rest, ".") if !found0 || v0 == "" { @@ -178,9 +178,9 @@ func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) { return v0, v2, true } -// DecodePipelinesByBranchSourceValue reads one value of the state pipelines under the form ".", refusing a field PipelinesByBranchSourceValue does not know and any text after the value. -func DecodePipelinesByBranchSourceValue(raw []byte) (PipelinesByBranchSourceValue, error) { - var v PipelinesByBranchSourceValue +// DecodePipelinesByBranchRevisionValue reads one value of the state pipelines under the form ".", refusing a field PipelinesByBranchRevisionValue does not know and any text after the value. +func DecodePipelinesByBranchRevisionValue(raw []byte) (PipelinesByBranchRevisionValue, error) { + var v PipelinesByBranchRevisionValue dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&v); err != nil { @@ -670,7 +670,7 @@ type Pipeline struct { // the modules the pipeline moves, its take-in's entries (design 47 H §746) Movers []Mover `json:"movers"` Outcome string `json:"outcome"` - // the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) + // the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) Pipeline string `json:"pipeline"` // for a pipeline a package's publish started, that publish (A-202) Published *Published `json:"published,omitempty"` @@ -720,7 +720,7 @@ type PipelineEnded struct { // per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2) Movers []MoverEnded `json:"movers"` Outcome string `json:"outcome"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // the catalog graph revision of the branch the pipeline runs for Revision int64 `json:"revision,omitempty"` @@ -753,7 +753,7 @@ type PipelineStarted struct { Call string `json:"call,omitempty"` // the pipeline's movers Movers []Mover `json:"movers"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // for a pipeline a package's publish started, that publish (A-202) Published *Published `json:"published,omitempty"` @@ -814,7 +814,7 @@ type StepEvent struct { // the module's build-input fingerprint at the commit Fingerprint string `json:"fingerprint,omitempty"` Module string `json:"module"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` Result StepResult `json:"result"` // the catalog graph revision of the branch the pipeline runs for @@ -966,7 +966,7 @@ type ToolchainUsed struct { Version string `json:"version"` } -// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72). +// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221). type Wait struct { Artifact string `json:"artifact,omitempty"` // the step's own ask, where it has one (design 47 G §731) @@ -983,7 +983,7 @@ type Wait struct { Module string `json:"module,omitempty"` // what the step waits on (design 47 *Health and going back*) On string `json:"on"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // the catalog graph revision of the branch the pipeline runs for Revision int64 `json:"revision,omitempty"` @@ -1024,7 +1024,7 @@ const ( // Waiting is a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72). type Waiting struct { - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` } @@ -1355,8 +1355,8 @@ type FullEvent = Fill // RoomEvent is the body of the event room. type RoomEvent = Fill -// PipelinesByBranchSourceValue is one value of the state pipelines under the key .. -type PipelinesByBranchSourceValue = Pipeline +// PipelinesByBranchRevisionValue is one value of the state pipelines under the key .. +type PipelinesByBranchRevisionValue = Pipeline // PipelinesByBranchSourcePublishedPackageVersionValue is one value of the state pipelines under the key ..published... type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go index 23b59e9c..4aaa6d53 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go @@ -45,7 +45,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")}, {name: "a take-in's pipeline kept, its steps naming their mover", subject: "", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, @@ -73,13 +73,13 @@ func TestFixtures(t *testing.T) { }, subjectIs: nil}, {name: "a pipeline waiting its turn, its seq given", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, {name: "a waiting pipeline names the movers a newer take-in took out of it", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json index cb8363f2..9229229f 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json @@ -1,7 +1,7 @@ [ { "name": "a take-in's pipeline kept, its steps naming their mover", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3589", "branch": "main", @@ -200,7 +200,7 @@ }, { "name": "a pipeline waiting its turn, its seq given", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3590", "branch": "main", @@ -224,7 +224,7 @@ }, { "name": "a waiting pipeline names the movers a newer take-in took out of it", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3590", "branch": "main", diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json index 5bf14a52..fd84b289 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json @@ -672,7 +672,7 @@ "events": [ { "name": "pipeline-started", - "subject": "pipeline-started..", + "subject": "pipeline-started..", "body": { "$ref": "#/defs/PipelineStarted" }, @@ -683,7 +683,7 @@ }, { "name": "pipeline-ended", - "subject": "pipeline-ended..", + "subject": "pipeline-ended..", "body": { "$ref": "#/defs/PipelineEnded" }, @@ -727,7 +727,7 @@ }, { "name": "wait-held", - "subject": "wait-held..", + "subject": "wait-held..", "body": { "$ref": "#/defs/Wait" }, @@ -738,7 +738,7 @@ }, { "name": "wait-ended", - "subject": "wait-ended..", + "subject": "wait-ended..", "body": { "$ref": "#/defs/Wait" }, @@ -778,11 +778,11 @@ "description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747), and one key per mover of a pipeline holding that mover's steps (H-A-187)", "key-forms": [ { - "key": ".", + "key": ".", "value": { "$ref": "#/defs/Pipeline" }, - "description": "a take-in's pipeline" + "description": "a pipeline of one catalog revision, keyed by the revision's number (design 47 H-A-213 (4), H-A-219); one kept from before under its source has the same form" }, { "key": "..published..", @@ -1199,7 +1199,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)" + "description": "the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)" }, "branch": { "type": "string", @@ -1313,7 +1313,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "branch": { "type": "string", @@ -1388,7 +1388,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "branch": { "type": "string", @@ -1495,7 +1495,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "module": { "type": "string" @@ -1550,7 +1550,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "module": { "type": "string", @@ -1635,7 +1635,7 @@ "description": "the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)" } }, - "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72)", + "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221)", "required": [ "v", "pipeline", @@ -1978,7 +1978,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" } }, "description": "a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72)", diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts index 748152b9..5c11adad 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -40,13 +40,13 @@ export declare const family: { readonly name: "mesh-pipeline-runner.events"; rea export declare const pipelineStartedStream: "SEAT_mesh-pipeline-runner_pipeline-started"; -export declare function pipelineStartedSubject(branch: string, source: string): string; +export declare function pipelineStartedSubject(branch: string, revision: string): string; export declare const pipelineStartedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*"; export declare const pipelineEndedStream: "SEAT_mesh-pipeline-runner_pipeline-ended"; -export declare function pipelineEndedSubject(branch: string, source: string): string; +export declare function pipelineEndedSubject(branch: string, revision: string): string; export declare const pipelineEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"; @@ -70,13 +70,13 @@ export declare const stepEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event. export declare const waitHeldStream: "SEAT_mesh-pipeline-runner_wait-held"; -export declare function waitHeldSubject(branch: string, source: string): string; +export declare function waitHeldSubject(branch: string, revision: string): string; export declare const waitHeldSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*"; export declare const waitEndedStream: "SEAT_mesh-pipeline-runner_wait-ended"; -export declare function waitEndedSubject(branch: string, source: string): string; +export declare function waitEndedSubject(branch: string, revision: string): string; export declare const waitEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*"; @@ -94,11 +94,11 @@ export declare const states: { readonly stepOutputs: "step-outputs"; }; -/** The key of one value of the state pipelines under the form ".". */ -export declare function pipelinesByBranchSourceKey(branch: string, source: string): string; +/** The key of one value of the state pipelines under the form ".". */ +export declare function pipelinesByBranchRevisionKey(branch: string, revision: string): string; -/** The parts of a key of the form ".", or undefined for a key of another form. */ -export declare function parsePipelinesByBranchSourceKey(_key: string): { branch: string; source: string } | undefined; +/** The parts of a key of the form ".", or undefined for a key of another form. */ +export declare function parsePipelinesByBranchRevisionKey(_key: string): { branch: string; revision: string } | undefined; /** The key of one value of the state pipelines under the form "..published..". */ export declare function pipelinesByBranchSourcePublishedPackageVersionKey(branch: string, source: string, packagePart: string, version: string): string; @@ -261,7 +261,7 @@ export interface Pipeline { /** the modules the pipeline moves, its take-in's entries (design 47 H §746) */ "movers": Mover[]; "outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled"; - /** the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */ + /** the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */ "pipeline": string; /** for a pipeline a package's publish started, that publish (A-202) */ "published"?: Published; @@ -300,7 +300,7 @@ export interface PipelineEnded { /** per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2) */ "movers": MoverEnded[]; "outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled"; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** the catalog graph revision of the branch the pipeline runs for */ "revision"?: number; @@ -322,7 +322,7 @@ export interface PipelineStarted { "call"?: string; /** the pipeline's movers */ "movers": Mover[]; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** for a pipeline a package's publish started, that publish (A-202) */ "published"?: Published; @@ -383,7 +383,7 @@ export interface StepEvent { /** the module's build-input fingerprint at the commit */ "fingerprint"?: string; "module": string; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; "result": StepResult; /** the catalog graph revision of the branch the pipeline runs for */ @@ -490,7 +490,7 @@ export interface ToolchainUsed { "version": string; } -/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72) */ +/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221) */ export interface Wait { "artifact"?: string; /** the step's own ask, where it has one (design 47 G §731) */ @@ -507,7 +507,7 @@ export interface Wait { "module"?: string; /** what the step waits on (design 47 *Health and going back*) */ "on": "person" | "build node" | "revision view" | "pipelines-at-once"; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** the catalog graph revision of the branch the pipeline runs for */ "revision"?: number; @@ -527,7 +527,7 @@ export interface Wait { /** a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72) */ export interface Waiting { - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; } @@ -800,8 +800,8 @@ export type FullEvent = Fill; /** is the body of the event room. */ export type RoomEvent = Fill; -/** is one value of the state pipelines under the key .. */ -export type PipelinesByBranchSourceValue = Pipeline; +/** is one value of the state pipelines under the key .. */ +export type PipelinesByBranchRevisionValue = Pipeline; /** is one value of the state pipelines under the key ..published... */ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline; diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js index 213a3df3..d8ff25e6 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js @@ -40,16 +40,16 @@ export const family = { name: "mesh-pipeline-runner.events", maxAgeDays: 7 }; export const pipelineStartedStream = "SEAT_mesh-pipeline-runner_pipeline-started"; -export function pipelineStartedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source; +export function pipelineStartedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision; } export const pipelineStartedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*"; export const pipelineEndedStream = "SEAT_mesh-pipeline-runner_pipeline-ended"; -export function pipelineEndedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source; +export function pipelineEndedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision; } export const pipelineEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"; @@ -80,16 +80,16 @@ export const stepEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.step-en export const waitHeldStream = "SEAT_mesh-pipeline-runner_wait-held"; -export function waitHeldSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source; +export function waitHeldSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision; } export const waitHeldSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*"; export const waitEndedStream = "SEAT_mesh-pipeline-runner_wait-ended"; -export function waitEndedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source; +export function waitEndedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision; } export const waitEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*"; @@ -108,23 +108,23 @@ export const states = { stepOutputs: "step-outputs", }; -/** The key of one value of the state pipelines under the form ".". */ -export function pipelinesByBranchSourceKey(branch, source) { - return branch + "." + source; +/** The key of one value of the state pipelines under the form ".". */ +export function pipelinesByBranchRevisionKey(branch, revision) { + return branch + "." + revision; } -/** The parts of a key of the form ".", or undefined for a key of another form. */ -export function parsePipelinesByBranchSourceKey(_key) { +/** The parts of a key of the form ".", or undefined for a key of another form. */ +export function parsePipelinesByBranchRevisionKey(_key) { let _rest = _key; const _at0 = _rest.indexOf("."); if (_at0 <= 0) return undefined; const branch = _rest.slice(0, _at0); _rest = _rest.slice(_at0 + 1); if (_rest === "") return undefined; - const source = _rest; + const revision = _rest; _rest = ""; if (_rest !== "") return undefined; - return { branch, source }; + return { branch, revision }; } /** The key of one value of the state pipelines under the form "..published..". */ diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index ec1e7717..4fc72a07 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -331,7 +331,7 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int } // The take-in's revision pinned for its one pipeline before any judge can remove it (§743, §749, A-449a). if len(movers) > 0 && ix.Pins != nil { - record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(src, 10))) + record := PipelineRecord(runner.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(src, 10))) if err := ix.pin(ctx, PinKey(branch, src), record); err != nil { ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err) return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go index 855191db..cecc84a8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go @@ -164,7 +164,7 @@ func (q *Queue) leave(key string) error { return nil } -// queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on . with step pipeline and +// queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on . with step pipeline and // on pipelines-at-once (design 47 H §748, H-A-52). func queueWait(p seat.Pipeline, at time.Time, how string) seat.Wait { return seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Revision: p.Revision, Run: p.Run, @@ -175,7 +175,7 @@ func (s *Starter) sayQueueWait(event string, p seat.Pipeline, at time.Time, how if s.Say == nil { return } - if err := s.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), queueWait(p, at, how)); err != nil { + if err := s.Say(runnerclient.RevisionSubject(event, p.Branch, subjectRevision(p)), queueWait(p, at, how)); err != nil { s.logf("%s of %s is not said: %v", event, p.Pipeline, err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index e6a0762b..668d2ed8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -206,7 +206,7 @@ func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) if raw == nil { return seat.Pipeline{}, false, nil } - p, err := seat.DecodePipelinesByBranchSourceValue(raw) + p, err := seat.DecodePipelinesByBranchRevisionValue(raw) if err != nil { return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 41f92321..1b2e0911 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -48,7 +48,13 @@ type Begin struct { // the branch escaped as a key's name is (design 47 H §746, H-A-213 (4)); a take-in's number is unchanged where it equalled // its source. A pipeline kept under its source before (`.`) has the same form and is read as it is. func Key(branch string, revision int64) string { - return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10)) + return seat.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10)) +} + +// subjectRevision is the number p's pipeline-started, pipeline-ended and waits are said under, .: its +// key's number (H-A-221). +func subjectRevision(p seat.Pipeline) int64 { + return keyRevision(Begin{Revision: p.Revision, Source: p.Source}) } // keyRevision is the number b's pipeline is keyed by (H-A-213 (4)): its revision, or its source where the start names no @@ -300,12 +306,12 @@ func started(p seat.Pipeline, at time.Time) seat.PipelineStarted { Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run} } -// sayStartedPipeline says p's pipeline-started on . (§746). +// sayStartedPipeline says p's pipeline-started on . (§746). func (s *Starter) sayStartedPipeline(p seat.Pipeline, at time.Time) { if s.Say == nil { return } - if err := s.Say(runnerclient.SourceSubject(seat.EventPipelineStarted, p.Branch, p.Source), started(p, at)); err != nil { + if err := s.Say(runnerclient.RevisionSubject(seat.EventPipelineStarted, p.Branch, subjectRevision(p)), started(p, at)); err != nil { s.logf("pipeline-started of %s is not said: %v", p.Pipeline, err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 3c5cbfe0..d3f5700b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -41,7 +41,7 @@ func TestStarts(t *testing.T) { if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Movers: []seat.Mover{{Module: "app"}}}); err != nil { t.Fatal(err) } - if b, r, ok := seat.ParsePipelinesByBranchSourceKey(Key("release.1", 3)); !ok || r != "3" || b == "release" { + if b, r, ok := seat.ParsePipelinesByBranchRevisionKey(Key("release.1", 3)); !ok || r != "3" || b == "release" { t.Errorf("the key of release.1 is %q", Key("release.1", 3)) } if _, err := s.Start(Begin{Branch: "main", Movers: []seat.Mover{{Module: "app"}}}); err == nil { @@ -151,7 +151,7 @@ func (failing) Create(string, []byte) error { return errors.New("bus away") } // R4/AC1 (design 47 H §746, acceptance test 112): a take-in.taken naming mesh-sdk and one importer starts one pipeline // keyed . by compare-and-set, its movers the event's entries, at the take-in's revision, and says one -// pipeline-started on . naming the call and the movers; R4/AC2: heard twice, the second writes nothing. A +// pipeline-started on . naming the call and the movers; R4/AC2: heard twice, the second writes nothing. A // take-in with no entries and one naming no source revision start nothing. func TestATakeInIsOnePipeline(t *testing.T) { store := created{} @@ -182,7 +182,7 @@ func TestATakeInIsOnePipeline(t *testing.T) { if m, _ := MoverOf(p, "mesh-sdk"); m.Commit != "c1" || m.Fingerprint != "src2:aa" || m.Repository != "o/sdk" || m.Tier != 1 { t.Errorf("the mover mesh-sdk = %+v", m) } - if subjects[0] != "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.7" || said[0].Call != ev.Call || len(said[0].Movers) != 2 { + if subjects[0] != "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.12" || said[0].Call != ev.Call || len(said[0].Movers) != 2 { t.Errorf("pipeline-started %+v on %s", said[0], subjects[0]) } if _, err := s.TakenIn(catalog.TakeInTaken{Branch: "main", Entries: ev.Entries}, view); err == nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 5ac10c50..40524451 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -441,7 +441,7 @@ func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) erro } // said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall; -// an end is said once, on . (design 47 H §746). The runner says no group-ended (§764). +// an end is said once, on . (design 47 H §746). The runner says no group-ended (§764). func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error { for _, i := range asked { if err := h.say(seat.EventWaitHeld, p, &p.Steps[i]); err != nil { @@ -489,7 +489,7 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error subject, ev := stepEvent(event, p, *step, at) return h.Say(subject, ev) case seat.EventPipelineEnded: - return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), pipelineEnded(p)) + return h.Say(runnerclient.RevisionSubject(event, p.Branch, subjectRevision(p)), pipelineEnded(p)) default: w := seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Revision: p.Revision, Source: p.Source, Run: p.Run, Step: seat.WaitStepApproval, On: seat.WaitOnPerson, Since: at} @@ -497,7 +497,7 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error m, _ := MoverOf(p, step.Module) w.Module, w.Commit, w.Fingerprint, w.Step, w.Ask = m.Module, m.Commit, m.Fingerprint, step.Step, step.Ask } - return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), w) + return h.Say(runnerclient.RevisionSubject(event, p.Branch, subjectRevision(p)), w) } }