diff --git a/modules/mesh-delivery-builder/builds/builds.go b/modules/mesh-delivery-builder/builds/builds.go index 8981c7dc..04c3c6cd 100644 --- a/modules/mesh-delivery-builder/builds/builds.go +++ b/modules/mesh-delivery-builder/builds/builds.go @@ -242,6 +242,11 @@ type BuilderBuild struct { // Imports are, on a done key, the mesh packages the built programs import, as the build module answered them // (BuildAnswer.Imports; novox/hq ADR 0593 §2): the builder carries them onto the build's end (BuildLogEntry.Imports). Imports []string `json:"imports,omitempty"` + // Packages are, on a done key only, the mesh packages' versions its Go build was built with, by module path, the ones + // the registry gave a package the entry named with none included (H-A-207): the build step's end reports them, so the + // runner asks the steps that waited for the pins with what the build was built against. A reader that does not know + // the field reads the build as reporting none. + Packages map[string]string `json:"packages,omitempty"` // Needs are, on a done key only, the reader capabilities (`form:`) the build's reference forms need of a // node-engine, beside the ones its manifest needs (novox/hq ADR 0618 §1). A reader that does not know the field // reads the build as needing none. diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index f4662d49..86d65f6c 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -843,6 +843,7 @@ type lane struct { tcVersion string // and the version it ran kept string // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty outOfStep []string // the generated files its build module's compare found out of step, passed or failed + pins map[string]string // the mesh packages' versions its Go ask was built with (H-A-207) err error } @@ -992,7 +993,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module used = nil } done := builds.BuilderBuild{State: builds.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used, - Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts)} // the forms it is written in (ADR 0618 §1) + Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts), Packages: pinsOf(lanes)} // the forms it is written in (ADR 0618 §1) for _, l := range lanes { if l.kept != "" { done.KeptOn = l.node // design 47 G §671: the build node that keeps the build @@ -1154,6 +1155,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes for _, fb := range req.Builds { delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465) } + l.pins = maps.Clone(ask.Packages) // kept with the done key, so its step's end reports them (H-A-207) } if id := asked.resume(language); id != "" { ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it @@ -1336,6 +1338,20 @@ func (b *Builder) takeOutOfStep(key string) []string { // importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593 // §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that // kept one reports only what it built. +// pinsOf is the mesh packages' versions the lanes were built with (H-A-207), nil when none was pinned. +func pinsOf(lanes []*lane) map[string]string { + var out map[string]string + for _, l := range lanes { + for p, v := range l.pins { + if out == nil { + out = map[string]string{} + } + out[p] = v + } + } + return out +} + func importsOf(lanes []*lane) []string { seen := map[string]bool{} for _, l := range lanes { diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index b19cbb25..efb91d8f 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -42,7 +42,18 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui b.askedRevision(run) var failed []string for _, e := range run.Entries { - outcome, said := b.buildEntry(ctx, e) + var pins map[string]string + outcome, said := b.buildEntry(ctx, e, &pins) + // The end reports the pins each entry was built with, so the runner asks its held test steps with them (H-A-207). + for path, version := range pins { + if version == "" { + continue + } + if end.Packages == nil { + end.Packages = map[string]string{} + } + end.Packages[path] = version + } switch outcome { case meshbuilds.StepEndedOutcomeCouldNotRun: return finish(outcome, said) @@ -77,8 +88,9 @@ func (b *Builder) askedRevision(run meshbuilds.StepRun) { } } -// buildEntry builds one entry at its fingerprint and waits for its end. -func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outcome, said string) { +// buildEntry builds one entry at its fingerprint and waits for its end; pins is set, when it is done, to the pins its +// done key keeps it was built with, the registry's included (H-A-207). +func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) { entry, err := b.StepEntry(ctx, e.Fingerprint) if err != nil { return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry %s at %s cannot be read: %v", e.Module, e.Fingerprint, err) @@ -100,6 +112,8 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc defer b.dropEnd(key, ended) if done, failedWords, found := b.keyEnd(ctx, key); found { if done { + // A key done before reports the pins it was built with, kept with it, never the ones a publish gives now. + *pins = b.keptPins(ctx, key) return b.passedWithItsVersion(ctx, entry) } return meshbuilds.StepEndedOutcomeFailed, failedWords @@ -110,6 +124,7 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc select { case log := <-ended: if log.Outcome == builds.BuildDone { + *pins = b.keptPins(ctx, key) return b.passedWithItsVersion(ctx, entry) } return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error) @@ -158,6 +173,20 @@ func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.Ca return meshbuilds.StepEndedOutcomePassed, "" } +// keptPins are the pins a done key keeps it was built with (H-A-207); none when it keeps none or cannot be read, so the +// runner ends the steps that waited for them could not run, loud. +func (b *Builder) keptPins(ctx context.Context, key string) map[string]string { + raw, _, ok, err := b.Claimer.Store.Get(ctx, key) + if err != nil || !ok { + return nil + } + v, err := builds.DecodeBuilderBuild(raw) + if err != nil || v.State != builds.BuildDone { + return nil + } + return v.Packages +} + // keyEnd reads the build key: whether it is done, and its error when it failed for good; found only for a key that has ended. func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed string, found bool) { raw, _, ok, err := b.Claimer.Store.Get(ctx, key) diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index ef35acf7..4b63b05a 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -190,6 +190,32 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) { !reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) { t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks) } + // The end reports the pins the build was built with, the one the publish gave included (H-A-207). + if !reflect.DeepEqual(end.Packages, map[string]string{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) { + t.Errorf("the end reports the pins %v; want the ones it was built with", end.Packages) + } + // A done key reports the pins it keeps it was built with, never the newer one a publish gives now (H-A-207). + r.b.PackagePublished(unpinnedX, "v0.1.5") + again := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}) + if again.Outcome != meshbuilds.StepEndedOutcomePassed || again.Packages[unpinnedX] != "v0.1.3" { + t.Errorf("the done key's end = %+v; want passed reporting x at v0.1.3, as built", again) + } + // A done key that keeps no pins reports none, so the runner ends the waiting steps loud. + v, err := builds.DecodeBuilderBuild(r.st.vals[r.key]) + if err != nil { + t.Fatal(err) + } + v.Packages = nil + raw, err := builds.EncodeBuilderBuild(v) + if err != nil { + t.Fatal(err) + } + r.st.vals[r.key] = raw + if bare := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); len(bare.Packages) != 0 { + t.Errorf("a done key keeping no pins reported %v; want none", bare.Packages) + } } func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) { diff --git a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go index dc45a781..179a6890 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go @@ -116,9 +116,11 @@ type StepEnded struct { // the node the builder ran it on, as in the subject Node string `json:"node"` // never passed for a build that built nothing - Outcome string `json:"outcome"` - Pipeline string `json:"pipeline"` - Rerun bool `json:"rerun,omitempty"` + Outcome string `json:"outcome"` + // the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207) + Packages map[string]string `json:"packages,omitempty"` + Pipeline string `json:"pipeline"` + Rerun bool `json:"rerun,omitempty"` // why it failed or could not run: the entry not built, the part not heard Said string `json:"said,omitempty"` Started *time.Time `json:"started,omitempty"` diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go index 7ed4f8fe..213116c3 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go @@ -905,8 +905,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go index d2136e78..391f8658 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go @@ -903,8 +903,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go index 47ca8f1d..bd89858a 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go @@ -903,8 +903,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/parts/build-step.json b/modules/mesh-delivery-contracts/parts/build-step.json index fcaebd62..0390aecb 100644 --- a/modules/mesh-delivery-contracts/parts/build-step.json +++ b/modules/mesh-delivery-contracts/parts/build-step.json @@ -1521,7 +1521,11 @@ }, "revision": { "type": "integer", - "description": "the source revision of the watched branch the pipeline runs for; absent for a run on request" + "description": "the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request" + }, + "source": { + "type": "integer", + "description": "the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request" }, "fingerprint": { "type": "string", diff --git a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json index 2caddbdf..e100e48c 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json @@ -367,6 +367,13 @@ "$ref": "#/defs/BuildEntry" }, "description": "each entry the step built, by module and fingerprint" + }, + "packages": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207)" } }, "required": [ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts index 68e2c00e..bf8e68c7 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts @@ -91,6 +91,8 @@ export interface StepEnded { "node": string; /** never passed for a build that built nothing */ "outcome": "passed" | "failed" | "nothing to run" | "could not run"; + /** the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207) */ + "packages"?: Record; "pipeline": string; "rerun"?: boolean; /** why it failed or could not run: the entry not built, the part not heard */ diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go index 26a92e9c..f0d87ef5 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go @@ -71,3 +71,28 @@ func TestPlaceLostRunsReadsEachCandidateNewest(t *testing.T) { t.Errorf("asked %+v, want the candidate's group alone", asked[1]) } } + +// H-A-213 (2), (4): a catalog-revision heard is handed whole to the start of a revision's pipeline; a start that fails is +// an error, so the bus delivers the event again, and a body that is no event is ended. +func TestARevisionHeardIsHandedToItsStart(t *testing.T) { + var got []catalog.CatalogRevisionEvent + fail := false + hear := revisedHear(func(ev catalog.CatalogRevisionEvent) (int, error) { + got = append(got, ev) + if fail { + return 0, errors.New("not read") + } + return 1, nil + }, func(string, ...any) {}) + data := []byte(`{"v":1,"kind":"trunk","branch":"main","revision":10,"source":41,"call-kind":"release","call":"release of mesh-sdk 0.1.17"}`) + if err := hear(t.Context(), data); err != nil || len(got) != 1 || got[0].Revision != 10 || got[0].CallKind != "release" || got[0].Source != 41 { + t.Fatalf("heard %+v (%v), want the release revision 10", got, err) + } + fail = true + if err := hear(t.Context(), data); err == nil { + t.Error("a start that failed is not an error, so the event is never heard again") + } + if err := hear(t.Context(), []byte("no event")); err != nil { + t.Errorf("a body that is no event = %v, want it ended", err) + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 9488c743..85891217 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -98,6 +98,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro var advancer atomic.Pointer[func(string)] var placer atomic.Pointer[func(catalog.CatalogRevisionEvent) error] var registrar atomic.Pointer[func(catalog.Registration)] + var reviser atomic.Pointer[func(catalog.CatalogRevisionEvent) (int, error)] rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) { if f := resumer.Load(); f != nil { (*f)(branch) @@ -115,6 +116,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if f := registrar.Load(); f != nil { (*f)(reg) } + }, func(ev catalog.CatalogRevisionEvent) (int, error) { + if f := reviser.Load(); f != nil { + return (*f)(ev) + } + return 0, fmt.Errorf("the revision %d of %s is heard before the runner can start its pipeline", ev.Revision, ev.Branch) }, logger.Printf) if err != nil { return err @@ -142,7 +148,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } logger.Printf("widens: each read bound %s, %d catalog revisions kept", readBound, revisionsKept) - hearer.Widens = widenerOf(calls.Call, readBound, revisionsKept, logger.Printf).Widens + widener := widenerOf(calls.Call, readBound, revisionsKept, logger.Printf) + hearer.Widens = widener.Widens hearer.LogLast = stepLog{ctx: ctx, js: js, module: cred.Module}.Last hearer.Where = routerRecordOf(cred.Module, calls.Call, calls.Ask) hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf} @@ -154,6 +161,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} hearer.AskSteps = starter.AskSteps hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } + hearer.AskPinHeld = func(key, module string, pins map[string]string) { + if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil { + logger.Printf("the steps of %s's %s that waited for its pins are not asked: %v", key, module, err) + } + } hearer.Release = func(key string) { if _, err := starter.Release(hearer.Store, key, rv.Catalog.View); err != nil { logger.Printf("the movers of %s waiting for a build are not placed: %v", key, err) @@ -197,6 +209,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } registrar.Store(®ister) + // A release's, a take-in again's or a recompute's revision starts one pipeline, its revision read by its number + // (H-A-213 (2), (4)). + revise := func(ev catalog.CatalogRevisionEvent) (int, error) { return starter.Revised(ev, widener.Revision) } + reviser.Store(&revise) starter.Store = store // The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in // the state queue, placed at the start from it and on every end. diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index 6b319931..8e9fb2d3 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -101,7 +101,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { } } answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs}) - answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries}) + answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 100, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries}) answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) // The router of the operator's asks holds the seat operator-channel; its record of each ask is in its state asks. const router = "operator-router" @@ -218,7 +218,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { } take := func(source int64, movers ...string) { t.Helper() - ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: 1, Call: fmt.Sprintf("%016x", source)} + ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: source, Call: fmt.Sprintf("%016x", source)} for _, m := range movers { ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go index 5be6db24..1f9365ce 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "github.com/nats-io/nats.go/jetstream" @@ -11,10 +12,10 @@ import ( "mesh-delivery-pipeline/internal/pipeline" ) -// hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one -// pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is -// written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard -// after a stop. +// hearPublished hears each package-published (design 47 H §746, §747): in the running pipeline of the revision that +// published it, the movers waiting for it are placed; it starts no pipeline (a release's revision does, H-A-213 (2)). The +// message is acknowledged once that is written, on a goroutine of its own, so a slow write holds no other event; the +// durable consumer delivers a publish heard after a stop. func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error { return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) { var ev builds.PackagePublished @@ -24,18 +25,38 @@ func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, l return } go func() { - n, err := starter.Published(ev, v.View) - if err != nil { - logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err) + if err := starter.Published(ev, v.View); err != nil { + logf("the publish of %s %s on %s: %v; heard again", ev.Artifact, ev.Version, ev.Branch, err) _ = m.Nak() return } - logf("the publish of %s %s on %s started %d pipelines", ev.Artifact, ev.Version, ev.Branch, n) + logf("the publish of %s %s on %s is heard", ev.Artifact, ev.Version, ev.Branch) _ = m.Ack() }() }) } +// revisedHear is the handler of catalog-revision that starts the pipeline of a revision whose call kind is not a +// take-in's (design 47 H-A-212, H-A-213 (2), H-A-217): every other revision is left to the views that hear it. An error +// has the event heard again; a second start of its key writes nothing. +func revisedHear(revised func(ev catalog.CatalogRevisionEvent) (int, error), logf func(string, ...any)) func(context.Context, []byte) error { + return func(_ context.Context, data []byte) error { + var ev catalog.CatalogRevisionEvent + if err := json.Unmarshal(data, &ev); err != nil { + logf("a catalog-revision is not read: %v", err) + return nil + } + n, err := revised(ev) + if err != nil { + return err + } + if n > 0 { + logf("the revision %d of %s (%s, %s) started %d pipeline", ev.Revision, ev.Branch, ev.CallKind, ev.Call, n) + } + return nil + } +} + // hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged // once it is written, and heard again when a write failed, so the key already written starts nothing twice. func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index d6915534..c4aa18c1 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -128,7 +128,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) if bs.Shared != nil { shared = bs.Shared(m.Data()) } - _, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared) + var pins map[string]string + if bs.Pins != nil { + pins = bs.Pins(m.Data()) + } + _, err = h.OnStepEndedPinned(bs, m.Subject(), ev, shared, pins) switch { case errors.Is(err, pipeline.ErrRefused): logf("%v", err) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index 95ed9670..e39fabf9 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -37,7 +37,7 @@ import ( // ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step // that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a // run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed; -// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended, +// a run that keeps saying its progress past its bound is not ended, // and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the // same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its // other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and @@ -88,7 +88,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil { + askStream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}) + if err != nil { t.Fatal(err) } @@ -116,6 +117,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) entries := []catalog.CatalogEntry{ {Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app", + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25"}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, {Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}}, {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, @@ -325,12 +327,23 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { // One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its // ask is taken (A-469b), its holder present and nothing failing it (A-466y). - const p3 = "main.3" + // A pipeline is keyed by its catalog revision (H-A-213 (4)): source 3's take-in of revision 1 is main.1. + const p3 = "main.1" takeIn("app", "c1", "src2:aa", 3, 1) placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747) passBuild(p3, "app", "c1", "src2:aa") unitRun, unit := runOfStep(p3, "app:program", "unit") integRun, integ := runOfStep(p3, "app:program", "integration") + // A test ask fetches with the entry's pins, as the builder gives them the build (H-A-203). + last, err := askStream.GetLastMsgForSubject(ctx, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner") + if err != nil { + t.Fatal(err) + } + var testAsk nodebuildgo.StepAsk + if err := json.Unmarshal(last.Data, &testAsk); err != nil || testAsk.Step == nil || + testAsk.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25" || testAsk.Step.Revision != 1 || testAsk.Step.Source != 3 { + t.Fatalf("the last test ask = %s (%v), want its fetch to carry the entry's pin, revision 1 and source 3 (H-A-219)", last.Data, err) + } if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 { t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers) } @@ -471,39 +484,20 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { // A second revision with the same digests asks its own runs, so a failed test runs again (§724). revision.Store(2) publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4}) - const p4, p5 = "main.4", "main.5" + const p4 = "main.2" takeIn("app", "c1", "src2:aa", 4, 2) placed(p4) run4, r4 := runOfStep(p4, "app:program", "integration") if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask { t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask) } - // A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads - // the run, the first is cancelled and leaves it, and the run's ask is not dropped. - takeIn("app", "c1", "src2:aa", 5, 2) - placed(p5) - waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 }) - if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" { - t.Fatalf("cancel: %s", e) - } - waitFor(t, ctx, logs, func() bool { - r, _ := runOf(run4) - return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5 - }) - if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask { - t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r) - } - mu.Lock() - dropped := append([]string(nil), drops...) - mu.Unlock() - if len(dropped) != 0 { - t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String()) - } + // A replaced reader leaving the ask to the other readers of another pipeline (§726) is the unit tests': one revision is + // one pipeline (H-A-213 (4)). // A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720). taken(r4.Ask) waitFor(t, ctx, logs, func() bool { - return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun && - step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun + return step(read(p4), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun && + step(read(p4), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun }) if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") { t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r) @@ -516,8 +510,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { // A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not // asked (§535, §647; A-470g, A-470h). - const web = "main.7" - takeIn("web", "w1", "src2:ww", 7, 2) + revision.Store(3) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 7}) + const web = "main.3" + takeIn("web", "w1", "src2:ww", 7, 3) waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 }) if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" { t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e) @@ -568,10 +564,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { t.Fatal(err) } } - revision.Store(3) - publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5}) - const p6 = "main.6" - takeIn("app", "c1", "src2:aa", 6, 3) + revision.Store(4) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 4, Kind: "trunk", Branch: "main", Source: 6}) + const p6 = "main.4" + takeIn("app", "c1", "src2:aa", 6, 4) waitFor(t, ctx, logs, func() bool { s := step(read(p6), "app:program", "unit") return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib") diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go index 3a620f52..440e2c3d 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -98,7 +98,7 @@ func (h *holderView) Holders(seatName string) ([]string, bool) { // after, then start hub once. func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State, call meshcatalogclient.Call, onReady, onAdvance func(branch string), candidate func(ctx context.Context, ev catalog.CatalogRevisionEvent) error, - registered func(reg catalog.Registration), logf func(string, ...any)) (*runnerViews, error) { + registered func(reg catalog.Registration), revised func(ev catalog.CatalogRevisionEvent) (int, error), logf func(string, ...any)) (*runnerViews, error) { findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf) if err != nil { return nil, err @@ -153,6 +153,11 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr if err := meshcatalogclient.WireCandidatesProcess(reader, candidate); err != nil { return nil, err } + // A trunk revision of a release, a take-in again or a recompute starts one pipeline of its own, read by its number + // (H-A-212, H-A-213 (2), H-A-217), heard on the same event's one subscription. + if err := reader.AdoptEvent(catalog.Seat, catalog.EventCatalogRevision, revisedHear(revised, logf)); err != nil { + return nil, err + } reader.Start(ctx, js, cred, status) return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings, Holders: &holderView{reader: reader, catalog: cv, logf: logf}}, nil } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go index 211738a7..12fef5af 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go @@ -166,7 +166,8 @@ func catalogRevision(call func(string, any) (json.RawMessage, error), kept *revi if h, ok := kept.get(branch, a.Revision.Revision); ok { return h, nil } - half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{}} + half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{}, + Call: a.Revision.Call, Movers: a.Revision.Movers} in.Revision = a.Revision.Revision // the pages of one revision, though a newer one is said between them } for _, e := range a.Entries { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go index fd32e2f8..60a7d333 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go @@ -231,11 +231,11 @@ func (a *Ager) newest(p seat.Pipeline) (bool, error) { if err != nil { return false, fmt.Errorf("the pipelines of %s are not listed: %w", p.Branch, err) } - // Newest for one of its movers: no later pipeline of the branch moves it. + // Newest for one of its movers: no pipeline of a later revision of the branch moves it (H-A-213 (4)). for _, m := range p.Movers { newer := false for _, k := range pkeys { - if q, _, err := a.Pipelines.Get(k); err == nil && q.Source > p.Source && Moves(q, m.Module) { + if q, _, err := a.Pipelines.Get(k); err == nil && keyRevision(Begin{Revision: q.Revision, Source: q.Source}) > keyRevision(Begin{Revision: p.Revision, Source: p.Source}) && Moves(q, m.Module) { newer = true break } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go index 2270238e..47da9a57 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go @@ -142,3 +142,23 @@ func TestOneStepEndAgesAtMostTwentyEntries(t *testing.T) { t.Fatalf("%d outputs deleted, sequence %d; want 20 and 20", len(f.outputs), f.aged["7d"]) } } + +// H-A-213 (4): a branch's newest pipeline for a mover is the one of the highest revision number, never of the highest source: +// a release revision of the same source is newer than its take-in's. +func TestTheNewestPipelineIsOfTheHighestRevision(t *testing.T) { + f := newAgedFixture() + for key, p := range map[string]seat.Pipeline{ + "main.41": {Pipeline: "main.41", Branch: "main", Source: 41, Revision: 41, Movers: []seat.Mover{{Module: "app"}}}, + "main.42": {Pipeline: "main.42", Branch: "main", Source: 41, Revision: 42, Movers: []seat.Mover{{Module: "app"}}}, + } { + f.pipes.memStore[key] = struct { + p seat.Pipeline + rev uint64 + }{p, 1} + } + for key, want := range map[string]bool{"main.41": false, "main.42": true} { + if got, err := f.ager.newest(f.pipes.memStore[key].p); got != want || err != nil { + t.Errorf("%s newest = %v (%v), want %v", key, got, err, want) + } + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go index abf01a7c..ee636c4a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go @@ -113,9 +113,19 @@ func TestAReleasedVersionIsTheMoversVersion(t *testing.T) { other := store[pkey] other.p.Pipeline, other.p.Branch, other.p.Movers = "beta.5", "beta", append([]seat.Mover(nil), other.p.Movers...) store["beta.5"] = other + // A pipeline is keyed by its revision (H-A-213 (4)): main.8, a later revision of source 5, is written too; main.6, of + // source 6, is not. + for key, source := range map[string]int64{"main.8": 5, "main.6": 6} { + again := store[pkey] + again.p.Pipeline, again.p.Source, again.p.Movers = key, source, append([]seat.Mover(nil), again.p.Movers...) + store[key] = again + } ev := catalog.VersionReleasedEvent{Module: "app", Version: "0.4.1", Source: 5} - if n, err := h.Released(ev, "main"); n != 1 || err != nil { - t.Fatalf("released wrote %d (%v), want app's mover of main.5", n, err) + if n, err := h.Released(ev, "main"); n != 2 || err != nil { + t.Fatalf("released wrote %d (%v), want app's mover of main.5 and main.8", n, err) + } + if m, _ := MoverOf(store["main.6"].p, "app"); m.Version != "" { + t.Errorf("main.6's mover = %+v, want no version: another source", m) } if m, _ := MoverOf(store[pkey].p, "app"); m.Version != "0.4.1" { t.Errorf("app's mover = %+v, want version 0.4.1", m) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go index 0065aabd..95f5536a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go @@ -70,14 +70,14 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if n, err := s.TakenIn(ev, view); n != 1 || err != nil { t.Fatalf("the take-in started %d (%v)", n, err) } - p, _, _ := mem.Get("main.4") + p, _, _ := mem.Get("main.7") if !placed(p, "mesh-sdk") || !placed(p, "builder") || placed(p, "app") || placed(p, "tool") { t.Fatalf("placed at the start: %+v, want mesh-sdk and builder only", p.Steps) } - if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); n != 0 || err != nil { - t.Fatalf("the publish started %d (%v), want none", n, err) + if err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); err != nil { + t.Fatalf("the publish is not heard: %v", err) } - p, _, _ = mem.Get("main.4") + p, _, _ = mem.Get("main.7") if sdk, _ := MoverOf(p, "mesh-sdk"); !placed(p, "app") || placed(p, "tool") || len(mem) != 1 || !slices.Contains(sdk.Published, "mesh-sdk:go") { t.Fatalf("after mesh-sdk's publish: %d pipelines, app placed %v, tool placed %v, mesh-sdk %+v", len(mem), placed(p, "app"), placed(p, "tool"), sdk) } @@ -87,22 +87,22 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { b := stepsOf(p, "builder")[0] bs := stepSeatOf(t, builds.Seat) end := func(outcome string) StepEnded { - e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.4","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`)) + e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.7","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`)) if err != nil { t.Fatal(err) } return e } - before := mem["main.4"] + before := mem["main.7"] if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("passed")); err != nil { t.Fatal(err) } - if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning { + if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning { t.Errorf("after builder's build passed, tool's steps = %+v, want placed and asked", stepsOf(p, "tool")) } // The same pipeline, builder's build failed: tool is placed with every step not asked, each saying its own step-ended // naming the dependency (H-A-101 (2)). - mem["main.4"] = before + mem["main.7"] = before var notAsked []seat.StepEvent s.Say = func(subject string, body any) error { if ev, ok := body.(seat.StepEvent); ok && strings.Contains(subject, ".step-ended.tool.") { @@ -113,7 +113,7 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("failed")); err != nil { t.Fatal(err) } - if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked { + if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked { t.Errorf("after builder's build failed, tool's steps = %+v, want placed not asked", stepsOf(p, "tool")) } if len(notAsked) == 0 || len(notAsked) != len(slices.DeleteFunc(stepsOf(p, "tool"), func(st seat.StepResult) bool { return st.Outcome != seat.StepResultOutcomeNotAsked })) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go index 455f8fa5..bd99d2c8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go @@ -51,7 +51,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing. if n, err := s.TakenIn(taken, view); n != 1 || err != nil { t.Fatalf("the take-in started %d (%v)", n, err) } - p, _, _ := store.Get("main.947") + p, _, _ := store.Get("main.357") if p.Revision != 357 || len(p.Steps) != 0 || len(asks.asks) != 0 { t.Fatalf("a take-in of revision 357 with the view at 356 started %+v with %d asks, want revision 357 and no step", p, len(asks.asks)) } @@ -60,7 +60,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing. } viewRevision = 357 n, err := s.Waiting(store.memStore, "main", view) - p, _, _ = store.Get("main.947") + p, _, _ = store.Get("main.357") if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Revision != 357 { t.Errorf("the view reaching 357 placed %d (%v): %d steps, %d asks at revision %d", n, err, len(p.Steps), len(asks.asks), p.Revision) } @@ -87,7 +87,7 @@ func TestAViewThatReachesTheRevisionDuringTheStartsPlacesTheHeldBackPipelines(t if n, err := s.TakenIn(taken, view); n != 1 || err != nil { t.Fatalf("started %d (%v)", n, err) } - if p, _, _ := store.Get("main.947"); len(p.Steps) == 0 || len(asks.asks) == 0 { + if p, _, _ := store.Get("main.357"); len(p.Steps) == 0 || len(asks.asks) == 0 { t.Errorf("the pipeline has %d steps and %d asks after the view reached 357 during the start, want them placed", len(p.Steps), len(asks.asks)) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index af5758d6..a7dbf327 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -3,7 +3,10 @@ package pipeline import ( "crypto/sha256" "encoding/hex" + "errors" "fmt" + "maps" + "slices" "sort" "strconv" "strings" @@ -140,6 +143,13 @@ func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[ steps = append(steps, s) continue } + // A Go step is asked only once every pin of its entry has a version (H-A-206): until then it waits on the + // publish, as the build does (H-A-165), and the publish re-pins the entry and moves the mover (H-A-204). + if path := unpinnedPackage(entry); path != "" && language == sdkstep.AskGo { + s.Seat, s.Outcome, s.Started, s.Why = bs, seat.StepResultOutcomeWaiting, &now, PinWait+path + steps = append(steps, s) + continue + } s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now s.Ask = AskID(startOf(p, now), p.Pipeline, m.Module, a.Ref, step) body := stepAsk(p, m, entry, a, language, step, s.Ask, asker, now) @@ -227,6 +237,132 @@ type Asked struct { Commit string } +// PinWait opens the why of a Go step held because its entry names a package with no version yet (H-A-206), the build's +// own wait form (H-A-165); the package's path follows. +const PinWait = "waits on the publish of " + +// unpinnedPackage is the first package, in path order, the entry names with no version, empty when every one has one. +func unpinnedPackage(e catalog.CatalogEntry) string { + for _, path := range slices.Sorted(maps.Keys(e.Packages)) { + if e.Packages[path] == "" { + return path + } + } + return "" +} + +// pinHeld says whether a step waits for its entry's pins (H-A-206). +func pinHeld(s seat.StepResult) bool { + return s.Outcome == seat.StepResultOutcomeWaiting && strings.HasPrefix(s.Why, PinWait) +} + +// endPinHeld ends each of build's mover's steps waiting for its entry's pins (H-A-206, H-A-207): as the build ended when +// it did not pass, so no step waits on a publish the build no longer waits for; could not run, loud, when the build passed +// and its end names no pins to run them with. It answers the steps it ended. +func endPinHeld(p *seat.Pipeline, build seat.StepResult, pins map[string]string, now time.Time) []int { + outcome, why := build.Outcome, "the build ended "+build.Outcome+" while this step waited: %s; "+build.Why + if stepPassed(build) { + if len(pins) > 0 { + return nil + } + outcome, why = seat.StepResultOutcomeCouldNotRun, "the build passed, but its end names no pins to run this step with (H-A-207); it %s" + } + var ended []int + for i := range p.Steps { + st := &p.Steps[i] + if st.Module != build.Module || !pinHeld(*st) { + continue + } + st.Outcome, st.Why, st.Ended = outcome, fmt.Sprintf(why, st.Why), &now + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) + ended = append(ended, i) + } + return ended +} + +// readRevision is revision's read, an error when none is given. +func readRevision(revision func(branch string, revision int64) (Half, error), branch string, n int64) (Half, error) { + if revision == nil { + return Half{}, errors.New("the runner reads no catalog revision") + } + return revision(branch, n) +} + +// heldOn says whether module has a step of p waiting for its entry's pins. +func heldOn(p seat.Pipeline, module string) bool { + return slices.ContainsFunc(p.Steps, func(st seat.StepResult) bool { return st.Module == module && pinHeld(st) }) +} + +// AskPinHeld asks the steps of pipeline key's mover module that waited for its entry's pins (H-A-206) with the pins its +// build passed with (H-A-207): each running under its own ask, its fetch the entry's as Place builds it with those pins. +// The entry is read at the pipeline's own catalog revision through revision, which keeps what it read (H-A-205), never +// the branch's newest: a revision that cannot be read, or whose entry is of another fingerprint than the mover's, or that +// names no artifact of the step, ends the step could not run, saying so. It answers how many it asked. +func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]string, revision func(branch string, revision int64) (Half, error)) (int, error) { + for { + p, rev, err := store.Get(key) + if err != nil { + return 0, fmt.Errorf("the pipeline %s is not read: %w", key, err) + } + m, moves := MoverOf(p, module) + if !moves || !heldOn(p, module) { + return 0, nil + } + var entry catalog.CatalogEntry + unread := "" + switch half, err := readRevision(revision, p.Branch, p.Revision); { + case err != nil: + unread = fmt.Sprintf("revision %d of %s, the pipeline's, cannot be read for its entry: %v", p.Revision, p.Branch, err) + case half.Entries[module].Fp != m.Fingerprint: + unread = fmt.Sprintf("revision %d of %s holds %s at %q, not the mover's %s", p.Revision, p.Branch, module, half.Entries[module].Fp, m.Fingerprint) + default: + entry = half.Entries[module] + } + entry.Packages = pins + now := s.now() + var asks []Asked + var ended []int + for i := range p.Steps { + st := &p.Steps[i] + if st.Module != module || !pinHeld(*st) { + continue + } + j := slices.IndexFunc(entry.Artifacts, func(a catalog.Artifact) bool { return a.Ref == st.Artifact }) + language, err := sdkstep.AskLanguageOfSeat(st.Seat) + if unread != "" || j < 0 || err != nil { + why := unread + if why == "" { + why = fmt.Sprintf("revision %d of %s names no artifact %s of %s to ask it of", p.Revision, p.Branch, st.Artifact, module) + } + st.Outcome, st.Ended = seat.StepResultOutcomeCouldNotRun, &now + st.Why = "its build passed with its pins, but " + why + " (H-A-207, H-A-205)" + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) + ended = append(ended, i) + continue + } + a := entry.Artifacts[j] + st.Outcome, st.Why, st.Started = seat.StepResultOutcomeRunning, "", &now + st.Ask = AskID(startOf(p, now), p.Pipeline, module, a.Ref, st.Step) + ask := Asked{Seat: st.Seat, ID: st.Ask, Ask: stepAsk(p, m, entry, a, language, st.Step, st.Ask, s.Name, now), Toolchain: a.Language, Commit: m.Commit} + if a.Package != nil { + ask.Package = a.Package.Name + } + asks = append(asks, ask) + } + if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return 0, fmt.Errorf("the pipeline %s is not written: %w", key, err) + } + s.logf("%s: %s's build passed with its pins: %d steps that waited for them are asked, %d could not run (H-A-207)", key, module, len(asks), len(ended)) + s.sayNotAsked(p, ended) + if s.Steps == nil || len(asks) == 0 { + return 0, nil + } + return len(asks), s.askSteps(key, asks) + } +} + func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk { // A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the // artifact's named paths in it when the entry has them. @@ -234,9 +370,21 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar if paths := artifactPaths(e, a.Ref); len(paths) > 0 { fetch.Paths = paths } + // The entry's pins, the same the builder gives the build (H-A-203, H-A-183): a test runs against what the build was + // built against, and the Go build module refuses a requirement of another repository with no pin (H-A-192). A + // package the entry names with no pin yet is left out, as the builder pins it only from its publish. + for path, version := range e.Packages { + if version == "" { + continue + } + if fetch.Packages == nil { + fetch.Packages = map[string]string{} + } + fetch.Packages[path] = version + } return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now, Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: m.Commit, Fetch: fetch, Fingerprint: m.Fingerprint, - Module: m.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}} + Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}} } // askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index d54f44a8..7f46c658 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -2,6 +2,8 @@ package pipeline import ( "encoding/json" + "fmt" + "reflect" "strconv" "strings" "testing" @@ -44,7 +46,7 @@ func placeFixture() (catalog.Registration, map[string]catalog.CatalogEntry) { Artifacts: map[string]catalog.StepStatuses{"tools": {Lint: "bypass"}}, }} return reg, map[string]catalog.CatalogEntry{ - "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, + "app": {Module: "app", Path: "modules/app", Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25"}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, {Ref: "app:web", Language: "rust"}}, Inputs: &catalog.EntryInputs{Paths: map[string][]string{"app:program": {"cmd/app"}}}}, "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, } @@ -100,10 +102,131 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { a := asks.asks[1].(nodebuildgo.StepAsk) if asks.at[1] != "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner" || a.Kind != "step" || a.Asker != "the-runner" || a.ID != p.Steps[1].Ask || asks.ids[1] != a.ID || a.Step.Commit != "c1" || a.Step.Fingerprint != "src2:aa" || a.Step.Pipeline != "main.4" || a.Step.Artifact != "app:program" || - a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Revision != 4 || + a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Source != 4 || len(a.Step.Fetch.Paths) != 1 || a.Step.Fetch.Paths[0] != "cmd/app" { t.Errorf("the ask = %+v at %s", a, asks.at[1]) } + // Every step ask fetches with the entry's pins, as the builder gives them the build (H-A-203, H-A-192). + for _, x := range asks.asks[1:] { + if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && (len(b.Step.Fetch.Packages) != 1 || b.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25") { + t.Errorf("the %s ask of %s fetches the pins %v, want the entry's", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages) + } + } + // An entry naming a package with no version yet asks no Go step: each waits on its publish, as the build does, said in + // the journal (H-A-206); the build is still asked. + waiting := map[string]catalog.CatalogEntry{} + for k, e := range entries { + waiting[k] = e + } + held := waiting["app"] + held.Packages = map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": ""} + held.Fp = "src2:ab" + waiting["app"] = held + var journal []string + heldAsks := &stepAsks{} + hs := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: heldAsks, Name: "the-runner", + Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }, + Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }} + if ok, err := hs.Start(Begin{Branch: "main", Source: 5, Revision: 3, Entries: waiting, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + for _, x := range heldAsks.asks { + if b, ok := x.(nodebuildgo.StepAsk); ok { + t.Errorf("the %s of %s is asked with a package unpinned", b.Step.Step, b.Step.Artifact) + } + } + var hp seat.Pipeline + if err := json.Unmarshal(store["main.3"], &hp); err != nil { + t.Fatal(err) + } + if st := hp.Steps[1]; st.Step != "lint" || st.Outcome != seat.StepResultOutcomeWaiting || st.Ask != "" || st.Why != "waits on the publish of git.novox.be/novox/x/go" { + t.Errorf("the held lint step = %+v; want it waiting on the publish", st) + } + // The build ending without passing past its bound ends each held step as it ended (H-A-206). + failed := hp.Steps[0] + failed.Outcome, failed.Why = seat.StepResultOutcomeFailed, "package git.novox.be/novox/x/go has no released version" + failedP := hp + failedP.Steps = append([]seat.StepResult(nil), hp.Steps...) + if ended := endPinHeld(&failedP, failed, nil, time.Now()); len(ended) == 0 || failedP.Steps[1].Outcome != seat.StepResultOutcomeFailed || failedP.Steps[1].Ended == nil { + t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", failedP.Steps[1], len(ended)) + } + // A build passed whose end names no pins ends them could not run, loud (H-A-207). + passed := hp.Steps[0] + passed.Outcome = seat.StepResultOutcomePassed + bare := hp + bare.Steps = append([]seat.StepResult(nil), hp.Steps...) + if ended := endPinHeld(&bare, passed, nil, time.Now()); len(ended) == 0 || bare.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || + !strings.Contains(bare.Steps[1].Why, "names no pins") { + t.Errorf("after a build passed naming no pins the held lint step = %+v; want could not run naming it", bare.Steps[1]) + } + // A build passed naming its pins asks the held steps with them: its end heard, the runner asks each with the pins the + // build was built with (H-A-207). + pinsBuilt := map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": "v0.1.4"} + ms := memStore{} + ms["main.5"] = struct { + p seat.Pipeline + rev uint64 + }{hp, 1} + // The entry is read at the pipeline's own revision (3), never the newer revision 4 that moved the module (H-A-205). + moved := held + moved.Fp, moved.Path = "src2:ac", "modules/moved" + revisions := func(branch string, n int64) (Half, error) { + switch n { + case 3: + return Half{Revision: 3, Entries: waiting}, nil + case 0, 4: + return Half{Revision: 4, Entries: map[string]catalog.CatalogEntry{"app": moved}}, nil + } + return Half{}, fmt.Errorf("no revision %d", n) + } + var askedWith map[string]string + h := &Hearer{Store: ms, Say: func(string, any) error { return nil }, Name: "the-runner", + AskPinHeld: func(key, module string, pins map[string]string) { + askedWith = pins + if _, err := hs.AskPinHeld(ms, key, module, pins, revisions); err != nil { + t.Errorf("ask held: %v", err) + } + }} + buildEnd := StepEnded{ID: hp.Steps[0].Ask, Pipeline: "main.5", Step: "build", Node: "n1", Outcome: seat.StepResultOutcomePassed, + Module: "app", Commit: "c1", Fingerprint: "src2:ab"} + if _, err := h.OnStepEndedPinned(StepSeats[3], StepSeats[3].Prefix+"n1", buildEnd, nil, pinsBuilt); err != nil { + t.Fatalf("the build's end: %v", err) + } + if !reflect.DeepEqual(askedWith, pinsBuilt) { + t.Errorf("the held steps were asked with %v; want the build's pins", askedWith) + } + var askedHeld int + for _, x := range heldAsks.asks { + if b, ok := x.(nodebuildgo.StepAsk); ok { + askedHeld++ + if !reflect.DeepEqual(b.Step.Fetch.Packages, pinsBuilt) { + t.Errorf("the %s of %s fetches %v; want the build's pins", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages) + } + if b.Step.Fetch.Dir != "modules/app" { + t.Errorf("the %s of %s fetches %s; want the pipeline's revision's directory", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Dir) + } + } + } + after, _, _ := ms.Get("main.5") + if askedHeld == 0 || after.Steps[1].Outcome == seat.StepResultOutcomeWaiting || after.Steps[1].Ask == "" { + t.Errorf("after the build passed %d held steps were asked, lint = %+v; want each asked", askedHeld, after.Steps[1]) + } + // A mover whose fingerprint the pipeline's revision does not hold ends its held steps could not run, naming it. + other := hp + other.Pipeline, other.Steps, other.Movers = "main.4", append([]seat.StepResult(nil), hp.Steps...), []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:zz", Repository: "o/app"}} + ms["main.4"] = struct { + p seat.Pipeline + rev uint64 + }{other, 1} + if n, err := hs.AskPinHeld(ms, "main.4", "app", pinsBuilt, revisions); n != 0 || err != nil { + t.Errorf("asked %d held steps of a mover its revision does not hold (%v); want none", n, err) + } + if o, _, _ := ms.Get("main.4"); o.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(o.Steps[1].Why, "not the mover's src2:zz") { + t.Errorf("the held lint step of a moved fingerprint = %+v; want could not run naming it", o.Steps[1]) + } + if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") { + t.Errorf("the journal says %q; want the held step named", journal) + } // An artifact that names no paths still fetches its module alone: the directory with no paths (principle 10, A-447). for _, x := range asks.asks[2:] { if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && b.Step.Artifact == "app:tools" && (b.Step.Fetch.Dir != "modules/app" || len(b.Step.Fetch.Paths) != 0) { @@ -409,7 +532,8 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }} start := func(source, revision int64) seat.Pipeline { t.Helper() - if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { + // Keyed by its source, so pipelines of one revision are several (one kept under its source before H-A-213 (4)). + if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { t.Fatalf("start %d = %v, %v", source, ok, err) } p, _, _ := store.Get(Key("main", source)) @@ -428,6 +552,12 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" { t.Fatalf("the test steps asked = %q, want the program's three alone", got) } + // Every step ask names the catalog revision in revision and the gate's source in source (H-A-219). + for _, a := range asks.asks { + if b, ok := a.(nodebuildgo.StepAsk); ok && (b.Step.Revision != 9 || b.Step.Source != 4) { + t.Errorf("the %s ask of %s names revision %d and source %d, want 9 and 4", b.Step.Step, b.Step.Artifact, b.Step.Revision, b.Step.Source) + } + } for _, level := range buildSteps { asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level) if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil { @@ -628,13 +758,13 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) { } } } - p, _, _ := mem.Get("main.4") + p, _, _ := mem.Get("main.7") if validates != 1 || compositions != 0 || stepNamed(p, "", seat.StepResultStepComposition).Outcome != seat.StepResultOutcomeWaiting { t.Fatalf("placed: %d validate and %d composition asks, steps %+v; want one validate, no composition yet", validates, compositions, p.Steps) } h := &Hearer{Store: mem, Name: "the-runner", Say: func(string, any) error { return nil }, AskSteps: s.AskSteps} bs := stepSeatOf(t, wants.Seat) - raw := `{"id":"` + validateAsk + `","pipeline":"main.4","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}` + raw := `{"id":"` + validateAsk + `","pipeline":"main.7","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}` ev, err := bs.Decode([]byte(raw)) if err != nil { t.Fatal(err) @@ -643,7 +773,7 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) { if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil { t.Fatal(err) } - p, _, _ = mem.Get("main.4") + p, _, _ = mem.Get("main.7") for _, st := range p.Steps { if st.Step == seat.StepResultStepValidate && st.Outcome != outcome { t.Errorf("%s: %s's validate is %s, want every mover's read from the one run", outcome, st.Module, st.Outcome) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go index 586b656e..54249bca 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go @@ -96,7 +96,7 @@ func queueFixture() (*Starter, memStore, memQueue, *[]string, *[]seat.Wait, *ste func takeIn(t *testing.T, s *Starter, view func(string) (Half, bool)) func(source int64, modules ...string) { return func(source int64, modules ...string) { t.Helper() - ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7} + ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: source} for _, m := range modules { ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go index 751fbf9a..1f0cc57a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go @@ -99,6 +99,10 @@ type Half struct { Revision int64 Source int64 Entries map[string]catalog.CatalogEntry + // Call and Movers are the revision's own, as the revision verb answers them (H-A-213): what made it and its movers; + // a view's half names none. + Call string + Movers []catalog.TakeInEntry } // Views is the runner's views of the catalog half, one per watched branch the registrations name (design 47 G §21, diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/released.go b/modules/mesh-delivery-pipeline/internal/pipeline/released.go index 76028fce..840ca598 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/released.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/released.go @@ -3,7 +3,6 @@ package pipeline import ( "errors" "fmt" - "strconv" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/keys" @@ -21,12 +20,13 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int, if branch != "" { b = keys.EscapeName(branch) } - source := strconv.FormatInt(ev.Source, 10) + // A pipeline is keyed by its revision (H-A-213 (4)), so the pipelines of a source are those naming it, of either key + // form a source's pipelines were kept under. var pkeys []string - for _, f := range []string{b + "." + source, b + "." + source + ".published.*.*"} { + for _, f := range []string{b + ".*", b + ".*.published.*.*"} { k, err := h.Store.Keys(f) if err != nil { - return 0, fmt.Errorf("the pipelines of source %s are not listed: %w", source, err) + return 0, fmt.Errorf("the pipelines of source %d are not listed: %w", ev.Source, err) } pkeys = append(pkeys, k...) } @@ -37,6 +37,9 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int, if err != nil { return wrote, fmt.Errorf("the pipeline %s is not read: %w", key, err) } + if p.Source != ev.Source || p.Run != "" { + break + } i := -1 for j, m := range p.Movers { if m.Module == ev.Module { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index cfb8cc48..41f92321 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "maps" "slices" "sort" "strconv" @@ -28,7 +27,7 @@ type Created interface { Create(key string, value []byte) error } -// Begin is what a start names: the pipeline's key (watched branch and source revision, design 47 H §746), the take-in's +// Begin is what a start names: the pipeline's key (watched branch and catalog revision, design 47 H §746, H-A-213), the take-in's // call and catalog graph revision, its movers, and the catalog half of that branch. type Begin struct { Branch string @@ -39,28 +38,30 @@ type Begin struct { Movers []seat.Mover // Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them. Entries map[string]catalog.CatalogEntry - // Published is the package's publish that started the pipeline, nil for any other start (§747). - Published *seat.Published - // Key is the pipeline's key when it is not . (a publish's, §747); empty for a take-in's. + // Key is the pipeline's key when it is not .; empty for every start the runner makes. Key string // unheld says the branch's catalog half is not held yet: the pipeline starts with no steps, placed once it is (A-336). unheld bool } -// Key is a take-in's pipeline key in the state pipelines, `.`, the branch escaped as a key's name is -// (design 47 H §746). -func Key(branch string, source int64) string { - return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10)) +// Key is a pipeline's key in the state pipelines, `.`, the catalog revision's number the indexer counts, +// the branch escaped as a key's name is (design 47 H §746, H-A-213 (4)); a take-in's number is unchanged where it equalled +// its source. A pipeline kept under its source before (`.`) has the same form and is read as it is. +func Key(branch string, revision int64) string { + return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10)) } -// PublishedKey is the key of a pipeline a package's publish starts, `..published..`, -// each part escaped as a key's name is (design 47 H §747). -func PublishedKey(branch string, source int64, pkg, version string) string { - return seat.PipelinesByBranchSourcePublishedPackageVersionKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), - keys.EscapeName(pkg), keys.EscapeName(version)) +// keyRevision is the number b's pipeline is keyed by (H-A-213 (4)): its revision, or its source where the start names no +// revision (a take-in.taken of an indexer whose revision numbers were its sources, H-A-211). +func keyRevision(b Begin) int64 { + if b.Revision > 0 { + return b.Revision + } + return b.Source } -// keyForms are the filters of every key form of the state pipelines: a take-in's, a run's and a publish's. +// keyForms are the filters of every key form of the state pipelines: a revision's, a run's, and a publish's kept from before +// a release revision started one pipeline (H-A-213 (2)). var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*"} // AllKeys are every pipeline's key, of each key form (§746, §747; H-A-83). @@ -189,7 +190,7 @@ func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]A } var none []string for _, st := range p.Steps { - if !open(st) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) { + if (!open(st) && !pinHeld(st)) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) { continue } if holders, read := s.Holders(st.Seat); read && len(holders) == 0 { @@ -282,7 +283,7 @@ func (s *Starter) sayStarted(key string, asks []Asked) { func newPipeline(b Begin, key string, now time.Time) seat.Pipeline { movers := append([]seat.Mover(nil), b.Movers...) return seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Call: b.Call, Revision: b.Revision, Movers: movers, - Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}, Published: b.Published} + Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}} } func (s *Starter) create(key string, p seat.Pipeline) error { @@ -337,6 +338,11 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog } steps, a := Place(*p, m, r, entries, now, s.Name) supersededWhy(*p, m.Module, entries, steps) + for _, st := range steps { + if pinHeld(st) { + s.logf("%s: %s's %s of %s is not asked: it %s, as its build does (H-A-206)", p.Pipeline, m.Module, st.Step, st.Artifact, st.Why) + } + } if failed != "" { why := "its dependency " + failed + " did not build" for k := range steps { @@ -490,7 +496,7 @@ func (s *Starter) Start(b Begin) (bool, error) { now := s.now() key := b.Key if key == "" { - key = Key(b.Branch, b.Source) + key = Key(b.Branch, keyRevision(b)) } if err := s.Fills.refuseStart(key); err != nil { return false, err @@ -887,14 +893,21 @@ func (s *Starter) Release(store Updater, key string, view func(branch string) (H }, s.Registration, view, "the publishes its movers waited for") } -// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the source -// revision that published it, and places the movers that waited for it (design 47 H §746, H-A-84). It answers that -// source's pipeline, read, and whether one is held. +// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the catalog +// revision that published it, found by the branch and revision the publish names (H-A-218), and places the movers that +// waited for it (design 47 H §746, H-A-84). It answers that pipeline, read, and whether one is held. func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch string) (Half, bool)) (seat.Pipeline, bool, error) { if s.Store == nil { return seat.Pipeline{}, false, nil } - key := Key(ev.Branch, ev.Source) + key, ok := Key(ev.Branch, ev.Revision), true + if ev.Revision <= 0 { + // A publish of an older builder names no revision: its source's lowest-numbered revision, the take-in's (H-A-218). + key, ok = s.sourcePipeline(ev.Branch, ev.Source) + } + if !ok { + return seat.Pipeline{}, false, nil + } for { p, rev, err := s.Store.Get(key) if err != nil { @@ -918,49 +931,81 @@ func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch st } } -// publishedKind is the edge a dependent builds on a published package by: B published before A builds, a package A -// imports (§53). -const publishedKind = 1 - -// Published hears a package-published (design 47 H §747, H-A-52): in the running pipeline of the source revision that -// published it, a mover that builds on it waits for it there (§746) and nothing starts; every module the catalog half -// holds a kind-1 edge from the package into that this pipeline does not move (a publish again of an older build, a -// dependent it does not move) is a mover of one pipeline keyed `..published..`, at the -// publishing source's catalog revision. The dependents come from the edges the runner holds, never from a comparison. -func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) (int, error) { - if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 { - return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev) - } - publishing, held, err := s.heardPublished(ev, view) +// sourcePipeline is the key of branch's pipeline of the lowest-numbered revision of source, the take-in's (H-A-213 (4), +// H-A-218): a pipeline is keyed by its revision, so one of a source is found by the source it names, never by its key. +func (s *Starter) sourcePipeline(branch string, source int64) (string, bool) { + pkeys, err := s.Store.Keys(keys.EscapeName(branch) + ".*") if err != nil { - return 0, err + s.logf("the pipelines of %s are not listed, so source %d's is not found: %v", branch, source, err) + return "", false } - half, ok := view(ev.Branch) - if !ok { - return 0, fmt.Errorf("the catalog half of %s is not held yet: the publish of %s %s is heard again once it is", ev.Branch, ev.Artifact, ev.Version) - } - var movers []seat.Mover - for _, module := range slices.Sorted(maps.Keys(half.Entries)) { - e := half.Entries[module] - if module == ev.Module || Moves(publishing, module) || !slices.ContainsFunc(e.Edges, func(x catalog.Edge) bool { - return x.Kind == publishedKind && x.From == ev.Artifact - }) { + found, low := "", int64(0) + for _, k := range pkeys { + p, _, err := s.Store.Get(k) + if err != nil || p.Branch != branch || p.Source != source || p.Run != "" { continue } - movers = append(movers, seat.Mover{Module: module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository}) + if r := keyRevision(Begin{Revision: p.Revision, Source: p.Source}); found == "" || r < low { + found, low = k, r + } } - if len(movers) == 0 { + return found, found != "" +} + +// Published hears a package-published (design 47 H §746, H-A-52): in the running pipeline of the revision that published +// it, a mover that builds on it waits for it there and is placed once it is heard. It starts no pipeline: a re-pin on a +// module's release is a catalog revision of its own, whose pipeline Revised starts (H-A-212, H-A-213 (2)). +func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) error { + if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 { + return fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev) + } + _, _, err := s.heardPublished(ev, view) + return err +} + +// revisedKinds are the call kinds of a catalog revision whose pipeline Revised starts: every kind but a take-in's, whose +// pipeline its take-in.taken starts (H-A-213 (2), H-A-217). +var revisedKinds = []string{catalog.CatalogRevisionEventCallKindRelease, catalog.CatalogRevisionEventCallKindTakeInAgain, + catalog.CatalogRevisionEventCallKindRecompute} + +// Revised starts, on a catalog-revision of the trunk kind whose call kind is release, take-in again or recompute, ONE +// pipeline keyed . by compare-and-set on its absence (design 47 H §746, H-A-212, H-A-213 (2) and (4), +// H-A-217): the revision is read by its number through the catalog seat's revision verb, never the newest half, and its +// movers are the revision's movers, its steps placed from the revision's entries. A revision of another kind, or one with +// no mover, starts nothing; a second hearing writes nothing. +func (s *Starter) Revised(ev catalog.CatalogRevisionEvent, read func(branch string, revision int64) (Half, error)) (int, error) { + if ev.Kind != "trunk" || !slices.Contains(revisedKinds, ev.CallKind) { return 0, nil } - revision := half.Revision - if held && publishing.Revision > 0 { - revision = publishing.Revision + branch, err := keys.UnescapeName(ev.Branch) + if err != nil || branch == "" || ev.Revision <= 0 { + return 0, fmt.Errorf("a catalog-revision names its watched branch and revision: branch %q revision %d", ev.Branch, ev.Revision) + } + if read == nil { + return 0, fmt.Errorf("the revision %d of %s cannot be read: the runner holds no revision reader", ev.Revision, branch) + } + half, err := read(branch, ev.Revision) + if err != nil { + return 0, fmt.Errorf("the revision %d of %s is not read, so its pipeline is not started: %w", ev.Revision, branch, err) + } + if half.Revision != ev.Revision { + return 0, fmt.Errorf("the revision verb answered revision %d of %s for %d", half.Revision, branch, ev.Revision) + } + if len(half.Movers) == 0 { + s.logf("the revision %d of %s (%s) names no mover: it starts no pipeline", ev.Revision, branch, ev.Call) + return 0, nil + } + movers := make([]seat.Mover, 0, len(half.Movers)) + for _, e := range half.Movers { + movers = append(movers, seat.Mover{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint, Repository: e.Repository}) } movers = Tiers(movers, half.Entries, s.logf) - started, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Call: publishing.Call, Revision: revision, Movers: movers, - Entries: half.Entries, Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version}, - Key: PublishedKey(ev.Branch, ev.Source, ev.Artifact, ev.Version)}) - if started { + call := half.Call + if call == "" { + call = ev.Call + } + ok, err := s.Start(Begin{Branch: branch, Source: half.Source, Call: call, Revision: half.Revision, Movers: movers, Entries: half.Entries}) + if ok { return 1, err } return 0, err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 019127c7..3c5cbfe0 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -11,6 +11,7 @@ import ( builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" ) type created map[string][]byte @@ -55,12 +56,11 @@ func TestStarts(t *testing.T) { } } -// R10/AC1 and AC2 (design 47 H §747, H-A-52): a package-published of a mover of its source's running pipeline whose -// dependent that pipeline moves starts nothing: the dependent waits there. A publish again of an older build starts one -// pipeline keyed ..published.., its movers the package's kind-1 dependents from the -// edges the runner holds, at the publishing source's revision, and says pipeline-started with the publish; an ordering -// edge starts nothing; heard twice, the second writes nothing. -func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) { +// H-A-212, H-A-213 (2) and (4), H-A-217: a package-published starts no pipeline, the movers waiting for it in its +// pipeline placed there; a trunk catalog-revision of a release, a take-in again or a recompute starts ONE pipeline keyed +// ., the revision read by its number, its movers the revision's, its source and call the revision's; +// heard twice the second writes nothing; a take-in's revision, a candidate's and one with no mover start nothing. +func TestARevisionOfAReleaseStartsOnePipeline(t *testing.T) { mem := memStore{} var said []seat.PipelineStarted s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(_ string, body any) error { @@ -70,37 +70,60 @@ func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) { return nil }} edges := []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 1}} - view := func(string) (Half, bool) { - return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{ - "mesh-sdk": {Module: "mesh-sdk"}, - "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges}, - "tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges}, - "ordered": {Module: "ordered", Edges: []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 4}}}}}, true + entries := map[string]catalog.CatalogEntry{ + "mesh-sdk": {Module: "mesh-sdk"}, + "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges}, + "tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges}} + view := func(string) (Half, bool) { return Half{Revision: 9, Entries: entries}, true } + ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 30, Module: "mesh-sdk", Version: "0.1.9"} + if err := s.Published(ev, view); err != nil || len(mem) != 0 { + t.Fatalf("a publish of an older build wrote %d pipelines (%v), want none", len(mem), err) } - mem[Key("main", 41)] = struct { - p seat.Pipeline - rev uint64 - }{seat.Pipeline{Pipeline: "main.41", Branch: "main", Source: 41, Revision: 8, Outcome: seat.PipelineOutcomeRunning, - Movers: []seat.Mover{{Module: "mesh-sdk"}, {Module: "app"}, {Module: "tool"}}}, 1} - ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"} - if n, err := s.Published(ev, view); n != 0 || err != nil || len(mem) != 1 { - t.Fatalf("a publish in a pipeline moving its dependents started %d (%v), %d pipelines", n, err, len(mem)) + var asked []int64 + movers := []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}, {Module: "tool", Commit: "c2", Fingerprint: "src2:bc"}} + read := func(branch string, revision int64) (Half, error) { + asked = append(asked, revision) + if branch != "release.1" { + return Half{}, fmt.Errorf("branch %q", branch) + } + r := Half{Revision: revision, Source: 41, Entries: entries, Call: "release of mesh-sdk 0.1.17"} + if revision != 12 { + r.Movers = movers + } + return r, nil } - // A publish again of an older build, at source 30: its pipeline is gone, so every kind-1 dependent is a mover. - ev.Source, ev.Version = 30, "0.1.9" + release := catalog.CatalogRevisionEvent{Kind: "trunk", Branch: keys.EscapeName("release.1"), Revision: 10, Source: 41, CallKind: "release", Call: "release of mesh-sdk 0.1.17"} for i := range 2 { - if n, err := s.Published(ev, view); n != 1-i || err != nil { + if n, err := s.Revised(release, read); n != 1-i || err != nil { t.Fatalf("hearing %d started %d (%v)", i+1, n, err) } } - key := PublishedKey("main", 30, "mesh-sdk:go", "0.1.9") - p, _, err := mem.Get(key) - if err != nil || len(p.Movers) != 2 || !Moves(p, "app") || !Moves(p, "tool") || Moves(p, "ordered") || p.Revision != 9 || - p.Published == nil || p.Published.Version != "0.1.9" || len(said) != 1 || said[0].Published == nil { - t.Fatalf("the pipeline %s = %+v (%v), said %+v", key, p, err, said) + p, _, err := mem.Get(Key("release.1", 10)) + if err != nil || len(mem) != 1 || len(p.Movers) != 2 || p.Revision != 10 || p.Source != 41 || p.Call != "release of mesh-sdk 0.1.17" || + len(said) != 1 || len(asked) != 2 || asked[0] != 10 { + t.Fatalf("the pipeline %s = %+v (%v), %d written, said %d, read %v", Key("release.1", 10), p, err, len(mem), len(said), asked) } - if app, _ := MoverOf(p, "app"); app.Commit != "c1" || app.Fingerprint != "src2:aa" || app.Repository != "o/app" { - t.Errorf("the mover app = %+v", app) + if app, _ := MoverOf(p, "app"); app.Fingerprint != "src2:ab" || app.Repository != "o/app" || app.Tier != 1 { + t.Errorf("the mover app = %+v, want the revision's", app) + } + for _, c := range []struct { + name string + ev catalog.CatalogRevisionEvent + want int + }{ + {"take-in again", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 11, CallKind: "take-in-again"}, 1}, + {"recompute with no mover", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 12, CallKind: "recompute"}, 0}, + {"recompute", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 13, CallKind: "recompute"}, 1}, + {"take-in", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 14, CallKind: "take-in"}, 0}, + {"candidate", catalog.CatalogRevisionEvent{Kind: "candidate", Branch: release.Branch, Revision: 15, CallKind: "release"}, 0}, + } { + if n, err := s.Revised(c.ev, read); n != c.want || err != nil { + t.Errorf("%s: started %d (%v), want %d", c.name, n, err, c.want) + } + } + if _, err := s.Revised(catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 16, CallKind: "release"}, + func(string, int64) (Half, error) { return Half{}, errors.New("no answer") }); err == nil { + t.Error("a revision that cannot be read is not an error, so it is never heard again") } } @@ -153,8 +176,8 @@ func TestATakeInIsOnePipeline(t *testing.T) { t.Fatalf("written %d pipelines and said %d starts, want one each", len(store), len(said)) } var p seat.Pipeline - if err := json.Unmarshal(store["main.7"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 { - t.Fatalf("the pipeline main.7 = %+v (%v)", p, err) + if err := json.Unmarshal(store["main.12"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 { + t.Fatalf("the pipeline main.12 = %+v (%v)", p, err) } if m, _ := MoverOf(p, "mesh-sdk"); m.Commit != "c1" || m.Fingerprint != "src2:aa" || m.Repository != "o/sdk" || m.Tier != 1 { t.Errorf("the mover mesh-sdk = %+v", m) @@ -243,25 +266,6 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T if n != 1 || err != nil || len(p.Steps) == 0 { t.Errorf("the half of next placed %d (%v): %d steps, want the take-in's pipeline placed", n, err, len(p.Steps)) } - // A-336: a package published heard before its branch's half is held starts nothing, heard again once it is: its - // dependents are the half's edges (§747). - held = false - app := entries["app"] - app.Edges = []catalog.Edge{{From: "mesh-sdk:go", To: "app:program", Kind: 1}} - entries["app"] = app - published := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "next", Source: 7, Module: "mesh-sdk", Version: "0.1.17"} - if n, err := s.Published(published, late); n != 0 || err == nil { - t.Fatalf("the publish before the half started %d (%v), want none and an error, heard again", n, err) - } - held = true - if n, err := s.Published(published, late); n != 1 || err != nil { - t.Fatalf("the publish with the half held started %d (%v)", n, err) - } - p, _, _ = store.Get(PublishedKey("next", 7, "mesh-sdk:go", "0.1.17")) - if len(p.Steps) == 0 { - t.Errorf("the publish's pipeline placed %d steps, want its steps", len(p.Steps)) - } - // A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running, // no step placed) is placed and asked as a newly started one when the half is held; no hold is read any more. var old seat.Pipeline @@ -318,8 +322,8 @@ func TestTakenInStartsOnlyItsEntries(t *testing.T) { if err != nil || n != 1 { t.Fatalf("the take-in started %d (%v), want its one entry", n, err) } - if p, _, err := mem.Get("main.9"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") { - t.Errorf("main.9 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err) + if p, _, err := mem.Get("main.12"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") { + t.Errorf("main.12 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err) } } @@ -335,3 +339,27 @@ func TestATakeInOfTheOldFormIsRefused(t *testing.T) { t.Errorf("the new form = %+v (%v)", ev, err) } } + +// H-A-218: a package-published is taken into the pipeline of the branch and revision it names; one naming no revision (an +// older builder) into its source's lowest-numbered revision, the take-in's. +func TestAPublishIsTakenIntoThePipelineOfItsRevision(t *testing.T) { + mem := memStore{} + for _, r := range []int64{7, 8} { + mem[Key("main", r)] = struct { + p seat.Pipeline + rev uint64 + }{seat.Pipeline{Pipeline: Key("main", r), Branch: "main", Source: 4, Revision: r, Outcome: seat.PipelineOutcomeRunning, + Movers: []seat.Mover{{Module: "mesh-sdk"}}}, 1} + } + s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(string, any) error { return nil }} + view := func(string) (Half, bool) { return Half{Revision: 8}, true } + published := func(key string) []string { m, _ := MoverOf(mem[key].p, "mesh-sdk"); return m.Published } + ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Revision: 8, Version: "0.1.18"} + if err := s.Published(ev, view); err != nil || len(published("main.8")) != 1 || len(published("main.7")) != 0 { + t.Fatalf("a publish of revision 8 (%v): main.8 %v, main.7 %v; want main.8's mover alone", err, published("main.8"), published("main.7")) + } + ev.Revision, ev.Artifact = 0, "mesh-sdk:ts" + if err := s.Published(ev, view); err != nil || len(published("main.7")) != 1 || len(published("main.8")) != 1 { + t.Errorf("a publish naming no revision (%v): main.7 %v, main.8 %v; want main.7's, the source's lowest", err, published("main.7"), published("main.8")) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 9b39cf26..5ac10c50 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -31,6 +31,8 @@ type StepSeat struct { // Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for // a seat whose ends name none. Shared func([]byte) []seat.Shared + // Pins reads the pins a build's end names it was built with (H-A-207); nil for a seat whose ends name none. + Pins func([]byte) map[string]string } // StepSeats are the three build seats, from their contracts. @@ -39,7 +41,7 @@ var StepSeats = []StepSeat{ {Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")}, {Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")}, // The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99). - {Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded}, + {Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded, Pins: buildPins}, // The seat mesh-wants answers the composition step (§22, §24). {Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared}, } @@ -77,6 +79,15 @@ func decodeWantsEnded(raw []byte) (StepEnded, error) { Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil } +// buildPins is the pins a build step's end of the seat mesh-builds names its entry was built with (H-A-207). +func buildPins(raw []byte) map[string]string { + var b builds.StepEnded + if json.Unmarshal(raw, &b) != nil { + return nil + } + return b.Packages +} + // decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one // entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is // refused (A-99). @@ -155,6 +166,9 @@ type Hearer struct { // Release places the movers of a pipeline whose awaited publishes are heard now, after a build step passed (a kind-3 // edge, §746; Starter.Release); nil places none. Release func(key string) + // AskPinHeld asks a mover's steps that waited for its entry's pins with the pins its build passed with (H-A-207; + // Starter.AskPinHeld); nil leaves them waiting. + AskPinHeld func(key, module string, pins map[string]string) // Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748; // Starter.Turn); nil takes none. Turn func(key string) @@ -197,6 +211,12 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi // of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726, // A-469m). func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) { + return h.OnStepEndedPinned(bs, subject, ev, shared, nil) +} + +// OnStepEndedPinned is OnStepEndedShared of a build's end naming the pins it was built with (H-A-207): a passed build's +// mover's steps that waited for its entry's pins are asked with them. +func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared, pins map[string]string) (seat.Pipeline, error) { node := strings.TrimPrefix(subject, bs.Prefix) if node == subject || node == "" || strings.Contains(node, ".") { return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat) @@ -272,6 +292,11 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh // Every mover's validate reads the one run (§745, H-A-51); its end asks each mover's composition, or ends it not // asked when validate did not pass. var due []Asked + askHeld := false + if s.Step == seat.StepResultStepBuild { + readers = append(readers, endPinHeld(&p, *s, pins, now)...) + askHeld = stepPassed(*s) && heldOn(p, s.Module) + } if s.Step == seat.StepResultStepValidate { readers = append(readers, sameValidate(&p, i, now)...) var notAsked []int @@ -324,6 +349,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh if release && h.Release != nil { h.Release(ev.Pipeline) } + if askHeld && h.AskPinHeld != nil { + h.AskPinHeld(ev.Pipeline, s.Module, pins) + } if h.AskSteps != nil && len(due) > 0 { if err := h.AskSteps(ev.Pipeline, due); err != nil { h.logf("the compositions of %s are written running and not asked, asked again at the next start: %v", ev.Pipeline, err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index 780009af..ba0712e4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -113,7 +113,7 @@ func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline { "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, } - if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { + if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { t.Fatalf("start = %v, %v", ok, err) } p, _, _ := f.store.Get(Key("main", source)) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 5264f76e..d86cec3b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -582,11 +582,11 @@ func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) { if !ok || half.Source <= 0 { return seat.StartAnswer{}, fmt.Errorf("the catalog half of %s holds no entry of %s at a source revision: nothing was started", in.Branch, in.Module) } - key := Key(in.Branch, half.Source) + key := Key(in.Branch, keyRevision(Begin{Revision: half.Revision, Source: half.Source})) if err := v.record(seat.VerbStart, "started the pipeline "+key+" by hand", in.Why); err != nil { return seat.StartAnswer{}, err } - // A hand start keys the newest revision's . with the module as its one mover (H-A-52). + // A hand start keys the newest revision's . with the module as its one mover (H-A-52, H-A-213 (4)). started, err := v.Starter.Start(Begin{Branch: in.Branch, Source: half.Source, Revision: half.Revision, Entries: half.Entries, Movers: []seat.Mover{{Module: in.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository, Tier: 1}}}) if err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go index d3a273a0..a2c8adae 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go @@ -287,7 +287,7 @@ func TestStartAndAskAgain(t *testing.T) { _, entries := placeFixture() v.View = func(string) (Half, bool) { return Half{Source: 7, Revision: 3, Entries: entries}, true } a, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "no pipeline for the head"}) - if err != nil || a.Pipeline != "main.7" || made["main.7"] == nil { + if err != nil || a.Pipeline != "main.3" || made["main.3"] == nil { t.Fatalf("start = %+v, %v; written %v", a, err, len(made)) } if _, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "again"}); err == nil {