diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go index 7ed4f8fe..213116c3 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go @@ -905,8 +905,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go index d2136e78..391f8658 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go @@ -903,8 +903,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go index 47ca8f1d..bd89858a 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go @@ -903,8 +903,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/parts/build-step.json b/modules/mesh-delivery-contracts/parts/build-step.json index fcaebd62..0390aecb 100644 --- a/modules/mesh-delivery-contracts/parts/build-step.json +++ b/modules/mesh-delivery-contracts/parts/build-step.json @@ -1521,7 +1521,11 @@ }, "revision": { "type": "integer", - "description": "the source revision of the watched branch the pipeline runs for; absent for a run on request" + "description": "the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request" + }, + "source": { + "type": "integer", + "description": "the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request" }, "fingerprint": { "type": "string", diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go index abf01a7c..ee636c4a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go @@ -113,9 +113,19 @@ func TestAReleasedVersionIsTheMoversVersion(t *testing.T) { other := store[pkey] other.p.Pipeline, other.p.Branch, other.p.Movers = "beta.5", "beta", append([]seat.Mover(nil), other.p.Movers...) store["beta.5"] = other + // A pipeline is keyed by its revision (H-A-213 (4)): main.8, a later revision of source 5, is written too; main.6, of + // source 6, is not. + for key, source := range map[string]int64{"main.8": 5, "main.6": 6} { + again := store[pkey] + again.p.Pipeline, again.p.Source, again.p.Movers = key, source, append([]seat.Mover(nil), again.p.Movers...) + store[key] = again + } ev := catalog.VersionReleasedEvent{Module: "app", Version: "0.4.1", Source: 5} - if n, err := h.Released(ev, "main"); n != 1 || err != nil { - t.Fatalf("released wrote %d (%v), want app's mover of main.5", n, err) + if n, err := h.Released(ev, "main"); n != 2 || err != nil { + t.Fatalf("released wrote %d (%v), want app's mover of main.5 and main.8", n, err) + } + if m, _ := MoverOf(store["main.6"].p, "app"); m.Version != "" { + t.Errorf("main.6's mover = %+v, want no version: another source", m) } if m, _ := MoverOf(store[pkey].p, "app"); m.Version != "0.4.1" { t.Errorf("app's mover = %+v, want version 0.4.1", m) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go index 0065aabd..73d1e020 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go @@ -70,14 +70,14 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if n, err := s.TakenIn(ev, view); n != 1 || err != nil { t.Fatalf("the take-in started %d (%v)", n, err) } - p, _, _ := mem.Get("main.4") + p, _, _ := mem.Get("main.7") if !placed(p, "mesh-sdk") || !placed(p, "builder") || placed(p, "app") || placed(p, "tool") { t.Fatalf("placed at the start: %+v, want mesh-sdk and builder only", p.Steps) } if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); n != 0 || err != nil { t.Fatalf("the publish started %d (%v), want none", n, err) } - p, _, _ = mem.Get("main.4") + p, _, _ = mem.Get("main.7") if sdk, _ := MoverOf(p, "mesh-sdk"); !placed(p, "app") || placed(p, "tool") || len(mem) != 1 || !slices.Contains(sdk.Published, "mesh-sdk:go") { t.Fatalf("after mesh-sdk's publish: %d pipelines, app placed %v, tool placed %v, mesh-sdk %+v", len(mem), placed(p, "app"), placed(p, "tool"), sdk) } @@ -87,22 +87,22 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { b := stepsOf(p, "builder")[0] bs := stepSeatOf(t, builds.Seat) end := func(outcome string) StepEnded { - e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.4","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`)) + e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.7","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`)) if err != nil { t.Fatal(err) } return e } - before := mem["main.4"] + before := mem["main.7"] if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("passed")); err != nil { t.Fatal(err) } - if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning { + if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning { t.Errorf("after builder's build passed, tool's steps = %+v, want placed and asked", stepsOf(p, "tool")) } // The same pipeline, builder's build failed: tool is placed with every step not asked, each saying its own step-ended // naming the dependency (H-A-101 (2)). - mem["main.4"] = before + mem["main.7"] = before var notAsked []seat.StepEvent s.Say = func(subject string, body any) error { if ev, ok := body.(seat.StepEvent); ok && strings.Contains(subject, ".step-ended.tool.") { @@ -113,7 +113,7 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("failed")); err != nil { t.Fatal(err) } - if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked { + if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked { t.Errorf("after builder's build failed, tool's steps = %+v, want placed not asked", stepsOf(p, "tool")) } if len(notAsked) == 0 || len(notAsked) != len(slices.DeleteFunc(stepsOf(p, "tool"), func(st seat.StepResult) bool { return st.Outcome != seat.StepResultOutcomeNotAsked })) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go index 455f8fa5..bd99d2c8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go @@ -51,7 +51,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing. if n, err := s.TakenIn(taken, view); n != 1 || err != nil { t.Fatalf("the take-in started %d (%v)", n, err) } - p, _, _ := store.Get("main.947") + p, _, _ := store.Get("main.357") if p.Revision != 357 || len(p.Steps) != 0 || len(asks.asks) != 0 { t.Fatalf("a take-in of revision 357 with the view at 356 started %+v with %d asks, want revision 357 and no step", p, len(asks.asks)) } @@ -60,7 +60,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing. } viewRevision = 357 n, err := s.Waiting(store.memStore, "main", view) - p, _, _ = store.Get("main.947") + p, _, _ = store.Get("main.357") if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Revision != 357 { t.Errorf("the view reaching 357 placed %d (%v): %d steps, %d asks at revision %d", n, err, len(p.Steps), len(asks.asks), p.Revision) } @@ -87,7 +87,7 @@ func TestAViewThatReachesTheRevisionDuringTheStartsPlacesTheHeldBackPipelines(t if n, err := s.TakenIn(taken, view); n != 1 || err != nil { t.Fatalf("started %d (%v)", n, err) } - if p, _, _ := store.Get("main.947"); len(p.Steps) == 0 || len(asks.asks) == 0 { + if p, _, _ := store.Get("main.357"); len(p.Steps) == 0 || len(asks.asks) == 0 { t.Errorf("the pipeline has %d steps and %d asks after the view reached 357 during the start, want them placed", len(p.Steps), len(asks.asks)) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 4fcc2aee..a7dbf327 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -384,7 +384,7 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar } return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now, Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: m.Commit, Fetch: fetch, Fingerprint: m.Fingerprint, - Module: m.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}} + Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}} } // askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index dd10438a..7f46c658 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -102,7 +102,7 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { a := asks.asks[1].(nodebuildgo.StepAsk) if asks.at[1] != "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner" || a.Kind != "step" || a.Asker != "the-runner" || a.ID != p.Steps[1].Ask || asks.ids[1] != a.ID || a.Step.Commit != "c1" || a.Step.Fingerprint != "src2:aa" || a.Step.Pipeline != "main.4" || a.Step.Artifact != "app:program" || - a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Revision != 4 || + a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Source != 4 || len(a.Step.Fetch.Paths) != 1 || a.Step.Fetch.Paths[0] != "cmd/app" { t.Errorf("the ask = %+v at %s", a, asks.at[1]) } @@ -136,7 +136,7 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { } } var hp seat.Pipeline - if err := json.Unmarshal(store["main.5"], &hp); err != nil { + if err := json.Unmarshal(store["main.3"], &hp); err != nil { t.Fatal(err) } if st := hp.Steps[1]; st.Step != "lint" || st.Outcome != seat.StepResultOutcomeWaiting || st.Ask != "" || st.Why != "waits on the publish of git.novox.be/novox/x/go" { @@ -213,15 +213,15 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { } // A mover whose fingerprint the pipeline's revision does not hold ends its held steps could not run, naming it. other := hp - other.Pipeline, other.Steps, other.Movers = "main.6", append([]seat.StepResult(nil), hp.Steps...), []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:zz", Repository: "o/app"}} - ms["main.6"] = struct { + other.Pipeline, other.Steps, other.Movers = "main.4", append([]seat.StepResult(nil), hp.Steps...), []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:zz", Repository: "o/app"}} + ms["main.4"] = struct { p seat.Pipeline rev uint64 }{other, 1} - if n, err := hs.AskPinHeld(ms, "main.6", "app", pinsBuilt, revisions); n != 0 || err != nil { + if n, err := hs.AskPinHeld(ms, "main.4", "app", pinsBuilt, revisions); n != 0 || err != nil { t.Errorf("asked %d held steps of a mover its revision does not hold (%v); want none", n, err) } - if o, _, _ := ms.Get("main.6"); o.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(o.Steps[1].Why, "not the mover's src2:zz") { + if o, _, _ := ms.Get("main.4"); o.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(o.Steps[1].Why, "not the mover's src2:zz") { t.Errorf("the held lint step of a moved fingerprint = %+v; want could not run naming it", o.Steps[1]) } if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") { @@ -532,7 +532,8 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }} start := func(source, revision int64) seat.Pipeline { t.Helper() - if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { + // Keyed by its source, so pipelines of one revision are several (one kept under its source before H-A-213 (4)). + if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { t.Fatalf("start %d = %v, %v", source, ok, err) } p, _, _ := store.Get(Key("main", source)) @@ -551,6 +552,12 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" { t.Fatalf("the test steps asked = %q, want the program's three alone", got) } + // Every step ask names the catalog revision in revision and the gate's source in source (H-A-219). + for _, a := range asks.asks { + if b, ok := a.(nodebuildgo.StepAsk); ok && (b.Step.Revision != 9 || b.Step.Source != 4) { + t.Errorf("the %s ask of %s names revision %d and source %d, want 9 and 4", b.Step.Step, b.Step.Artifact, b.Step.Revision, b.Step.Source) + } + } for _, level := range buildSteps { asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level) if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil { @@ -751,13 +758,13 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) { } } } - p, _, _ := mem.Get("main.4") + p, _, _ := mem.Get("main.7") if validates != 1 || compositions != 0 || stepNamed(p, "", seat.StepResultStepComposition).Outcome != seat.StepResultOutcomeWaiting { t.Fatalf("placed: %d validate and %d composition asks, steps %+v; want one validate, no composition yet", validates, compositions, p.Steps) } h := &Hearer{Store: mem, Name: "the-runner", Say: func(string, any) error { return nil }, AskSteps: s.AskSteps} bs := stepSeatOf(t, wants.Seat) - raw := `{"id":"` + validateAsk + `","pipeline":"main.4","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}` + raw := `{"id":"` + validateAsk + `","pipeline":"main.7","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}` ev, err := bs.Decode([]byte(raw)) if err != nil { t.Fatal(err) @@ -766,7 +773,7 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) { if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil { t.Fatal(err) } - p, _, _ = mem.Get("main.4") + p, _, _ = mem.Get("main.7") for _, st := range p.Steps { if st.Step == seat.StepResultStepValidate && st.Outcome != outcome { t.Errorf("%s: %s's validate is %s, want every mover's read from the one run", outcome, st.Module, st.Outcome) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go index 586b656e..54249bca 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go @@ -96,7 +96,7 @@ func queueFixture() (*Starter, memStore, memQueue, *[]string, *[]seat.Wait, *ste func takeIn(t *testing.T, s *Starter, view func(string) (Half, bool)) func(source int64, modules ...string) { return func(source int64, modules ...string) { t.Helper() - ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7} + ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: source} for _, m := range modules { ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/released.go b/modules/mesh-delivery-pipeline/internal/pipeline/released.go index 76028fce..840ca598 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/released.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/released.go @@ -3,7 +3,6 @@ package pipeline import ( "errors" "fmt" - "strconv" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/keys" @@ -21,12 +20,13 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int, if branch != "" { b = keys.EscapeName(branch) } - source := strconv.FormatInt(ev.Source, 10) + // A pipeline is keyed by its revision (H-A-213 (4)), so the pipelines of a source are those naming it, of either key + // form a source's pipelines were kept under. var pkeys []string - for _, f := range []string{b + "." + source, b + "." + source + ".published.*.*"} { + for _, f := range []string{b + ".*", b + ".*.published.*.*"} { k, err := h.Store.Keys(f) if err != nil { - return 0, fmt.Errorf("the pipelines of source %s are not listed: %w", source, err) + return 0, fmt.Errorf("the pipelines of source %d are not listed: %w", ev.Source, err) } pkeys = append(pkeys, k...) } @@ -37,6 +37,9 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int, if err != nil { return wrote, fmt.Errorf("the pipeline %s is not read: %w", key, err) } + if p.Source != ev.Source || p.Run != "" { + break + } i := -1 for j, m := range p.Movers { if m.Module == ev.Module { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index cb039452..7f99aa83 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -28,7 +28,7 @@ type Created interface { Create(key string, value []byte) error } -// Begin is what a start names: the pipeline's key (watched branch and source revision, design 47 H §746), the take-in's +// Begin is what a start names: the pipeline's key (watched branch and catalog revision, design 47 H §746, H-A-213), the take-in's // call and catalog graph revision, its movers, and the catalog half of that branch. type Begin struct { Branch string @@ -41,16 +41,26 @@ type Begin struct { Entries map[string]catalog.CatalogEntry // Published is the package's publish that started the pipeline, nil for any other start (§747). Published *seat.Published - // Key is the pipeline's key when it is not . (a publish's, §747); empty for a take-in's. + // Key is the pipeline's key when it is not . (a publish's, §747); empty for a take-in's. Key string // unheld says the branch's catalog half is not held yet: the pipeline starts with no steps, placed once it is (A-336). unheld bool } -// Key is a take-in's pipeline key in the state pipelines, `.`, the branch escaped as a key's name is -// (design 47 H §746). -func Key(branch string, source int64) string { - return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10)) +// Key is a pipeline's key in the state pipelines, `.`, the catalog revision's number the indexer counts, +// the branch escaped as a key's name is (design 47 H §746, H-A-213 (4)); a take-in's number is unchanged where it equalled +// its source. A pipeline kept under its source before (`.`) has the same form and is read as it is. +func Key(branch string, revision int64) string { + return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10)) +} + +// keyRevision is the number b's pipeline is keyed by (H-A-213 (4)): its revision, or its source where the start names no +// revision (a take-in.taken of an indexer whose revision numbers were its sources, H-A-211). +func keyRevision(b Begin) int64 { + if b.Revision > 0 { + return b.Revision + } + return b.Source } // PublishedKey is the key of a pipeline a package's publish starts, `..published..`, @@ -495,7 +505,7 @@ func (s *Starter) Start(b Begin) (bool, error) { now := s.now() key := b.Key if key == "" { - key = Key(b.Branch, b.Source) + key = Key(b.Branch, keyRevision(b)) } if err := s.Fills.refuseStart(key); err != nil { return false, err @@ -899,7 +909,10 @@ func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch st if s.Store == nil { return seat.Pipeline{}, false, nil } - key := Key(ev.Branch, ev.Source) + key, ok := s.sourcePipeline(ev.Branch, ev.Source) + if !ok { + return seat.Pipeline{}, false, nil + } for { p, rev, err := s.Store.Get(key) if err != nil { @@ -923,6 +936,27 @@ func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch st } } +// sourcePipeline is the key of branch's pipeline of the lowest-numbered revision of source, the take-in's (H-A-213 (4), +// H-A-218): a pipeline is keyed by its revision, so one of a source is found by the source it names, never by its key. +func (s *Starter) sourcePipeline(branch string, source int64) (string, bool) { + pkeys, err := s.Store.Keys(keys.EscapeName(branch) + ".*") + if err != nil { + s.logf("the pipelines of %s are not listed, so source %d's is not found: %v", branch, source, err) + return "", false + } + found, low := "", int64(0) + for _, k := range pkeys { + p, _, err := s.Store.Get(k) + if err != nil || p.Branch != branch || p.Source != source || p.Run != "" { + continue + } + if r := keyRevision(Begin{Revision: p.Revision, Source: p.Source}); found == "" || r < low { + found, low = k, r + } + } + return found, found != "" +} + // publishedKind is the edge a dependent builds on a published package by: B published before A builds, a package A // imports (§53). const publishedKind = 1 diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 019127c7..9418cfca 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -153,8 +153,8 @@ func TestATakeInIsOnePipeline(t *testing.T) { t.Fatalf("written %d pipelines and said %d starts, want one each", len(store), len(said)) } var p seat.Pipeline - if err := json.Unmarshal(store["main.7"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 { - t.Fatalf("the pipeline main.7 = %+v (%v)", p, err) + if err := json.Unmarshal(store["main.12"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 { + t.Fatalf("the pipeline main.12 = %+v (%v)", p, err) } if m, _ := MoverOf(p, "mesh-sdk"); m.Commit != "c1" || m.Fingerprint != "src2:aa" || m.Repository != "o/sdk" || m.Tier != 1 { t.Errorf("the mover mesh-sdk = %+v", m) @@ -318,8 +318,8 @@ func TestTakenInStartsOnlyItsEntries(t *testing.T) { if err != nil || n != 1 { t.Fatalf("the take-in started %d (%v), want its one entry", n, err) } - if p, _, err := mem.Get("main.9"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") { - t.Errorf("main.9 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err) + if p, _, err := mem.Get("main.12"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") { + t.Errorf("main.12 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index 780009af..ba0712e4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -113,7 +113,7 @@ func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline { "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, } - if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { + if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { t.Fatalf("start = %v, %v", ok, err) } p, _, _ := f.store.Get(Key("main", source)) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 5264f76e..d86cec3b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -582,11 +582,11 @@ func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) { if !ok || half.Source <= 0 { return seat.StartAnswer{}, fmt.Errorf("the catalog half of %s holds no entry of %s at a source revision: nothing was started", in.Branch, in.Module) } - key := Key(in.Branch, half.Source) + key := Key(in.Branch, keyRevision(Begin{Revision: half.Revision, Source: half.Source})) if err := v.record(seat.VerbStart, "started the pipeline "+key+" by hand", in.Why); err != nil { return seat.StartAnswer{}, err } - // A hand start keys the newest revision's . with the module as its one mover (H-A-52). + // A hand start keys the newest revision's . with the module as its one mover (H-A-52, H-A-213 (4)). started, err := v.Starter.Start(Begin{Branch: in.Branch, Source: half.Source, Revision: half.Revision, Entries: half.Entries, Movers: []seat.Mover{{Module: in.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository, Tier: 1}}}) if err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go index d3a273a0..a2c8adae 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go @@ -287,7 +287,7 @@ func TestStartAndAskAgain(t *testing.T) { _, entries := placeFixture() v.View = func(string) (Half, bool) { return Half{Source: 7, Revision: 3, Entries: entries}, true } a, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "no pipeline for the head"}) - if err != nil || a.Pipeline != "main.7" || made["main.7"] == nil { + if err != nil || a.Pipeline != "main.3" || made["main.3"] == nil { t.Fatalf("start = %+v, %v; written %v", a, err, len(made)) } if _, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "again"}); err == nil {