From a926a40eed397cf8372a697ba443dc6d383a7cbd Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 22:55:36 +0100 Subject: [PATCH 01/30] Design 47 H1: every step ask fetches with the entry's pins, as the build does (H-A-203) The runner's lint, unit and integration asks carried no pins, only the build ask did, so the Go build module, which refuses a requirement of another repository with no pin, could not run twelve test steps of the contracts mover (main.45). stepAsk now puts the entry's packages into the fetch in the form the entry keeps and the builder sends, so a test runs against what the build was built against; a package the entry names with no pin yet is left out. Builds: design 47 H H-A-203, H-A-192, H-A-183 --- .../testruns_integration_test.go | 14 +++++++++++++- .../internal/pipeline/place.go | 12 ++++++++++++ .../internal/pipeline/place_test.go | 9 ++++++++- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index 95ed9670..a0aa6696 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -88,7 +88,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil { + askStream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}) + if err != nil { t.Fatal(err) } @@ -116,6 +117,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) entries := []catalog.CatalogEntry{ {Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app", + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25"}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, {Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}}, {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, @@ -331,6 +333,16 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { passBuild(p3, "app", "c1", "src2:aa") unitRun, unit := runOfStep(p3, "app:program", "unit") integRun, integ := runOfStep(p3, "app:program", "integration") + // A test ask fetches with the entry's pins, as the builder gives them the build (H-A-203). + last, err := askStream.GetLastMsgForSubject(ctx, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner") + if err != nil { + t.Fatal(err) + } + var testAsk nodebuildgo.StepAsk + if err := json.Unmarshal(last.Data, &testAsk); err != nil || testAsk.Step == nil || + testAsk.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25" { + t.Fatalf("the last test ask = %s (%v), want its fetch to carry the entry's pin", last.Data, err) + } if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 { t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index af5758d6..4110b3d1 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -234,6 +234,18 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar if paths := artifactPaths(e, a.Ref); len(paths) > 0 { fetch.Paths = paths } + // The entry's pins, the same the builder gives the build (H-A-203, H-A-183): a test runs against what the build was + // built against, and the Go build module refuses a requirement of another repository with no pin (H-A-192). A + // package the entry names with no pin yet is left out, as the builder pins it only from its publish. + for path, version := range e.Packages { + if version == "" { + continue + } + if fetch.Packages == nil { + fetch.Packages = map[string]string{} + } + fetch.Packages[path] = version + } return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now, Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: m.Commit, Fetch: fetch, Fingerprint: m.Fingerprint, Module: m.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index d54f44a8..21b3047e 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -44,7 +44,7 @@ func placeFixture() (catalog.Registration, map[string]catalog.CatalogEntry) { Artifacts: map[string]catalog.StepStatuses{"tools": {Lint: "bypass"}}, }} return reg, map[string]catalog.CatalogEntry{ - "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, + "app": {Module: "app", Path: "modules/app", Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": ""}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, {Ref: "app:web", Language: "rust"}}, Inputs: &catalog.EntryInputs{Paths: map[string][]string{"app:program": {"cmd/app"}}}}, "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, } @@ -104,6 +104,13 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { len(a.Step.Fetch.Paths) != 1 || a.Step.Fetch.Paths[0] != "cmd/app" { t.Errorf("the ask = %+v at %s", a, asks.at[1]) } + // Every step ask fetches with the entry's pins, as the builder gives them the build; a package with no pin yet is left + // out (H-A-203, H-A-192). + for _, x := range asks.asks[1:] { + if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && (len(b.Step.Fetch.Packages) != 1 || b.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25") { + t.Errorf("the %s ask of %s fetches the pins %v, want the entry's", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages) + } + } // An artifact that names no paths still fetches its module alone: the directory with no paths (principle 10, A-447). for _, x := range asks.asks[2:] { if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && b.Step.Artifact == "app:tools" && (b.Step.Fetch.Dir != "modules/app" || len(b.Step.Fetch.Paths) != 0) { -- 2.54.0 From 0418a2b94b751985083f8ec1b73197fd9f2be356 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 22:57:26 +0100 Subject: [PATCH 02/30] Design 47 H1: an entry's packages never leave out a requirement Build asks were refused for a requirement with no pin: the import form's versions name only what the last build imported, and the version line leaves out a package none publishes, so the pins kept only those. The pins now name every other-repository package of the module's go.mod graph (its kept packages), at the newest published version of its accepted series, or with no version while none is published, so the builder waits for it. Builds: design 47 H H-A-204, H-A-201, H-A-165, H-A-167 --- .../internal/index/package_trees_test.go | 2 +- .../internal/index/repin_test.go | 32 +++++++++++++++++++ .../internal/index/toolchains.go | 4 +-- .../internal/index/versions.go | 18 ++++++++--- 4 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/repin_test.go diff --git a/modules/mesh-delivery-indexer/internal/index/package_trees_test.go b/modules/mesh-delivery-indexer/internal/index/package_trees_test.go index 999bb220..06c1fc06 100644 --- a/modules/mesh-delivery-indexer/internal/index/package_trees_test.go +++ b/modules/mesh-delivery-indexer/internal/index/package_trees_test.go @@ -137,7 +137,7 @@ func TestTheEntryPinsEveryRequirementOfTheModuleGraph(t *testing.T) { t.Errorf("packages = %+v, want the client's requirement and state's mesh-sdk, and no module of the repository", m.Inputs2) } builds := "git.novox.be/novox/mesh-delivery/modules/b/builds" - pins := releasedPins(sdkstep.PackageVersions{builds: "v0.1.106", conditions: "v0.1.4"}, nil, catalogue.Manifest{}, "novox/mesh-delivery") + pins := releasedPins(sdkstep.PackageVersions{builds: "v0.1.106", conditions: "v0.1.4"}, nil, catalogue.Manifest{}, "novox/mesh-delivery", nil) if _, pinned := pins[builds]; pinned || len(pins) != 1 { t.Errorf("pins = %v, want %s only, never a module of the repository", pins, conditions) } diff --git a/modules/mesh-delivery-indexer/internal/index/repin_test.go b/modules/mesh-delivery-indexer/internal/index/repin_test.go new file mode 100644 index 00000000..0e76a74e --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/repin_test.go @@ -0,0 +1,32 @@ +package index + +import ( + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Design 47 H-A-204, H-A-201, H-A-165: an entry's packages name every requirement of its module graph of another +// repository: one the form's versions leave out (an import form naming only what the build imported) is pinned by its +// series' newest published version, and one none publishes yet is kept with no version, never left out. +func TestAnEntryNamesEveryRequirementPinnedOrWaiting(t *testing.T) { + const ( + wireguard = "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go" + busgrants = "git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants" + sdk = "git.novox.be/novox/mesh-sdk/go" + ) + series := map[string]SeriesPackage{"wg": {Name: "mesh-wireguard/go", GoModule: wireguard, Series: catalogue.DefaultSeries, + Newest: &meshcatalog.PackagesByPackageSeriesValueNewest{Version: "0.1.47"}}} + pins := releasedPins(sdkstep.PackageVersions{sdk: "c0ffee"}, series, catalogue.Manifest{}, "novox/mesh-catalog", []string{busgrants, sdk, wireguard}) + want := sdkstep.PackageVersions{sdk: "", wireguard: "v0.1.47", busgrants: ""} + if len(pins) != len(want) { + t.Fatalf("pins = %v, want %v", pins, want) + } + for p, v := range want { + if got, named := pins[p]; !named || got != v { + t.Errorf("the pin of %s is %q (named %v), want %q", p, got, named, v) + } + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index f8c1eb67..09161367 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -298,7 +298,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.fp, r.none = f.fp, f.none if r.fp != "" { // a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167) - r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository) + r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository, kept.Packages) r.tcs = toolchainsUsed(toolchains) } } @@ -517,7 +517,7 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, if f.none != "" || f.fp == "" { return "", nil, false } - fp, pkgs := pinnedOf(f, series, k.manifest, k.held.Repository) + fp, pkgs := pinnedOf(f, series, k.manifest, k.held.Repository, kept.Packages) return fp, pkgs, fp == k.held.Fp2 } diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 9500d302..c35b7752 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -376,12 +376,20 @@ func isFloor(raw []byte) bool { // release number is never a pin. A package with none published in that series is kept with no version, so its kind-1 // edge stands (H §746) and its build waits for the publish (H-A-165). A module of the module's own repository is never // pinned, whatever a build reported: it is built from the tree (design 47 H-A-198, H-A-201, H-A-202). nil for none. -func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest, repository string) sdkstep.PackageVersions { - if len(versions) == 0 { +// +// required are the mesh packages the module's go.mod graph requires (its kept packages, H-A-201): each of another +// repository is named whether or not the form's versions name it, with no version while none is published, never left +// out, so the builder waits for its publish instead of the build module refusing (design 47 H-A-204, H-A-165). +func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest, repository string, required []string) sdkstep.PackageVersions { + names := slices.Collect(maps.Keys(versions)) + if repository != "" { + names = append(names, fingerprint.OtherPackages(required, repository)...) + } + if len(names) == 0 { return nil } out := sdkstep.PackageVersions{} - for p := range versions { + for _, p := range names { if repo, _, ok := trunk.PackageRepository(p); ok && repository != "" && trunk.SameRepository(repo, repository) { continue } @@ -403,8 +411,8 @@ func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPack // form's fingerprint takes them (design 47 H-A-167, H-A-172; ADR 0573 §4) and a re-pin moves it, the module a mover // built again against the release; a fingerprint with no pin is the form's as it is. The held fingerprint of a build that // stands is kept while its pins are the ones it was built with, else its import form takes the new pins. -func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, repository string) (string, sdkstep.PackageVersions) { - pins := releasedPins(f.versions, series, m, repository) +func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, repository string, required []string) (string, sdkstep.PackageVersions) { + pins := releasedPins(f.versions, series, m, repository, required) if f.stands { if maps.Equal(f.built, map[string]string(pins)) { return f.fp, pins -- 2.54.0 From 94202fb65175ea51bbc99c96932643eb6ec2a579 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 22:58:06 +0100 Subject: [PATCH 03/30] Design 47 H1: a release re-pins every entry naming its package A release re-pinned only the entries whose pins already named one of its packages, so an entry whose module did not change and whose pins left the package out was never re-pinned. Every kept entry whose pins or kept requirements name a package of the released module is now re-pinned, of any repository and changed or not; its fingerprint moves with the pin. Builds: design 47 H H-A-204, H-A-165, H-A-167, H-A-201 --- .../internal/index/repin_test.go | 26 +++++++++++++++++++ .../internal/index/toolchains.go | 8 +++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-indexer/internal/index/repin_test.go b/modules/mesh-delivery-indexer/internal/index/repin_test.go index 0e76a74e..80ed4dfd 100644 --- a/modules/mesh-delivery-indexer/internal/index/repin_test.go +++ b/modules/mesh-delivery-indexer/internal/index/repin_test.go @@ -1,8 +1,12 @@ package index import ( + "context" + "slices" + "strings" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -30,3 +34,25 @@ func TestAnEntryNamesEveryRequirementPinnedOrWaiting(t *testing.T) { } } } + +// Design 47 H-A-204, H-A-165: a release re-pins every kept entry naming one of its packages, of any repository and +// changed or not, by its pins or by its kept requirements alone: here g, of another repository, holds no pin of mesh-sdk. +func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) { + objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)} + ix, mods, _ := ptpIndexer(t, objects) + f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, nil) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + g, _ := moduleOf(t, mods, "g") + if g.Inputs2 == nil || !slices.Contains(g.Inputs2.Packages, "git.novox.be/novox/mesh-sdk/go") { + t.Fatalf("g = %+v, want mesh-sdk among its kept requirements", g.Inputs2) + } + g.Packages = nil + mods.put("g", mustEncode(t, g)) + mods.put("mesh-sdk", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: strings.Repeat("1", 40), + Path: meshcatalogclient.RootPath, None2: None2Pending})) + if n, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil || n != 1 { + t.Errorf("a release of mesh-sdk re-pinned %d modules (%v), want g, which requires it with no pin", n, err) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 09161367..01d1e03b 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -609,9 +609,15 @@ func (ix *Indexer) Repin(ctx context.Context, module string) (int, error) { if err != nil { return 0, err } + // Every kept entry naming a package of module is re-pinned, of any repository, changed or not: by its pins, and by its + // kept requirements, which name a package it holds no pin of yet (design 47 H-A-204, H-A-165, H-A-167). only := map[string]bool{} for key, m := range held { - for p := range m.Packages { + names := slices.Collect(maps.Keys(m.Packages)) + if m.Inputs2 != nil { + names = append(names, m.Inputs2.Packages...) + } + for _, p := range names { if moduleOfKey(fingerprint.HolderOf(p, held)) == module { only[key] = true } -- 2.54.0 From e3fed7751558298365c74fa7c899fb72037deb17 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 22:58:30 +0100 Subject: [PATCH 04/30] Design 47 H1: a Go step waits for its entry's pins as the build does (H-A-206) An entry may name a package with no version yet, and the runner cannot pin it from a publish it does not hear, so a step asked then is refused by the Go build module. The placing now asks a Go artifact's lint, unit and integration steps only when every pin of the entry has a version; until then each stands waiting, its why "waits on the publish of " as the build's wait reads, said in the journal and read by show. When the mover's build ends without passing, the held steps end as it ended; the publish re-pins the entry and moves the mover, whose pipeline runs them. A held step still counts in the judgement of a seat with no holder. Builds: design 47 H H-A-206, H-A-203, H-A-165 --- .../internal/pipeline/place.go | 48 +++++++++++++++++++ .../internal/pipeline/place_test.go | 44 +++++++++++++++-- .../internal/pipeline/start.go | 7 ++- .../internal/pipeline/steps.go | 3 ++ 4 files changed, 98 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 4110b3d1..d978de57 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -4,6 +4,8 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "maps" + "slices" "sort" "strconv" "strings" @@ -140,6 +142,13 @@ func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[ steps = append(steps, s) continue } + // A Go step is asked only once every pin of its entry has a version (H-A-206): until then it waits on the + // publish, as the build does (H-A-165), and the publish re-pins the entry and moves the mover (H-A-204). + if path := unpinnedPackage(entry); path != "" && language == sdkstep.AskGo { + s.Seat, s.Outcome, s.Started, s.Why = bs, seat.StepResultOutcomeWaiting, &now, PinWait+path + steps = append(steps, s) + continue + } s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now s.Ask = AskID(startOf(p, now), p.Pipeline, m.Module, a.Ref, step) body := stepAsk(p, m, entry, a, language, step, s.Ask, asker, now) @@ -227,6 +236,45 @@ type Asked struct { Commit string } +// PinWait opens the why of a Go step held because its entry names a package with no version yet (H-A-206), the build's +// own wait form (H-A-165); the package's path follows. +const PinWait = "waits on the publish of " + +// unpinnedPackage is the first package, in path order, the entry names with no version, empty when every one has one. +func unpinnedPackage(e catalog.CatalogEntry) string { + for _, path := range slices.Sorted(maps.Keys(e.Packages)) { + if e.Packages[path] == "" { + return path + } + } + return "" +} + +// pinHeld says whether a step waits for its entry's pins (H-A-206). +func pinHeld(s seat.StepResult) bool { + return s.Outcome == seat.StepResultOutcomeWaiting && strings.HasPrefix(s.Why, PinWait) +} + +// endPinHeld ends, as the mover's build ended without passing, each of its steps waiting for its entry's pins (H-A-206): +// with the build's outcome and why, so no step waits on a publish the build no longer waits for. It answers the steps it +// ended; a build that passed ends none, the publish re-pinning the entry and moving the mover. +func endPinHeld(p *seat.Pipeline, build seat.StepResult, now time.Time) []int { + if stepPassed(build) { + return nil + } + var ended []int + for i := range p.Steps { + st := &p.Steps[i] + if st.Module != build.Module || !pinHeld(*st) { + continue + } + st.Outcome, st.Why, st.Ended = build.Outcome, "the build ended "+build.Outcome+" while this step waited: "+st.Why+"; "+build.Why, &now + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) + ended = append(ended, i) + } + return ended +} + func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk { // A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the // artifact's named paths in it when the entry has them. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index 21b3047e..0ffc885f 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -2,6 +2,7 @@ package pipeline import ( "encoding/json" + "fmt" "strconv" "strings" "testing" @@ -44,7 +45,7 @@ func placeFixture() (catalog.Registration, map[string]catalog.CatalogEntry) { Artifacts: map[string]catalog.StepStatuses{"tools": {Lint: "bypass"}}, }} return reg, map[string]catalog.CatalogEntry{ - "app": {Module: "app", Path: "modules/app", Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": ""}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, + "app": {Module: "app", Path: "modules/app", Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25"}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, {Ref: "app:web", Language: "rust"}}, Inputs: &catalog.EntryInputs{Paths: map[string][]string{"app:program": {"cmd/app"}}}}, "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, } @@ -104,13 +105,50 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { len(a.Step.Fetch.Paths) != 1 || a.Step.Fetch.Paths[0] != "cmd/app" { t.Errorf("the ask = %+v at %s", a, asks.at[1]) } - // Every step ask fetches with the entry's pins, as the builder gives them the build; a package with no pin yet is left - // out (H-A-203, H-A-192). + // Every step ask fetches with the entry's pins, as the builder gives them the build (H-A-203, H-A-192). for _, x := range asks.asks[1:] { if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && (len(b.Step.Fetch.Packages) != 1 || b.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25") { t.Errorf("the %s ask of %s fetches the pins %v, want the entry's", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages) } } + // An entry naming a package with no version yet asks no Go step: each waits on its publish, as the build does, said in + // the journal (H-A-206); the build is still asked. + waiting := map[string]catalog.CatalogEntry{} + for k, e := range entries { + waiting[k] = e + } + held := waiting["app"] + held.Packages = map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": ""} + waiting["app"] = held + var journal []string + heldAsks := &stepAsks{} + hs := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: heldAsks, Name: "the-runner", + Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }, + Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }} + if ok, err := hs.Start(Begin{Branch: "main", Source: 5, Entries: waiting, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + for _, x := range heldAsks.asks { + if b, ok := x.(nodebuildgo.StepAsk); ok { + t.Errorf("the %s of %s is asked with a package unpinned", b.Step.Step, b.Step.Artifact) + } + } + var hp seat.Pipeline + if err := json.Unmarshal(store["main.5"], &hp); err != nil { + t.Fatal(err) + } + if st := hp.Steps[1]; st.Step != "lint" || st.Outcome != seat.StepResultOutcomeWaiting || st.Ask != "" || st.Why != "waits on the publish of git.novox.be/novox/x/go" { + t.Errorf("the held lint step = %+v; want it waiting on the publish", st) + } + // The build ending without passing past its bound ends each held step as it ended (H-A-206). + failed := hp.Steps[0] + failed.Outcome, failed.Why = seat.StepResultOutcomeFailed, "package git.novox.be/novox/x/go has no released version" + if ended := endPinHeld(&hp, failed, time.Now()); len(ended) == 0 || hp.Steps[1].Outcome != seat.StepResultOutcomeFailed || hp.Steps[1].Ended == nil { + t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", hp.Steps[1], len(ended)) + } + if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") { + t.Errorf("the journal says %q; want the held step named", journal) + } // An artifact that names no paths still fetches its module alone: the directory with no paths (principle 10, A-447). for _, x := range asks.asks[2:] { if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && b.Step.Artifact == "app:tools" && (b.Step.Fetch.Dir != "modules/app" || len(b.Step.Fetch.Paths) != 0) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index cfb8cc48..cb039452 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -189,7 +189,7 @@ func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]A } var none []string for _, st := range p.Steps { - if !open(st) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) { + if (!open(st) && !pinHeld(st)) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) { continue } if holders, read := s.Holders(st.Seat); read && len(holders) == 0 { @@ -337,6 +337,11 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog } steps, a := Place(*p, m, r, entries, now, s.Name) supersededWhy(*p, m.Module, entries, steps) + for _, st := range steps { + if pinHeld(st) { + s.logf("%s: %s's %s of %s is not asked: it %s, as its build does (H-A-206)", p.Pipeline, m.Module, st.Step, st.Artifact, st.Why) + } + } if failed != "" { why := "its dependency " + failed + " did not build" for k := range steps { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 9b39cf26..854af132 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -272,6 +272,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh // Every mover's validate reads the one run (§745, H-A-51); its end asks each mover's composition, or ends it not // asked when validate did not pass. var due []Asked + if s.Step == seat.StepResultStepBuild { + readers = append(readers, endPinHeld(&p, *s, now)...) + } if s.Step == seat.StepResultStepValidate { readers = append(readers, sameValidate(&p, i, now)...) var notAsked []int -- 2.54.0 From 0fcc614e25b95128880fd96e8ffd0fac7b89f332 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 22:59:31 +0100 Subject: [PATCH 05/30] Build an ask of an older kept revision by reading it by number, never waiting (H-A-205) Two pipelines of consecutive revisions ran at once (main.45, 22:5x CET): the picture held source revision 46, whose entries had moved on, so the older pipeline's builds waited for a picture that only moves forward and failed "revision 45 of main not in the builder's picture within 2m0s". An ask of a source revision below the newest the picture holds now reads its catalog revision by number through the catalog seat's verb revision (kept while its pipeline runs, H-A-147) and is built when that revision is of its source and names every entry it asks; only a revision above the newest is waited for (H-A-155). Builds: design 47 H H-A-205, H-A-147, H-A-155 --- .../cmd/mesh-delivery-builder/keptrevision.go | 47 +++++++++++++++++++ .../cmd/mesh-delivery-builder/main.go | 2 + .../internal/build/revisionwait.go | 32 +++++++++++++ .../internal/build/run.go | 4 ++ .../internal/build/step_test.go | 38 ++++++++++++++- 5 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/keptrevision.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/keptrevision.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/keptrevision.go new file mode 100644 index 00000000..f9d6e55e --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/keptrevision.go @@ -0,0 +1,47 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// keptRevisionVia reads one kept revision of a branch by its number through the catalog seat's verb revision, page by page +// (design 47 H H-A-205, H-A-147): the source revision it was taken in from and the fingerprint of each module it names. +func keptRevisionVia(call toolCaller) func(context.Context, string, int64) (int64, map[string]string, error) { + return func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) { + fps := map[string]string{} + var source int64 + after := "" + for { + raw, err := call(meshcatalogclient.RevisionVerb, meshcatalog.RevisionInput{Branch: keys.EscapeName(branch), Revision: revision, After: after}) + if err != nil { + return 0, nil, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return 0, nil, fmt.Errorf("%s answered: %s", meshcatalogclient.RevisionVerb, said) + } + var a meshcatalog.RevisionAnswer + if err := json.Unmarshal(answer, &a); err != nil { + return 0, nil, fmt.Errorf("%s answered no revision: %w", meshcatalogclient.RevisionVerb, err) + } + if a.NoneYet || a.Revision == nil || a.Revision.Revision != revision { + return 0, nil, fmt.Errorf("%s answered no revision %d of %s", meshcatalogclient.RevisionVerb, revision, branch) + } + source = a.Revision.Source + for _, e := range a.Entries { + fps[e.Module] = e.Fp + } + if a.Next == "" { + return source, fps, nil + } + after = a.Next + } + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index a5167640..0faf4eaf 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -469,6 +469,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // a holder of the seat that is down holds no other part of it). half.b = b b.Picture = half.picture + // An ask of a revision below the newest the picture holds is read by number from the catalog seat (H-A-205). + b.KeptRevision = keptRevisionVia(calls.Call) branches.Held = func(string, meshcatalog.CatalogRevision) { b.PictureHeld() } // a revision held wakes a step ask's wait (H-A-162) half.ready = func() { if err := b.Sweep(ctx, half.pictured()...); err != nil { diff --git a/modules/mesh-delivery-builder/internal/build/revisionwait.go b/modules/mesh-delivery-builder/internal/build/revisionwait.go index 2a052bbc..2ae3c745 100644 --- a/modules/mesh-delivery-builder/internal/build/revisionwait.go +++ b/modules/mesh-delivery-builder/internal/build/revisionwait.go @@ -20,6 +20,11 @@ import ( // revision its view holds (PictureHeld, H-A-162), never a // timer or a poll, up to RevisionWait (the setting revision-wait-minutes); past it the ask fails, naming the revision and the // branch, and nothing is built. A step ask with no source (a run on request) does not wait. +// +// **The picture holds every kept revision, not only the newest** (design 47 H H-A-205, H-A-147): an ask of a source revision +// below the newest the picture holds, whose entries the picture no longer names, asks the catalog seat's verb revision for +// its catalog revision by number, once, and is built when that kept revision is of its source and names every entry it +// asks; it never waits, since the picture only moves on. Only a revision above the newest is waited for. // DefaultRevisionWait is the bound of a step ask's wait for its revision, its setting revision-wait-minutes' default (H-A-156). const DefaultRevisionWait = 2 * time.Minute @@ -68,12 +73,39 @@ func (b *Builder) holdsRevision(branch string, source int64, entries []meshbuild return true } +// keptRevisionHolds answers no outcome when the catalog keeps the ask's revision (its catalog revision by number, of its +// source revision) and that revision names every entry of the ask; else failed or could not run, with its words. +func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, held int64) (outcome, said string) { + below := fmt.Sprintf("revision %d of %s is below the %d the builder's picture holds", run.Source, run.Branch, held) + if b.KeptRevision == nil || run.Revision <= 0 { + return meshbuilds.StepEndedOutcomeFailed, below + ", and the ask names no catalog revision to read it by" + } + source, fps, err := b.KeptRevision(ctx, run.Branch, run.Revision) + if err != nil { + b.logf("%s: %s, and the catalog's revision %d cannot be read: %v", run.Pipeline, below, run.Revision, err) + return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s, and the catalog's revision %d cannot be read: %v", below, run.Revision, err) + } + if source != run.Source { + return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d is of source revision %d", below, run.Revision, source) + } + for _, e := range run.Entries { + if fps[e.Module] != e.Fingerprint { + return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d names %s at %q, not %s", below, run.Revision, e.Module, fps[e.Module], e.Fingerprint) + } + } + b.logf("%s: %s; the catalog keeps revision %d with its entries: the build goes on", run.Pipeline, below, run.Revision) + return "", "" +} + // awaitRevision returns no outcome once the builder's picture of the ask's branch holds its source revision; past the bound the // step's outcome failed, at the builder's stop could not run, each with its words. func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (outcome, said string) { if b.Picture == nil || run.Source <= 0 || run.Branch == "" { return "", "" } + if held, ok := b.Picture(run.Branch); ok && run.Source < held && !b.holdsRevision(run.Branch, run.Source, run.Entries) { + return b.keptRevisionHolds(ctx, run, held) + } bound := b.revisionWait() notHeld := fmt.Sprintf("revision %d of %s not in the builder's picture", run.Source, run.Branch) wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8) diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ba6f3e3c..4bfb6df4 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -110,6 +110,10 @@ type Builder struct { // Picture answers the source revision the builder's picture of a watched branch holds, false while it holds none (design 47 // H H-A-155, H-A-141): a step ask naming a newer one waits for it. Nil waits for nothing. Picture func(branch string) (source int64, held bool) + // KeptRevision asks the catalog seat's verb revision for one kept revision of a branch by its number (design 47 H + // H-A-205, H-A-147): the source revision it was taken in from and the fingerprint of each module it names. A step ask of + // a source revision below the newest the picture holds is built when that kept revision names its entries. Nil reads none. + KeptRevision func(ctx context.Context, branch string, revision int64) (source int64, fingerprints map[string]string, err error) // RevisionWait bounds a step ask's wait for its revision, its setting revision-wait-minutes (H-A-156); DefaultRevisionWait // when 0. RevisionWait time.Duration diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 6a777cf2..2bf0847a 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -119,6 +119,7 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T) } t.Run("H-A-155: an ask waits for the picture to take its revision in, then builds with its version reserved", aStepAskWaitsForItsRevision) t.Run("H-A-155: past the bound the ask fails naming its revision, and nothing is built", aStepAskPastItsRevisionBoundFails) + t.Run("H-A-205: an ask of a revision below the picture's newest is read by number and built", anOlderKeptRevisionIsReadByNumber) t.Run("H-A-162: a revision held that changes no entry wakes the ask", aRevisionHeldWakesTheAsk) t.Run("H-A-165: a package with no pin waits for its release and is built with it", aPackageWithNoPinWaitsForItsRelease) t.Run("H-A-165: past its bound a package with no release fails the step", aPackageWithNoReleaseFailsPastItsBound) @@ -329,6 +330,11 @@ func aStepAskWaitsForItsRevision(t *testing.T) { held.Store(40) asked := make(chan struct{}, 1) r, res, entry := revisionRig(t, &held, asked) + var keptReads atomic.Int64 // H-A-205: a revision above the newest is waited for, never read by number + r.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { + keptReads.Add(1) + return 0, nil, errors.New("not asked") + } ended := make(chan meshbuilds.StepEnded, 1) go func() { ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41, @@ -353,8 +359,36 @@ func aStepAskWaitsForItsRevision(t *testing.T) { t.Fatal("the ask was not woken by the entry the picture took in") } r.b.Wait() - if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 { - t.Errorf("end = %+v, reserved from %+v; want passed with one version reserved from revision 41", end, res.sources) + if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 || keptReads.Load() != 0 { + t.Errorf("end = %+v, reserved from %+v, kept revisions read %d; want passed with one version reserved from revision 41, none read", end, res.sources, keptReads.Load()) + } +} + +// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, whose entry of m moved +// on, while the older pipeline asks the build of revision 45. Its kept catalog revision is read by number and the build runs; +// a kept revision naming another fingerprint fails the step, naming it. +func anOlderKeptRevisionIsReadByNumber(t *testing.T) { + var held atomic.Int64 + held.Store(46) + r, _, _ := revisionRig(t, &held, make(chan struct{}, 1)) + r.b.RevisionWait = time.Nanosecond // a wait would fail at once: the older revision is never waited for + var read []string + r.b.KeptRevision = func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) { + read = append(read, fmt.Sprintf("%s %d", branch, revision)) + return 45, map[string]string{"m": fp1}, nil + } + run := meshbuilds.StepRun{Pipeline: "main.45.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 45, Source: 45, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}} + end := r.b.BuildStep(t.Context(), run) + r.b.Wait() + if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 45"}) { + t.Errorf("end = %+v, kept revisions read %v; want passed, revision 45 of main read once", end, read) + } + r.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { + return 45, map[string]string{"m": "src2:other"}, nil + } + if end := r.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") { + t.Errorf("end = %+v; want failed naming the fingerprint the kept revision names", end) } } -- 2.54.0 From 014f7b7c2f39ee72d639f3a9e8f766685f626ae1 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:03:45 +0100 Subject: [PATCH 06/30] Reserve the version of an ask admitted by its kept revision (H-A-205) An ask of an older kept revision was admitted, but after a builder restart its fingerprint had never been heard on the branch, so its build neither reserved a version nor was checked against one and passed silently. Each asked fingerprint is now recorded as the branch's (its source revision, and first as the module's registration says), without moving what the picture heard, so the newer fingerprint's build is not superseded. Builds: design 47 H H-A-205, H-A-155 --- .../cmd/mesh-delivery-builder/main.go | 1 + .../internal/build/revisionwait.go | 13 +++++++++++++ .../mesh-delivery-builder/internal/build/run.go | 4 ++++ .../internal/build/step_test.go | 15 +++++++++++---- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 0faf4eaf..60097089 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -471,6 +471,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro b.Picture = half.picture // An ask of a revision below the newest the picture holds is read by number from the catalog seat (H-A-205). b.KeptRevision = keptRevisionVia(calls.Call) + b.FirstBranch = half.first branches.Held = func(string, meshcatalog.CatalogRevision) { b.PictureHeld() } // a revision held wakes a step ask's wait (H-A-162) half.ready = func() { if err := b.Sweep(ctx, half.pictured()...); err != nil { diff --git a/modules/mesh-delivery-builder/internal/build/revisionwait.go b/modules/mesh-delivery-builder/internal/build/revisionwait.go index 2ae3c745..274ec20c 100644 --- a/modules/mesh-delivery-builder/internal/build/revisionwait.go +++ b/modules/mesh-delivery-builder/internal/build/revisionwait.go @@ -93,6 +93,19 @@ func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d names %s at %q, not %s", below, run.Revision, e.Module, fps[e.Module], e.Fingerprint) } } + // Each asked fingerprint is recorded as the branch's, so its build reserves its version (H-A-155) as one the picture + // took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded. + b.mu.Lock() + for _, e := range run.Entries { + key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint) + if err != nil || b.origin[key].Branch != "" { + continue + } + first := b.FirstBranch == nil || b.FirstBranch(run.Branch, e.Module) + b.trunkFP[key] = true + b.origin[key] = Origin{Branch: run.Branch, Source: run.Source, First: first} + } + b.mu.Unlock() b.logf("%s: %s; the catalog keeps revision %d with its entries: the build goes on", run.Pipeline, below, run.Revision) return "", "" } diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 4bfb6df4..b87cfd1a 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -114,6 +114,10 @@ type Builder struct { // H-A-205, H-A-147): the source revision it was taken in from and the fingerprint of each module it names. A step ask of // a source revision below the newest the picture holds is built when that kept revision names its entries. Nil reads none. KeptRevision func(ctx context.Context, branch string, revision int64) (source int64, fingerprints map[string]string, err error) + // FirstBranch says whether branch is the first watched branch of the module's registration, the one whose builds reserve + // a version (design 47 G §371): an ask admitted by its kept revision is recorded as that branch's (H-A-205, H-A-155). Nil + // takes every branch as the first, as an origin never heard is. + FirstBranch func(branch, module string) bool // RevisionWait bounds a step ask's wait for its revision, its setting revision-wait-minutes (H-A-156); DefaultRevisionWait // when 0. RevisionWait time.Duration diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 2bf0847a..2a0e23dd 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -370,8 +370,9 @@ func aStepAskWaitsForItsRevision(t *testing.T) { func anOlderKeptRevisionIsReadByNumber(t *testing.T) { var held atomic.Int64 held.Store(46) - r, _, _ := revisionRig(t, &held, make(chan struct{}, 1)) - r.b.RevisionWait = time.Nanosecond // a wait would fail at once: the older revision is never waited for + r, res, _ := revisionRig(t, &held, make(chan struct{}, 1)) // a picture at 46 that never heard 45 (a restart) + r.b.RevisionWait = time.Nanosecond // a wait would fail at once: the older revision is never waited for + r.b.FirstBranch = func(branch, module string) bool { return branch == "main" && module == "m" } var read []string r.b.KeptRevision = func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) { read = append(read, fmt.Sprintf("%s %d", branch, revision)) @@ -384,10 +385,16 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) { if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 45"}) { t.Errorf("end = %+v, kept revisions read %v; want passed, revision 45 of main read once", end, read) } - r.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { + // Its fingerprint is recorded as the branch's: its build reserves its version from source revision 45 (H-A-155). + if len(res.sources) != 1 || res.sources[0].Revision != 45 { + t.Errorf("reserved from %+v; want one version reserved from revision 45", res.sources) + } + r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1)) + r2.b.RevisionWait = time.Nanosecond + r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { return 45, map[string]string{"m": "src2:other"}, nil } - if end := r.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") { + if end := r2.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") { t.Errorf("end = %+v; want failed naming the fingerprint the kept revision names", end) } } -- 2.54.0 From b0d2d22002302df7dfb26b6745ba0138bd13c73b Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:04:07 +0100 Subject: [PATCH 07/30] Design 47 H1: indexer_entry reads one entry's pins and paths at a revision A reader twice could not read one entry's pins because the whole revision (1.3 MB) exceeds what the bus carries; the tool answers one module's packages, paths per artifact, fingerprint and manifest digest, at the newest trunk revision (read as entryOf reads it) or at a given one from its kept list and that one entry object. Builds: design 47 H H-A-204 (hq f977c0e2); tools-are-how-a-module-is-spoken-to --- .../cmd/mesh-delivery-indexer-tools/entry.go | 85 +++++++++++++++ .../mesh-delivery-indexer-tools/entry_test.go | 100 ++++++++++++++++++ .../cmd/mesh-delivery-indexer-tools/main.go | 2 +- .../internal/index/catalog_revisions.go | 48 +++++++++ .../internal/index/entry_verb.go | 2 + .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 1 + 7 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go new file mode 100644 index 00000000..8eb969dd --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "fmt" + "strconv" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-indexer/internal/index" +) + +// entryAnswer is what indexer_entry answers: the trunk revision read, the entry's packages (the pins: each other-repository +// package with the version it builds against, empty while none is published), its named paths by artifact, its fingerprint +// and its manifest's digest. +type entryAnswer struct { + Module string `json:"module"` + Revision int64 `json:"revision"` + Packages map[string]string `json:"packages"` + Paths map[string][]string `json:"paths"` + Fp string `json:"fp,omitempty"` + ManifestDigest string `json:"manifest-digest,omitempty"` +} + +// entryReadTool is indexer_entry (H-A-204; tools-are-how-a-module-is-spoken-to) over the indexer's catalog-revisions objects: +// one module's entry pins and paths at a trunk revision, small, so a pin can be read without the revision whole (1.3 MB, +// more than the bus carries). +func entryReadTool(o index.Objects) stdio.Tool { + return stdio.Tool{Name: index.EntryPinsTool, + Description: "Read one module's catalog graph entry pins: the packages it builds against with the version each is pinned to " + + "(an empty version while none is published), the paths each artifact is built from, its fingerprint and its manifest " + + "digest, at the newest trunk revision or at a trunk revision given; a small answer, for when the revision whole is too " + + "large for the bus (H-A-204).", + Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module whose entry is read"}, + "revision": map[string]any{"type": "integer", "description": "the trunk revision; none reads the newest"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + module = strings.TrimSpace(module) + if module == "" { + return nil, fmt.Errorf("%s names the module", index.EntryPinsTool) + } + revision, err := revisionArg(args["revision"]) + if err != nil { + return nil, err + } + rev, e, err := index.TrunkEntryAt(context.Background(), o, module, revision) + if err != nil { + return nil, err + } + out := entryAnswer{Module: e.Module, Revision: rev.Revision, Packages: e.Packages, Paths: map[string][]string{}, + Fp: e.Fp, ManifestDigest: e.ManifestDigest} + if out.Packages == nil { + out.Packages = map[string]string{} + } + if e.Inputs != nil && e.Inputs.Paths != nil { + out.Paths = e.Inputs.Paths + } + return out, nil + }} +} + +// revisionArg is the revision argument as a number: absent is 0 (the newest); a number or its digits as a string. +func revisionArg(v any) (int64, error) { + switch r := v.(type) { + case nil: + return 0, nil + case float64: + if r < 0 || r != float64(int64(r)) { + return 0, fmt.Errorf("revision %v is not a trunk revision's number", r) + } + return int64(r), nil + case string: + if strings.TrimSpace(r) == "" { + return 0, nil + } + n, err := strconv.ParseInt(strings.TrimSpace(r), 10, 64) + if err != nil || n < 0 { + return 0, fmt.Errorf("revision %q is not a trunk revision's number", r) + } + return n, nil + } + return 0, fmt.Errorf("revision %v is not a trunk revision's number", v) +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go new file mode 100644 index 00000000..aefb9942 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + + "mesh-delivery-indexer/internal/index" +) + +// memObjects is catalog-revisions in memory. +type memObjects map[string][]byte + +func (m memObjects) Put(_ context.Context, name string, data []byte) error { + m[name] = data + return nil +} +func (m memObjects) Get(_ context.Context, name string) ([]byte, error) { + d, ok := m[name] + if !ok { + return nil, index.ErrNoObject + } + return d, nil +} +func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil } +func (m memObjects) Names(context.Context) ([]string, error) { + var out []string + for n := range m { + out = append(out, n) + } + return out, nil +} +func (memObjects) Fill(context.Context) (int64, int64, error) { return 0, 0, nil } + +// TestEntryTool: indexer_entry answers one entry's pins, paths, fingerprint and manifest digest, small, at the newest trunk +// revision and at a given one read from its kept object; an unknown module or revision is an error (H-A-204). +func TestEntryTool(t *testing.T) { + o := memObjects{} + keep := func(revision int64, entries ...meshcatalog.CatalogEntry) { + rev := meshcatalog.CatalogRevision{Branch: index.TrunkBranch, Revision: revision} + for _, e := range entries { + e.Manifest = map[string]any{"module": e.Module, "big": strings.Repeat("x", 64<<10)} // the manifest whole is never answered + data, _ := json.Marshal(e) + digest := e.Module + "-" + e.Fp + o[index.EntryObject(digest)] = data + rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: e.Module, Digest: digest}) + } + data, _ := json.Marshal(rev) + o[index.BranchObject(index.TrunkBranch, revision)] = data + } + paths := &meshcatalog.EntryInputs{Paths: map[string][]string{"mesh-grants:program": {"modules/mesh-grants", "sdk"}}} + keep(7, meshcatalog.CatalogEntry{Module: "mesh-grants", Fp: "src2:old", ManifestDigest: "sha256:a", Inputs: paths, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26"}}, meshcatalog.CatalogEntry{Module: "other", Fp: "src2:o"}) + keep(9, meshcatalog.CatalogEntry{Module: "mesh-grants", Fp: "src2:new", ManifestDigest: "sha256:b", Inputs: paths, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.27", "busgrants": ""}}) + tool := entryReadTool(o) + if tool.Name != "indexer_entry" { + t.Fatalf("name %s", tool.Name) + } + for _, c := range []struct { + name string + args map[string]any + want entryAnswer + err string + }{ + {"newest", map[string]any{"module": "mesh-grants"}, entryAnswer{Module: "mesh-grants", Revision: 9, Fp: "src2:new", ManifestDigest: "sha256:b", + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.27", "busgrants": ""}, Paths: paths.Paths}, ""}, + {"given", map[string]any{"module": "mesh-grants", "revision": float64(7)}, entryAnswer{Module: "mesh-grants", Revision: 7, Fp: "src2:old", ManifestDigest: "sha256:a", + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26"}, Paths: paths.Paths}, ""}, + {"given as digits, no paths", map[string]any{"module": "other", "revision": "7"}, entryAnswer{Module: "other", Revision: 7, Fp: "src2:o", + Packages: map[string]string{}, Paths: map[string][]string{}}, ""}, + {"not in the newest", map[string]any{"module": "other"}, entryAnswer{}, "holds no module other"}, + {"revision not kept", map[string]any{"module": "mesh-grants", "revision": float64(8)}, entryAnswer{}, "revision 8 is not kept"}, + {"no module", map[string]any{}, entryAnswer{}, "names the module"}, + } { + t.Run(c.name, func(t *testing.T) { + got, err := tool.Run(c.args) + if c.err != "" { + if err == nil || !strings.Contains(err.Error(), c.err) { + t.Fatalf("error %v, want %q", err, c.err) + } + return + } + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, c.want) { + t.Fatalf("answer %+v, want %+v", got, c.want) + } + data, _ := json.Marshal(got) + if len(data) >= 8<<10 { + t.Fatalf("answer of %d bytes, want under 8 KB", len(data)) + } + }) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index c4311fa4..a2eba94a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -193,7 +193,7 @@ func main() { tools = append(tools, catalogTools(trunk.State, kept, watched)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). - tools = append(tools, index.RevisionTool(revisions, kept), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), + tools = append(tools, index.RevisionTool(revisions, kept), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), entryReadTool(revisions), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}), index.InputsReadTool(keptRead{stdio.State(index.InputsState)}, stateobjects.ForState(index.InputsState).Objects)) tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index ec1e7717..b26e2ba7 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -210,6 +210,54 @@ func readNewestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Ca } } +// TrunkEntryAt is a module's entry in a trunk revision, with that revision's list (H-A-204): for revision 0 the newest trunk +// revision read as entryOf reads it (readNewestOf); else the kept list of that trunk revision and only the module's entry +// object, so one entry is read without the revision whole. +func TrunkEntryAt(ctx context.Context, o Objects, module string, revision int64) (meshcatalog.CatalogRevision, meshcatalog.CatalogEntry, error) { + if revision == 0 { + rev, entries, found, err := readNewestOf(ctx, o, TrunkBranch) + if err != nil { + return rev, meshcatalog.CatalogEntry{}, err + } + if !found { + return rev, meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet") + } + for _, e := range entries { + if e.Module == module { + return rev, e, nil + } + } + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d holds no module %s", rev.Revision, module) + } + name := BranchObject(TrunkBranch, revision) + var rev meshcatalog.CatalogRevision + data, err := o.Get(ctx, name) + if errors.Is(err, ErrNoObject) { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d is not kept", revision) + } + if err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s: %w", name, err) + } + if err := json.Unmarshal(data, &rev); err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s: %w", name, err) + } + for _, ref := range rev.Entries { + if ref.Module != module { + continue + } + data, err := o.Get(ctx, EntryObject(ref.Digest)) + if err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s, the entry of %s (%s): %w", name, module, ref.Digest, err) + } + var e meshcatalog.CatalogEntry + if err := json.Unmarshal(data, &e); err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s, the entry of %s: %w", name, module, err) + } + return rev, e, nil + } + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d holds no module %s", revision, module) +} + // changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name. func changedEntries(before, after []meshcatalog.EntryRef) []string { was := map[string]string{} diff --git a/modules/mesh-delivery-indexer/internal/index/entry_verb.go b/modules/mesh-delivery-indexer/internal/index/entry_verb.go index 96aee4e8..e310ac84 100644 --- a/modules/mesh-delivery-indexer/internal/index/entry_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/entry_verb.go @@ -21,6 +21,8 @@ const ( EntryIndexState = "entry-index" // EntryIndexTool is the tool that reads a fingerprint's key and its object's digest (class read). EntryIndexTool = "indexer_entries" + // EntryPinsTool is the tool that reads one module's entry pins and paths at a trunk revision (class read; H-A-204). + EntryPinsTool = "indexer_entry" ) // EntryIndexValue is one key of entry-index: the digest of the entry object of the fingerprint the key names. diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 8a24e6af..6a4fc4c4 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -205,9 +205,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build index.FingerprintsAgainTool, // issue 1966 // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, - index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 + index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204 index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 42 || !slices.Contains(m.Tools, "modules") { + if !reflect.DeepEqual(m.Tools, want) || len(want) != 43 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 0eeaa80c..155327c1 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -57,6 +57,7 @@ "indexer_recompute", "indexer_drop_candidate_graph", "indexer_entries", + "indexer_entry", "indexer_named_drop", "indexer_proposals", "indexer_proposal_drop", -- 2.54.0 From cb26645614e1178cecd85e9ef34be3cf28a389ad Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:05:08 +0100 Subject: [PATCH 08/30] Design 47 H1: the build step's end reports the pins it was built with (H-A-207) A test step held for an entry's empty pin had nothing to run with once the build passed: the builder took the pin from the registry and kept it to itself. The seat mesh-builds' step end gains packages, and the builder sets it from the pins each entry was built with, the registry's included, so the runner asks the held steps with what the build was built against. The seat's definition changes: its Go and TypeScript contracts are generated again. Builds: design 47 H H-A-207 --- .../internal/build/step.go | 19 ++++++++++++++++--- .../internal/build/step_test.go | 4 ++++ .../go/seats/meshbuilds/contract.go | 8 +++++--- .../seats/mesh-builds/seat.json | 7 +++++++ .../ts/seats/meshbuilds/index.d.ts | 2 ++ 5 files changed, 34 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index 09687285..cfff2cac 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -41,7 +41,18 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui } var failed []string for _, e := range run.Entries { - outcome, said := b.buildEntry(ctx, e) + var pins map[string]string + outcome, said := b.buildEntry(ctx, e, &pins) + // The end reports the pins each entry was built with, so the runner asks its held test steps with them (H-A-207). + for path, version := range pins { + if version == "" { + continue + } + if end.Packages == nil { + end.Packages = map[string]string{} + } + end.Packages[path] = version + } switch outcome { case meshbuilds.StepEndedOutcomeCouldNotRun: return finish(outcome, said) @@ -55,8 +66,9 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui return finish(meshbuilds.StepEndedOutcomePassed, "") } -// buildEntry builds one entry at its fingerprint and waits for its end. -func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outcome, said string) { +// buildEntry builds one entry at its fingerprint and waits for its end; pins is set to the pins it is built with, the +// entry's own and those the registry gave a package the entry named with none (H-A-207). +func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) { entry, err := b.StepEntry(ctx, e.Fingerprint) if err != nil { return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry %s at %s cannot be read: %v", e.Module, e.Fingerprint, err) @@ -70,6 +82,7 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc return outcome, said } entry.Packages = pinned + *pins = pinned key, err := builds.BuilderBuildKey(entry.Module, entry.Fp) if err != nil { return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error() diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 6a777cf2..08bbbedd 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -189,6 +189,10 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) { !reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) { t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks) } + // The end reports the pins the build was built with, the one the publish gave included (H-A-207). + if !reflect.DeepEqual(end.Packages, map[string]string{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) { + t.Errorf("the end reports the pins %v; want the ones it was built with", end.Packages) + } } func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) { diff --git a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go index 836f14e0..4cc3285b 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go @@ -114,9 +114,11 @@ type StepEnded struct { // the node the builder ran it on, as in the subject Node string `json:"node"` // never passed for a build that built nothing - Outcome string `json:"outcome"` - Pipeline string `json:"pipeline"` - Rerun bool `json:"rerun,omitempty"` + Outcome string `json:"outcome"` + // the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207) + Packages map[string]string `json:"packages,omitempty"` + Pipeline string `json:"pipeline"` + Rerun bool `json:"rerun,omitempty"` // why it failed or could not run: the entry not built, the part not heard Said string `json:"said,omitempty"` Started *time.Time `json:"started,omitempty"` diff --git a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json index d10eec4f..fc3d717d 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json @@ -363,6 +363,13 @@ "$ref": "#/defs/BuildEntry" }, "description": "each entry the step built, by module and fingerprint" + }, + "packages": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207)" } }, "required": [ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts index d142d6d9..947f2e52 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts @@ -89,6 +89,8 @@ export interface StepEnded { "node": string; /** never passed for a build that built nothing */ "outcome": "passed" | "failed" | "nothing to run" | "could not run"; + /** the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207) */ + "packages"?: Record; "pipeline": string; "rerun"?: boolean; /** why it failed or could not run: the entry not built, the part not heard */ -- 2.54.0 From 64cba95a101d8d8e3bf4501fae7d1e3496a5f64f Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:07:33 +0100 Subject: [PATCH 09/30] Design 47 H1: a held test step is asked with the pins its build passed with (H-A-207) When the build of an entry with an empty pin passed, its held lint, unit and integration steps had no pin to run with and nothing ended them, so a pipeline stood running and, one pipeline at a time, the queue stopped. The runner reads the pins the build's end names and, when the build passed, asks each held step with them, its fetch the entry's as placed; a passed build whose end names no pins ends the held steps could not run, naming why, as does an artifact the branch's half no longer names. Builds: design 47 H H-A-207, H-A-206, H-A-203 --- .../cmd/mesh-delivery-pipeline/main.go | 5 ++ .../cmd/mesh-delivery-pipeline/steps.go | 6 +- .../internal/pipeline/place.go | 82 +++++++++++++++++-- .../internal/pipeline/place_test.go | 53 +++++++++++- .../internal/pipeline/steps.go | 29 ++++++- 5 files changed, 164 insertions(+), 11 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 9488c743..8d0fdd22 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -154,6 +154,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} hearer.AskSteps = starter.AskSteps hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } + hearer.AskPinHeld = func(key, module string, pins map[string]string) { + if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, rv.Catalog.View); err != nil { + logger.Printf("the steps of %s's %s that waited for its pins are not asked: %v", key, module, err) + } + } hearer.Release = func(key string) { if _, err := starter.Release(hearer.Store, key, rv.Catalog.View); err != nil { logger.Printf("the movers of %s waiting for a build are not placed: %v", key, err) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index d6915534..c4aa18c1 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -128,7 +128,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) if bs.Shared != nil { shared = bs.Shared(m.Data()) } - _, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared) + var pins map[string]string + if bs.Pins != nil { + pins = bs.Pins(m.Data()) + } + _, err = h.OnStepEndedPinned(bs, m.Subject(), ev, shared, pins) switch { case errors.Is(err, pipeline.ErrRefused): logf("%v", err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index d978de57..5c096fe9 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -3,6 +3,7 @@ package pipeline import ( "crypto/sha256" "encoding/hex" + "errors" "fmt" "maps" "slices" @@ -255,12 +256,16 @@ func pinHeld(s seat.StepResult) bool { return s.Outcome == seat.StepResultOutcomeWaiting && strings.HasPrefix(s.Why, PinWait) } -// endPinHeld ends, as the mover's build ended without passing, each of its steps waiting for its entry's pins (H-A-206): -// with the build's outcome and why, so no step waits on a publish the build no longer waits for. It answers the steps it -// ended; a build that passed ends none, the publish re-pinning the entry and moving the mover. -func endPinHeld(p *seat.Pipeline, build seat.StepResult, now time.Time) []int { +// endPinHeld ends each of build's mover's steps waiting for its entry's pins (H-A-206, H-A-207): as the build ended when +// it did not pass, so no step waits on a publish the build no longer waits for; could not run, loud, when the build passed +// and its end names no pins to run them with. It answers the steps it ended. +func endPinHeld(p *seat.Pipeline, build seat.StepResult, pins map[string]string, now time.Time) []int { + outcome, why := build.Outcome, "the build ended "+build.Outcome+" while this step waited: %s; "+build.Why if stepPassed(build) { - return nil + if len(pins) > 0 { + return nil + } + outcome, why = seat.StepResultOutcomeCouldNotRun, "the build passed, but its end names no pins to run this step with (H-A-207); it %s" } var ended []int for i := range p.Steps { @@ -268,13 +273,78 @@ func endPinHeld(p *seat.Pipeline, build seat.StepResult, now time.Time) []int { if st.Module != build.Module || !pinHeld(*st) { continue } - st.Outcome, st.Why, st.Ended = build.Outcome, "the build ended "+build.Outcome+" while this step waited: "+st.Why+"; "+build.Why, &now + st.Outcome, st.Why, st.Ended = outcome, fmt.Sprintf(why, st.Why), &now p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) ended = append(ended, i) } return ended } +// heldOn says whether module has a step of p waiting for its entry's pins. +func heldOn(p seat.Pipeline, module string) bool { + return slices.ContainsFunc(p.Steps, func(st seat.StepResult) bool { return st.Module == module && pinHeld(st) }) +} + +// AskPinHeld asks the steps of pipeline key's mover module that waited for its entry's pins (H-A-206) with the pins its +// build passed with (H-A-207): each running under its own ask, its fetch the entry's as Place builds it with those pins. +// A step whose artifact the branch's half no longer names ends could not run, saying so. It answers how many it asked. +func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]string, view func(branch string) (Half, bool)) (int, error) { + for { + p, rev, err := store.Get(key) + if err != nil { + return 0, fmt.Errorf("the pipeline %s is not read: %w", key, err) + } + m, moves := MoverOf(p, module) + if !moves || !heldOn(p, module) { + return 0, nil + } + var entry catalog.CatalogEntry + if view != nil { + if half, ok := view(p.Branch); ok { + entry = half.Entries[module] + } + } + entry.Packages = pins + now := s.now() + var asks []Asked + var ended []int + for i := range p.Steps { + st := &p.Steps[i] + if st.Module != module || !pinHeld(*st) { + continue + } + j := slices.IndexFunc(entry.Artifacts, func(a catalog.Artifact) bool { return a.Ref == st.Artifact }) + language, err := sdkstep.AskLanguageOfSeat(st.Seat) + if j < 0 || err != nil { + st.Outcome, st.Ended = seat.StepResultOutcomeCouldNotRun, &now + st.Why = fmt.Sprintf("its build passed with its pins, but the catalog half of %s names no artifact %s of %s to ask it of (H-A-207)", p.Branch, st.Artifact, module) + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) + ended = append(ended, i) + continue + } + a := entry.Artifacts[j] + st.Outcome, st.Why, st.Started = seat.StepResultOutcomeRunning, "", &now + st.Ask = AskID(startOf(p, now), p.Pipeline, module, a.Ref, st.Step) + ask := Asked{Seat: st.Seat, ID: st.Ask, Ask: stepAsk(p, m, entry, a, language, st.Step, st.Ask, s.Name, now), Toolchain: a.Language, Commit: m.Commit} + if a.Package != nil { + ask.Package = a.Package.Name + } + asks = append(asks, ask) + } + if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return 0, fmt.Errorf("the pipeline %s is not written: %w", key, err) + } + s.logf("%s: %s's build passed with its pins: %d steps that waited for them are asked, %d could not run (H-A-207)", key, module, len(asks), len(ended)) + s.sayNotAsked(p, ended) + if s.Steps == nil || len(asks) == 0 { + return 0, nil + } + return len(asks), s.askSteps(key, asks) + } +} + func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk { // A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the // artifact's named paths in it when the entry has them. diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index 0ffc885f..f152f0a0 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -3,6 +3,7 @@ package pipeline import ( "encoding/json" "fmt" + "reflect" "strconv" "strings" "testing" @@ -143,8 +144,56 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { // The build ending without passing past its bound ends each held step as it ended (H-A-206). failed := hp.Steps[0] failed.Outcome, failed.Why = seat.StepResultOutcomeFailed, "package git.novox.be/novox/x/go has no released version" - if ended := endPinHeld(&hp, failed, time.Now()); len(ended) == 0 || hp.Steps[1].Outcome != seat.StepResultOutcomeFailed || hp.Steps[1].Ended == nil { - t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", hp.Steps[1], len(ended)) + failedP := hp + failedP.Steps = append([]seat.StepResult(nil), hp.Steps...) + if ended := endPinHeld(&failedP, failed, nil, time.Now()); len(ended) == 0 || failedP.Steps[1].Outcome != seat.StepResultOutcomeFailed || failedP.Steps[1].Ended == nil { + t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", failedP.Steps[1], len(ended)) + } + // A build passed whose end names no pins ends them could not run, loud (H-A-207). + passed := hp.Steps[0] + passed.Outcome = seat.StepResultOutcomePassed + bare := hp + bare.Steps = append([]seat.StepResult(nil), hp.Steps...) + if ended := endPinHeld(&bare, passed, nil, time.Now()); len(ended) == 0 || bare.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || + !strings.Contains(bare.Steps[1].Why, "names no pins") { + t.Errorf("after a build passed naming no pins the held lint step = %+v; want could not run naming it", bare.Steps[1]) + } + // A build passed naming its pins asks the held steps with them: its end heard, the runner asks each with the pins the + // build was built with (H-A-207). + pinsBuilt := map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": "v0.1.4"} + ms := memStore{} + ms["main.5"] = struct { + p seat.Pipeline + rev uint64 + }{hp, 1} + var askedWith map[string]string + h := &Hearer{Store: ms, Say: func(string, any) error { return nil }, Name: "the-runner", + AskPinHeld: func(key, module string, pins map[string]string) { + askedWith = pins + if _, err := hs.AskPinHeld(ms, key, module, pins, func(string) (Half, bool) { return Half{Entries: waiting}, true }); err != nil { + t.Errorf("ask held: %v", err) + } + }} + buildEnd := StepEnded{ID: hp.Steps[0].Ask, Pipeline: "main.5", Step: "build", Node: "n1", Outcome: seat.StepResultOutcomePassed, + Module: "app", Commit: "c1", Fingerprint: "src2:ab"} + if _, err := h.OnStepEndedPinned(StepSeats[3], StepSeats[3].Prefix+"n1", buildEnd, nil, pinsBuilt); err != nil { + t.Fatalf("the build's end: %v", err) + } + if !reflect.DeepEqual(askedWith, pinsBuilt) { + t.Errorf("the held steps were asked with %v; want the build's pins", askedWith) + } + var askedHeld int + for _, x := range heldAsks.asks { + if b, ok := x.(nodebuildgo.StepAsk); ok { + askedHeld++ + if !reflect.DeepEqual(b.Step.Fetch.Packages, pinsBuilt) { + t.Errorf("the %s of %s fetches %v; want the build's pins", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages) + } + } + } + after, _, _ := ms.Get("main.5") + if askedHeld == 0 || after.Steps[1].Outcome == seat.StepResultOutcomeWaiting || after.Steps[1].Ask == "" { + t.Errorf("after the build passed %d held steps were asked, lint = %+v; want each asked", askedHeld, after.Steps[1]) } if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") { t.Errorf("the journal says %q; want the held step named", journal) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 854af132..5ac10c50 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -31,6 +31,8 @@ type StepSeat struct { // Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for // a seat whose ends name none. Shared func([]byte) []seat.Shared + // Pins reads the pins a build's end names it was built with (H-A-207); nil for a seat whose ends name none. + Pins func([]byte) map[string]string } // StepSeats are the three build seats, from their contracts. @@ -39,7 +41,7 @@ var StepSeats = []StepSeat{ {Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")}, {Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")}, // The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99). - {Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded}, + {Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded, Pins: buildPins}, // The seat mesh-wants answers the composition step (§22, §24). {Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared}, } @@ -77,6 +79,15 @@ func decodeWantsEnded(raw []byte) (StepEnded, error) { Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil } +// buildPins is the pins a build step's end of the seat mesh-builds names its entry was built with (H-A-207). +func buildPins(raw []byte) map[string]string { + var b builds.StepEnded + if json.Unmarshal(raw, &b) != nil { + return nil + } + return b.Packages +} + // decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one // entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is // refused (A-99). @@ -155,6 +166,9 @@ type Hearer struct { // Release places the movers of a pipeline whose awaited publishes are heard now, after a build step passed (a kind-3 // edge, §746; Starter.Release); nil places none. Release func(key string) + // AskPinHeld asks a mover's steps that waited for its entry's pins with the pins its build passed with (H-A-207; + // Starter.AskPinHeld); nil leaves them waiting. + AskPinHeld func(key, module string, pins map[string]string) // Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748; // Starter.Turn); nil takes none. Turn func(key string) @@ -197,6 +211,12 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi // of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726, // A-469m). func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) { + return h.OnStepEndedPinned(bs, subject, ev, shared, nil) +} + +// OnStepEndedPinned is OnStepEndedShared of a build's end naming the pins it was built with (H-A-207): a passed build's +// mover's steps that waited for its entry's pins are asked with them. +func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared, pins map[string]string) (seat.Pipeline, error) { node := strings.TrimPrefix(subject, bs.Prefix) if node == subject || node == "" || strings.Contains(node, ".") { return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat) @@ -272,8 +292,10 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh // Every mover's validate reads the one run (§745, H-A-51); its end asks each mover's composition, or ends it not // asked when validate did not pass. var due []Asked + askHeld := false if s.Step == seat.StepResultStepBuild { - readers = append(readers, endPinHeld(&p, *s, now)...) + readers = append(readers, endPinHeld(&p, *s, pins, now)...) + askHeld = stepPassed(*s) && heldOn(p, s.Module) } if s.Step == seat.StepResultStepValidate { readers = append(readers, sameValidate(&p, i, now)...) @@ -327,6 +349,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh if release && h.Release != nil { h.Release(ev.Pipeline) } + if askHeld && h.AskPinHeld != nil { + h.AskPinHeld(ev.Pipeline, s.Module, pins) + } if h.AskSteps != nil && len(due) > 0 { if err := h.AskSteps(ev.Pipeline, due); err != nil { h.logf("the compositions of %s are written running and not asked, asked again at the next start: %v", ev.Pipeline, err) -- 2.54.0 From e29aa8aca9139128bffacf6d6305e397d00390f3 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:12:29 +0100 Subject: [PATCH 10/30] Design 47 H1: a done build key keeps the pins it was built with, and its step's end reports those (H-A-207) A build step whose key was done before reported the pins it resolved at that moment, so a publish since would have the runner ask the waiting test steps against a version the build was never built with. The done key's record keeps the versions its Go ask was built with, beside its digests, and the step's end reports the record's; a done key keeping none reports none, so the runner ends those steps could not run, loud. Builds: design 47 H H-A-207, H-A-203 --- .../mesh-delivery-builder/builds/builds.go | 5 +++++ .../internal/build/run.go | 18 ++++++++++++++- .../internal/build/step.go | 22 ++++++++++++++++--- .../internal/build/step_test.go | 22 +++++++++++++++++++ 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-builder/builds/builds.go b/modules/mesh-delivery-builder/builds/builds.go index 8981c7dc..04c3c6cd 100644 --- a/modules/mesh-delivery-builder/builds/builds.go +++ b/modules/mesh-delivery-builder/builds/builds.go @@ -242,6 +242,11 @@ type BuilderBuild struct { // Imports are, on a done key, the mesh packages the built programs import, as the build module answered them // (BuildAnswer.Imports; novox/hq ADR 0593 §2): the builder carries them onto the build's end (BuildLogEntry.Imports). Imports []string `json:"imports,omitempty"` + // Packages are, on a done key only, the mesh packages' versions its Go build was built with, by module path, the ones + // the registry gave a package the entry named with none included (H-A-207): the build step's end reports them, so the + // runner asks the steps that waited for the pins with what the build was built against. A reader that does not know + // the field reads the build as reporting none. + Packages map[string]string `json:"packages,omitempty"` // Needs are, on a done key only, the reader capabilities (`form:`) the build's reference forms need of a // node-engine, beside the ones its manifest needs (novox/hq ADR 0618 §1). A reader that does not know the field // reads the build as needing none. diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ba6f3e3c..b893e9a1 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -833,6 +833,7 @@ type lane struct { tcVersion string // and the version it ran kept string // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty outOfStep []string // the generated files its build module's compare found out of step, passed or failed + pins map[string]string // the mesh packages' versions its Go ask was built with (H-A-207) err error } @@ -982,7 +983,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module used = nil } done := builds.BuilderBuild{State: builds.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used, - Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts)} // the forms it is written in (ADR 0618 §1) + Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts), Packages: pinsOf(lanes)} // the forms it is written in (ADR 0618 §1) for _, l := range lanes { if l.kept != "" { done.KeptOn = l.node // design 47 G §671: the build node that keeps the build @@ -1144,6 +1145,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes for _, fb := range req.Builds { delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465) } + l.pins = maps.Clone(ask.Packages) // kept with the done key, so its step's end reports them (H-A-207) } if id := asked.resume(language); id != "" { ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it @@ -1326,6 +1328,20 @@ func (b *Builder) takeOutOfStep(key string) []string { // importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593 // §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that // kept one reports only what it built. +// pinsOf is the mesh packages' versions the lanes were built with (H-A-207), nil when none was pinned. +func pinsOf(lanes []*lane) map[string]string { + var out map[string]string + for _, l := range lanes { + for p, v := range l.pins { + if out == nil { + out = map[string]string{} + } + out[p] = v + } + } + return out +} + func importsOf(lanes []*lane) []string { seen := map[string]bool{} for _, l := range lanes { diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index cfff2cac..bd24c5a3 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -66,8 +66,8 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui return finish(meshbuilds.StepEndedOutcomePassed, "") } -// buildEntry builds one entry at its fingerprint and waits for its end; pins is set to the pins it is built with, the -// entry's own and those the registry gave a package the entry named with none (H-A-207). +// buildEntry builds one entry at its fingerprint and waits for its end; pins is set, when it is done, to the pins its +// done key keeps it was built with, the registry's included (H-A-207). func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) { entry, err := b.StepEntry(ctx, e.Fingerprint) if err != nil { @@ -82,7 +82,6 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins return outcome, said } entry.Packages = pinned - *pins = pinned key, err := builds.BuilderBuildKey(entry.Module, entry.Fp) if err != nil { return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error() @@ -91,6 +90,8 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins defer b.dropEnd(key, ended) if done, failedWords, found := b.keyEnd(ctx, key); found { if done { + // A key done before reports the pins it was built with, kept with it, never the ones a publish gives now. + *pins = b.keptPins(ctx, key) return b.passedWithItsVersion(ctx, entry) } return meshbuilds.StepEndedOutcomeFailed, failedWords @@ -101,6 +102,7 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins select { case log := <-ended: if log.Outcome == builds.BuildDone { + *pins = b.keptPins(ctx, key) return b.passedWithItsVersion(ctx, entry) } return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error) @@ -149,6 +151,20 @@ func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.Ca return meshbuilds.StepEndedOutcomePassed, "" } +// keptPins are the pins a done key keeps it was built with (H-A-207); none when it keeps none or cannot be read, so the +// runner ends the steps that waited for them could not run, loud. +func (b *Builder) keptPins(ctx context.Context, key string) map[string]string { + raw, _, ok, err := b.Claimer.Store.Get(ctx, key) + if err != nil || !ok { + return nil + } + v, err := builds.DecodeBuilderBuild(raw) + if err != nil || v.State != builds.BuildDone { + return nil + } + return v.Packages +} + // keyEnd reads the build key: whether it is done, and its error when it failed for good; found only for a key that has ended. func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed string, found bool) { raw, _, ok, err := b.Claimer.Store.Get(ctx, key) diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 08bbbedd..c70f1ae4 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -193,6 +193,28 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) { if !reflect.DeepEqual(end.Packages, map[string]string{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) { t.Errorf("the end reports the pins %v; want the ones it was built with", end.Packages) } + // A done key reports the pins it keeps it was built with, never the newer one a publish gives now (H-A-207). + r.b.PackagePublished(unpinnedX, "v0.1.5") + again := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}) + if again.Outcome != meshbuilds.StepEndedOutcomePassed || again.Packages[unpinnedX] != "v0.1.3" { + t.Errorf("the done key's end = %+v; want passed reporting x at v0.1.3, as built", again) + } + // A done key that keeps no pins reports none, so the runner ends the waiting steps loud. + v, err := builds.DecodeBuilderBuild(r.st.vals[r.key]) + if err != nil { + t.Fatal(err) + } + v.Packages = nil + raw, err := builds.EncodeBuilderBuild(v) + if err != nil { + t.Fatal(err) + } + r.st.vals[r.key] = raw + if bare := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); len(bare.Packages) != 0 { + t.Errorf("a done key keeping no pins reported %v; want none", bare.Packages) + } } func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) { -- 2.54.0 From c418f4903846d723fc5130c420439ea9c5f97506 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:13:47 +0100 Subject: [PATCH 11/30] Design 47 H1: held test steps are asked from the pipeline's own catalog revision (H-A-205, H-A-207) The runner built the asks of the steps that waited for their pins from the branch's newest catalog half, so a newer revision that moved the module changed what a passed build's tests fetched. The entry is now read at the pipeline's own revision through the runner's revision reader, which keeps what it read; a revision that cannot be read, or whose entry is of another fingerprint than the mover's, ends the held steps could not run, naming why. Builds: design 47 H H-A-205, H-A-207, H-A-203 --- .../cmd/mesh-delivery-pipeline/main.go | 5 +-- .../internal/pipeline/place.go | 34 ++++++++++++++----- .../internal/pipeline/place_test.go | 33 ++++++++++++++++-- 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 8d0fdd22..80a28898 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -142,7 +142,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } logger.Printf("widens: each read bound %s, %d catalog revisions kept", readBound, revisionsKept) - hearer.Widens = widenerOf(calls.Call, readBound, revisionsKept, logger.Printf).Widens + widener := widenerOf(calls.Call, readBound, revisionsKept, logger.Printf) + hearer.Widens = widener.Widens hearer.LogLast = stepLog{ctx: ctx, js: js, module: cred.Module}.Last hearer.Where = routerRecordOf(cred.Module, calls.Call, calls.Ask) hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf} @@ -155,7 +156,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro hearer.AskSteps = starter.AskSteps hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } hearer.AskPinHeld = func(key, module string, pins map[string]string) { - if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, rv.Catalog.View); err != nil { + if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil { logger.Printf("the steps of %s's %s that waited for its pins are not asked: %v", key, module, err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 5c096fe9..4fcc2aee 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -280,6 +280,14 @@ func endPinHeld(p *seat.Pipeline, build seat.StepResult, pins map[string]string, return ended } +// readRevision is revision's read, an error when none is given. +func readRevision(revision func(branch string, revision int64) (Half, error), branch string, n int64) (Half, error) { + if revision == nil { + return Half{}, errors.New("the runner reads no catalog revision") + } + return revision(branch, n) +} + // heldOn says whether module has a step of p waiting for its entry's pins. func heldOn(p seat.Pipeline, module string) bool { return slices.ContainsFunc(p.Steps, func(st seat.StepResult) bool { return st.Module == module && pinHeld(st) }) @@ -287,8 +295,10 @@ func heldOn(p seat.Pipeline, module string) bool { // AskPinHeld asks the steps of pipeline key's mover module that waited for its entry's pins (H-A-206) with the pins its // build passed with (H-A-207): each running under its own ask, its fetch the entry's as Place builds it with those pins. -// A step whose artifact the branch's half no longer names ends could not run, saying so. It answers how many it asked. -func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]string, view func(branch string) (Half, bool)) (int, error) { +// The entry is read at the pipeline's own catalog revision through revision, which keeps what it read (H-A-205), never +// the branch's newest: a revision that cannot be read, or whose entry is of another fingerprint than the mover's, or that +// names no artifact of the step, ends the step could not run, saying so. It answers how many it asked. +func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]string, revision func(branch string, revision int64) (Half, error)) (int, error) { for { p, rev, err := store.Get(key) if err != nil { @@ -299,10 +309,14 @@ func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string] return 0, nil } var entry catalog.CatalogEntry - if view != nil { - if half, ok := view(p.Branch); ok { - entry = half.Entries[module] - } + unread := "" + switch half, err := readRevision(revision, p.Branch, p.Revision); { + case err != nil: + unread = fmt.Sprintf("revision %d of %s, the pipeline's, cannot be read for its entry: %v", p.Revision, p.Branch, err) + case half.Entries[module].Fp != m.Fingerprint: + unread = fmt.Sprintf("revision %d of %s holds %s at %q, not the mover's %s", p.Revision, p.Branch, module, half.Entries[module].Fp, m.Fingerprint) + default: + entry = half.Entries[module] } entry.Packages = pins now := s.now() @@ -315,9 +329,13 @@ func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string] } j := slices.IndexFunc(entry.Artifacts, func(a catalog.Artifact) bool { return a.Ref == st.Artifact }) language, err := sdkstep.AskLanguageOfSeat(st.Seat) - if j < 0 || err != nil { + if unread != "" || j < 0 || err != nil { + why := unread + if why == "" { + why = fmt.Sprintf("revision %d of %s names no artifact %s of %s to ask it of", p.Revision, p.Branch, st.Artifact, module) + } st.Outcome, st.Ended = seat.StepResultOutcomeCouldNotRun, &now - st.Why = fmt.Sprintf("its build passed with its pins, but the catalog half of %s names no artifact %s of %s to ask it of (H-A-207)", p.Branch, st.Artifact, module) + st.Why = "its build passed with its pins, but " + why + " (H-A-207, H-A-205)" p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) ended = append(ended, i) continue diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index f152f0a0..dd10438a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -120,13 +120,14 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { } held := waiting["app"] held.Packages = map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": ""} + held.Fp = "src2:ab" waiting["app"] = held var journal []string heldAsks := &stepAsks{} hs := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: heldAsks, Name: "the-runner", Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }, Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }} - if ok, err := hs.Start(Begin{Branch: "main", Source: 5, Entries: waiting, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}}); !ok || err != nil { + if ok, err := hs.Start(Begin{Branch: "main", Source: 5, Revision: 3, Entries: waiting, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}}); !ok || err != nil { t.Fatalf("start = %v, %v", ok, err) } for _, x := range heldAsks.asks { @@ -166,11 +167,23 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { p seat.Pipeline rev uint64 }{hp, 1} + // The entry is read at the pipeline's own revision (3), never the newer revision 4 that moved the module (H-A-205). + moved := held + moved.Fp, moved.Path = "src2:ac", "modules/moved" + revisions := func(branch string, n int64) (Half, error) { + switch n { + case 3: + return Half{Revision: 3, Entries: waiting}, nil + case 0, 4: + return Half{Revision: 4, Entries: map[string]catalog.CatalogEntry{"app": moved}}, nil + } + return Half{}, fmt.Errorf("no revision %d", n) + } var askedWith map[string]string h := &Hearer{Store: ms, Say: func(string, any) error { return nil }, Name: "the-runner", AskPinHeld: func(key, module string, pins map[string]string) { askedWith = pins - if _, err := hs.AskPinHeld(ms, key, module, pins, func(string) (Half, bool) { return Half{Entries: waiting}, true }); err != nil { + if _, err := hs.AskPinHeld(ms, key, module, pins, revisions); err != nil { t.Errorf("ask held: %v", err) } }} @@ -189,12 +202,28 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { if !reflect.DeepEqual(b.Step.Fetch.Packages, pinsBuilt) { t.Errorf("the %s of %s fetches %v; want the build's pins", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages) } + if b.Step.Fetch.Dir != "modules/app" { + t.Errorf("the %s of %s fetches %s; want the pipeline's revision's directory", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Dir) + } } } after, _, _ := ms.Get("main.5") if askedHeld == 0 || after.Steps[1].Outcome == seat.StepResultOutcomeWaiting || after.Steps[1].Ask == "" { t.Errorf("after the build passed %d held steps were asked, lint = %+v; want each asked", askedHeld, after.Steps[1]) } + // A mover whose fingerprint the pipeline's revision does not hold ends its held steps could not run, naming it. + other := hp + other.Pipeline, other.Steps, other.Movers = "main.6", append([]seat.StepResult(nil), hp.Steps...), []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:zz", Repository: "o/app"}} + ms["main.6"] = struct { + p seat.Pipeline + rev uint64 + }{other, 1} + if n, err := hs.AskPinHeld(ms, "main.6", "app", pinsBuilt, revisions); n != 0 || err != nil { + t.Errorf("asked %d held steps of a mover its revision does not hold (%v); want none", n, err) + } + if o, _, _ := ms.Get("main.6"); o.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(o.Steps[1].Why, "not the mover's src2:zz") { + t.Errorf("the held lint step of a moved fingerprint = %+v; want could not run naming it", o.Steps[1]) + } if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") { t.Errorf("the journal says %q; want the held step named", journal) } -- 2.54.0 From 7199824e9de1fef7f42e64341d0f84d576efe101 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:16:25 +0100 Subject: [PATCH 12/30] Design 47 H1: indexer_take_in_again re-reads a repository at its kept merge A repository with no merge since the walk of the module graph keeps its modules' direct requirements only, and no tool took it in again. The tool asks the indexer, through the state takeins, to read the repository's trees at the merge its taken key keeps and apply them as a take-in does, whose recompute of the modules state is the one a publish re-pin runs; kept requirements and pins gain the transitive packages. An unknown repository, or one never taken in, is refused in the answer and as the tool's error. It keeps no catalog revision yet: H-A-211's revision follows with the re-pin's. Builds: design 47 H H-A-208, H-A-210, H-A-201 (hq a7b876c1, d7d18ef0) --- .../cmd/mesh-delivery-indexer/sourcemoved.go | 8 +++ .../cmd/mesh-delivery-indexer/takein.go | 5 ++ .../internal/index/held.go | 26 +++++++ .../internal/index/take_in_again.go | 69 ++++++++++++++++++ .../internal/index/take_in_again_test.go | 72 +++++++++++++++++++ .../internal/index/takein.go | 2 + .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 1 + 8 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/take_in_again.go create mode 100644 modules/mesh-delivery-indexer/internal/index/take_in_again_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go index cde7d759..4a42bcdd 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go @@ -286,6 +286,14 @@ func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn { return r } +// TakeInAgain takes a repository in again at its kept merge commit (indexer_take_in_again, H-A-208, H-A-210), one take-in +// again at a time beside the held ones. +func (a actor) TakeInAgain(ctx context.Context, repository string) index.TakenIn { + a.s.again.Lock() + defer a.s.again.Unlock() + return a.Indexer.TakeInAgain(ctx, repository) +} + // HeldSkip ends a branch's held source revision unwritten (indexer_take_in_skip): take-in.taken says it skipped. func (a actor) HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn { r := index.TakenIn{Module: branch} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 14625a36..31c7f4a8 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -24,6 +24,8 @@ type takerIn interface { // HeldAgain and HeldSkip act on a watched branch's held source-moved (design 47 H §744). HeldAgain(ctx context.Context, branch string) index.TakenIn HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn + // TakeInAgain takes a repository in again at its kept merge commit (H-A-208, H-A-210). + TakeInAgain(ctx context.Context, repository string) index.TakenIn } // takenIn answers one value of the takeins state (a hand act, issue 1945): a value not yet answered is acted on now and @@ -53,6 +55,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream } else if a.Held != "" { asked, r = "take-in again of a held take-in", ix.HeldAgain(ctx, a.Held) a.Module = a.Held + } else if a.Again != "" { + asked, r = "take-in again of a repository", ix.TakeInAgain(ctx, a.Again) + a.Module = a.Again } else if a.Registry != nil { asked, r = "registrations' act "+a.Registry.Verb, ix.RegistryAct(ctx, a) } else if a.Recompute { diff --git a/modules/mesh-delivery-indexer/internal/index/held.go b/modules/mesh-delivery-indexer/internal/index/held.go index c947f68e..5e8774ad 100644 --- a/modules/mesh-delivery-indexer/internal/index/held.go +++ b/modules/mesh-delivery-indexer/internal/index/held.go @@ -260,6 +260,32 @@ func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error) } return a.Result, nil }}, + {Name: TakeInAgainTool, + Description: "Take a repository in again now at the merge commit the indexer keeps of it (design 47 H-A-208, H-A-210): its " + + "module graph is read again (each module's requirements, transitively) and the modules state recomputed as a " + + "package's publish re-pins it, so a changed take-in rule reaches a repository with no merge since; pins and " + + "fingerprints move, no catalog revision is kept and the gate is not asked. An unknown repository is refused.", + Input: map[string]any{"type": "object", "required": []string{"repository"}, "properties": map[string]any{ + "repository": map[string]any{"type": "string", "description": "the repository as owner/name, such as novox/mesh-host"}, + "why": map[string]any{"type": "string", "description": "why it is taken in again now"}}}, + Run: func(args map[string]any) (any, error) { + repository := str(args, "repository") + if repository == "" { + return nil, errors.New("name the repository (repository) as owner/name") + } + data, err := json.Marshal(TakeInAsk{V: 1, Again: repository, Why: str(args, "why"), By: stdio.CallerOf(args), At: now().UTC()}) + if err != nil { + return nil, err + } + a, err := ask("again."+keys.EscapeName(repository), data) + if err != nil { + return nil, err + } + if a.Result != nil && !a.Result.Stored { + return a.Result, fmt.Errorf("%s is not taken in again: %s", repository, a.Result.Answer) + } + return a.Result, nil + }}, {Name: TakeInSkipTool, Description: "End a watched branch's held source-moved unwritten (design 47 H §744): it is acknowledged, take-in.taken is said " + "with no entries and skipped, and the hand act is recorded through mesh-hand-acts before it is asked. indexer_pending " + diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again.go b/modules/mesh-delivery-indexer/internal/index/take_in_again.go new file mode 100644 index 00000000..ed37a307 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again.go @@ -0,0 +1,69 @@ +package index + +import ( + "context" + "errors" + "fmt" + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// TakeInAgainTool takes a repository in again at its kept merge commit (design 47 H-A-208, H-A-210): a rule change of the +// take-in (such as the walk of the module graph, H-A-201) reaches a repository with no merge since. +const TakeInAgainTool = "indexer_take_in_again" + +// TakeInAgain re-reads the repository's module graph at the merge commit its taken key keeps (wholeAt, the reading of every +// take-in: the kept requirements of each of its modules, H-A-201, H-A-157) and applies it as a take-in does, whose recompute +// of the modules state is the one a publish re-pin runs (recomputeLocked): pins and fingerprints move as they would there. +// It keeps no catalog revision and asks no gate (H-A-210). A repository with no taken key is refused, said in the answer. +func (ix *Indexer) TakeInAgain(ctx context.Context, repository string) TakenIn { + repository = strings.TrimSpace(repository) + r := TakenIn{Module: repository, Repository: repository} + refuse := func(format string, a ...any) TakenIn { + r.Answer = "refused: " + fmt.Sprintf(format, a...) + ix.logf("take-in again of %s: %s", repository, r.Answer) + return r + } + if ix.Taken == nil || ix.Modules == nil { + return refuse("the indexer keeps no taken merges or no modules, so nothing is taken in again") + } + key, err := trunk.RepositoryKey(repository) + if err != nil { + return refuse("%q names no repository as owner/name", repository) + } + value, _, found, err := ix.Taken.Get(ctx, key) + if errors.Is(err, process.ErrNotYet) { + return refuse("the state taken is not on the bus yet") + } + if err != nil { + return refuse("taken.%s cannot be read: %v", key, err) + } + if !found { + return refuse("the indexer keeps no merge of %s (no key taken.%s): an unknown repository, or one never taken in", repository, key) + } + t, err := takein.DecodeTaken(value) + if err != nil || t.Merge == "" { + return refuse("taken.%s names no merge commit", key) + } + ix.pendingMu.Lock() // one take-in step at a time (pending.go) + defer ix.pendingMu.Unlock() + f, err := ix.wholeAt(ctx, repository, t.Merge) + if err != nil { + return refuse("%v", err) + } + f.Branch = TrunkBranch + applied, err := ix.ApplyMessage(ctx, f) + if err != nil { + return refuse("taking %s at %.12s in again: %v", repository, t.Merge, err) + } + r.Stored, r.Commit = true, t.Merge + r.Answer = fmt.Sprintf("taken in again at %.12s: %d modules written", t.Merge, len(applied.Written)) + if len(applied.Written) > 0 { + r.Answer += " (" + strings.Join(applied.Written, ", ") + ")" + } + ix.logf("take-in again of %s: %s", repository, r.Answer) + return r +} diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go new file mode 100644 index 00000000..0ac11588 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go @@ -0,0 +1,72 @@ +package index + +import ( + "context" + "maps" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Design 47 H-A-208, H-A-210: a repository taken in before the walk of the module graph (H-A-201) keeps its modules' direct +// requirements only; indexer_take_in_again re-reads it at its kept merge commit and recomputes the modules state, so the +// module's kept requirements and pins gain the transitive package. An unknown repository is refused. +func TestATakeInAgainReachesARepositoryWithNoMergeSince(t *testing.T) { + const conditions = "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)} + ix, mods, _ := ptpIndexer(t, objects) + ix.Taken = newMem() + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + direct := map[string]string{"modules/g": "module g\n\nrequire git.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero + "\n"} + walked := map[string]string{"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n\nrequire " + conditions + ptpZero + "\n"} + goMods := direct // before the rule: the client's go.mod reads as requiring nothing + ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) { + if c, ok := goMods[dir]; ok { + return []byte(c), nil + } + if dir == "modules/c/client" { + return []byte("module git.novox.be/novox/mesh-delivery/modules/c/client\n"), nil + } + return []byte(ptpGoMods[dir]), nil + } + f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, + map[string]map[string]string{"modules/g": {"go.sum": avahiGoSum}}) + var readAt []string + ix.Read = func(_ context.Context, repository, sha, _ string) (RepoRead, error) { + readAt = append(readAt, repository+"@"+sha) + return RepoRead{Root: f.Root, Trees: f.Trees, Manifests: f.Manifests, Locks: f.Locks, Packages: f.Packages}, nil + } + ctx := context.Background() + if err := ix.Apply(ctx, f); err != nil { + t.Fatal(err) + } + if m, _ := moduleOf(t, mods, "g"); m.Inputs2 == nil || slices.Contains(m.Inputs2.Packages, conditions) || strings.Contains(m.None2, conditions) { + t.Fatalf("before: packages = %+v (%s), want no transitive requirement yet", m.Inputs2, m.None2) + } + if r := ix.TakeInAgain(ctx, "novox/unknown"); r.Stored || !strings.HasPrefix(r.Answer, "refused:") { + t.Errorf("unknown repository: %+v, want refused", r) + } + goMods = maps.Clone(direct) + maps.Copy(goMods, walked) + r := ix.TakeInAgain(ctx, "novox/mesh-delivery") + if !r.Stored || r.Commit != merge { + t.Fatalf("take-in again = %+v, want stored at the kept merge %s", r, merge) + } + if len(readAt) == 0 || readAt[0] != "novox/mesh-delivery@"+merge { + t.Errorf("read at %v, want the kept merge commit", readAt) + } + m, _ := moduleOf(t, mods, "g") + if m.Inputs2 == nil || !slices.Contains(m.Inputs2.Packages, conditions) { + t.Errorf("after: kept requirements = %+v, want %s", m.Inputs2, conditions) + } + // The recompute pins it: nothing is published here, so the fingerprint waits on it by name (H-A-161), and the pins the + // entry names from the kept requirements hold it with no version until it is (H-A-204). + if !strings.Contains(m.None2, conditions) { + t.Errorf("after: the recompute's reason %q, want it waiting on %s", m.None2, conditions) + } + if v, named := releasedPins(nil, nil, catalogue.Manifest{}, "novox/mesh-delivery", m.Inputs2.Packages)[conditions]; !named || v != "" { + t.Errorf("after: pins name %s %v (%q), want it named, waiting", conditions, named, v) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index 934e19c9..3cba48d0 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -46,6 +46,8 @@ type TakeInAsk struct { HeldSource int64 `json:"held-source,omitempty"` Skip bool `json:"skip,omitempty"` NamedMember string `json:"named-member,omitempty"` + // Again names the repository taken in again at its kept merge commit (indexer_take_in_again, take_in_again.go; H-A-208, H-A-210). + Again string `json:"again,omitempty"` // RunDrop asks one run on request, by its group run-, to be dropped with its candidate (indexer_run_drop, runs_tools.go, A-194). RunDrop string `json:"run-drop,omitempty"` // Registry asks one of the registrations' acts (registrations.go): a trusted change of one, the verb registrations, diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 6a4fc4c4..81ef37ba 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -206,8 +206,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204 - index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 43 || !slices.Contains(m.Tools, "modules") { + index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) + if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 155327c1..bc75c22c 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -67,6 +67,7 @@ "indexer_pin_drop", "indexer_pending", "indexer_take_in_pending", + "indexer_take_in_again", "indexer_take_in_skip", "indexer_runs", "indexer_run_drop" -- 2.54.0 From 64d5490ea270c96f78b79b327c25503198fa4965 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:21:23 +0100 Subject: [PATCH 13/30] Design 47 H1 (3): a publish whose recompute fails is not acknowledged The registry's published is offered again when the recompute of its users fails, as a failed series write already was. Builds: design 47 H S15 (H-pins-spec-and-gaps item 3), H-A-167 --- .../cmd/mesh-delivery-indexer/series.go | 8 ++- .../cmd/mesh-delivery-indexer/series_test.go | 61 +++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go index ff1b77f2..8ac23cb9 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go @@ -22,8 +22,8 @@ var ( ) // heardVersion reads one message of the consumer as a version, false for any other subject. A package's is kept in its -// series; one that does not decode is said and acknowledged; a write that fails is left unacknowledged, so the bus gives -// it again. +// series; one that does not decode is said and acknowledged; a write that fails, or the recompute of its users that a +// publish starts, is left unacknowledged, so the bus gives it again. func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) bool { var h index.HeardVersion switch seat, ok := publishedSeats[m.Subject()]; { @@ -53,8 +53,10 @@ func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf } if kept && h.Published { // A series' newest moved: its users' fingerprints take it (DLV-I6), in the same turn as the publish is heard. + // A recompute that fails is not acknowledged, so the publish is offered again (design 47 H S15, H-A-167). if err := ix.Recompute(ctx); err != nil { - logf("writing the modules again after %s %s was published: %v", h.Package, h.Version, err) + logf("writing the modules again after %s %s was published failed, so the bus gives it again: %v", h.Package, h.Version, err) + return true } } _ = m.Ack() diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go new file mode 100644 index 00000000..85ddbb54 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry" + + "mesh-delivery-indexer/internal/index" +) + +// ackedMsg is a message that counts its acknowledgements. +type ackedMsg struct { + jetstream.Msg + subject string + data []byte + acked int +} + +func (a *ackedMsg) Subject() string { return a.subject } +func (a *ackedMsg) Data() []byte { return a.data } +func (a *ackedMsg) Ack() error { a.acked++; return nil } + +// unlistedStore is a store whose listing fails: a recompute over it fails. +type unlistedStore struct{ memStore } + +func (unlistedStore) Keys(context.Context) ([]string, error) { + return nil, errors.New("the bus does not answer") +} + +// TestHeardVersionRecomputeFailed: a publish whose recompute of its users fails is not acknowledged, so the bus gives +// it again (design 47 H S15, the pin specification's item 3); one whose recompute passes is. +func TestHeardVersionRecomputeFailed(t *testing.T) { + data, err := json.Marshal(gopackageregistry.PublishedEvent{Kind: "package", Name: "git.example/novox/lib", Version: "0.1.4"}) + if err != nil { + t.Fatal(err) + } + for _, c := range []struct { + name string + modules index.Store + acked int + }{ + {"recompute fails", &unlistedStore{memStore{vals: map[string][]byte{}}}, 0}, + {"recompute passes", &memStore{vals: map[string][]byte{}}, 1}, + } { + t.Run(c.name, func(t *testing.T) { + ix := &index.Indexer{Packages: &memStore{vals: map[string][]byte{}}, Modules: c.modules, Manifests: &memStore{vals: map[string][]byte{}}} + m := &ackedMsg{subject: gopackageregistry.SubjectPublished, data: data} + if !heardVersion(context.Background(), ix, m, t.Logf) { + t.Fatal("the registry's published is not taken as a version") + } + if m.acked != c.acked { + t.Errorf("acknowledged %d times, want %d", m.acked, c.acked) + } + }) + } +} -- 2.54.0 From aee7e4c26c40ac5bd197a3a86fe0204450c5ed64 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:24:16 +0100 Subject: [PATCH 14/30] mesh-delivery-sdk: an entry's packages may name an empty version (H item 4, the SDK's half) The indexer names a package of another repository with no version published yet by an empty version, which waits for its publish; PackageVersions.Check refused it, so a run on request of such a module failed its candidate check. CheckEntry is the entry form, allowing an empty version, while Check (an ask's pins) still refuses one. Builds: design 47 H S8, S9, L10 (H-pins-spec-and-gaps section 8 item 4); H-A-204, H-A-165 --- .../mesh-delivery-sdk/buildstep/ask_test.go | 22 +++++++++++++++++++ modules/mesh-delivery-sdk/buildstep/steps.go | 17 +++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-sdk/buildstep/ask_test.go b/modules/mesh-delivery-sdk/buildstep/ask_test.go index 5467bf23..e996d4c2 100644 --- a/modules/mesh-delivery-sdk/buildstep/ask_test.go +++ b/modules/mesh-delivery-sdk/buildstep/ask_test.go @@ -87,6 +87,7 @@ func TestAskCheckAndRoundTrip(t *testing.T) { {"no path", func(a *StepAsk) { a.Build.Path = "" }, "names no directory"}, {"short commit", func(a *StepAsk) { a.Build.Fetch.Commit = "abc" }, "full commit"}, {"bad package pin", func(a *StepAsk) { a.Packages = PackageVersions{"m": "main"} }, "neither a full commit"}, + {"empty package pin", func(a *StepAsk) { a.Packages = PackageVersions{"m/x": ""} }, "neither a full commit"}, } { a := goodBuild() c.edit(&a) @@ -94,6 +95,27 @@ func TestAskCheckAndRoundTrip(t *testing.T) { t.Errorf("%s: %v, want %q", c.name, err, c.want) } } + // Design 47 H, S8 (H-A-204, H-A-165): a catalog entry's packages name a package with no version published yet by an + // empty version, which waits for its publish; the entry form allows it, an ask's pins never do (above). + for _, c := range []struct { + name string + p PackageVersions + entry string // what CheckEntry's refusal holds, "" for none + pinsToo bool // Check (an ask's pins) passes it as well + }{ + {name: "an empty version", p: PackageVersions{"m/x": "", "m/y": "v0.1.3"}}, + {name: "versions only", p: PackageVersions{"m/y": "v0.1.3", "m/z": testCommit}, pinsToo: true}, + {name: "a version that is no version", p: PackageVersions{"m/x": "main"}, entry: "neither a full commit"}, + {name: "no module path", p: PackageVersions{"": ""}, entry: "no module path"}, + } { + err := c.p.CheckEntry() + if (err == nil) != (c.entry == "") || (err != nil && !strings.Contains(err.Error(), c.entry)) { + t.Errorf("%s: CheckEntry answered %v, want %q", c.name, err, c.entry) + } + if err := c.p.Check(); (err == nil) != c.pinsToo { + t.Errorf("%s: Check answered %v, want a pass %v", c.name, err, c.pinsToo) + } + } // 1921 (B0, G): a run ask goes only on the Go seat's check stream, alone, builds a clean package at a full commit // over heads named once at full commits, and names its input on a seat's verb by a digest within its bound. for _, c := range []struct { diff --git a/modules/mesh-delivery-sdk/buildstep/steps.go b/modules/mesh-delivery-sdk/buildstep/steps.go index 84b4e0db..33a4fa3a 100644 --- a/modules/mesh-delivery-sdk/buildstep/steps.go +++ b/modules/mesh-delivery-sdk/buildstep/steps.go @@ -184,16 +184,27 @@ var ( ) // PackageVersions are the versions of the mesh packages a build or a check builds against, by package (a Go module's -// path): each a full commit or a Go semantic version. +// path): each a full commit or a Go semantic version. A catalog entry's packages may also name a package with an empty +// version, one of another repository with no version published yet in its series, which waits for its publish (design +// 47 H, S8, S9; H-A-204, H-A-165): CheckEntry allows it, Check (an ask's pins) never. type PackageVersions map[string]string // Check refuses a package that is no module path or a version that is neither a full commit nor a Go semantic -// version, naming it. -func (p PackageVersions) Check() error { +// version, naming it: an ask's pins, an empty version among them. +func (p PackageVersions) Check() error { return p.check(false) } + +// CheckEntry is Check for a catalog entry's packages: an empty version is allowed, and stands for a package not +// published yet, whose build waits for its publish (design 47 H, S8; H-A-204, H-A-165). +func (p PackageVersions) CheckEntry() error { return p.check(true) } + +func (p PackageVersions) check(emptyAllowed bool) error { for pkg, version := range p { if pkg == "" || strings.ContainsAny(pkg, " \t\n@") || strings.HasPrefix(pkg, "/") { return fmt.Errorf("buildstep: the package %q is no module path", pkg) } + if version == "" && emptyAllowed { + continue + } if !askCommitOK.MatchString(version) && !PackageSemver.MatchString(version) { return fmt.Errorf("buildstep: the package %s names the version %q, which is neither a full commit nor a Go semantic version (v0.1.15-feat-x.2)", pkg, version) } -- 2.54.0 From 52f7a5b6f61435ff54f2be228f81f4eedb4ddea3 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:22:13 +0100 Subject: [PATCH 15/30] Design 47 H1 (4): a candidate's entry may name a package with no version yet An entry names an unpublished package with an empty version, never left out, so a run on request's candidate checks its packages as an entry's (CheckEntry), not as an ask's. Built against the SDK head aee7e4c2 (fix/47-H1-sdk-empty-pin), which goes in the same merge call. Builds: design 47 H S8 (H-pins-spec-and-gaps item 4, L10), H-A-204 --- .../internal/index/candidate_values.go | 4 ++- .../internal/index/candidate_values_test.go | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/candidate_values_test.go diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_values.go b/modules/mesh-delivery-indexer/internal/index/candidate_values.go index 62cc8b33..f443f807 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidate_values.go +++ b/modules/mesh-delivery-indexer/internal/index/candidate_values.go @@ -39,7 +39,9 @@ func checkCandidateModule(m meshcatalog.CandidatesByGroupValuePullsItemFpsValue) return fmt.Errorf("index: the named path %q names no git object", p) } } - if err := sdkstep.PackageVersions(m.Packages).Check(); err != nil { + // An entry names a package with no published version yet with an empty version (design 47 H S8, H-A-204): it is checked + // as an entry's packages, not as an ask's. + if err := sdkstep.PackageVersions(m.Packages).CheckEntry(); err != nil { return err } if m.Fp2 != "" { diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_values_test.go b/modules/mesh-delivery-indexer/internal/index/candidate_values_test.go new file mode 100644 index 00000000..fa25038a --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/candidate_values_test.go @@ -0,0 +1,30 @@ +package index + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" +) + +// TestCheckCandidateModulePins: an entry's package with no published version (an empty pin) passes a run on request's +// candidate; a pin that is no version is refused (design 47 H S8, the pin specification's item 4). +func TestCheckCandidateModulePins(t *testing.T) { + for _, c := range []struct { + name string + packages map[string]string + ok bool + }{ + {"an empty pin", map[string]string{"git.example/novox/lib/go": ""}, true}, + {"a version", map[string]string{"git.example/novox/lib/go": "v0.1.4"}, true}, + {"no version", map[string]string{"git.example/novox/lib/go": "latest"}, false}, + } { + t.Run(c.name, func(t *testing.T) { + m := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Fp2: "src2:" + strings.Repeat("ab", 32), Packages: c.packages} + err := checkCandidateModule(m) + if (err == nil) != c.ok { + t.Errorf("checkCandidateModule(%v) = %v, want ok %v", c.packages, err, c.ok) + } + }) + } +} -- 2.54.0 From 47ea1303320c008993928590454247d7f942bdef Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:31:47 +0100 Subject: [PATCH 16/30] Design 47 H1 (1): a release re-pin keeps the next catalog revision of the same source on every watched branch A branch's revision number is the indexer's own count (the newest + 1); the gate's source is the revision's field source, which the take-in's dedupe and source-to-take read. A catalog revision carries its call (the seat mesh-catalog's CatalogRevision and catalog-revision gain call, regenerated by seatgen). A re-pin on a module's release runs on every watched branch holding a module that names its package (the first on the modules state, another on its newest revision's picture), and keeps, where a fingerprint moved, the branch's next revision of the same source with the call 'release of ' and the moved modules as its movers, pinned for the release's pipeline, and says catalog-revision; no take-in.taken. Builds: design 47 H S11, S12, S13, S14 (H-pins-spec-and-gaps item 1), H-A-211, H-A-212, H-A-213 (1)-(4: the pin of every revision a pipeline runs on), H-A-147 --- .../go/seats/meshcatalog/contract.go | 4 + .../go/seats/meshcatalog/contract_test.go | 4 +- .../seats/mesh-catalog/fixtures/events.json | 1 + .../seats/mesh-catalog/fixtures/states.json | 3 +- .../seats/mesh-catalog/seat.json | 8 ++ .../ts/seats/meshcatalog/index.d.ts | 4 + .../takein_integration_test.go | 50 ++++---- .../internal/index/catalog_branches_test.go | 24 ++-- .../internal/index/catalog_revisions.go | 92 ++++++++++----- .../internal/index/entry_verb_test.go | 2 +- .../internal/index/pins_test.go | 68 +++++------ .../internal/index/repin_test.go | 46 +++++++- .../internal/index/takein_source.go | 81 +++++++++---- .../internal/index/takein_source_test.go | 29 ++--- .../internal/index/toolchains.go | 111 ++++++++++++++++-- .../internal/index/toolchains_test.go | 18 +-- .../internal/index/versions.go | 2 +- .../internal/index/wanted_names_test.go | 4 +- 18 files changed, 389 insertions(+), 162 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go index 261c6f66..993c9448 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go @@ -715,6 +715,8 @@ type CatalogRevision struct { Base int64 `json:"base,omitempty"` // the watched branch the revision is of Branch string `json:"branch,omitempty"` + // what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's + Call string `json:"call,omitempty"` // the seat definitions it was judged with, each by its digest, read whole through seat-definitions Definitions []CatalogRevisionDefinitionsItem `json:"definitions,omitempty"` // one per module, by name @@ -1301,6 +1303,8 @@ type CatalogRevisionEvent struct { At *time.Time `json:"at,omitempty"` // the watched branch the revision is of, escaped as a key name is Branch string `json:"branch,omitempty"` + // what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's + Call string `json:"call,omitempty"` // the modules whose entries differ from the revision before it of the same kind Changed []string `json:"changed,omitempty"` // how many entries it lists diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go index e8f85fa8..beb81baa 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go @@ -17,7 +17,7 @@ func TestFixtures(t *testing.T) { decode func([]byte) (any, error) subjectIs *regexp.Regexp }{ - {name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", + {name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")}, {name: "a candidate's revision kept", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}", decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")}, @@ -59,7 +59,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v ModulesAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "the module verb (captured: xclip)", subject: "", given: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", wire: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", decode: func(b []byte) (any, error) { var v ModuleAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, - {name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", + {name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", decode: func(b []byte) (any, error) { var v CatalogRevisionsByBranchRevisionValue err := json.Unmarshal(b, &v) diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json index fd3678df..5c8e30db 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json @@ -8,6 +8,7 @@ "revision": 12, "kind": "trunk", "source": 3589, + "call": "release of mesh-sdk 0.1.27", "entries": 2, "changed": [ "xclip" diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json index e537ca82..750ba355 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json @@ -209,7 +209,8 @@ "repository": "novox/mesh-catalog" } ], - "base": 9 + "base": 9, + "call": "release of mesh-sdk 0.1.27" } }, { diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json index e9db48fc..1e5687bc 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json @@ -657,6 +657,10 @@ "source": { "type": "integer" }, + "call": { + "type": "string", + "description": "what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's" + }, "group": { "type": "string" }, @@ -1871,6 +1875,10 @@ "type": "integer", "description": "the source revision it was taken in from; for a candidate, the source revision its heads are merged onto" }, + "call": { + "type": "string", + "description": "what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's" + }, "group": { "type": "string", "description": "a candidate's group" diff --git a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts index b3d0c697..5877f3fb 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts @@ -277,6 +277,8 @@ export interface CatalogRevision { "base"?: number; /** the watched branch the revision is of */ "branch"?: string; + /** what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's */ + "call"?: string; /** the seat definitions it was judged with, each by its digest, read whole through seat-definitions */ "definitions"?: CatalogRevisionDefinitionsItem[]; /** one per module, by name */ @@ -796,6 +798,8 @@ export interface CatalogRevisionEvent { "at"?: string; /** the watched branch the revision is of, escaped as a key name is */ "branch"?: string; + /** what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's */ + "call"?: string; /** the modules whose entries differ from the revision before it of the same kind */ "changed"?: string[]; /** how many entries it lists */ diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index c42a2a37..0020713c 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "os" "slices" "strconv" @@ -447,37 +448,42 @@ func TestIndexerWhole(t *testing.T) { // The first revision: no mover, no pin; its entries are the base. sayMoved(t, ctx, js, "main", 10, c10, true) - if first := heard.next(t, "main", 10); len(first.Entries) != 0 || first.Revision != 10 { - t.Fatalf("the first revision said %+v, want revision 10 with no entries", first) + // A revision's number is the indexer's own count, the branch's newest + 1 (design 47 H-A-213 (1)): the later take-ins of + // sources 11 to 13 are counted on from the number of source 10's. + first := heard.next(t, "main", 10) + r := func(source int64) int64 { return first.Revision + source - 10 } + rn := func(source int64) string { return fmt.Sprintf("main.%d", r(source)) } + if len(first.Entries) != 0 || first.Revision <= 0 || first.Source != 10 { + t.Fatalf("the first revision said %+v, want a revision of source 10 with no entries", first) } - untilKey(t, ctx, pins, index.PinKey("main", 10), absent) + untilKey(t, ctx, pins, index.PinKey("main", r(10)), absent) // One take-in, one revision carrying its movers, pinned for its pipeline; said again from the kept movers. sayMoved(t, ctx, js, "main", 11, c11, false) taken := heard.next(t, "main", 11) if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Commit != c11 || taken.Entries[0].Fingerprint == "" || - taken.Entries[0].Repository != "novox/mesh-catalog" || taken.Revision != 11 || taken.Call == "" { + taken.Entries[0].Repository != "novox/mesh-catalog" || taken.Revision != r(11) || taken.Call == "" { t.Fatalf("take-in.taken of main 11 = %+v, want avahi at %s as its one mover", taken, c11) } - rev, _, err := index.ReadRevision(ctx, objects, "main.11") + rev, _, err := index.ReadRevision(ctx, objects, rn(11)) if err != nil || len(rev.Movers) != 1 || rev.Movers[0] != taken.Entries[0] { t.Fatalf("main.11 = %+v (%v), want it kept carrying the movers said", rev, err) } - untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.11")) - untilKey(t, ctx, pins, index.PinKey("main", 10), pinned("pipeline.main.11")) // validate's base, H-A-147 + untilKey(t, ctx, pins, index.PinKey("main", r(11)), pinned("pipeline."+rn(11))) + untilKey(t, ctx, pins, index.PinKey("main", r(10)), pinned("pipeline."+rn(11))) // validate's base, H-A-147 info, err := objects.open(ctx) if err != nil { t.Fatal(err) } - before, err := info.GetInfo(ctx, "main.11") + before, err := info.GetInfo(ctx, rn(11)) if err != nil { t.Fatal(err) } sayMoved(t, ctx, js, "main", 11, c11, false) - if again := heard.next(t, "main", 11); len(again.Entries) != 1 || again.Entries[0] != taken.Entries[0] || again.Revision != 11 { + if again := heard.next(t, "main", 11); len(again.Entries) != 1 || again.Entries[0] != taken.Entries[0] || again.Revision != r(11) { t.Errorf("the source-moved given again said %+v, want the kept movers %+v", again, taken.Entries) } - if after, err := info.GetInfo(ctx, "main.11"); err != nil || after.NUID != before.NUID { + if after, err := info.GetInfo(ctx, rn(11)); err != nil || after.NUID != before.NUID { t.Errorf("main.11 was written again (%v): a source-moved given again keeps no second revision", err) } @@ -485,11 +491,11 @@ func TestIndexerWhole(t *testing.T) { // (H-A-147), which the take-in pinned for it. sayMoved(t, ctx, js, "main", 12, c12, false) heard.next(t, "main", 12) - if err := pins.Delete(ctx, index.PinKey("main", 12)); err != nil { // the take-in's own pin, so pipeline-started alone writes it below + if err := pins.Delete(ctx, index.PinKey("main", r(12))); err != nil { // the take-in's own pin, so pipeline-started alone writes it below t.Fatal(err) } - sayPipeline(t, ctx, js, false, "main", 12) - untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12")) + sayPipeline(t, ctx, js, false, "main", r(12)) + untilKey(t, ctx, pins, index.PinKey("main", r(12)), pinned("pipeline."+rn(12))) // A run on request: its candidate and its key written on run-started, from avahi's files at the run's commit (A-160, A-174). manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") if err != nil { @@ -515,7 +521,7 @@ func TestIndexerWhole(t *testing.T) { heard.acked(t, "main") } restart() - for _, name := range []string{"main.10", "main.11"} { + for _, name := range []string{rn(10), rn(11)} { if _, _, err := index.ReadRevision(ctx, objects, name); err != nil { t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err) } @@ -553,27 +559,27 @@ func TestIndexerWhole(t *testing.T) { t.Errorf("the held source-moved's consumer %+v (%v), want nothing unacknowledged", info, err) } setCap(t, ctx, js, 1<<30) // a larger cap alone wakes nothing: no timer takes a held take-in in - sayPipeline(t, ctx, js, true, "main", 11) + sayPipeline(t, ctx, js, true, "main", r(11)) if got := heard.next(t, "main", 13); len(got.Entries) != 1 || got.Entries[0].Commit != c13 { t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13) } untilKey(t, ctx, held, keys.EscapeName("main"), absent) // main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170). - untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi"))) - untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins + untilKey(t, ctx, pins, index.PinKey("main", r(11)), pinned("pipeline."+rn(12), index.ReleaseRecord("main", "avahi"))) + untilRevisionGone(t, ctx, js, rn(10)) // the end freed both of main.11's pins // main.12's passed pipeline releases avahi again; its pipeline-wanted, the seat mesh-wants' (its wants written to the end), // frees main.11's older release, and main.11 goes (H-A-168, H-A-170, H-A-173). - sayPipeline(t, ctx, js, true, "main", 12) - untilKey(t, ctx, pins, index.PinKey("main", 12), func(v []byte) bool { + sayPipeline(t, ctx, js, true, "main", r(12)) + untilKey(t, ctx, pins, index.PinKey("main", r(12)), func(v []byte) bool { var p index.PinValue return v != nil && json.Unmarshal(v, &p) == nil && slices.Contains(p.Records, index.ReleaseRecord("main", "avahi")) }) - wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: "main.12", Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning}) + wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: rn(12), Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning}) if _, err := js.Publish(ctx, meshwants.SubjectPipelineWanted("main", "12"), wantedRaw); err != nil { t.Fatal(err) } - untilKey(t, ctx, pins, index.PinKey("main", 11), absent) - untilRevisionGone(t, ctx, js, "main.11") + untilKey(t, ctx, pins, index.PinKey("main", r(11)), absent) + untilRevisionGone(t, ctx, js, rn(11)) if n := len(heard.pending); n != 0 { t.Errorf("take-in.pending said %d more times for one held source revision, want once", n) } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go index 5e239931..574de13d 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go @@ -36,13 +36,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { for _, n := range names { have[n] = true } - for _, want := range []string{"main.5", "feature/x=2ey.7", "feature/x=2ey.8"} { + for _, want := range []string{"main.1", "feature/x=2ey.1", "feature/x=2ey.2"} { if !have[want] { t.Errorf("the objects hold %v, want %s", names, want) } } - if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 8 { - t.Errorf("said %+v, want main 5, then the escaped branch 7 and 8", said) + if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 2 || said[2].Source != 8 { + t.Errorf("said %+v, want main 1 of source 5, then the escaped branch 1 and 2 of sources 7 and 8", said) } answer := func(branch string, revision int64) meshcatalog.RevisionAnswer { a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Revision: revision}) @@ -51,14 +51,14 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { } return a } - if a := answer("main", 0); a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" { + if a := answer("main", 0); a.Revision.Revision != 1 || a.Revision.Source != 5 || a.Entries[0].Commit != "c1" { t.Errorf("main answered %+v", a) } - if a := answer("feature/x=2ey", 0); a.Revision.Revision != 8 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" { - t.Errorf("the branch's newest answered %+v, want its revision 8 at c3", a.Revision) + if a := answer("feature/x=2ey", 0); a.Revision.Revision != 2 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" { + t.Errorf("the branch's newest answered %+v, want its revision 2 at c3", a.Revision) } - if a := answer("feature/x=2ey", 7); a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" { - t.Errorf("the branch's revision 7 answered %+v", a.Revision) + if a := answer("feature/x=2ey", 1); a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" { + t.Errorf("the branch's revision 1 answered %+v", a.Revision) } if a := answer("beta", 0); !a.NoneYet { t.Errorf("a branch with no revision answered %+v, want none-yet", a) @@ -68,13 +68,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) { return Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Source: source}) } - if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" { + if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Revision.Source != 5 || a.Entries[0].Commit != "c1" { t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err) } - if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" { - t.Errorf("the branch's source 7 answered %+v (%v), want its revision 7 at c2", a.Revision, err) + if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" { + t.Errorf("the branch's source 7 answered %+v (%v), want its revision 1 at c2", a.Revision, err) } - if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 8 { + if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 { t.Errorf("the branch's source 8 answered %+v (%v)", a.Revision, err) } if _, err := bySource("feature/x=2ey", 9); err == nil { diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index b26e2ba7..b528d1a5 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "slices" "strconv" "strings" @@ -87,6 +88,37 @@ func BranchRevisionOf(branch, name string) (int64, bool) { return r, err == nil && r > 0 } +// watchedBranches are the watched branches the bucket keeps a revision of, the first included when it keeps one, sorted: +// each name . but an entry's, a candidate's and the earlier forms' (earlierForm). +func (ix *Indexer) watchedBranches(ctx context.Context) ([]string, error) { + if ix.Revisions == nil { + return nil, nil + } + names, err := ix.Revisions.Names(ctx) + if err != nil { + return nil, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err) + } + seen := map[string]bool{} + for _, n := range names { + if strings.HasPrefix(n, "entry.") || strings.HasPrefix(n, "candidate.") || earlierForm(n) { + continue + } + i := strings.LastIndexByte(n, '.') + if i <= 0 { + continue + } + if r, err := strconv.ParseInt(n[i+1:], 10, 64); err != nil || r <= 0 { + continue + } + branch, err := keys.UnescapeName(n[:i]) + if err != nil { + continue + } + seen[branch] = true + } + return slices.Sorted(maps.Keys(seen)), nil +} + // TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one. func TrunkRevisionOf(name string) (int64, bool) { return BranchRevisionOf(TrunkBranch, name) } @@ -314,17 +346,22 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata } // KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design -// 47 H §743, H-A-32): ., the trunk's included, numbered by the source, and says catalog-revision naming the -// branch. movers are the take-in's: kept on the revision (H-A-92), and the revision pinned for the take-in's pipeline, -// pipeline.., in the same write when there is one (A-449a), with the branch's previous revision (H-A-147). A revision already kept under that name is not written -// again (read before write, under the branch's one consumer); ErrObjectsFull is a store too full to keep it. +// 47 H §743, H-A-32), as keepBranchEntries does with no call. func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error { - return ix.keepBranchEntries(ctx, branch, src, entries, movers) + _, err := ix.keepBranchEntries(ctx, branch, src, "", entries, movers) + return err } -func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error { +// keepBranchEntries keeps entries as the next catalog revision of a watched branch, ., the trunk's included: n is +// the indexer's own count, the branch's newest revision + 1, and src the gate's source revision it was read at, its field +// source (design 47 H-A-213 (1)); call is what made it (H-A-212, H-A-213 (3)): a take-in's the gate's call, a re-pin's +// "release of ", a take-in again's "take-in again of ". It says catalog-revision naming the +// branch, its source and its call. movers are kept on the revision (H-A-92), and the revision pinned for its one pipeline, +// pipeline.., in the same write when there is one (A-449a, H-A-213 (4)), with the branch's previous revision +// (H-A-147). It answers the number it kept; ErrObjectsFull is a store too full to keep it. +func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, call string, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) (int64, error) { if ix.Revisions == nil { - return nil + return 0, nil } if branch == "" { branch = TrunkBranch @@ -334,28 +371,23 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int newest, _, err := newestOf(ctx, ix.Revisions, branch) if errors.Is(err, process.ErrNotYet) { ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects) - return err + return 0, err } if err != nil { ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err) - return fmt.Errorf("the newest catalog graph revision: %w", err) - } - if _, err := ix.Revisions.Get(ctx, BranchObject(branch, src)); err == nil { - ix.logf("catalog graph: revision %s is kept already: not written again", BranchObject(branch, src)) - return nil - } else if !errors.Is(err, ErrNoObject) { - return fmt.Errorf("revision %s: %w", BranchObject(branch, src), err) + return 0, fmt.Errorf("the newest catalog graph revision: %w", err) } + number := newest.Revision + 1 entries = WithEdges(ix.withActs(ctx, entries)) // the acting rules and the edges between artifact references are written here, once, for every reader (A-208) at := ix.now().UTC() - rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: src, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries) + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: number, Kind: "trunk", Branch: branch, Source: src, Call: call, At: &at}, entries) if err != nil { - return err + return 0, err } rev.Movers = movers // The base of the validate step's case before: the newest revision kept before this write, which the take-in pins for its // pipeline below, named on the revision in this same write (H-A-147, H-A-151, H-A-154); none on the branch's first take-in. - pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0 && newest.Revision != src + pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0 if pinsBase { rev.Base = newest.Revision } @@ -367,32 +399,32 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull) ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity) ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused}) - return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull) + return 0, fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull) } if err := ix.indexEntries(ctx, rev, entries); err != nil { ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) - return err + return 0, err } if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) - return err + return 0, err } // The take-in's revision pinned for its one pipeline before any judge can remove it (§743, §749, A-449a). if len(movers) > 0 && ix.Pins != nil { - record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(src, 10))) - if err := ix.pin(ctx, PinKey(branch, src), record); err != nil { - ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err) - return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err) + record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(number, 10))) + if err := ix.pin(ctx, PinKey(branch, number), record); err != nil { + ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, number), record, err) + return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, number), err) } // The branch's previous revision, the newest kept before this write, pinned for the same pipeline: it is the base of // the validate step's case before (H-A-108 (3), H-A-147), kept as long as the pipeline that needs it. if pinsBase { if err := ix.pin(ctx, PinKey(branch, rev.Base), record); err != nil { ix.logf("pins: the previous revision %s is not pinned for %s: %v", BranchObject(branch, newest.Revision), record, err) - return fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err) + return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err) } } else { - ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, src), record) + ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, number), record) } } // The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it. @@ -412,14 +444,14 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 { ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity}) } - ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed)) + ix.logf("catalog graph: revision %d of source revision %d kept (%s), %d entries, %d changed", rev.Revision, src, call, len(rev.Entries), len(changed)) if ix.SayCatalogRevision != nil { if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind, - Branch: keys.EscapeName(branch), Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil { + Branch: keys.EscapeName(branch), Source: src, Call: call, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil { ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err) } } - return nil + return number, nil } // HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held). diff --git a/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go b/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go index 5de12ff7..de25475f 100644 --- a/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go +++ b/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go @@ -49,7 +49,7 @@ func TestAKeptRevisionIndexesItsEntries(t *testing.T) { ctx := context.Background() fp := "src2:" + strings.Repeat("d", 64) ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }} - if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { + if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } if got, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: fp}); err != nil || got.Entry.Module != "avahi" { diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index 4a255ebd..8f8a0389 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -122,25 +122,25 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te } ix.JudgePins(ctx) } - take(10) - end(10, runner.PipelineEndedOutcomePassed, "0.1.10") - take(11) - end(11, runner.PipelineEndedOutcomeFailed, "") - want("a passed release past a later failed pipeline", "main.10", "main.11") - take(12) - end(12, runner.PipelineEndedOutcomePassed, "") - wanted(12, meshwants.PipelineWantedOutcomeRunning) - want("a passed pipeline releasing no version frees no older release", "main.10", "main.12") - take(13) - end(13, runner.PipelineEndedOutcomePassed, "0.1.13") - want("two releases, the unreleased one between them gone", "main.10", "main.13") - wanted(13, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing - want("a tier-late pipeline-wanted", "main.10", "main.13") - wanted(13, meshwants.PipelineWantedOutcomeRunning) - wanted(13, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more - want("the newer release written everywhere", "main.13") - if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { - t.Errorf("the pin of main.13 holds %s, want its release alone", v) + take(4) + end(4, runner.PipelineEndedOutcomePassed, "0.1.10") + take(5) + end(5, runner.PipelineEndedOutcomeFailed, "") + want("a passed release past a later failed pipeline", "main.4", "main.5") + take(6) + end(6, runner.PipelineEndedOutcomePassed, "") + wanted(6, meshwants.PipelineWantedOutcomeRunning) + want("a passed pipeline releasing no version frees no older release", "main.4", "main.6") + take(7) + end(7, runner.PipelineEndedOutcomePassed, "0.1.13") + want("two releases, the unreleased one between them gone", "main.4", "main.7") + wanted(7, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing + want("a tier-late pipeline-wanted", "main.4", "main.7") + wanted(7, meshwants.PipelineWantedOutcomeRunning) + wanted(7, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more + want("the newer release written everywhere", "main.7") + if v, _, _, _ := ix.Pins.Get(ctx, "main.7"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { + t.Errorf("the pin of main.7 holds %s, want its release alone", v) } } @@ -168,21 +168,22 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { before []int64 // sources taken in with no mover before the take-in of source 5 whileRuns []string afterTheEnd []string - base int64 // the base main.5 names (H-A-154) + at int64 // the number the take-in of source 5 keeps: the branch's newest + 1 (H-A-213 (1)) + base int64 // the base that revision names (H-A-154) passed bool // main.5's pipeline passed: its release pins main.5 unreleased bool // its mover released no version there, so nothing pins main.5 (H-A-173) then []int64 // sources taken in with no mover after the end released []string }{ - {name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4}, - {name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2}, - {name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}}, - {name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, - base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}}, - {name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, - base: 4, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.6"}}, - {name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, - base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}}, + {name: "the previous revision kept and pinned", before: []int64{3, 4}, at: 3, whileRuns: []string{"main.2", "main.3"}, afterTheEnd: []string{"main.3"}, base: 2}, + {name: "the newest kept before, whatever its source", before: []int64{2}, at: 2, whileRuns: []string{"main.1", "main.2"}, afterTheEnd: []string{"main.2"}, base: 1}, + {name: "no revision before it pins only its own", at: 1, whileRuns: []string{"main.1"}, afterTheEnd: []string{"main.1"}}, + {name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"}, + base: 1, passed: true, then: []int64{6}, afterTheEnd: []string{"main.2", "main.3"}, released: []string{"main.2"}}, + {name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"}, + base: 1, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.3"}}, + {name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"}, + base: 1, then: []int64{6}, afterTheEnd: []string{"main.3"}}, } { t.Run(tc.name, func(t *testing.T) { ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(), @@ -206,18 +207,19 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { keep(s, nil) } keep(5, mover) - if rev, _, err := ReadRevision(ctx, ix.Revisions, "main.5"); err != nil || rev.Base != tc.base { - t.Errorf("main.5 names the base %d (%v), want %d, the revision its take-in pinned (H-A-154)", rev.Base, err, tc.base) + at := fmt.Sprintf("main.%d", tc.at) + if rev, _, err := ReadRevision(ctx, ix.Revisions, at); err != nil || rev.Base != tc.base || rev.Source != 5 { + t.Errorf("%s names the source %d and the base %d (%v), want 5 and %d, the revision its take-in pinned (H-A-154)", at, rev.Source, rev.Base, err, tc.base) } if got := kept(); !slices.Equal(got, tc.whileRuns) { t.Errorf("while the pipeline runs: kept %v, want %v", got, tc.whileRuns) } for _, k := range tc.whileRuns { - if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5"}) { + if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline." + at}) { t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v) } } - end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed, + end := runner.PipelineEnded{Branch: "main", Pipeline: at, Revision: tc.at, Outcome: runner.PipelineEndedOutcomeFailed, Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: "0.1.5"}}} if tc.unreleased { end.Movers[0].Version = "" diff --git a/modules/mesh-delivery-indexer/internal/index/repin_test.go b/modules/mesh-delivery-indexer/internal/index/repin_test.go index 80ed4dfd..4a0f9ab4 100644 --- a/modules/mesh-delivery-indexer/internal/index/repin_test.go +++ b/modules/mesh-delivery-indexer/internal/index/repin_test.go @@ -37,11 +37,19 @@ func TestAnEntryNamesEveryRequirementPinnedOrWaiting(t *testing.T) { // Design 47 H-A-204, H-A-165: a release re-pins every kept entry naming one of its packages, of any repository and // changed or not, by its pins or by its kept requirements alone: here g, of another repository, holds no pin of mesh-sdk. +// H-A-211, H-A-212, H-A-213: on every watched branch holding g, the re-pin keeps the branch's next revision, of the same +// source as the one before it, with the call "release of mesh-sdk 0.1.27" and g as its mover at its new fingerprint, and +// says catalog-revision with that call. func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) { + ctx := context.Background() objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)} ix, mods, _ := ptpIndexer(t, objects) + ix.Revisions, ix.Packages = newMemObjects(1<<20), newMem() + reportToolchain(t, ix, "mesh-build-go", "n1", "go", "1.26.0") + var said []meshcatalog.CatalogRevisionEvent + ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { said = append(said, e); return nil } f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, nil) - if err := ix.Apply(context.Background(), f); err != nil { + if err := ix.Apply(ctx, f); err != nil { t.Fatal(err) } g, _ := moduleOf(t, mods, "g") @@ -52,7 +60,39 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) { mods.put("g", mustEncode(t, g)) mods.put("mesh-sdk", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath, None2: None2Pending})) - if n, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil || n != 1 { - t.Errorf("a release of mesh-sdk re-pinned %d modules (%v), want g, which requires it with no pin", n, err) + entries, err := ix.viewEntries(ctx) + if err != nil { + t.Fatal(err) + } + sources := map[string]int64{"main": 5, "feature/x": 9} + for branch, src := range sources { + if err := ix.KeepBranchEntries(ctx, branch, src, entries, nil); err != nil { + t.Fatal(err) + } + } + said = nil + if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: "git.novox.be/novox/mesh-sdk/go", Version: "0.1.27", Published: true}); err != nil { + t.Fatal(err) + } + if n, err := ix.Repin(ctx, "mesh-sdk", "0.1.27"); err != nil || n != 2 { + t.Errorf("a release of mesh-sdk re-pinned %d modules (%v), want g on each of the two branches, which requires it with no pin", n, err) + } + now, _ := moduleOf(t, mods, "g") + if now.Fp2 == "" || now.Fp2 == g.Fp2 || now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.27" { + t.Fatalf("g after the re-pin = %s pinned %v, want a new fingerprint pinned to v0.1.27 (was %s)", now.Fp2, now.Packages, g.Fp2) + } + for branch, src := range sources { + rev, kept, found, err := readNewestOf(ctx, ix.Revisions, branch) + if err != nil || !found || rev.Revision != 2 || rev.Source != src || rev.Call != "release of mesh-sdk 0.1.27" || + len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != now.Fp2 { + t.Errorf("%s's newest = %+v (%v), want revision 2 of source %d, its call the release, g its mover at %s", branch, rev, err, src, now.Fp2) + continue + } + if i := slices.IndexFunc(kept, func(e meshcatalog.CatalogEntry) bool { return e.Module == "g" }); i < 0 || kept[i].Fp != now.Fp2 { + t.Errorf("%s's newest holds no entry of g at %s", branch, now.Fp2) + } + } + if len(said) != 2 || said[0].Call != "release of mesh-sdk 0.1.27" || said[1].Call != said[0].Call { + t.Errorf("said %+v, want catalog-revision of each branch with the release's call", said) } } diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source.go b/modules/mesh-delivery-indexer/internal/index/takein_source.go index 0b4d4947..07110af8 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source.go @@ -2,9 +2,11 @@ package index import ( "context" + "encoding/json" "errors" "fmt" "maps" + "slices" "strconv" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -25,9 +27,10 @@ func TakenID(branch string, source int64) string { // TakeInSource takes one source revision of a watched branch in, as the gate's source-moved names it (design 47 H §743): for each // repository it moves, the trees at the named commit through the seat git (wholeAt) and the fingerprints of the modules // registered for it; its movers (moversOf) against the branch's newest catalog revision; one catalog revision of the branch, -// ., carrying the movers and pinned for the take-in's pipeline when it has one (KeepBranchEntries); and +// the branch's next, . of source , carrying the movers and pinned for the take-in's pipeline when it has one +// (keepBranchEntries, H-A-213); and // take-in.taken once, with branch, source, revision, call and one entry per mover; a branch's first revision (start) has no -// mover (§768). A revision already kept under that name (a +// mover (§768). A revision already kept of that source (a // restart before the acknowledgement) is said again from its kept movers, recomputing nothing (H-A-92). The first watched // branch is taken in on the indexer's own view; another on its newest revision's, computed and kept only as that revision. // An error leaves the source-moved held, written into the state held behind its branch's earlier ones (ErrHeldUnwritten when @@ -57,13 +60,13 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() - name := BranchObject(e.Branch, e.Source) - if kept, _, err := ReadRevision(ctx, ix.Revisions, name); err == nil { - taken.Revision, taken.Entries = kept.Revision, kept.Movers - ix.logf("take-in: %s is kept already: take-in.taken said again from its %d movers", name, len(kept.Movers)) - return taken, ix.sayTaken(ctx, taken) - } else if !errors.Is(err, ErrNoObject) { + // A source revision kept already is found by the revision's field source, never by its number (H-A-213 (1)). + if kept, found, err := takenOfSource(ctx, ix.Revisions, e.Branch, e.Source); err != nil { return taken, err + } else if found { + taken.Revision, taken.Entries = kept.Revision, kept.Movers + ix.logf("take-in: source revision %d of %s is kept already as %s: take-in.taken said again from its %d movers", e.Source, e.Branch, RevisionObject(kept), len(kept.Movers)) + return taken, ix.sayTaken(ctx, taken) } _, before, _, err := readNewestOf(ctx, ix.Revisions, e.Branch) if err != nil { @@ -106,14 +109,15 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE if e.Start { movers = nil // a branch's first revision moves nothing: its entries are the base later take-ins are held against (§768, H-A-19) } - if err := ix.KeepBranchEntries(ctx, e.Branch, e.Source, entries, movers); err != nil { + number, err := ix.keepBranchEntries(ctx, e.Branch, e.Source, e.Call, entries, movers) + if err != nil { if errors.Is(err, ErrObjectsFull) { ix.heldFull.Store(true) // room is said once the store has room again (RoomAgain), and wakes the take-in (§744) } return taken, err } - taken.Revision, taken.Entries = e.Source, movers - ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, name, len(e.Heads), len(movers)) + taken.Revision, taken.Entries = number, movers + ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, BranchObject(e.Branch, number), len(e.Heads), len(movers)) return taken, ix.sayTaken(ctx, taken) } @@ -157,22 +161,57 @@ func (ix *Indexer) SourceToTake(ctx context.Context, branch string, a meshdelive if ix.Revisions == nil { return e, false, errors.New("the indexer keeps no catalog revisions") } - names, err := ix.Revisions.Names(ctx) + // The gate's source a branch is taken in at is its newest revision's field source, never the revision's number (H-A-213 (1)). + newest, found, err := newestOf(ctx, ix.Revisions, branch) if err != nil { - return e, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err) + return e, false, fmt.Errorf("the newest catalog revision of %s: %w", branch, err) } - var kept int64 - for _, n := range names { - if r, ok := BranchRevisionOf(branch, n); ok && r > kept { - kept = r - } - } - if kept >= a.Revision { + if found && newest.Source >= a.Revision { return e, false, nil } - e.Start = kept == 0 + e.Start = !found for _, r := range a.Repositories { e.Heads = append(e.Heads, meshdelivery.SourceMovedEventHeadsItem{Repository: r.Repository, Commit: r.Head}) } return e, true, nil } + +// takenOfSource is the revision a take-in of source src kept on branch: of the branch's revisions whose field source is src, +// the lowest numbered (a re-pin or a take-in again keeps the same source on a later number, H-A-211, H-A-213). The revisions +// are read from the newest down while their source is not below src; none is false. +func takenOfSource(ctx context.Context, o Objects, branch string, src int64) (meshcatalog.CatalogRevision, bool, error) { + names, err := o.Names(ctx) + if err != nil { + return meshcatalog.CatalogRevision{}, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err) + } + var numbers []int64 + for _, n := range names { + if r, ok := BranchRevisionOf(branch, n); ok { + numbers = append(numbers, r) + } + } + slices.Sort(numbers) + var kept meshcatalog.CatalogRevision + found := false + for i := len(numbers) - 1; i >= 0; i-- { + name := BranchObject(branch, numbers[i]) + data, err := o.Get(ctx, name) + if errors.Is(err, ErrNoObject) { + continue // removed under the read by the pin judge + } + if err != nil { + return kept, false, fmt.Errorf("revision %s: %w", name, err) + } + var rev meshcatalog.CatalogRevision + if err := json.Unmarshal(data, &rev); err != nil { + return kept, false, fmt.Errorf("%s: %w", name, err) + } + if rev.Source < src { + break + } + if rev.Source == src { + kept, found = rev, true + } + } + return kept, found, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go index d36933e0..be9c567b 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -43,7 +43,8 @@ func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTa return ix, objects, said, f } -// Design 47 H §743, H-A-32, H-A-92: a take-in writes one catalog revision . carrying its movers, pins it for the +// Design 47 H §743, H-A-32, H-A-92, H-A-213 (1): a take-in writes one catalog revision ., n the branch's newest + 1 +// and its source the gate's, carrying its movers, pins it for the // take-in's pipeline and says take-in.taken once; a redelivery writes nothing again and says the same; a revision with no mover // pins nothing and says no entries; the old form is refused. func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { @@ -56,19 +57,19 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { t.Fatal(err) } if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Fingerprint == "" || taken.Entries[0].Commit != f.Merge || - taken.Revision != 7 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call { - t.Fatalf("taken %+v, want revision 7 of main with avahi as its one mover", taken) + taken.Revision != 1 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call { + t.Fatalf("taken %+v, want revision 1 (the branch's first) of source 7 of main with avahi as its one mover", taken) } - rev, _, err := ReadRevision(ctx, objects, "main.7") + rev, _, err := ReadRevision(ctx, objects, "main.1") if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 { - t.Fatalf("main.7 = %+v (%v), want it kept carrying the movers", rev, err) + t.Fatalf("main.1 = %+v (%v), want it kept carrying the movers", rev, err) } - if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 7)); !found { - t.Error("main.7 is not pinned for its pipeline") + if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 1)); !found { + t.Error("main.1 is not pinned for its pipeline") } else { var pv PinValue - if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.7"}) { - t.Errorf("main.7 is pinned by %s, want pipeline.main.7", v) + if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.1"}) { + t.Errorf("main.1 is pinned by %s, want pipeline.main.1", v) } } puts := objects.puts @@ -79,14 +80,14 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { same := moved same.Source, same.Call = 8, "fedcba9876543210" none, err := ix.TakeInSource(ctx, same) - if err != nil || len(none.Entries) != 0 || none.Revision != 8 { - t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 8 with no entries", none, err) + if err != nil || len(none.Entries) != 0 || none.Revision != 2 || none.Source != 8 { + t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 2 of source 8 with no entries", none, err) } - if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 8)); found { + if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 2)); found { t.Error("a take-in with no mover pinned its revision") } - if _, _, err := ReadRevision(ctx, objects, "main.8"); err != nil { - t.Errorf("main.8 is not kept: %v", err) + if _, _, err := ReadRevision(ctx, objects, "main.2"); err != nil { + t.Errorf("main.2 is not kept: %v", err) } if len(*said) != 3 { t.Errorf("said %d take-in.taken, want 3", len(*said)) diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 01d1e03b..a6680672 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -598,19 +598,70 @@ func importsOf(pkgs sdkstep.PackageVersions, repository string) bool { return false } +// ReleaseCall is the call of the catalog revision a re-pin keeps: the release of a module at a version (design 47 H-A-213 (2)). +func ReleaseCall(module, version string) string { return "release of " + module + " " + version } + // Repin moves the pins of every module whose packages hold a package of module (a held module's directory holds it) to -// module's newest released version, the pass of H-A-161 for those modules alone: a version-released of module starts it -// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It answers how many it named. -func (ix *Indexer) Repin(ctx context.Context, module string) (int, error) { +// module's newest released version, the pass of H-A-161 for those modules alone: a release of module at version starts it +// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It runs on every watched branch +// holding such a module: the first on the indexer's own modules state, another on its newest revision's picture (as a +// take-in does), and keeps on each whose re-pinned fingerprints moved the branch's next catalog revision, of the same +// source as its newest, with the call ReleaseCall and those modules as its movers, pinned for the release's pipeline, and +// says catalog-revision; no take-in.taken (H-A-211, H-A-212, H-A-213). It answers how many modules it named. +func (ix *Indexer) Repin(ctx context.Context, module, version string) (int, error) { if ix.Modules == nil || ix.Manifests == nil { return 0, nil } - held, err := ix.heldModules(ctx) + ix.pendingMu.Lock() // one take-in step at a time, so the branch's newest stays the one re-pinned (pending.go) + defer ix.pendingMu.Unlock() + branches, err := ix.watchedBranches(ctx) if err != nil { return 0, err } - // Every kept entry naming a package of module is re-pinned, of any repository, changed or not: by its pins, and by its - // kept requirements, which name a package it holds no pin of yet (design 47 H-A-204, H-A-165, H-A-167). + if !slices.Contains(branches, TrunkBranch) { + branches = append([]string{TrunkBranch}, branches...) // the modules state is re-pinned whether a revision is kept or not + } + named := 0 + for _, branch := range branches { + x := ix + if !isTrunkBranch(branch) { + view, nodes, _, err := BranchView(ctx, ix.Revisions, branch) + if err != nil { + return named, err + } + d := ix.dry() + d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) + d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) } + x = d.ix + } + only, err := x.repinned(ctx, module) + if err != nil { + return named, err + } + if len(only) == 0 { + continue + } + named += len(only) + x.recompute.Lock() + err = x.recomputeLocked(ctx, nil, nil, "", only) + x.recompute.Unlock() + if err != nil { + return named, err + } + if err := ix.keepRepinned(ctx, x, branch, ReleaseCall(module, version), only); err != nil { + return named, err + } + } + return named, nil +} + +// repinned are the modules the indexer's modules state holds whose packages name a package of module: by their pins, and +// by their kept requirements, which name a package they hold no pin of yet (design 47 H-A-204, H-A-165, H-A-167). +func (ix *Indexer) repinned(ctx context.Context, module string) (map[string]bool, error) { + held, err := ix.heldModules(ctx) + if err != nil { + return nil, err + } only := map[string]bool{} for key, m := range held { names := slices.Collect(maps.Keys(m.Packages)) @@ -623,10 +674,48 @@ func (ix *Indexer) Repin(ctx context.Context, module string) (int, error) { } } } - if len(only) == 0 { - return 0, nil + return only, nil +} + +// keepRepinned keeps x's view, re-pinned for the modules only names, as branch's next catalog revision of the source of its +// newest, with call, when a fingerprint of them moved against that newest; its movers are those that moved (H-A-211, +// H-A-212, H-A-213). A branch with no revision kept yet keeps none: there is no source to keep it of. +func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch, call string, only map[string]bool) error { + if ix.Revisions == nil { + return nil } - ix.recompute.Lock() - defer ix.recompute.Unlock() - return len(only), ix.recomputeLocked(ctx, nil, nil, "", only) + newest, before, found, err := readNewestOf(ctx, ix.Revisions, branch) + if err != nil { + return fmt.Errorf("the newest catalog revision of %s: %w", branch, err) + } + if !found { + ix.logf("re-pin (%s): %s keeps no revision yet, so none is kept for it", call, branch) + return nil + } + entries, err := x.viewEntries(ctx) + if err != nil { + return err + } + was := make(map[string]string, len(before)) + for _, e := range before { + was[e.Module] = e.Fp + } + var movers []meshcatalog.TakeInEntry + for _, key := range slices.Sorted(maps.Keys(only)) { + e, ok := entries[key] + if !ok || e.Fp == "" || was[e.Module] == e.Fp { + continue + } + movers = append(movers, meshcatalog.TakeInEntry{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository}) + } + if len(movers) == 0 { + ix.logf("re-pin (%s): no fingerprint of %s moved, so no revision is kept", call, branch) + return nil + } + number, err := ix.keepBranchEntries(ctx, branch, newest.Source, call, entries, movers) + if err != nil { + return err + } + ix.logf("re-pin (%s): %s kept as revision %d of source %d, %d movers", call, branch, number, newest.Source, len(movers)) + return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 561b544a..059aa977 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -504,7 +504,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // } return first.KeepBranchEntries(context.Background(), TrunkBranch, source, entries, nil) } - if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 6 || + if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 || firstSaid[0].Entries != int64(len(builderModules(t))) { t.Fatalf("the take-in said %+v (%v), want trunk revision 1 with every module", firstSaid, err) } @@ -519,7 +519,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if err := takeIn(7); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) { t.Errorf("the next take-in after a write said %+v (%v), want revision 7 naming avahi", firstSaid, err) } - if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 7 { + if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 2 { t.Errorf("the view at a restart is read at %d (%v), want 7", at, err) } else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") { t.Errorf("the view at a restart lost the verdict: %s", raw) @@ -536,21 +536,21 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Errorf("the first-view tools %+v serve no modules", FirstViewTools) } if a, err := views.FirstView(testSelf, ModulesState, trunkStates.State(ModulesState), trunkStates.State(views.MarksState), "", time.Now()); err != nil || - a.Revision != 7 || len(a.Values) != len(builderModules(t)) { - t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 7", len(a.Values), a.Revision, err) + a.Revision != 2 || len(a.Values) != len(builderModules(t)) { + t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 2", len(a.Values), a.Revision, err) } else if m, err := meshcatalogclient.DecodeModule(a.Values[viewKey]); err != nil || m.Manifest == "" { t.Errorf("the first view's avahi = %s (%v), want the module's value naming its manifest", a.Values[viewKey], err) } - if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":7`) { - t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 7", e, ok, err) + if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) { + t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err) } // A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never // one revision read per key (about 11 s a page live, 2026-11-14). reads := 0 keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}} if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil || - a.Revision != 7 || len(a.Values) != len(builderModules(t)) || a.Next != "" { - t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 7 in one page", len(a.Values), a.Revision, a.Next, err) + a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" { + t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err) } if _, _, err := keptView.State("graph").Get("avahi"); err != nil || reads != 1 { t.Errorf("the kept view read the revision %d times (%v), want once", reads, err) @@ -822,7 +822,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { if _, err := ix.HearVersion(context.Background(), HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: "0.2.0", Commit: strings.Repeat("1", 40), GoModule: sdk, Published: true}); err != nil { t.Fatal(err) } - if _, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil { // the pass for avahi alone: mesh-tools is held stale here + if _, err := ix.Repin(context.Background(), "mesh-sdk", "0.1.27"); err != nil { // the pass for avahi alone: mesh-tools is held stale here t.Fatal(err) } if b, _ := moduleOf(t, mods, "avahi"); b.Fp2 != a.Fp2 || b.Packages[sdk] != "v0.1.18" { diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index c35b7752..e764bdad 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -451,7 +451,7 @@ func PinsOf(packages map[string]string) sdkstep.PackageVersions { // repin is Repin of a release of module at version, said in the journal; its failure is the caller's error. func (ix *Indexer) repin(ctx context.Context, module, version string) error { - n, err := ix.Repin(ctx, module) + n, err := ix.Repin(ctx, module, version) if err != nil { return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err) } diff --git a/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go b/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go index cdc1fc3f..d7a136cd 100644 --- a/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go +++ b/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go @@ -102,7 +102,7 @@ func TestAnEntryNoRevisionListsStaysWhileAWantNamesItAndGoesAfterTheWantMovesOn( n := NewNamed(newMem(), nil) ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }} n.Changed = ix.RemoveUnlisted - if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { + if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { // the revision that listed it is removed @@ -213,7 +213,7 @@ func TestNoEntryIsRemovedWhileAStartReadWasEmptyOrAnEventWasNotApplied(t *testin fp := namedFp("c") setup := func(n *Named) (*Indexer, func() bool) { ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }} - if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { + if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { -- 2.54.0 From 1b1fd9a7116404b9c5e33fa51738a359dd3c2d2d Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:33:50 +0100 Subject: [PATCH 17/30] Design 47 H1 (2): a take-in again keeps the trunk's next catalog revision of the same source The take-in again keeps what it recomputed as a re-pin does: the trunk's next revision of the newest's source, its call 'take-in again of ', its movers every module whose fingerprint moved, pinned for its pipeline, said as catalog-revision. Builds: design 47 H-A-208, H-A-210 as H-A-211 and H-A-212 amend it, H-A-213 (1), (3) --- .../internal/index/take_in_again.go | 11 ++++- .../internal/index/take_in_again_test.go | 44 ++++++++++++++----- .../internal/index/toolchains.go | 12 +++-- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again.go b/modules/mesh-delivery-indexer/internal/index/take_in_again.go index ed37a307..acc92faf 100644 --- a/modules/mesh-delivery-indexer/internal/index/take_in_again.go +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again.go @@ -15,10 +15,14 @@ import ( // take-in (such as the walk of the module graph, H-A-201) reaches a repository with no merge since. const TakeInAgainTool = "indexer_take_in_again" +// TakeInAgainCall is the call of the catalog revision a take-in again keeps (design 47 H-A-212, H-A-213 (3)). +func TakeInAgainCall(repository string) string { return "take-in again of " + repository } + // TakeInAgain re-reads the repository's module graph at the merge commit its taken key keeps (wholeAt, the reading of every // take-in: the kept requirements of each of its modules, H-A-201, H-A-157) and applies it as a take-in does, whose recompute // of the modules state is the one a publish re-pin runs (recomputeLocked): pins and fingerprints move as they would there. -// It keeps no catalog revision and asks no gate (H-A-210). A repository with no taken key is refused, said in the answer. +// It keeps the result as the trunk's next catalog revision of the same source, its call TakeInAgainCall (H-A-212, H-A-213), +// and asks no gate. A repository with no taken key is refused, said in the answer. func (ix *Indexer) TakeInAgain(ctx context.Context, repository string) TakenIn { repository = strings.TrimSpace(repository) r := TakenIn{Module: repository, Repository: repository} @@ -59,6 +63,11 @@ func (ix *Indexer) TakeInAgain(ctx context.Context, repository string) TakenIn { if err != nil { return refuse("taking %s at %.12s in again: %v", repository, t.Merge, err) } + // The take-in again keeps its revision as a re-pin does: the branch's next, of the same source, its call naming the + // repository, its movers every module whose fingerprint moved (design 47 H-A-212, H-A-213). + if err := ix.keepRepinned(ctx, ix, TrunkBranch, TakeInAgainCall(repository), nil); err != nil { + return refuse("taken in again at %.12s, but its catalog revision is not kept: %v", t.Merge, err) + } r.Stored, r.Commit = true, t.Merge r.Answer = fmt.Sprintf("taken in again at %.12s: %d modules written", t.Merge, len(applied.Written)) if len(applied.Written) > 0 { diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go index 0ac11588..37dd7915 100644 --- a/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go @@ -7,17 +7,19 @@ import ( "strings" "testing" - "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) // Design 47 H-A-208, H-A-210: a repository taken in before the walk of the module graph (H-A-201) keeps its modules' direct // requirements only; indexer_take_in_again re-reads it at its kept merge commit and recomputes the modules state, so the -// module's kept requirements and pins gain the transitive package. An unknown repository is refused. +// module's kept requirements and pins gain the transitive package, and keeps the trunk's next catalog revision of the same +// source with the call "take-in again of " (H-A-212, H-A-213). An unknown repository is refused. func TestATakeInAgainReachesARepositoryWithNoMergeSince(t *testing.T) { const conditions = "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)} ix, mods, _ := ptpIndexer(t, objects) - ix.Taken = newMem() + ix.Taken, ix.Revisions, ix.Packages = newMem(), newMemObjects(1<<20), newMem() reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") direct := map[string]string{"modules/g": "module g\n\nrequire git.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero + "\n"} walked := map[string]string{"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n\nrequire " + conditions + ptpZero + "\n"} @@ -39,11 +41,29 @@ func TestATakeInAgainReachesARepositoryWithNoMergeSince(t *testing.T) { return RepoRead{Root: f.Root, Trees: f.Trees, Manifests: f.Manifests, Locks: f.Locks, Packages: f.Packages}, nil } ctx := context.Background() + for p, v := range map[string]string{conditions: "0.1.4", "git.novox.be/novox/mesh-sdk/go": "0.1.27"} { + if _, err := ix.HearVersion(ctx, HeardVersion{Package: p, GoModule: p, Version: v, Published: true}); err != nil { + t.Fatal(err) + } + } + // the modules that hold the two packages, so their published versions pin them (fingerprint.HolderOf) + mods.put("mesh-sdk", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: strings.Repeat("1", 40), + Path: meshcatalogclient.RootPath, None2: None2Pending})) + mods.put("meshconditionsclient", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-contracts", Commit: strings.Repeat("2", 40), + Path: "health-and-repair/go/clients/meshconditionsclient", None2: None2Pending})) if err := ix.Apply(ctx, f); err != nil { t.Fatal(err) } - if m, _ := moduleOf(t, mods, "g"); m.Inputs2 == nil || slices.Contains(m.Inputs2.Packages, conditions) || strings.Contains(m.None2, conditions) { - t.Fatalf("before: packages = %+v (%s), want no transitive requirement yet", m.Inputs2, m.None2) + before, _ := moduleOf(t, mods, "g") + if before.Inputs2 == nil || slices.Contains(before.Inputs2.Packages, conditions) || strings.Contains(before.None2, conditions) { + t.Fatalf("before: packages = %+v (%s), want no transitive requirement yet", before.Inputs2, before.None2) + } + entries, err := ix.viewEntries(ctx) + if err != nil { + t.Fatal(err) + } + if err := ix.KeepBranchEntries(ctx, TrunkBranch, 4, entries, nil); err != nil { + t.Fatal(err) } if r := ix.TakeInAgain(ctx, "novox/unknown"); r.Stored || !strings.HasPrefix(r.Answer, "refused:") { t.Errorf("unknown repository: %+v, want refused", r) @@ -61,12 +81,14 @@ func TestATakeInAgainReachesARepositoryWithNoMergeSince(t *testing.T) { if m.Inputs2 == nil || !slices.Contains(m.Inputs2.Packages, conditions) { t.Errorf("after: kept requirements = %+v, want %s", m.Inputs2, conditions) } - // The recompute pins it: nothing is published here, so the fingerprint waits on it by name (H-A-161), and the pins the - // entry names from the kept requirements hold it with no version until it is (H-A-204). - if !strings.Contains(m.None2, conditions) { - t.Errorf("after: the recompute's reason %q, want it waiting on %s", m.None2, conditions) + // The recompute pins it at its published version, which moves the fingerprint (H-A-167, H-A-204), and the take-in again + // keeps the trunk's next revision of the same source, its call naming the repository, g its mover (H-A-212, H-A-213). + if m.Fp2 == "" || m.Fp2 == before.Fp2 || m.Packages[conditions] != "v0.1.4" { + t.Errorf("after: g = %s (%s) pinned %v, want a new fingerprint pinned to v0.1.4 (was %s)", m.Fp2, m.None2, m.Packages, before.Fp2) } - if v, named := releasedPins(nil, nil, catalogue.Manifest{}, "novox/mesh-delivery", m.Inputs2.Packages)[conditions]; !named || v != "" { - t.Errorf("after: pins name %s %v (%q), want it named, waiting", conditions, named, v) + rev, _, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) + if err != nil || !found || rev.Revision != 2 || rev.Source != 4 || rev.Call != "take-in again of novox/mesh-delivery" || + len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != m.Fp2 { + t.Errorf("the newest trunk revision = %+v (%v), want revision 2 of source 4 with the take-in again's call and g its mover", rev, err) } } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index a6680672..4c9dca54 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -677,9 +677,9 @@ func (ix *Indexer) repinned(ctx context.Context, module string) (map[string]bool return only, nil } -// keepRepinned keeps x's view, re-pinned for the modules only names, as branch's next catalog revision of the source of its -// newest, with call, when a fingerprint of them moved against that newest; its movers are those that moved (H-A-211, -// H-A-212, H-A-213). A branch with no revision kept yet keeps none: there is no source to keep it of. +// keepRepinned keeps x's view, re-pinned for the modules only names (every module for nil), as branch's next catalog revision +// of the source of its newest, with call, when a fingerprint of them moved against that newest; its movers are those that +// moved (H-A-211, H-A-212, H-A-213). A branch with no revision kept yet keeps none: there is no source to keep it of. func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch, call string, only map[string]bool) error { if ix.Revisions == nil { return nil @@ -701,7 +701,11 @@ func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch, call st was[e.Module] = e.Fp } var movers []meshcatalog.TakeInEntry - for _, key := range slices.Sorted(maps.Keys(only)) { + moved := slices.Sorted(maps.Keys(only)) + if only == nil { // a take-in again: every module whose fingerprint moved (H-A-212) + moved = slices.Sorted(maps.Keys(entries)) + } + for _, key := range moved { e, ok := entries[key] if !ok || e.Fp == "" || was[e.Module] == e.Fp { continue -- 2.54.0 From 83e4470757402e1264ef5500103ec88d3e0e10e2 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:39:53 +0100 Subject: [PATCH 18/30] Design 47 H1 (1), review round 1: a source answers its take-in's revision The revision verb's source input, and the pipeline-wanted that frees pins by it, answer the lowest-numbered revision of a source (the take-in's), never a later re-pin's of the same source. TestIndexerWhole asks the exact first number: the bed seeds main.1 of source 0, so source 10's is 2 (no source is kept twice). Builds: design 47 H-A-213 (1), (4), H-A-211 --- .../mesh-delivery-indexer/takein_integration_test.go | 4 ++-- .../internal/index/catalog_branches_test.go | 7 +++++++ .../internal/index/revision_verb.go | 12 +++++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index 0020713c..3055a5ac 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -453,8 +453,8 @@ func TestIndexerWhole(t *testing.T) { first := heard.next(t, "main", 10) r := func(source int64) int64 { return first.Revision + source - 10 } rn := func(source int64) string { return fmt.Sprintf("main.%d", r(source)) } - if len(first.Entries) != 0 || first.Revision <= 0 || first.Source != 10 { - t.Fatalf("the first revision said %+v, want a revision of source 10 with no entries", first) + if len(first.Entries) != 0 || first.Revision != 2 || first.Source != 10 { + t.Fatalf("the first revision said %+v, want revision 2 (the bed's seed is main.1, of source 0) of source 10 with no entries", first) } untilKey(t, ctx, pins, index.PinKey("main", r(10)), absent) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go index 574de13d..31fe824f 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go @@ -77,6 +77,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 { t.Errorf("the branch's source 8 answered %+v (%v)", a.Revision, err) } + // H-A-213: a re-pin's revision of source 8 answers below the take-in's: source 8 answers revision 2. + if _, err := ix.keepBranchEntries(ctx, "feature/x.y", 8, "release of mesh-sdk 0.1.27", tip("c4"), nil); err != nil { + t.Fatal(err) + } + if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 || a.Entries[0].Commit != "c3" { + t.Errorf("the branch's source 8 after a re-pin answered %+v (%v), want the take-in's revision 2", a.Revision, err) + } if _, err := bySource("feature/x=2ey", 9); err == nil { t.Error("a source no revision was taken in from was answered") } diff --git a/modules/mesh-delivery-indexer/internal/index/revision_verb.go b/modules/mesh-delivery-indexer/internal/index/revision_verb.go index 20402883..2667fbb4 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_verb.go @@ -167,18 +167,24 @@ func revisionOfSource(ctx context.Context, o Objects, branch string, source int6 } } slices.SortFunc(found, func(a, b int64) int { return int(b - a) }) + // A re-pin or a take-in again keeps a later revision of the same source (H-A-211, H-A-213): the source's own revision is + // the lowest numbered of it, the take-in's. + lowest := "" for _, r := range found { name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: r}) rev, _, err := ReadRevision(ctx, o, name) if err != nil { return "", err } - if rev.Source == source { - return name, nil - } if rev.Source < source { break } + if rev.Source == source { + lowest = name + } + } + if lowest != "" { + return lowest, nil } return "", fmt.Errorf("no revision of %q was taken in from the source revision %d", branch, source) } -- 2.54.0 From 9b3a728f9e572c12f8bbb723a9ee92f34af507f4 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:43:33 +0100 Subject: [PATCH 19/30] Design 47 H1 (8): a catalog revision's call names its kind, one of three constants The seat mesh-catalog's CatalogRevision and catalog-revision gain call-kind, an enumeration the seat's schema validates (take-in, release, take-in-again), with module and version for a release and repository for a take-in again, beside call, the words for a person (regenerated by seatgen, additive). The indexer's three writers set them: a take-in's from the gate's call, a re-pin's on a module's release, a take-in again's; no reader parses the words. Builds: design 47 H H-A-213 (3), H-A-214, H-A-215 --- .../go/seats/meshcatalog/contract.go | 38 +++++++++++++++-- .../go/seats/meshcatalog/contract_test.go | 4 +- .../seats/mesh-catalog/fixtures/events.json | 3 ++ .../seats/mesh-catalog/fixtures/states.json | 5 ++- .../seats/mesh-catalog/seat.json | 42 +++++++++++++++++++ .../ts/seats/meshcatalog/index.d.ts | 16 +++++++ .../internal/index/catalog_branches_test.go | 2 +- .../internal/index/catalog_revisions.go | 34 ++++++++++++--- .../internal/index/repin_test.go | 4 +- .../internal/index/take_in_again.go | 2 +- .../internal/index/take_in_again_test.go | 1 + .../internal/index/takein_source.go | 2 +- .../internal/index/takein_source_test.go | 3 +- .../internal/index/toolchains.go | 10 ++--- 14 files changed, 144 insertions(+), 22 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go index 993c9448..9029f776 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go @@ -717,6 +717,8 @@ type CatalogRevision struct { Branch string `json:"branch,omitempty"` // what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's Call string `json:"call,omitempty"` + // the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215): a take-in's, a re-pin's on a module's release, a take-in again's; absent for a candidate's + CallKind string `json:"call-kind,omitempty"` // the seat definitions it was judged with, each by its digest, read whole through seat-definitions Definitions []CatalogRevisionDefinitionsItem `json:"definitions,omitempty"` // one per module, by name @@ -727,8 +729,12 @@ type CatalogRevision struct { Heads string `json:"heads,omitempty"` // trunk or candidate Kind string `json:"kind"` + // for call-kind release, the module released (H-A-215) + Module string `json:"module,omitempty"` // the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover Movers []TakeInEntry `json:"movers,omitempty"` + // for call-kind take-in-again, the repository taken in again (H-A-215) + Repository string `json:"repository,omitempty"` // the catalog graph revision's number, counted by its holder Revision int64 `json:"revision"` // the source revision it was taken in from; for a candidate, the source revision its heads are merged onto @@ -737,8 +743,17 @@ type CatalogRevision struct { Trunk []Head `json:"trunk,omitempty"` // the layout version V int64 `json:"v,omitempty"` + // for call-kind release, the version released (H-A-215) + Version string `json:"version,omitempty"` } +// The values of CatalogRevision's call-kind. +const ( + CatalogRevisionCallKindTakeIn = "take-in" + CatalogRevisionCallKindRelease = "release" + CatalogRevisionCallKindTakeInAgain = "take-in-again" +) + // Edge is one edge of the catalog graph between artifact references, B before A (ADR 0666 section 14): kind 1 B published before A builds (a package A imports), 2 B published before A delivers (fetched by reference at its start), 3 B delivered before A builds (its build module and toolchain), 4 B delivered before A delivers (a provision, bus state, a format's reader), 5 B has acted for A's new version (an acting rule), 6 B runs a version that serves A (a verb or a manifest word A uses). type Edge struct { // declared in the artifact entry's needs of A's manifest, where nothing else says it @@ -1305,6 +1320,8 @@ type CatalogRevisionEvent struct { Branch string `json:"branch,omitempty"` // what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's Call string `json:"call,omitempty"` + // the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215); absent for a candidate's + CallKind string `json:"call-kind,omitempty"` // the modules whose entries differ from the revision before it of the same kind Changed []string `json:"changed,omitempty"` // how many entries it lists @@ -1312,12 +1329,25 @@ type CatalogRevisionEvent struct { Group string `json:"group,omitempty"` Heads string `json:"heads,omitempty"` // trunk or candidate: a candidate's moves no running part - Kind string `json:"kind"` - Revision int64 `json:"revision"` - Source int64 `json:"source"` - V int64 `json:"v"` + Kind string `json:"kind"` + // for call-kind release, the module released (H-A-215) + Module string `json:"module,omitempty"` + // for call-kind take-in-again, the repository taken in again (H-A-215) + Repository string `json:"repository,omitempty"` + Revision int64 `json:"revision"` + Source int64 `json:"source"` + V int64 `json:"v"` + // for call-kind release, the version released (H-A-215) + Version string `json:"version,omitempty"` } +// The values of CatalogRevisionEvent's call-kind. +const ( + CatalogRevisionEventCallKindTakeIn = "take-in" + CatalogRevisionEventCallKindRelease = "release" + CatalogRevisionEventCallKindTakeInAgain = "take-in-again" +) + // FillingEvent is the body of the event filling. type FillingEvent = Fill diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go index beb81baa..5258e5ea 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go @@ -17,7 +17,7 @@ func TestFixtures(t *testing.T) { decode func([]byte) (any, error) subjectIs *regexp.Regexp }{ - {name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", + {name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"revision\":12,\"source\":3589,\"v\":1,\"version\":\"0.1.27\"}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"revision\":12,\"source\":3589,\"v\":1,\"version\":\"0.1.27\"}", decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")}, {name: "a candidate's revision kept", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}", decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")}, @@ -59,7 +59,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v ModulesAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "the module verb (captured: xclip)", subject: "", given: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", wire: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", decode: func(b []byte) (any, error) { var v ModuleAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, - {name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", + {name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1,\"version\":\"0.1.27\"}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1,\"version\":\"0.1.27\"}", decode: func(b []byte) (any, error) { var v CatalogRevisionsByBranchRevisionValue err := json.Unmarshal(b, &v) diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json index 5c8e30db..1c0b4dd0 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json @@ -9,6 +9,9 @@ "kind": "trunk", "source": 3589, "call": "release of mesh-sdk 0.1.27", + "call-kind": "release", + "module": "mesh-sdk", + "version": "0.1.27", "entries": 2, "changed": [ "xclip" diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json index 750ba355..965b8ff6 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json @@ -210,7 +210,10 @@ } ], "base": 9, - "call": "release of mesh-sdk 0.1.27" + "call": "release of mesh-sdk 0.1.27", + "call-kind": "release", + "module": "mesh-sdk", + "version": "0.1.27" } }, { diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json index 1e5687bc..e0195d64 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json @@ -661,6 +661,27 @@ "type": "string", "description": "what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's" }, + "call-kind": { + "type": "string", + "enum": [ + "take-in", + "release", + "take-in-again" + ], + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215); absent for a candidate's" + }, + "module": { + "type": "string", + "description": "for call-kind release, the module released (H-A-215)" + }, + "version": { + "type": "string", + "description": "for call-kind release, the version released (H-A-215)" + }, + "repository": { + "type": "string", + "description": "for call-kind take-in-again, the repository taken in again (H-A-215)" + }, "group": { "type": "string" }, @@ -1879,6 +1900,27 @@ "type": "string", "description": "what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's" }, + "call-kind": { + "type": "string", + "enum": [ + "take-in", + "release", + "take-in-again" + ], + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215): a take-in's, a re-pin's on a module's release, a take-in again's; absent for a candidate's" + }, + "module": { + "type": "string", + "description": "for call-kind release, the module released (H-A-215)" + }, + "version": { + "type": "string", + "description": "for call-kind release, the version released (H-A-215)" + }, + "repository": { + "type": "string", + "description": "for call-kind take-in-again, the repository taken in again (H-A-215)" + }, "group": { "type": "string", "description": "a candidate's group" diff --git a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts index 5877f3fb..1fcf459d 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts @@ -279,6 +279,8 @@ export interface CatalogRevision { "branch"?: string; /** what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's */ "call"?: string; + /** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215): a take-in's, a re-pin's on a module's release, a take-in again's; absent for a candidate's */ + "call-kind"?: "take-in" | "release" | "take-in-again"; /** the seat definitions it was judged with, each by its digest, read whole through seat-definitions */ "definitions"?: CatalogRevisionDefinitionsItem[]; /** one per module, by name */ @@ -289,8 +291,12 @@ export interface CatalogRevision { "heads"?: string; /** trunk or candidate */ "kind": string; + /** for call-kind release, the module released (H-A-215) */ + "module"?: string; /** the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover */ "movers"?: TakeInEntry[]; + /** for call-kind take-in-again, the repository taken in again (H-A-215) */ + "repository"?: string; /** the catalog graph revision's number, counted by its holder */ "revision": number; /** the source revision it was taken in from; for a candidate, the source revision its heads are merged onto */ @@ -299,6 +305,8 @@ export interface CatalogRevision { "trunk"?: Head[]; /** the layout version */ "v"?: number; + /** for call-kind release, the version released (H-A-215) */ + "version"?: string; } /** one edge of the catalog graph between artifact references, B before A (ADR 0666 section 14): kind 1 B published before A builds (a package A imports), 2 B published before A delivers (fetched by reference at its start), 3 B delivered before A builds (its build module and toolchain), 4 B delivered before A delivers (a provision, bus state, a format's reader), 5 B has acted for A's new version (an acting rule), 6 B runs a version that serves A (a verb or a manifest word A uses) */ @@ -800,6 +808,8 @@ export interface CatalogRevisionEvent { "branch"?: string; /** what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's */ "call"?: string; + /** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215); absent for a candidate's */ + "call-kind"?: "take-in" | "release" | "take-in-again"; /** the modules whose entries differ from the revision before it of the same kind */ "changed"?: string[]; /** how many entries it lists */ @@ -808,9 +818,15 @@ export interface CatalogRevisionEvent { "heads"?: string; /** trunk or candidate: a candidate's moves no running part */ "kind": string; + /** for call-kind release, the module released (H-A-215) */ + "module"?: string; + /** for call-kind take-in-again, the repository taken in again (H-A-215) */ + "repository"?: string; "revision": number; "source": number; "v": number; + /** for call-kind release, the version released (H-A-215) */ + "version"?: string; } /** is the body of the event filling. */ diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go index 31fe824f..66319e48 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go @@ -78,7 +78,7 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { t.Errorf("the branch's source 8 answered %+v (%v)", a.Revision, err) } // H-A-213: a re-pin's revision of source 8 answers below the take-in's: source 8 answers revision 2. - if _, err := ix.keepBranchEntries(ctx, "feature/x.y", 8, "release of mesh-sdk 0.1.27", tip("c4"), nil); err != nil { + if _, err := ix.keepBranchEntries(ctx, "feature/x.y", 8, releaseCall("mesh-sdk", "0.1.27"), tip("c4"), nil); err != nil { t.Fatal(err) } if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 || a.Entries[0].Commit != "c3" { diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index b528d1a5..2110f08f 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -348,7 +348,7 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata // KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design // 47 H §743, H-A-32), as keepBranchEntries does with no call. func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error { - _, err := ix.keepBranchEntries(ctx, branch, src, "", entries, movers) + _, err := ix.keepBranchEntries(ctx, branch, src, revisionCall{}, entries, movers) return err } @@ -359,7 +359,7 @@ func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int // branch, its source and its call. movers are kept on the revision (H-A-92), and the revision pinned for its one pipeline, // pipeline.., in the same write when there is one (A-449a, H-A-213 (4)), with the branch's previous revision // (H-A-147). It answers the number it kept; ErrObjectsFull is a store too full to keep it. -func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, call string, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) (int64, error) { +func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, call revisionCall, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) (int64, error) { if ix.Revisions == nil { return 0, nil } @@ -380,7 +380,8 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int number := newest.Revision + 1 entries = WithEdges(ix.withActs(ctx, entries)) // the acting rules and the edges between artifact references are written here, once, for every reader (A-208) at := ix.now().UTC() - rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: number, Kind: "trunk", Branch: branch, Source: src, Call: call, At: &at}, entries) + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: number, Kind: "trunk", Branch: branch, Source: src, Call: call.Call, + CallKind: call.Kind, Module: call.Module, Version: call.Version, Repository: call.Repository, At: &at}, entries) if err != nil { return 0, err } @@ -444,16 +445,39 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 { ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity}) } - ix.logf("catalog graph: revision %d of source revision %d kept (%s), %d entries, %d changed", rev.Revision, src, call, len(rev.Entries), len(changed)) + ix.logf("catalog graph: revision %d of source revision %d kept (%s), %d entries, %d changed", rev.Revision, src, call.Call, len(rev.Entries), len(changed)) if ix.SayCatalogRevision != nil { if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind, - Branch: keys.EscapeName(branch), Source: src, Call: call, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil { + Branch: keys.EscapeName(branch), Source: src, Call: call.Call, CallKind: call.Kind, Module: call.Module, Version: call.Version, + Repository: call.Repository, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil { ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err) } } return number, nil } +// revisionCall is what made a catalog revision (design 47 H-A-213 (3), H-A-215): call, the words for a person; Kind, one of the +// seat mesh-catalog's three call kinds, which a reader reads instead of the words; Module and Version for a release, Repository +// for a take-in again. The zero value names no call (a revision kept by hand or a test). +type revisionCall struct { + Call, Kind, Module, Version, Repository string +} + +// takeInCall is a take-in's: the gate's call its source revision names. +func takeInCall(call string) revisionCall { + return revisionCall{Call: call, Kind: meshcatalog.CatalogRevisionCallKindTakeIn} +} + +// releaseCall is a re-pin's on the release of module at version (H-A-213 (2)). +func releaseCall(module, version string) revisionCall { + return revisionCall{Call: ReleaseCall(module, version), Kind: meshcatalog.CatalogRevisionCallKindRelease, Module: module, Version: version} +} + +// takeInAgainCall is a take-in again's of repository (H-A-212). +func takeInAgainCall(repository string) revisionCall { + return revisionCall{Call: TakeInAgainCall(repository), Kind: meshcatalog.CatalogRevisionCallKindTakeInAgain, Repository: repository} +} + // HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held). const HeldObjectsFull = "objects full" diff --git a/modules/mesh-delivery-indexer/internal/index/repin_test.go b/modules/mesh-delivery-indexer/internal/index/repin_test.go index 4a0f9ab4..584c3708 100644 --- a/modules/mesh-delivery-indexer/internal/index/repin_test.go +++ b/modules/mesh-delivery-indexer/internal/index/repin_test.go @@ -84,6 +84,7 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) { for branch, src := range sources { rev, kept, found, err := readNewestOf(ctx, ix.Revisions, branch) if err != nil || !found || rev.Revision != 2 || rev.Source != src || rev.Call != "release of mesh-sdk 0.1.27" || + rev.CallKind != meshcatalog.CatalogRevisionCallKindRelease || rev.Module != "mesh-sdk" || rev.Version != "0.1.27" || // H-A-215 len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != now.Fp2 { t.Errorf("%s's newest = %+v (%v), want revision 2 of source %d, its call the release, g its mover at %s", branch, rev, err, src, now.Fp2) continue @@ -92,7 +93,8 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) { t.Errorf("%s's newest holds no entry of g at %s", branch, now.Fp2) } } - if len(said) != 2 || said[0].Call != "release of mesh-sdk 0.1.27" || said[1].Call != said[0].Call { + if len(said) != 2 || said[0].Call != "release of mesh-sdk 0.1.27" || said[1].Call != said[0].Call || + said[0].CallKind != meshcatalog.CatalogRevisionEventCallKindRelease || said[1].Module != "mesh-sdk" || said[1].Version != "0.1.27" { t.Errorf("said %+v, want catalog-revision of each branch with the release's call", said) } } diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again.go b/modules/mesh-delivery-indexer/internal/index/take_in_again.go index acc92faf..e27a23f1 100644 --- a/modules/mesh-delivery-indexer/internal/index/take_in_again.go +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again.go @@ -65,7 +65,7 @@ func (ix *Indexer) TakeInAgain(ctx context.Context, repository string) TakenIn { } // The take-in again keeps its revision as a re-pin does: the branch's next, of the same source, its call naming the // repository, its movers every module whose fingerprint moved (design 47 H-A-212, H-A-213). - if err := ix.keepRepinned(ctx, ix, TrunkBranch, TakeInAgainCall(repository), nil); err != nil { + if err := ix.keepRepinned(ctx, ix, TrunkBranch, takeInAgainCall(repository), nil); err != nil { return refuse("taken in again at %.12s, but its catalog revision is not kept: %v", t.Merge, err) } r.Stored, r.Commit = true, t.Merge diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go index 37dd7915..1e114d53 100644 --- a/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go @@ -88,6 +88,7 @@ func TestATakeInAgainReachesARepositoryWithNoMergeSince(t *testing.T) { } rev, _, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) if err != nil || !found || rev.Revision != 2 || rev.Source != 4 || rev.Call != "take-in again of novox/mesh-delivery" || + rev.CallKind != meshcatalog.CatalogRevisionCallKindTakeInAgain || rev.Repository != "novox/mesh-delivery" || // H-A-215 len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != m.Fp2 { t.Errorf("the newest trunk revision = %+v (%v), want revision 2 of source 4 with the take-in again's call and g its mover", rev, err) } diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source.go b/modules/mesh-delivery-indexer/internal/index/takein_source.go index 07110af8..667eb11a 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source.go @@ -109,7 +109,7 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE if e.Start { movers = nil // a branch's first revision moves nothing: its entries are the base later take-ins are held against (§768, H-A-19) } - number, err := ix.keepBranchEntries(ctx, e.Branch, e.Source, e.Call, entries, movers) + number, err := ix.keepBranchEntries(ctx, e.Branch, e.Source, takeInCall(e.Call), entries, movers) if err != nil { if errors.Is(err, ErrObjectsFull) { ix.heldFull.Store(true) // room is said once the store has room again (RoomAgain), and wakes the take-in (§744) diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go index be9c567b..2e439e85 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -61,7 +61,8 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { t.Fatalf("taken %+v, want revision 1 (the branch's first) of source 7 of main with avahi as its one mover", taken) } rev, _, err := ReadRevision(ctx, objects, "main.1") - if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 { + if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 || + rev.Call != moved.Call || rev.CallKind != meshcatalog.CatalogRevisionCallKindTakeIn { // H-A-215 t.Fatalf("main.1 = %+v (%v), want it kept carrying the movers", rev, err) } if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 1)); !found { diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 4c9dca54..b0043487 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -648,7 +648,7 @@ func (ix *Indexer) Repin(ctx context.Context, module, version string) (int, erro if err != nil { return named, err } - if err := ix.keepRepinned(ctx, x, branch, ReleaseCall(module, version), only); err != nil { + if err := ix.keepRepinned(ctx, x, branch, releaseCall(module, version), only); err != nil { return named, err } } @@ -680,7 +680,7 @@ func (ix *Indexer) repinned(ctx context.Context, module string) (map[string]bool // keepRepinned keeps x's view, re-pinned for the modules only names (every module for nil), as branch's next catalog revision // of the source of its newest, with call, when a fingerprint of them moved against that newest; its movers are those that // moved (H-A-211, H-A-212, H-A-213). A branch with no revision kept yet keeps none: there is no source to keep it of. -func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch, call string, only map[string]bool) error { +func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch string, call revisionCall, only map[string]bool) error { if ix.Revisions == nil { return nil } @@ -689,7 +689,7 @@ func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch, call st return fmt.Errorf("the newest catalog revision of %s: %w", branch, err) } if !found { - ix.logf("re-pin (%s): %s keeps no revision yet, so none is kept for it", call, branch) + ix.logf("re-pin (%s): %s keeps no revision yet, so none is kept for it", call.Call, branch) return nil } entries, err := x.viewEntries(ctx) @@ -713,13 +713,13 @@ func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch, call st movers = append(movers, meshcatalog.TakeInEntry{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository}) } if len(movers) == 0 { - ix.logf("re-pin (%s): no fingerprint of %s moved, so no revision is kept", call, branch) + ix.logf("re-pin (%s): no fingerprint of %s moved, so no revision is kept", call.Call, branch) return nil } number, err := ix.keepBranchEntries(ctx, branch, newest.Source, call, entries, movers) if err != nil { return err } - ix.logf("re-pin (%s): %s kept as revision %d of source %d, %d movers", call, branch, number, newest.Source, len(movers)) + ix.logf("re-pin (%s): %s kept as revision %d of source %d, %d movers", call.Call, branch, number, newest.Source, len(movers)) return nil } -- 2.54.0 From 07dc14bf533dccbc636e555b154378343984de98 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:46:25 +0100 Subject: [PATCH 20/30] Design 47 H1 (6): pipeline-wanted names the pipeline's revision, and the indexer frees that revision's pins The seat mesh-wants' pipeline-wanted gains revision, the pipeline's catalog revision number (seatgen regenerated, additive). The assigner keeps the source the pipeline-ended names in its tier progress and says it beside the revision, the number of the pipeline's key; a progress kept before the field says the key's number as both. The indexer frees the pins of the revision the event names, not of another revision of the same source, and for an event naming none takes the source's lowest-numbered revision, the take-in's. Builds: design 47 H H-A-213 (4), H-A-214, H-A-216 --- .../cmd/mesh-delivery-assigner/pipelines.go | 2 +- .../cmd/mesh-delivery-assigner/tiers.go | 36 +++++++++++-------- .../cmd/mesh-delivery-assigner/tiers_test.go | 19 +++++++--- .../go/seats/meshwants/contract.go | 8 +++-- .../go/seats/meshwants/contract_test.go | 2 +- .../seats/mesh-wants/fixtures/events.json | 5 +-- .../seats/mesh-wants/seat.json | 6 +++- .../ts/seats/meshwants/index.d.ts | 4 ++- .../cmd/mesh-delivery-indexer/named_test.go | 11 +++--- .../cmd/mesh-delivery-indexer/runs.go | 4 +-- .../internal/index/pins.go | 22 ++++++++---- .../internal/index/pins_test.go | 18 ++++++++-- 12 files changed, 94 insertions(+), 43 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index 0ca00286..2d160479 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -169,7 +169,7 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) { changed := false var kept []tierMover defer func() { - if err := e.startTiers(p.Pipeline, kept); err != nil { + if err := e.startTiers(p.Pipeline, p.Source, kept); err != nil { e.logf("the tier progress of %s is not written: %v", p.Pipeline, err) } }() diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go index 5211d564..5c45c24f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go @@ -42,11 +42,14 @@ type tierMover struct { Tier int64 `json:"tier"` } -// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, and the movers. +// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, the movers, and the source +// revision its pipeline-ended names, said in pipeline-wanted beside the key's revision (design 47 H-A-214, H-A-216); zero in a +// value kept before the field, read as the key's number. type tierProgress struct { Tier int64 `json:"tier"` Since time.Time `json:"since"` Movers []tierMover `json:"movers"` + Source int64 `json:"source,omitempty"` } // tierKey is the key of pipeline's tier progress; pipeline is the runner's key `.`, its branch escaped. @@ -60,7 +63,7 @@ func isTierKey(key string) bool { // startTiers writes the tier progress of a passed pipeline whose movers' versions are now kept, its first tier the lowest of // theirs, since now. A pipeline heard again writes nothing. -func (e *pipelineEnds) startTiers(pipeline string, movers []tierMover) error { +func (e *pipelineEnds) startTiers(pipeline string, source int64, movers []tierMover) error { if len(movers) == 0 { return nil } @@ -68,7 +71,7 @@ func (e *pipelineEnds) startTiers(pipeline string, movers []tierMover) error { for _, m := range movers { first = min(first, m.Tier) } - value, err := json.Marshal(tierProgress{Tier: first, Since: e.clock().UTC(), Movers: movers}) + value, err := json.Marshal(tierProgress{Tier: first, Since: e.clock().UTC(), Movers: movers, Source: source}) if err != nil { return err } @@ -131,7 +134,7 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st continue } pipeline := strings.TrimPrefix(key, tierPrefix) - branchEsc, source, _ := strings.Cut(pipeline, ".") + branchEsc, number, _ := strings.Cut(pipeline, ".") branch, err := keys.UnescapeName(branchEsc) if err != nil { continue @@ -144,7 +147,7 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st notRunning := p.notRunning(n, placedOn, pipeline, branch, t, t.Tier) late := !now.Before(t.Since.Add(bound)) if late { - p.tierLate(pipeline, branch, source, t, notRunning) + p.tierLate(pipeline, branch, number, t, notRunning) break } if len(notRunning) > 0 { @@ -165,10 +168,10 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st p.logf("the tier progress %s is not deleted: %v", key, err) break } - p.sayPipelineWanted(pipeline, branch, source, meshwants.PipelineWantedOutcomeRunning, nil) + p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeRunning, nil) break } - value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers}) + value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers, Source: t.Source}) if err == nil { err = p.ends.st.swap(key, value, rev) } @@ -217,7 +220,7 @@ func (p *passer) notRunning(n assign.Nodes, placedOn func(module, branch string) // tierLate ends a pipeline's delivery at a tier past tier-bound: the finding tier-late is raised and pipeline-wanted says tier-late, // once each; the finding clears once the late movers run or a later pipeline moved them. The progress stays, so no later tier is // written, until no later tier's mover is kept at this pipeline any more. -func (p *passer) tierLate(pipeline, branch, source string, t tierProgress, notRunning []meshwants.NotRunning) { +func (p *passer) tierLate(pipeline, branch, number string, t tierProgress, notRunning []meshwants.NotRunning) { o := tierLateFinding(pipeline, t.Tier, notRunning) key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind) if len(notRunning) > 0 { @@ -231,7 +234,7 @@ func (p *passer) tierLate(pipeline, branch, source string, t tierProgress, notRu p.said[key+".raised"] = true delete(p.said, key+".cleared") p.logf("pipeline %s: tier %d is late (tier-bound %s): %s; no later tier is written", pipeline, t.Tier, p.tierBoundOf(), o.Summary) - p.sayPipelineWanted(pipeline, branch, source, meshwants.PipelineWantedOutcomeTierLate, notRunning) + p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeTierLate, notRunning) } } else if !p.said[key+".cleared"] { if p.findings.Delete != nil { @@ -271,11 +274,16 @@ func tierLateFinding(pipeline string, tier int64, notRunning []meshwants.NotRunn Source: "assigner." + kindTierLate} } -// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome). -func (p *passer) sayPipelineWanted(pipeline, branch, source, outcome string, notRunning []meshwants.NotRunning) { - var src int64 - _, _ = fmt.Sscan(source, &src) - body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: src, Outcome: outcome, NotRunning: notRunning} +// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome): its +// revision the number of the pipeline's key `.`, its source the one the pipeline-ended named (design 47 +// H-A-214, H-A-216), the key's number for a progress kept before it held one. +func (p *passer) sayPipelineWanted(pipeline, branch, number string, source int64, outcome string, notRunning []meshwants.NotRunning) { + var revision int64 + _, _ = fmt.Sscan(number, &revision) + if source <= 0 { + source = revision + } + body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: source, Revision: revision, Outcome: outcome, NotRunning: notRunning} event := meshwants.EventPipelineWanted + "." + pipeline if err := p.wantsSay(event, body, "pipeline-wanted."+pipeline+"."+outcome); err != nil { p.logf("pipeline-wanted of %s is not said: %v", pipeline, err) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go index 39d9bb39..9937f775 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go @@ -62,8 +62,11 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { p.w.mu.Unlock() p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now } - if _, err := p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed), - mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))); err != nil { + // The pipeline of revision 5 (its key's number) of source 4: a re-pin's revision (H-A-213, H-A-216). + e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed), + mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed)) + e.Source = 4 + if _, err := p.ends.take(e); err != nil { t.Fatal(err) } wanted := func(node, module string) bool { @@ -88,7 +91,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web}) _ = p.w.hearRunning("running.g14.web", raw) p.wants(context.Background()) - if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || len(findings) != 0 { + if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || said[0].Revision != 5 || said[0].Source != 4 || len(findings) != 0 { t.Errorf("every tier runs: said %+v, findings %v", said, findings) } if _, _, ok := p.ends.tiersOf(trunkBranch + ".5"); ok { @@ -108,8 +111,14 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") { t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings) } - if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" { - t.Errorf("pipeline-wanted tier-late is not said once: %+v", said) + if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" || + said[0].Revision != 5 || said[0].Source != 4 { + t.Errorf("pipeline-wanted tier-late is not said once, naming revision 5 of source 4: %+v", said) + } + // A tier progress kept before it held its source says the key's number as both (H-A-216). + p.sayPipelineWanted(trunkBranch+".5", trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil) + if len(said) != 2 || said[1].Revision != 5 || said[1].Source != 5 { + t.Errorf("a progress with no source: said %+v, want revision and source 5", said) } if wanted("g14", "web") || wanted("ace", "web") { t.Error("a consumer's want is written after its provider's tier was late") diff --git a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go index a4dc8aa9..44c5c076 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go @@ -1001,10 +1001,12 @@ type PipelineWanted struct { // for tier-late, the late tier's movers not running and the machines they do not run on NotRunning []NotRunning `json:"not-running,omitempty"` Outcome string `json:"outcome"` - // the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746) + // the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746, H-A-213 (4)) Pipeline string `json:"pipeline"` - Source int64 `json:"source"` - V int64 `json:"v"` + // the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event + Revision int64 `json:"revision,omitempty"` + Source int64 `json:"source"` + V int64 `json:"v"` } // The values of PipelineWanted's outcome. diff --git a/modules/mesh-delivery-contracts/go/seats/meshwants/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshwants/contract_test.go index daeed530..d623d162 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshwants/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshwants/contract_test.go @@ -29,7 +29,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v WantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wanted\\.[^.*> ]+\\.[^.*> ]+$")}, {name: "a want removed, its node left or its module unplaced", subject: "mesh.seat.mesh-wants.event.wanted.n1.xclip", given: "{\"at\":\"2026-11-09T19:05:00Z\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"module\":\"xclip\",\"node\":\"n1\",\"removed\":true,\"revision\":41,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:05:00Z\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"module\":\"xclip\",\"node\":\"n1\",\"removed\":true,\"revision\":41,\"v\":1}", decode: func(b []byte) (any, error) { var v WantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wanted\\.[^.*> ]+\\.[^.*> ]+$")}, - {name: "a pipeline's wants all run", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3589", given: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3589\",\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3589\",\"source\":3589,\"v\":1}", + {name: "a pipeline's wants all run", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", given: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3590\",\"revision\":3590,\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3590\",\"revision\":3590,\"source\":3589,\"v\":1}", decode: func(b []byte) (any, error) { var v PipelineWantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.pipeline-wanted\\.[^.*> ]+\\.[^.*> ]+$")}, {name: "a pipeline's tier late, naming the movers and machines not running", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", given: "{\"branch\":\"main\",\"not-running\":[{\"machines\":[\"anchor\",\"laptop\"],\"module\":\"nats\"}],\"outcome\":\"tier-late\",\"pipeline\":\"main.3590\",\"source\":3590,\"v\":1}", wire: "{\"branch\":\"main\",\"not-running\":[{\"machines\":[\"anchor\",\"laptop\"],\"module\":\"nats\"}],\"outcome\":\"tier-late\",\"pipeline\":\"main.3590\",\"source\":3590,\"v\":1}", decode: func(b []byte) (any, error) { var v PipelineWantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.pipeline-wanted\\.[^.*> ]+\\.[^.*> ]+$")}, diff --git a/modules/mesh-delivery-contracts/seats/mesh-wants/fixtures/events.json b/modules/mesh-delivery-contracts/seats/mesh-wants/fixtures/events.json index feeace4e..3381500a 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-wants/fixtures/events.json +++ b/modules/mesh-delivery-contracts/seats/mesh-wants/fixtures/events.json @@ -65,12 +65,13 @@ { "name": "a pipeline's wants all run", "of": "event:pipeline-wanted", - "subject": "mesh.seat.mesh-wants.event.pipeline-wanted.main.3589", + "subject": "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", "given": { "v": 1, - "pipeline": "main.3589", + "pipeline": "main.3590", "branch": "main", "source": 3589, + "revision": 3590, "outcome": "running" } }, diff --git a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json index f09ae1f9..e55cf587 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json @@ -1725,7 +1725,7 @@ }, "pipeline": { "type": "string", - "description": "the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746)" + "description": "the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746, H-A-213 (4))" }, "branch": { "type": "string" @@ -1733,6 +1733,10 @@ "source": { "type": "integer" }, + "revision": { + "type": "integer", + "description": "the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event" + }, "outcome": { "type": "string", "enum": [ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts index 0b53c871..e7bf8b47 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts @@ -412,8 +412,10 @@ export interface PipelineWanted { /** for tier-late, the late tier's movers not running and the machines they do not run on */ "not-running"?: NotRunning[]; "outcome": "running" | "tier-late"; - /** the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746) */ + /** the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746, H-A-213 (4)) */ "pipeline": string; + /** the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event */ + "revision"?: number; "source": number; "v": number; } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index c57388a2..cc42bf67 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -90,7 +90,7 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 }{ {name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"}, {name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"}, - {name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.12","branch":"main","source":12,"outcome":"tier-late"}`, wanted: true, acked: 1}, + {name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.13","branch":"main","source":12,"revision":13,"outcome":"tier-late"}`, wanted: true, acked: 1}, {name: "not json", body: `nope`, termd: 1}, {name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"}, {name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"}, @@ -100,8 +100,8 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 switch { case row.wanted: heardPipelineWanted(context.Background(), ix, m, t.Logf) - if ix.wanted != "main.12 tier-late" { - t.Errorf("%s: was handed %q, want main.12 tier-late", row.name, ix.wanted) + if ix.wanted != "main.12.13 tier-late" { // its revision handed beside its source (H-A-214) + t.Errorf("%s: was handed %q, want main.12.13 tier-late", row.name, ix.wanted) } case row.ended: heardPipelineEnded(context.Background(), ix, m, t.Logf) @@ -169,8 +169,11 @@ func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarte return nil } -func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64, outcome string) error { +func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source, revision int64, outcome string) error { f.wanted = fmt.Sprintf("%s.%d %s", branch, source, outcome) + if revision > 0 { + f.wanted = fmt.Sprintf("%s.%d.%d %s", branch, source, revision, outcome) + } return f.err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go index fdbf67aa..1c095858 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go @@ -21,7 +21,7 @@ type runTaker interface { StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error - HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error + HeardPipelineWanted(ctx context.Context, branch string, source, revision int64, outcome string) error JudgePins(ctx context.Context) } @@ -99,7 +99,7 @@ func heardPipelineWanted(ctx context.Context, ix runTaker, m delivered, logf fun _ = m.Term() return } - err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Outcome) + err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Revision, e.Outcome) settleRun(m, err, "pipeline-wanted of "+e.Pipeline, logf) if err == nil { ix.JudgePins(ctx) diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 33816fa8..92a543d2 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -119,8 +119,9 @@ func ReleaseRecord(branch, module string) string { // every revision of the branch below it; a mover the revision holds no release of (superseded, no version released) keeps its // older release's revision. Outcome tier-late frees nothing: its later tiers' wants are not written, though their release // records are held, and the same pipeline's later running frees them. A pipeline whose revision is not kept frees nothing and -// says so; one applied twice changes nothing. -func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error { +// says so; one applied twice changes nothing. The pipeline's revision is the one the event names (design 47 H-A-214); an event +// naming none (revision zero, said before the field) is of the source's lowest-numbered revision, the take-in's. +func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source, revision int64, outcome string) error { if ix.Pins == nil || ix.Revisions == nil { return errors.New("the indexer keeps no state " + PinsState) } @@ -128,12 +129,21 @@ func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, sourc ix.logf("pins: pipeline-wanted of %s %d is %s: it frees no release until its wants are written everywhere (H-A-174)", branch, source, outcome) return nil } - name, err := revisionOfSource(ctx, ix.Revisions, branch, source) - if err != nil { - ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err) - return nil + var name string + if revision > 0 { + name = RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: revision}) + } else { + var err error + if name, err = revisionOfSource(ctx, ix.Revisions, branch, source); err != nil { + ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err) + return nil + } } rev, _, err := ReadRevision(ctx, ix.Revisions, name) + if revision > 0 && errors.Is(err, ErrNoObject) { + ix.logf("pins: pipeline-wanted of %s revision %d frees no release: %s is not kept", branch, revision, name) + return nil + } if err != nil { return fmt.Errorf("%s: %w", name, err) } diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index 8f8a0389..c6de5147 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -106,18 +106,20 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te t.Fatal(err) } } - end := func(source int64, outcome, version string) { + endOf := func(source, revision int64, outcome, version string) { t.Helper() - e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome, + e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", revision), Revision: revision, Source: source, Outcome: outcome, Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: version}}} if err := ix.HeardPins(ctx, nil, &e); err != nil { t.Fatal(err) } ix.JudgePins(ctx) } + end := func(source int64, outcome, version string) { t.Helper(); endOf(source, source, outcome, version) } + // An event naming no revision (said before the field) is of the source's lowest-numbered revision, the take-in's (H-A-214). wanted := func(source int64, outcome string) { t.Helper() - if err := ix.HeardPipelineWanted(ctx, "main", source, outcome); err != nil { + if err := ix.HeardPipelineWanted(ctx, "main", source, 0, outcome); err != nil { t.Fatal(err) } ix.JudgePins(ctx) @@ -142,6 +144,16 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te if v, _, _, _ := ix.Pins.Get(ctx, "main.7"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { t.Errorf("the pin of main.7 holds %s, want its release alone", v) } + // H-A-214: a re-pin's revision 8 of the same source 7 releases 0.1.14; its pipeline-wanted names revision 8, so the release + // on revision 8 frees the older one on revision 7, which the source's lowest-numbered revision would not. + take(7) + endOf(7, 8, runner.PipelineEndedOutcomePassed, "0.1.14") + want("a re-pin's release beside the take-in's", "main.7", "main.8") + if err := ix.HeardPipelineWanted(ctx, "main", 7, 8, meshwants.PipelineWantedOutcomeRunning); err != nil { + t.Fatal(err) + } + ix.JudgePins(ctx) + want("the re-pin's release written everywhere", "main.8") } func mustRecords(t *testing.T, v []byte) []string { -- 2.54.0 From 04f54d777354e1203efc80a74b8947bd5bc57812 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:48:08 +0100 Subject: [PATCH 21/30] Design 47 H1 (7): a registry's published recomputes no module The indexer kept a registry's published in its series and then recomputed every module, moving fingerprints with no kept catalog revision beside the re-pin a release's package-published starts. It keeps the version in its series and recomputes nothing: a fingerprint moves only with a kept revision, one per module release, kept by the re-pin. The seat mesh-wants' pipeline-wanted subject names its last token revision, the number of the runner's key (its form unchanged). Builds: design 47 H H-A-211, H-A-213 (2), H-A-213 (4) (the subject's token), H-pins-spec-and-gaps leftover L3 --- .../go/seats/meshwants/contract.go | 6 ++-- .../seats/mesh-wants/seat.json | 2 +- .../ts/seats/meshwants/index.d.ts | 2 +- .../ts/seats/meshwants/index.js | 4 +-- .../cmd/mesh-delivery-indexer/series.go | 13 ++------ .../cmd/mesh-delivery-indexer/series_test.go | 33 +++++++------------ 6 files changed, 22 insertions(+), 38 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go index 44c5c076..0df94fed 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go @@ -83,9 +83,9 @@ const SubjectWantedAll = "mesh.seat.mesh-wants.event.wanted.*.*" // StreamPipelineWanted is the stream pipeline-wanted is kept in. const StreamPipelineWanted = "SEAT_mesh-wants_pipeline-wanted" -// SubjectPipelineWanted is where pipeline-wanted is published, on mesh.seat.mesh-wants.event.pipeline-wanted... -func SubjectPipelineWanted(branch, source string) string { - return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + source +// SubjectPipelineWanted is where pipeline-wanted is published, on mesh.seat.mesh-wants.event.pipeline-wanted... +func SubjectPipelineWanted(branch, revision string) string { + return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + revision } // SubjectPipelineWantedAll hears every pipeline-wanted. diff --git a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json index e55cf587..b827436e 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json @@ -434,7 +434,7 @@ }, { "name": "pipeline-wanted", - "subject": "pipeline-wanted..", + "subject": "pipeline-wanted..", "keep": { "max-age-days": 30, "max-mib": 64 diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts index e7bf8b47..d4ef255a 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts @@ -50,7 +50,7 @@ export declare const wantedSubjectAll: "mesh.seat.mesh-wants.event.wanted.*.*"; export declare const pipelineWantedStream: "SEAT_mesh-wants_pipeline-wanted"; -export declare function pipelineWantedSubject(branch: string, source: string): string; +export declare function pipelineWantedSubject(branch: string, revision: string): string; export declare const pipelineWantedSubjectAll: "mesh.seat.mesh-wants.event.pipeline-wanted.*.*"; diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js index a544df9a..b0c21d4b 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js @@ -52,8 +52,8 @@ export const wantedSubjectAll = "mesh.seat.mesh-wants.event.wanted.*.*"; export const pipelineWantedStream = "SEAT_mesh-wants_pipeline-wanted"; -export function pipelineWantedSubject(branch, source) { - return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + source; +export function pipelineWantedSubject(branch, revision) { + return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + revision; } export const pipelineWantedSubjectAll = "mesh.seat.mesh-wants.event.pipeline-wanted.*.*"; diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go index 8ac23cb9..bdfc2a5d 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go @@ -22,8 +22,9 @@ var ( ) // heardVersion reads one message of the consumer as a version, false for any other subject. A package's is kept in its -// series; one that does not decode is said and acknowledged; a write that fails, or the recompute of its users that a -// publish starts, is left unacknowledged, so the bus gives it again. +// series; one that does not decode is said and acknowledged; a write that fails is left unacknowledged, so the bus gives it +// again. It recomputes no module: a fingerprint moves only with a kept catalog revision, a release's by the re-pin its +// package-published starts, one revision per module release (design 47 H-A-211, H-A-213 (2)). func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) bool { var h index.HeardVersion switch seat, ok := publishedSeats[m.Subject()]; { @@ -51,14 +52,6 @@ func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf if kept { logf("packages: %s %s kept in its series (published %v)", h.Package, h.Version, h.Published) } - if kept && h.Published { - // A series' newest moved: its users' fingerprints take it (DLV-I6), in the same turn as the publish is heard. - // A recompute that fails is not acknowledged, so the publish is offered again (design 47 H S15, H-A-167). - if err := ix.Recompute(ctx); err != nil { - logf("writing the modules again after %s %s was published failed, so the bus gives it again: %v", h.Package, h.Version, err) - return true - } - } _ = m.Ack() return true } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go index 85ddbb54..c58fb27f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go @@ -32,30 +32,21 @@ func (unlistedStore) Keys(context.Context) ([]string, error) { return nil, errors.New("the bus does not answer") } -// TestHeardVersionRecomputeFailed: a publish whose recompute of its users fails is not acknowledged, so the bus gives -// it again (design 47 H S15, the pin specification's item 3); one whose recompute passes is. -func TestHeardVersionRecomputeFailed(t *testing.T) { +// TestHeardVersionMovesNoFingerprint: a registry's published is kept in its series and acknowledged, and recomputes no +// module: a fingerprint moves only with a kept revision, a release's by its re-pin on the package-published (design 47 +// H-A-211, H-A-213 (2); the pin specification's leftover L3). The modules state here fails any recompute. +func TestHeardVersionMovesNoFingerprint(t *testing.T) { data, err := json.Marshal(gopackageregistry.PublishedEvent{Kind: "package", Name: "git.example/novox/lib", Version: "0.1.4"}) if err != nil { t.Fatal(err) } - for _, c := range []struct { - name string - modules index.Store - acked int - }{ - {"recompute fails", &unlistedStore{memStore{vals: map[string][]byte{}}}, 0}, - {"recompute passes", &memStore{vals: map[string][]byte{}}, 1}, - } { - t.Run(c.name, func(t *testing.T) { - ix := &index.Indexer{Packages: &memStore{vals: map[string][]byte{}}, Modules: c.modules, Manifests: &memStore{vals: map[string][]byte{}}} - m := &ackedMsg{subject: gopackageregistry.SubjectPublished, data: data} - if !heardVersion(context.Background(), ix, m, t.Logf) { - t.Fatal("the registry's published is not taken as a version") - } - if m.acked != c.acked { - t.Errorf("acknowledged %d times, want %d", m.acked, c.acked) - } - }) + ix := &index.Indexer{Packages: &memStore{vals: map[string][]byte{}}, Modules: &unlistedStore{memStore{vals: map[string][]byte{}}}, + Manifests: &memStore{vals: map[string][]byte{}}} + m := &ackedMsg{subject: gopackageregistry.SubjectPublished, data: data} + if !heardVersion(context.Background(), ix, m, t.Logf) { + t.Fatal("the registry's published is not taken as a version") + } + if m.acked != 1 { + t.Errorf("acknowledged %d times, want once: a publish recomputes nothing that could fail", m.acked) } } -- 2.54.0 From 595510ef38f6cef39766fb10d89447dde8a2b819 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:50:50 +0100 Subject: [PATCH 22/30] Design 47 H1 (9): a recompute that moves a fingerprint keeps a revision of kind recompute Every whole recompute of the modules state (a toolchain reported, the toolchains read, the assignments of a build module, the indexer's start, a hand act) moved fingerprints with no kept catalog revision. It now runs as a re-pin does: when at least one fingerprint moved against the trunk's newest revision, it keeps the trunk's next revision of the same source, its movers the moved modules, its call-kind recompute (the seat mesh-catalog's enumeration gains it, seatgen regenerated) and its call naming the reason; one that moves none keeps nothing. Builds: design 47 H H-A-211, H-A-213 (2), H-A-217 --- .../go/seats/meshcatalog/contract.go | 6 ++++-- .../seats/mesh-catalog/seat.json | 10 +++++---- .../ts/seats/meshcatalog/index.d.ts | 8 +++---- .../cmd/mesh-delivery-indexer/main.go | 4 ++-- .../internal/index/catalog_revisions.go | 9 ++++++++ .../internal/index/revision_tools.go | 2 +- .../internal/index/take_in_again_test.go | 18 ++++++++++++++++ .../internal/index/toolchains.go | 21 +++++++++++++++---- 8 files changed, 61 insertions(+), 17 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go index 9029f776..1ad5e63d 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go @@ -717,7 +717,7 @@ type CatalogRevision struct { Branch string `json:"branch,omitempty"` // what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's Call string `json:"call,omitempty"` - // the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215): a take-in's, a re-pin's on a module's release, a take-in again's; absent for a candidate's + // the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's CallKind string `json:"call-kind,omitempty"` // the seat definitions it was judged with, each by its digest, read whole through seat-definitions Definitions []CatalogRevisionDefinitionsItem `json:"definitions,omitempty"` @@ -752,6 +752,7 @@ const ( CatalogRevisionCallKindTakeIn = "take-in" CatalogRevisionCallKindRelease = "release" CatalogRevisionCallKindTakeInAgain = "take-in-again" + CatalogRevisionCallKindRecompute = "recompute" ) // Edge is one edge of the catalog graph between artifact references, B before A (ADR 0666 section 14): kind 1 B published before A builds (a package A imports), 2 B published before A delivers (fetched by reference at its start), 3 B delivered before A builds (its build module and toolchain), 4 B delivered before A delivers (a provision, bus state, a format's reader), 5 B has acted for A's new version (an acting rule), 6 B runs a version that serves A (a verb or a manifest word A uses). @@ -1320,7 +1321,7 @@ type CatalogRevisionEvent struct { Branch string `json:"branch,omitempty"` // what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's Call string `json:"call,omitempty"` - // the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215); absent for a candidate's + // the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's CallKind string `json:"call-kind,omitempty"` // the modules whose entries differ from the revision before it of the same kind Changed []string `json:"changed,omitempty"` @@ -1346,6 +1347,7 @@ const ( CatalogRevisionEventCallKindTakeIn = "take-in" CatalogRevisionEventCallKindRelease = "release" CatalogRevisionEventCallKindTakeInAgain = "take-in-again" + CatalogRevisionEventCallKindRecompute = "recompute" ) // FillingEvent is the body of the event filling. diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json index e0195d64..164a7576 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json @@ -666,9 +666,10 @@ "enum": [ "take-in", "release", - "take-in-again" + "take-in-again", + "recompute" ], - "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215); absent for a candidate's" + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's" }, "module": { "type": "string", @@ -1905,9 +1906,10 @@ "enum": [ "take-in", "release", - "take-in-again" + "take-in-again", + "recompute" ], - "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215): a take-in's, a re-pin's on a module's release, a take-in again's; absent for a candidate's" + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's" }, "module": { "type": "string", diff --git a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts index 1fcf459d..1bef6feb 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts @@ -279,8 +279,8 @@ export interface CatalogRevision { "branch"?: string; /** what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's */ "call"?: string; - /** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215): a take-in's, a re-pin's on a module's release, a take-in again's; absent for a candidate's */ - "call-kind"?: "take-in" | "release" | "take-in-again"; + /** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's */ + "call-kind"?: "take-in" | "release" | "take-in-again" | "recompute"; /** the seat definitions it was judged with, each by its digest, read whole through seat-definitions */ "definitions"?: CatalogRevisionDefinitionsItem[]; /** one per module, by name */ @@ -808,8 +808,8 @@ export interface CatalogRevisionEvent { "branch"?: string; /** what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's */ "call"?: string; - /** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215); absent for a candidate's */ - "call-kind"?: "take-in" | "release" | "take-in-again"; + /** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's */ + "call-kind"?: "take-in" | "release" | "take-in-again" | "recompute"; /** the modules whose entries differ from the revision before it of the same kind */ "changed"?: string[]; /** how many entries it lists */ diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index bcf46813..76a24231 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -433,7 +433,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro done() done = step("following the assignments") followAssignments(ctx, reader, seatHolders.holders(), process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views reads nothing and indexer_first_view_again is not heard", logger.Printf), cred, js, gate, func(ctx context.Context) { - if err := ix.Recompute(ctx); err != nil { + if err := ix.RecomputeKept(ctx, "the assignments of a build module"); err != nil { logger.Printf("writing the modules again after the assignments of a build module changed: %v", err) } }, logger.Printf) @@ -449,7 +449,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // the toolchain. return ix.ToolchainReported(ctx) }, - recompute: ix.Recompute}, func() []string { return buildModuleNames(ix.BuildModules()) }, logger) + recompute: func(ctx context.Context) error { return ix.RecomputeKept(ctx, "the toolchains read") }}, func() []string { return buildModuleNames(ix.BuildModules()) }, logger) if err := tools.start(); err != nil { return err } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 2110f08f..548e8c93 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -478,6 +478,15 @@ func takeInAgainCall(repository string) revisionCall { return revisionCall{Call: TakeInAgainCall(repository), Kind: meshcatalog.CatalogRevisionCallKindTakeInAgain, Repository: repository} } +// recomputeCall is a recompute's that moved a fingerprint, its call naming the reason (H-A-217). +func recomputeCall(reason string) revisionCall { + return revisionCall{Call: RecomputeCall(reason), Kind: meshcatalog.CatalogRevisionCallKindRecompute} +} + +// RecomputeCall is the call of the catalog revision a recompute keeps: its reason, such as the assignments of a build module, +// a toolchain or a hand act (design 47 H-A-217). +func RecomputeCall(reason string) string { return "recompute on " + reason } + // HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held). const HeldObjectsFull = "objects full" diff --git a/modules/mesh-delivery-indexer/internal/index/revision_tools.go b/modules/mesh-delivery-indexer/internal/index/revision_tools.go index 279e41e8..cd98aba6 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_tools.go @@ -272,7 +272,7 @@ func (ix *Indexer) RecomputeByHand(ctx context.Context, by, why string) TakenIn r.Answer = "the newest catalog graph revision cannot be read: " + err.Error() return r } - if err := ix.Recompute(ctx); err != nil { + if err := ix.RecomputeKept(ctx, fmt.Sprintf("a hand act by %s (%s)", by, why)); err != nil { r.Answer = "the recompute failed: " + err.Error() return r } diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go index 1e114d53..1fe65e2d 100644 --- a/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go @@ -92,4 +92,22 @@ func TestATakeInAgainReachesARepositoryWithNoMergeSince(t *testing.T) { len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != m.Fp2 { t.Errorf("the newest trunk revision = %+v (%v), want revision 2 of source 4 with the take-in again's call and g its mover", rev, err) } + // H-A-217: a recompute that moves no fingerprint keeps nothing; one that moves one (a toolchain reported) keeps the trunk's + // next revision of the same source, its call-kind recompute, its call naming the reason, the moved modules its movers. + if err := ix.RecomputeKept(ctx, "the assignments of a build module"); err != nil { + t.Fatal(err) + } + if rev, _, _, _ := readNewestOf(ctx, ix.Revisions, TrunkBranch); rev.Revision != 2 { + t.Errorf("a recompute that moved nothing kept revision %d (%s), want none", rev.Revision, rev.Call) + } + reportToolchainRecipe(t, ix, "mesh-build-go", "novox", "go", "go1.27.1", "2") // a recipe change moves a Go module at once + moved, _ := moduleOf(t, mods, "g") + rev, _, found, err = readNewestOf(ctx, ix.Revisions, TrunkBranch) + if err != nil || !found || rev.Revision != 3 || rev.Source != 4 || rev.CallKind != meshcatalog.CatalogRevisionCallKindRecompute || + rev.Call != RecomputeCall("a toolchain reported") || !slices.ContainsFunc(rev.Movers, func(e meshcatalog.TakeInEntry) bool { + return e.Module == "g" && e.Fingerprint == moved.Fp2 && moved.Fp2 != m.Fp2 + }) { + t.Errorf("after a toolchain reported: the newest trunk revision = %+v (%v), want revision 3 of source 4, kind recompute, g its mover at %s (was %s)", + rev, err, moved.Fp2, m.Fp2) + } } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index b0043487..dfba512a 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -79,9 +79,22 @@ func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value return ix.ToolchainReported(ctx) } -// ToolchainReported is what a changed toolchain moves: the modules written again (Recompute). +// ToolchainReported is what a changed toolchain moves: the modules written again, kept as a revision (RecomputeKept). func (ix *Indexer) ToolchainReported(ctx context.Context) error { - return ix.Recompute(ctx) + return ix.RecomputeKept(ctx, "a toolchain reported") +} + +// RecomputeKept is Recompute kept as a re-pin is (design 47 H-A-217): when it moved at least one fingerprint against the +// trunk's newest revision, it keeps the trunk's next revision of the same source, its movers the modules that moved, its +// call-kind recompute and its call naming reason, pinned for its pipeline, and says catalog-revision; one that moves none +// keeps nothing. Recompute writes the trunk's modules state, so only the trunk's revision is kept. +func (ix *Indexer) RecomputeKept(ctx context.Context, reason string) error { + ix.pendingMu.Lock() // one take-in step at a time, so the trunk's newest stays the one compared against (pending.go) + defer ix.pendingMu.Unlock() + if err := ix.Recompute(ctx); err != nil { + return err + } + return ix.keepRepinned(ctx, ix, TrunkBranch, recomputeCall(reason), nil) } // agreed is the toolchain of each language every reporting node agrees on, or why there is none. @@ -544,7 +557,7 @@ func (ix *Indexer) RecomputeAtStart(ctx context.Context) error { ix.logf("no toolchain is read yet: the new-form fingerprints are computed when one is") return nil } - return ix.Recompute(ctx) + return ix.RecomputeKept(ctx, "the indexer's start") } // fillPaths gives every module whose key names no directory (a key written before the indexer learned paths) the one its @@ -702,7 +715,7 @@ func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch string, } var movers []meshcatalog.TakeInEntry moved := slices.Sorted(maps.Keys(only)) - if only == nil { // a take-in again: every module whose fingerprint moved (H-A-212) + if only == nil { // a take-in again or a recompute: every module whose fingerprint moved (H-A-212, H-A-217) moved = slices.Sorted(maps.Keys(entries)) } for _, key := range moved { -- 2.54.0 From 189d75ad40cc594043f6aa1ed1986e2a3174ade8 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:02:48 +0100 Subject: [PATCH 23/30] Design 47 H1 (5a): a pipeline is keyed by its catalog revision, and every step ask names revision and source One source now has several catalog revisions, so a pipeline keyed by its source could not tell them apart. A pipeline is keyed .; one of a source (a publish's, a version-released's) is found by the source it names, never by its key, so pipelines kept under their source are read as they are. The Go, Node and image step asks name the catalog revision in revision and the gate's source in a new optional source (the build-step part of the seats node-build-go, node-build-node and node-build-image, seatgen regenerated). Builds: design 47 H H-A-211, H-A-213 (4), H-A-219 --- .../go/seats/nodebuildgo/contract.go | 4 +- .../go/seats/nodebuildimage/contract.go | 4 +- .../go/seats/nodebuildnode/contract.go | 4 +- .../parts/build-step.json | 6 ++- .../seats/nodebuildgo/parts/build-step.d.ts | 4 +- .../nodebuildimage/parts/build-step.d.ts | 4 +- .../seats/nodebuildnode/parts/build-step.d.ts | 4 +- .../internal/pipeline/ended_test.go | 14 +++++- .../internal/pipeline/order_test.go | 14 +++--- .../internal/pipeline/pins_test.go | 6 +-- .../internal/pipeline/place.go | 2 +- .../internal/pipeline/place_test.go | 27 ++++++---- .../internal/pipeline/queue_test.go | 2 +- .../internal/pipeline/released.go | 11 ++-- .../internal/pipeline/start.go | 50 ++++++++++++++++--- .../internal/pipeline/start_test.go | 8 +-- .../internal/pipeline/testruns_test.go | 2 +- .../internal/pipeline/verbs.go | 4 +- .../internal/pipeline/verbs_test.go | 2 +- 19 files changed, 121 insertions(+), 51 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go index 7ed4f8fe..213116c3 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go @@ -905,8 +905,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go index d2136e78..391f8658 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go @@ -903,8 +903,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go index 47ca8f1d..bd89858a 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go @@ -903,8 +903,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/parts/build-step.json b/modules/mesh-delivery-contracts/parts/build-step.json index fcaebd62..0390aecb 100644 --- a/modules/mesh-delivery-contracts/parts/build-step.json +++ b/modules/mesh-delivery-contracts/parts/build-step.json @@ -1521,7 +1521,11 @@ }, "revision": { "type": "integer", - "description": "the source revision of the watched branch the pipeline runs for; absent for a run on request" + "description": "the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request" + }, + "source": { + "type": "integer", + "description": "the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request" }, "fingerprint": { "type": "string", diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go index abf01a7c..ee636c4a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go @@ -113,9 +113,19 @@ func TestAReleasedVersionIsTheMoversVersion(t *testing.T) { other := store[pkey] other.p.Pipeline, other.p.Branch, other.p.Movers = "beta.5", "beta", append([]seat.Mover(nil), other.p.Movers...) store["beta.5"] = other + // A pipeline is keyed by its revision (H-A-213 (4)): main.8, a later revision of source 5, is written too; main.6, of + // source 6, is not. + for key, source := range map[string]int64{"main.8": 5, "main.6": 6} { + again := store[pkey] + again.p.Pipeline, again.p.Source, again.p.Movers = key, source, append([]seat.Mover(nil), again.p.Movers...) + store[key] = again + } ev := catalog.VersionReleasedEvent{Module: "app", Version: "0.4.1", Source: 5} - if n, err := h.Released(ev, "main"); n != 1 || err != nil { - t.Fatalf("released wrote %d (%v), want app's mover of main.5", n, err) + if n, err := h.Released(ev, "main"); n != 2 || err != nil { + t.Fatalf("released wrote %d (%v), want app's mover of main.5 and main.8", n, err) + } + if m, _ := MoverOf(store["main.6"].p, "app"); m.Version != "" { + t.Errorf("main.6's mover = %+v, want no version: another source", m) } if m, _ := MoverOf(store[pkey].p, "app"); m.Version != "0.4.1" { t.Errorf("app's mover = %+v, want version 0.4.1", m) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go index 0065aabd..73d1e020 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go @@ -70,14 +70,14 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if n, err := s.TakenIn(ev, view); n != 1 || err != nil { t.Fatalf("the take-in started %d (%v)", n, err) } - p, _, _ := mem.Get("main.4") + p, _, _ := mem.Get("main.7") if !placed(p, "mesh-sdk") || !placed(p, "builder") || placed(p, "app") || placed(p, "tool") { t.Fatalf("placed at the start: %+v, want mesh-sdk and builder only", p.Steps) } if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); n != 0 || err != nil { t.Fatalf("the publish started %d (%v), want none", n, err) } - p, _, _ = mem.Get("main.4") + p, _, _ = mem.Get("main.7") if sdk, _ := MoverOf(p, "mesh-sdk"); !placed(p, "app") || placed(p, "tool") || len(mem) != 1 || !slices.Contains(sdk.Published, "mesh-sdk:go") { t.Fatalf("after mesh-sdk's publish: %d pipelines, app placed %v, tool placed %v, mesh-sdk %+v", len(mem), placed(p, "app"), placed(p, "tool"), sdk) } @@ -87,22 +87,22 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { b := stepsOf(p, "builder")[0] bs := stepSeatOf(t, builds.Seat) end := func(outcome string) StepEnded { - e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.4","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`)) + e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.7","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`)) if err != nil { t.Fatal(err) } return e } - before := mem["main.4"] + before := mem["main.7"] if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("passed")); err != nil { t.Fatal(err) } - if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning { + if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning { t.Errorf("after builder's build passed, tool's steps = %+v, want placed and asked", stepsOf(p, "tool")) } // The same pipeline, builder's build failed: tool is placed with every step not asked, each saying its own step-ended // naming the dependency (H-A-101 (2)). - mem["main.4"] = before + mem["main.7"] = before var notAsked []seat.StepEvent s.Say = func(subject string, body any) error { if ev, ok := body.(seat.StepEvent); ok && strings.Contains(subject, ".step-ended.tool.") { @@ -113,7 +113,7 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("failed")); err != nil { t.Fatal(err) } - if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked { + if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked { t.Errorf("after builder's build failed, tool's steps = %+v, want placed not asked", stepsOf(p, "tool")) } if len(notAsked) == 0 || len(notAsked) != len(slices.DeleteFunc(stepsOf(p, "tool"), func(st seat.StepResult) bool { return st.Outcome != seat.StepResultOutcomeNotAsked })) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go index 455f8fa5..bd99d2c8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go @@ -51,7 +51,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing. if n, err := s.TakenIn(taken, view); n != 1 || err != nil { t.Fatalf("the take-in started %d (%v)", n, err) } - p, _, _ := store.Get("main.947") + p, _, _ := store.Get("main.357") if p.Revision != 357 || len(p.Steps) != 0 || len(asks.asks) != 0 { t.Fatalf("a take-in of revision 357 with the view at 356 started %+v with %d asks, want revision 357 and no step", p, len(asks.asks)) } @@ -60,7 +60,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing. } viewRevision = 357 n, err := s.Waiting(store.memStore, "main", view) - p, _, _ = store.Get("main.947") + p, _, _ = store.Get("main.357") if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Revision != 357 { t.Errorf("the view reaching 357 placed %d (%v): %d steps, %d asks at revision %d", n, err, len(p.Steps), len(asks.asks), p.Revision) } @@ -87,7 +87,7 @@ func TestAViewThatReachesTheRevisionDuringTheStartsPlacesTheHeldBackPipelines(t if n, err := s.TakenIn(taken, view); n != 1 || err != nil { t.Fatalf("started %d (%v)", n, err) } - if p, _, _ := store.Get("main.947"); len(p.Steps) == 0 || len(asks.asks) == 0 { + if p, _, _ := store.Get("main.357"); len(p.Steps) == 0 || len(asks.asks) == 0 { t.Errorf("the pipeline has %d steps and %d asks after the view reached 357 during the start, want them placed", len(p.Steps), len(asks.asks)) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index 4fcc2aee..a7dbf327 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -384,7 +384,7 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar } return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now, Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: m.Commit, Fetch: fetch, Fingerprint: m.Fingerprint, - Module: m.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}} + Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}} } // askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index dd10438a..7f46c658 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -102,7 +102,7 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { a := asks.asks[1].(nodebuildgo.StepAsk) if asks.at[1] != "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner" || a.Kind != "step" || a.Asker != "the-runner" || a.ID != p.Steps[1].Ask || asks.ids[1] != a.ID || a.Step.Commit != "c1" || a.Step.Fingerprint != "src2:aa" || a.Step.Pipeline != "main.4" || a.Step.Artifact != "app:program" || - a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Revision != 4 || + a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Source != 4 || len(a.Step.Fetch.Paths) != 1 || a.Step.Fetch.Paths[0] != "cmd/app" { t.Errorf("the ask = %+v at %s", a, asks.at[1]) } @@ -136,7 +136,7 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { } } var hp seat.Pipeline - if err := json.Unmarshal(store["main.5"], &hp); err != nil { + if err := json.Unmarshal(store["main.3"], &hp); err != nil { t.Fatal(err) } if st := hp.Steps[1]; st.Step != "lint" || st.Outcome != seat.StepResultOutcomeWaiting || st.Ask != "" || st.Why != "waits on the publish of git.novox.be/novox/x/go" { @@ -213,15 +213,15 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { } // A mover whose fingerprint the pipeline's revision does not hold ends its held steps could not run, naming it. other := hp - other.Pipeline, other.Steps, other.Movers = "main.6", append([]seat.StepResult(nil), hp.Steps...), []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:zz", Repository: "o/app"}} - ms["main.6"] = struct { + other.Pipeline, other.Steps, other.Movers = "main.4", append([]seat.StepResult(nil), hp.Steps...), []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:zz", Repository: "o/app"}} + ms["main.4"] = struct { p seat.Pipeline rev uint64 }{other, 1} - if n, err := hs.AskPinHeld(ms, "main.6", "app", pinsBuilt, revisions); n != 0 || err != nil { + if n, err := hs.AskPinHeld(ms, "main.4", "app", pinsBuilt, revisions); n != 0 || err != nil { t.Errorf("asked %d held steps of a mover its revision does not hold (%v); want none", n, err) } - if o, _, _ := ms.Get("main.6"); o.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(o.Steps[1].Why, "not the mover's src2:zz") { + if o, _, _ := ms.Get("main.4"); o.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(o.Steps[1].Why, "not the mover's src2:zz") { t.Errorf("the held lint step of a moved fingerprint = %+v; want could not run naming it", o.Steps[1]) } if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") { @@ -532,7 +532,8 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }} start := func(source, revision int64) seat.Pipeline { t.Helper() - if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { + // Keyed by its source, so pipelines of one revision are several (one kept under its source before H-A-213 (4)). + if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { t.Fatalf("start %d = %v, %v", source, ok, err) } p, _, _ := store.Get(Key("main", source)) @@ -551,6 +552,12 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" { t.Fatalf("the test steps asked = %q, want the program's three alone", got) } + // Every step ask names the catalog revision in revision and the gate's source in source (H-A-219). + for _, a := range asks.asks { + if b, ok := a.(nodebuildgo.StepAsk); ok && (b.Step.Revision != 9 || b.Step.Source != 4) { + t.Errorf("the %s ask of %s names revision %d and source %d, want 9 and 4", b.Step.Step, b.Step.Artifact, b.Step.Revision, b.Step.Source) + } + } for _, level := range buildSteps { asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level) if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil { @@ -751,13 +758,13 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) { } } } - p, _, _ := mem.Get("main.4") + p, _, _ := mem.Get("main.7") if validates != 1 || compositions != 0 || stepNamed(p, "", seat.StepResultStepComposition).Outcome != seat.StepResultOutcomeWaiting { t.Fatalf("placed: %d validate and %d composition asks, steps %+v; want one validate, no composition yet", validates, compositions, p.Steps) } h := &Hearer{Store: mem, Name: "the-runner", Say: func(string, any) error { return nil }, AskSteps: s.AskSteps} bs := stepSeatOf(t, wants.Seat) - raw := `{"id":"` + validateAsk + `","pipeline":"main.4","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}` + raw := `{"id":"` + validateAsk + `","pipeline":"main.7","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}` ev, err := bs.Decode([]byte(raw)) if err != nil { t.Fatal(err) @@ -766,7 +773,7 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) { if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil { t.Fatal(err) } - p, _, _ = mem.Get("main.4") + p, _, _ = mem.Get("main.7") for _, st := range p.Steps { if st.Step == seat.StepResultStepValidate && st.Outcome != outcome { t.Errorf("%s: %s's validate is %s, want every mover's read from the one run", outcome, st.Module, st.Outcome) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go index 586b656e..54249bca 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go @@ -96,7 +96,7 @@ func queueFixture() (*Starter, memStore, memQueue, *[]string, *[]seat.Wait, *ste func takeIn(t *testing.T, s *Starter, view func(string) (Half, bool)) func(source int64, modules ...string) { return func(source int64, modules ...string) { t.Helper() - ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7} + ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: source} for _, m := range modules { ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/released.go b/modules/mesh-delivery-pipeline/internal/pipeline/released.go index 76028fce..840ca598 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/released.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/released.go @@ -3,7 +3,6 @@ package pipeline import ( "errors" "fmt" - "strconv" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/keys" @@ -21,12 +20,13 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int, if branch != "" { b = keys.EscapeName(branch) } - source := strconv.FormatInt(ev.Source, 10) + // A pipeline is keyed by its revision (H-A-213 (4)), so the pipelines of a source are those naming it, of either key + // form a source's pipelines were kept under. var pkeys []string - for _, f := range []string{b + "." + source, b + "." + source + ".published.*.*"} { + for _, f := range []string{b + ".*", b + ".*.published.*.*"} { k, err := h.Store.Keys(f) if err != nil { - return 0, fmt.Errorf("the pipelines of source %s are not listed: %w", source, err) + return 0, fmt.Errorf("the pipelines of source %d are not listed: %w", ev.Source, err) } pkeys = append(pkeys, k...) } @@ -37,6 +37,9 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int, if err != nil { return wrote, fmt.Errorf("the pipeline %s is not read: %w", key, err) } + if p.Source != ev.Source || p.Run != "" { + break + } i := -1 for j, m := range p.Movers { if m.Module == ev.Module { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index cb039452..7f99aa83 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -28,7 +28,7 @@ type Created interface { Create(key string, value []byte) error } -// Begin is what a start names: the pipeline's key (watched branch and source revision, design 47 H §746), the take-in's +// Begin is what a start names: the pipeline's key (watched branch and catalog revision, design 47 H §746, H-A-213), the take-in's // call and catalog graph revision, its movers, and the catalog half of that branch. type Begin struct { Branch string @@ -41,16 +41,26 @@ type Begin struct { Entries map[string]catalog.CatalogEntry // Published is the package's publish that started the pipeline, nil for any other start (§747). Published *seat.Published - // Key is the pipeline's key when it is not . (a publish's, §747); empty for a take-in's. + // Key is the pipeline's key when it is not . (a publish's, §747); empty for a take-in's. Key string // unheld says the branch's catalog half is not held yet: the pipeline starts with no steps, placed once it is (A-336). unheld bool } -// Key is a take-in's pipeline key in the state pipelines, `.`, the branch escaped as a key's name is -// (design 47 H §746). -func Key(branch string, source int64) string { - return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10)) +// Key is a pipeline's key in the state pipelines, `.`, the catalog revision's number the indexer counts, +// the branch escaped as a key's name is (design 47 H §746, H-A-213 (4)); a take-in's number is unchanged where it equalled +// its source. A pipeline kept under its source before (`.`) has the same form and is read as it is. +func Key(branch string, revision int64) string { + return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10)) +} + +// keyRevision is the number b's pipeline is keyed by (H-A-213 (4)): its revision, or its source where the start names no +// revision (a take-in.taken of an indexer whose revision numbers were its sources, H-A-211). +func keyRevision(b Begin) int64 { + if b.Revision > 0 { + return b.Revision + } + return b.Source } // PublishedKey is the key of a pipeline a package's publish starts, `..published..`, @@ -495,7 +505,7 @@ func (s *Starter) Start(b Begin) (bool, error) { now := s.now() key := b.Key if key == "" { - key = Key(b.Branch, b.Source) + key = Key(b.Branch, keyRevision(b)) } if err := s.Fills.refuseStart(key); err != nil { return false, err @@ -899,7 +909,10 @@ func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch st if s.Store == nil { return seat.Pipeline{}, false, nil } - key := Key(ev.Branch, ev.Source) + key, ok := s.sourcePipeline(ev.Branch, ev.Source) + if !ok { + return seat.Pipeline{}, false, nil + } for { p, rev, err := s.Store.Get(key) if err != nil { @@ -923,6 +936,27 @@ func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch st } } +// sourcePipeline is the key of branch's pipeline of the lowest-numbered revision of source, the take-in's (H-A-213 (4), +// H-A-218): a pipeline is keyed by its revision, so one of a source is found by the source it names, never by its key. +func (s *Starter) sourcePipeline(branch string, source int64) (string, bool) { + pkeys, err := s.Store.Keys(keys.EscapeName(branch) + ".*") + if err != nil { + s.logf("the pipelines of %s are not listed, so source %d's is not found: %v", branch, source, err) + return "", false + } + found, low := "", int64(0) + for _, k := range pkeys { + p, _, err := s.Store.Get(k) + if err != nil || p.Branch != branch || p.Source != source || p.Run != "" { + continue + } + if r := keyRevision(Begin{Revision: p.Revision, Source: p.Source}); found == "" || r < low { + found, low = k, r + } + } + return found, found != "" +} + // publishedKind is the edge a dependent builds on a published package by: B published before A builds, a package A // imports (§53). const publishedKind = 1 diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 019127c7..9418cfca 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -153,8 +153,8 @@ func TestATakeInIsOnePipeline(t *testing.T) { t.Fatalf("written %d pipelines and said %d starts, want one each", len(store), len(said)) } var p seat.Pipeline - if err := json.Unmarshal(store["main.7"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 { - t.Fatalf("the pipeline main.7 = %+v (%v)", p, err) + if err := json.Unmarshal(store["main.12"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 { + t.Fatalf("the pipeline main.12 = %+v (%v)", p, err) } if m, _ := MoverOf(p, "mesh-sdk"); m.Commit != "c1" || m.Fingerprint != "src2:aa" || m.Repository != "o/sdk" || m.Tier != 1 { t.Errorf("the mover mesh-sdk = %+v", m) @@ -318,8 +318,8 @@ func TestTakenInStartsOnlyItsEntries(t *testing.T) { if err != nil || n != 1 { t.Fatalf("the take-in started %d (%v), want its one entry", n, err) } - if p, _, err := mem.Get("main.9"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") { - t.Errorf("main.9 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err) + if p, _, err := mem.Get("main.12"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") { + t.Errorf("main.12 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index 780009af..ba0712e4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -113,7 +113,7 @@ func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline { "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, } - if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { + if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { t.Fatalf("start = %v, %v", ok, err) } p, _, _ := f.store.Get(Key("main", source)) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 5264f76e..d86cec3b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -582,11 +582,11 @@ func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) { if !ok || half.Source <= 0 { return seat.StartAnswer{}, fmt.Errorf("the catalog half of %s holds no entry of %s at a source revision: nothing was started", in.Branch, in.Module) } - key := Key(in.Branch, half.Source) + key := Key(in.Branch, keyRevision(Begin{Revision: half.Revision, Source: half.Source})) if err := v.record(seat.VerbStart, "started the pipeline "+key+" by hand", in.Why); err != nil { return seat.StartAnswer{}, err } - // A hand start keys the newest revision's . with the module as its one mover (H-A-52). + // A hand start keys the newest revision's . with the module as its one mover (H-A-52, H-A-213 (4)). started, err := v.Starter.Start(Begin{Branch: in.Branch, Source: half.Source, Revision: half.Revision, Entries: half.Entries, Movers: []seat.Mover{{Module: in.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository, Tier: 1}}}) if err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go index d3a273a0..a2c8adae 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go @@ -287,7 +287,7 @@ func TestStartAndAskAgain(t *testing.T) { _, entries := placeFixture() v.View = func(string) (Half, bool) { return Half{Source: 7, Revision: 3, Entries: entries}, true } a, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "no pipeline for the head"}) - if err != nil || a.Pipeline != "main.7" || made["main.7"] == nil { + if err != nil || a.Pipeline != "main.3" || made["main.3"] == nil { t.Fatalf("start = %+v, %v; written %v", a, err, len(made)) } if _, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "again"}); err == nil { -- 2.54.0 From 91b5e8d0866ad6d54618969bbcd66ef530dcc631 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:07:55 +0100 Subject: [PATCH 24/30] Design 47 H1 (5b): a release, take-in again or recompute revision starts one pipeline, read by its number A re-pin on a module's release, a take-in again and a recompute that moves a fingerprint each keep a catalog revision of their own. The runner hears a trunk catalog-revision of those kinds, reads that revision by its number through the catalog seat's revision verb, and starts one pipeline keyed . with the revision's movers. A package-published no longer starts per-artifact pipelines: it only places the movers waiting for it. A branch's newest pipeline for a mover is the one of the highest revision number. Builds: design 47 H H-A-212, H-A-213 (2) and (4), H-A-217 --- .../mesh-delivery-pipeline/candidate_test.go | 25 +++++ .../cmd/mesh-delivery-pipeline/main.go | 10 ++ .../cmd/mesh-delivery-pipeline/start.go | 37 +++++-- .../cmd/mesh-delivery-pipeline/views.go | 7 +- .../cmd/mesh-delivery-pipeline/widens.go | 3 +- .../internal/pipeline/aged.go | 4 +- .../internal/pipeline/aged_test.go | 20 ++++ .../internal/pipeline/order_test.go | 4 +- .../internal/pipeline/registrations.go | 4 + .../internal/pipeline/start.go | 102 +++++++++--------- .../internal/pipeline/start_test.go | 102 +++++++++--------- 11 files changed, 205 insertions(+), 113 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go index 26a92e9c..f0d87ef5 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go @@ -71,3 +71,28 @@ func TestPlaceLostRunsReadsEachCandidateNewest(t *testing.T) { t.Errorf("asked %+v, want the candidate's group alone", asked[1]) } } + +// H-A-213 (2), (4): a catalog-revision heard is handed whole to the start of a revision's pipeline; a start that fails is +// an error, so the bus delivers the event again, and a body that is no event is ended. +func TestARevisionHeardIsHandedToItsStart(t *testing.T) { + var got []catalog.CatalogRevisionEvent + fail := false + hear := revisedHear(func(ev catalog.CatalogRevisionEvent) (int, error) { + got = append(got, ev) + if fail { + return 0, errors.New("not read") + } + return 1, nil + }, func(string, ...any) {}) + data := []byte(`{"v":1,"kind":"trunk","branch":"main","revision":10,"source":41,"call-kind":"release","call":"release of mesh-sdk 0.1.17"}`) + if err := hear(t.Context(), data); err != nil || len(got) != 1 || got[0].Revision != 10 || got[0].CallKind != "release" || got[0].Source != 41 { + t.Fatalf("heard %+v (%v), want the release revision 10", got, err) + } + fail = true + if err := hear(t.Context(), data); err == nil { + t.Error("a start that failed is not an error, so the event is never heard again") + } + if err := hear(t.Context(), []byte("no event")); err != nil { + t.Errorf("a body that is no event = %v, want it ended", err) + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 80a28898..85891217 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -98,6 +98,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro var advancer atomic.Pointer[func(string)] var placer atomic.Pointer[func(catalog.CatalogRevisionEvent) error] var registrar atomic.Pointer[func(catalog.Registration)] + var reviser atomic.Pointer[func(catalog.CatalogRevisionEvent) (int, error)] rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) { if f := resumer.Load(); f != nil { (*f)(branch) @@ -115,6 +116,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if f := registrar.Load(); f != nil { (*f)(reg) } + }, func(ev catalog.CatalogRevisionEvent) (int, error) { + if f := reviser.Load(); f != nil { + return (*f)(ev) + } + return 0, fmt.Errorf("the revision %d of %s is heard before the runner can start its pipeline", ev.Revision, ev.Branch) }, logger.Printf) if err != nil { return err @@ -203,6 +209,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } registrar.Store(®ister) + // A release's, a take-in again's or a recompute's revision starts one pipeline, its revision read by its number + // (H-A-213 (2), (4)). + revise := func(ev catalog.CatalogRevisionEvent) (int, error) { return starter.Revised(ev, widener.Revision) } + reviser.Store(&revise) starter.Store = store // The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in // the state queue, placed at the start from it and on every end. diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go index 5be6db24..1f9365ce 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "github.com/nats-io/nats.go/jetstream" @@ -11,10 +12,10 @@ import ( "mesh-delivery-pipeline/internal/pipeline" ) -// hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one -// pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is -// written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard -// after a stop. +// hearPublished hears each package-published (design 47 H §746, §747): in the running pipeline of the revision that +// published it, the movers waiting for it are placed; it starts no pipeline (a release's revision does, H-A-213 (2)). The +// message is acknowledged once that is written, on a goroutine of its own, so a slow write holds no other event; the +// durable consumer delivers a publish heard after a stop. func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error { return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) { var ev builds.PackagePublished @@ -24,18 +25,38 @@ func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, l return } go func() { - n, err := starter.Published(ev, v.View) - if err != nil { - logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err) + if err := starter.Published(ev, v.View); err != nil { + logf("the publish of %s %s on %s: %v; heard again", ev.Artifact, ev.Version, ev.Branch, err) _ = m.Nak() return } - logf("the publish of %s %s on %s started %d pipelines", ev.Artifact, ev.Version, ev.Branch, n) + logf("the publish of %s %s on %s is heard", ev.Artifact, ev.Version, ev.Branch) _ = m.Ack() }() }) } +// revisedHear is the handler of catalog-revision that starts the pipeline of a revision whose call kind is not a +// take-in's (design 47 H-A-212, H-A-213 (2), H-A-217): every other revision is left to the views that hear it. An error +// has the event heard again; a second start of its key writes nothing. +func revisedHear(revised func(ev catalog.CatalogRevisionEvent) (int, error), logf func(string, ...any)) func(context.Context, []byte) error { + return func(_ context.Context, data []byte) error { + var ev catalog.CatalogRevisionEvent + if err := json.Unmarshal(data, &ev); err != nil { + logf("a catalog-revision is not read: %v", err) + return nil + } + n, err := revised(ev) + if err != nil { + return err + } + if n > 0 { + logf("the revision %d of %s (%s, %s) started %d pipeline", ev.Revision, ev.Branch, ev.CallKind, ev.Call, n) + } + return nil + } +} + // hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged // once it is written, and heard again when a write failed, so the key already written starts nothing twice. func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go index 3a620f52..440e2c3d 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -98,7 +98,7 @@ func (h *holderView) Holders(seatName string) ([]string, bool) { // after, then start hub once. func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State, call meshcatalogclient.Call, onReady, onAdvance func(branch string), candidate func(ctx context.Context, ev catalog.CatalogRevisionEvent) error, - registered func(reg catalog.Registration), logf func(string, ...any)) (*runnerViews, error) { + registered func(reg catalog.Registration), revised func(ev catalog.CatalogRevisionEvent) (int, error), logf func(string, ...any)) (*runnerViews, error) { findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf) if err != nil { return nil, err @@ -153,6 +153,11 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr if err := meshcatalogclient.WireCandidatesProcess(reader, candidate); err != nil { return nil, err } + // A trunk revision of a release, a take-in again or a recompute starts one pipeline of its own, read by its number + // (H-A-212, H-A-213 (2), H-A-217), heard on the same event's one subscription. + if err := reader.AdoptEvent(catalog.Seat, catalog.EventCatalogRevision, revisedHear(revised, logf)); err != nil { + return nil, err + } reader.Start(ctx, js, cred, status) return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings, Holders: &holderView{reader: reader, catalog: cv, logf: logf}}, nil } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go index 211738a7..12fef5af 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go @@ -166,7 +166,8 @@ func catalogRevision(call func(string, any) (json.RawMessage, error), kept *revi if h, ok := kept.get(branch, a.Revision.Revision); ok { return h, nil } - half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{}} + half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{}, + Call: a.Revision.Call, Movers: a.Revision.Movers} in.Revision = a.Revision.Revision // the pages of one revision, though a newer one is said between them } for _, e := range a.Entries { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go index fd32e2f8..60a7d333 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go @@ -231,11 +231,11 @@ func (a *Ager) newest(p seat.Pipeline) (bool, error) { if err != nil { return false, fmt.Errorf("the pipelines of %s are not listed: %w", p.Branch, err) } - // Newest for one of its movers: no later pipeline of the branch moves it. + // Newest for one of its movers: no pipeline of a later revision of the branch moves it (H-A-213 (4)). for _, m := range p.Movers { newer := false for _, k := range pkeys { - if q, _, err := a.Pipelines.Get(k); err == nil && q.Source > p.Source && Moves(q, m.Module) { + if q, _, err := a.Pipelines.Get(k); err == nil && keyRevision(Begin{Revision: q.Revision, Source: q.Source}) > keyRevision(Begin{Revision: p.Revision, Source: p.Source}) && Moves(q, m.Module) { newer = true break } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go index 2270238e..47da9a57 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go @@ -142,3 +142,23 @@ func TestOneStepEndAgesAtMostTwentyEntries(t *testing.T) { t.Fatalf("%d outputs deleted, sequence %d; want 20 and 20", len(f.outputs), f.aged["7d"]) } } + +// H-A-213 (4): a branch's newest pipeline for a mover is the one of the highest revision number, never of the highest source: +// a release revision of the same source is newer than its take-in's. +func TestTheNewestPipelineIsOfTheHighestRevision(t *testing.T) { + f := newAgedFixture() + for key, p := range map[string]seat.Pipeline{ + "main.41": {Pipeline: "main.41", Branch: "main", Source: 41, Revision: 41, Movers: []seat.Mover{{Module: "app"}}}, + "main.42": {Pipeline: "main.42", Branch: "main", Source: 41, Revision: 42, Movers: []seat.Mover{{Module: "app"}}}, + } { + f.pipes.memStore[key] = struct { + p seat.Pipeline + rev uint64 + }{p, 1} + } + for key, want := range map[string]bool{"main.41": false, "main.42": true} { + if got, err := f.ager.newest(f.pipes.memStore[key].p); got != want || err != nil { + t.Errorf("%s newest = %v (%v), want %v", key, got, err, want) + } + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go index 73d1e020..95f5536a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go @@ -74,8 +74,8 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if !placed(p, "mesh-sdk") || !placed(p, "builder") || placed(p, "app") || placed(p, "tool") { t.Fatalf("placed at the start: %+v, want mesh-sdk and builder only", p.Steps) } - if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); n != 0 || err != nil { - t.Fatalf("the publish started %d (%v), want none", n, err) + if err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); err != nil { + t.Fatalf("the publish is not heard: %v", err) } p, _, _ = mem.Get("main.7") if sdk, _ := MoverOf(p, "mesh-sdk"); !placed(p, "app") || placed(p, "tool") || len(mem) != 1 || !slices.Contains(sdk.Published, "mesh-sdk:go") { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go index 751fbf9a..1f0cc57a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go @@ -99,6 +99,10 @@ type Half struct { Revision int64 Source int64 Entries map[string]catalog.CatalogEntry + // Call and Movers are the revision's own, as the revision verb answers them (H-A-213): what made it and its movers; + // a view's half names none. + Call string + Movers []catalog.TakeInEntry } // Views is the runner's views of the catalog half, one per watched branch the registrations name (design 47 G §21, diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 7f99aa83..7d1a50ea 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "maps" "slices" "sort" "strconv" @@ -39,9 +38,7 @@ type Begin struct { Movers []seat.Mover // Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them. Entries map[string]catalog.CatalogEntry - // Published is the package's publish that started the pipeline, nil for any other start (§747). - Published *seat.Published - // Key is the pipeline's key when it is not . (a publish's, §747); empty for a take-in's. + // Key is the pipeline's key when it is not .; empty for every start the runner makes. Key string // unheld says the branch's catalog half is not held yet: the pipeline starts with no steps, placed once it is (A-336). unheld bool @@ -63,14 +60,8 @@ func keyRevision(b Begin) int64 { return b.Source } -// PublishedKey is the key of a pipeline a package's publish starts, `..published..`, -// each part escaped as a key's name is (design 47 H §747). -func PublishedKey(branch string, source int64, pkg, version string) string { - return seat.PipelinesByBranchSourcePublishedPackageVersionKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), - keys.EscapeName(pkg), keys.EscapeName(version)) -} - -// keyForms are the filters of every key form of the state pipelines: a take-in's, a run's and a publish's. +// keyForms are the filters of every key form of the state pipelines: a revision's, a run's, and a publish's kept from before +// a release revision started one pipeline (H-A-213 (2)). var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*"} // AllKeys are every pipeline's key, of each key form (§746, §747; H-A-83). @@ -292,7 +283,7 @@ func (s *Starter) sayStarted(key string, asks []Asked) { func newPipeline(b Begin, key string, now time.Time) seat.Pipeline { movers := append([]seat.Mover(nil), b.Movers...) return seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Call: b.Call, Revision: b.Revision, Movers: movers, - Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}, Published: b.Published} + Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}} } func (s *Starter) create(key string, p seat.Pipeline) error { @@ -957,49 +948,60 @@ func (s *Starter) sourcePipeline(branch string, source int64) (string, bool) { return found, found != "" } -// publishedKind is the edge a dependent builds on a published package by: B published before A builds, a package A -// imports (§53). -const publishedKind = 1 - -// Published hears a package-published (design 47 H §747, H-A-52): in the running pipeline of the source revision that -// published it, a mover that builds on it waits for it there (§746) and nothing starts; every module the catalog half -// holds a kind-1 edge from the package into that this pipeline does not move (a publish again of an older build, a -// dependent it does not move) is a mover of one pipeline keyed `..published..`, at the -// publishing source's catalog revision. The dependents come from the edges the runner holds, never from a comparison. -func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) (int, error) { +// Published hears a package-published (design 47 H §746, H-A-52): in the running pipeline of the revision that published +// it, a mover that builds on it waits for it there and is placed once it is heard. It starts no pipeline: a re-pin on a +// module's release is a catalog revision of its own, whose pipeline Revised starts (H-A-212, H-A-213 (2)). +func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) error { if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 { - return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev) + return fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev) } - publishing, held, err := s.heardPublished(ev, view) - if err != nil { - return 0, err - } - half, ok := view(ev.Branch) - if !ok { - return 0, fmt.Errorf("the catalog half of %s is not held yet: the publish of %s %s is heard again once it is", ev.Branch, ev.Artifact, ev.Version) - } - var movers []seat.Mover - for _, module := range slices.Sorted(maps.Keys(half.Entries)) { - e := half.Entries[module] - if module == ev.Module || Moves(publishing, module) || !slices.ContainsFunc(e.Edges, func(x catalog.Edge) bool { - return x.Kind == publishedKind && x.From == ev.Artifact - }) { - continue - } - movers = append(movers, seat.Mover{Module: module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository}) - } - if len(movers) == 0 { + _, _, err := s.heardPublished(ev, view) + return err +} + +// revisedKinds are the call kinds of a catalog revision whose pipeline Revised starts: every kind but a take-in's, whose +// pipeline its take-in.taken starts (H-A-213 (2), H-A-217). +var revisedKinds = []string{catalog.CatalogRevisionEventCallKindRelease, catalog.CatalogRevisionEventCallKindTakeInAgain, + catalog.CatalogRevisionEventCallKindRecompute} + +// Revised starts, on a catalog-revision of the trunk kind whose call kind is release, take-in again or recompute, ONE +// pipeline keyed . by compare-and-set on its absence (design 47 H §746, H-A-212, H-A-213 (2) and (4), +// H-A-217): the revision is read by its number through the catalog seat's revision verb, never the newest half, and its +// movers are the revision's movers, its steps placed from the revision's entries. A revision of another kind, or one with +// no mover, starts nothing; a second hearing writes nothing. +func (s *Starter) Revised(ev catalog.CatalogRevisionEvent, read func(branch string, revision int64) (Half, error)) (int, error) { + if ev.Kind != "trunk" || !slices.Contains(revisedKinds, ev.CallKind) { return 0, nil } - revision := half.Revision - if held && publishing.Revision > 0 { - revision = publishing.Revision + branch, err := keys.UnescapeName(ev.Branch) + if err != nil || branch == "" || ev.Revision <= 0 { + return 0, fmt.Errorf("a catalog-revision names its watched branch and revision: branch %q revision %d", ev.Branch, ev.Revision) + } + if read == nil { + return 0, fmt.Errorf("the revision %d of %s cannot be read: the runner holds no revision reader", ev.Revision, branch) + } + half, err := read(branch, ev.Revision) + if err != nil { + return 0, fmt.Errorf("the revision %d of %s is not read, so its pipeline is not started: %w", ev.Revision, branch, err) + } + if half.Revision != ev.Revision { + return 0, fmt.Errorf("the revision verb answered revision %d of %s for %d", half.Revision, branch, ev.Revision) + } + if len(half.Movers) == 0 { + s.logf("the revision %d of %s (%s) names no mover: it starts no pipeline", ev.Revision, branch, ev.Call) + return 0, nil + } + movers := make([]seat.Mover, 0, len(half.Movers)) + for _, e := range half.Movers { + movers = append(movers, seat.Mover{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint, Repository: e.Repository}) } movers = Tiers(movers, half.Entries, s.logf) - started, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Call: publishing.Call, Revision: revision, Movers: movers, - Entries: half.Entries, Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version}, - Key: PublishedKey(ev.Branch, ev.Source, ev.Artifact, ev.Version)}) - if started { + call := half.Call + if call == "" { + call = ev.Call + } + ok, err := s.Start(Begin{Branch: branch, Source: half.Source, Call: call, Revision: half.Revision, Movers: movers, Entries: half.Entries}) + if ok { return 1, err } return 0, err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 9418cfca..7baa680c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -11,6 +11,7 @@ import ( builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" ) type created map[string][]byte @@ -55,12 +56,11 @@ func TestStarts(t *testing.T) { } } -// R10/AC1 and AC2 (design 47 H §747, H-A-52): a package-published of a mover of its source's running pipeline whose -// dependent that pipeline moves starts nothing: the dependent waits there. A publish again of an older build starts one -// pipeline keyed ..published.., its movers the package's kind-1 dependents from the -// edges the runner holds, at the publishing source's revision, and says pipeline-started with the publish; an ordering -// edge starts nothing; heard twice, the second writes nothing. -func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) { +// H-A-212, H-A-213 (2) and (4), H-A-217: a package-published starts no pipeline, the movers waiting for it in its +// pipeline placed there; a trunk catalog-revision of a release, a take-in again or a recompute starts ONE pipeline keyed +// ., the revision read by its number, its movers the revision's, its source and call the revision's; +// heard twice the second writes nothing; a take-in's revision, a candidate's and one with no mover start nothing. +func TestARevisionOfAReleaseStartsOnePipeline(t *testing.T) { mem := memStore{} var said []seat.PipelineStarted s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(_ string, body any) error { @@ -70,37 +70,60 @@ func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) { return nil }} edges := []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 1}} - view := func(string) (Half, bool) { - return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{ - "mesh-sdk": {Module: "mesh-sdk"}, - "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges}, - "tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges}, - "ordered": {Module: "ordered", Edges: []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 4}}}}}, true + entries := map[string]catalog.CatalogEntry{ + "mesh-sdk": {Module: "mesh-sdk"}, + "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges}, + "tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges}} + view := func(string) (Half, bool) { return Half{Revision: 9, Entries: entries}, true } + ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 30, Module: "mesh-sdk", Version: "0.1.9"} + if err := s.Published(ev, view); err != nil || len(mem) != 0 { + t.Fatalf("a publish of an older build wrote %d pipelines (%v), want none", len(mem), err) } - mem[Key("main", 41)] = struct { - p seat.Pipeline - rev uint64 - }{seat.Pipeline{Pipeline: "main.41", Branch: "main", Source: 41, Revision: 8, Outcome: seat.PipelineOutcomeRunning, - Movers: []seat.Mover{{Module: "mesh-sdk"}, {Module: "app"}, {Module: "tool"}}}, 1} - ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"} - if n, err := s.Published(ev, view); n != 0 || err != nil || len(mem) != 1 { - t.Fatalf("a publish in a pipeline moving its dependents started %d (%v), %d pipelines", n, err, len(mem)) + var asked []int64 + movers := []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}, {Module: "tool", Commit: "c2", Fingerprint: "src2:bc"}} + read := func(branch string, revision int64) (Half, error) { + asked = append(asked, revision) + if branch != "release.1" { + return Half{}, fmt.Errorf("branch %q", branch) + } + r := Half{Revision: revision, Source: 41, Entries: entries, Call: "release of mesh-sdk 0.1.17"} + if revision != 12 { + r.Movers = movers + } + return r, nil } - // A publish again of an older build, at source 30: its pipeline is gone, so every kind-1 dependent is a mover. - ev.Source, ev.Version = 30, "0.1.9" + release := catalog.CatalogRevisionEvent{Kind: "trunk", Branch: keys.EscapeName("release.1"), Revision: 10, Source: 41, CallKind: "release", Call: "release of mesh-sdk 0.1.17"} for i := range 2 { - if n, err := s.Published(ev, view); n != 1-i || err != nil { + if n, err := s.Revised(release, read); n != 1-i || err != nil { t.Fatalf("hearing %d started %d (%v)", i+1, n, err) } } - key := PublishedKey("main", 30, "mesh-sdk:go", "0.1.9") - p, _, err := mem.Get(key) - if err != nil || len(p.Movers) != 2 || !Moves(p, "app") || !Moves(p, "tool") || Moves(p, "ordered") || p.Revision != 9 || - p.Published == nil || p.Published.Version != "0.1.9" || len(said) != 1 || said[0].Published == nil { - t.Fatalf("the pipeline %s = %+v (%v), said %+v", key, p, err, said) + p, _, err := mem.Get(Key("release.1", 10)) + if err != nil || len(mem) != 1 || len(p.Movers) != 2 || p.Revision != 10 || p.Source != 41 || p.Call != "release of mesh-sdk 0.1.17" || + len(said) != 1 || len(asked) != 2 || asked[0] != 10 { + t.Fatalf("the pipeline %s = %+v (%v), %d written, said %d, read %v", Key("release.1", 10), p, err, len(mem), len(said), asked) } - if app, _ := MoverOf(p, "app"); app.Commit != "c1" || app.Fingerprint != "src2:aa" || app.Repository != "o/app" { - t.Errorf("the mover app = %+v", app) + if app, _ := MoverOf(p, "app"); app.Fingerprint != "src2:ab" || app.Repository != "o/app" || app.Tier != 1 { + t.Errorf("the mover app = %+v, want the revision's", app) + } + for _, c := range []struct { + name string + ev catalog.CatalogRevisionEvent + want int + }{ + {"take-in again", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 11, CallKind: "take-in-again"}, 1}, + {"recompute with no mover", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 12, CallKind: "recompute"}, 0}, + {"recompute", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 13, CallKind: "recompute"}, 1}, + {"take-in", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 14, CallKind: "take-in"}, 0}, + {"candidate", catalog.CatalogRevisionEvent{Kind: "candidate", Branch: release.Branch, Revision: 15, CallKind: "release"}, 0}, + } { + if n, err := s.Revised(c.ev, read); n != c.want || err != nil { + t.Errorf("%s: started %d (%v), want %d", c.name, n, err, c.want) + } + } + if _, err := s.Revised(catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 16, CallKind: "release"}, + func(string, int64) (Half, error) { return Half{}, errors.New("no answer") }); err == nil { + t.Error("a revision that cannot be read is not an error, so it is never heard again") } } @@ -243,25 +266,6 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T if n != 1 || err != nil || len(p.Steps) == 0 { t.Errorf("the half of next placed %d (%v): %d steps, want the take-in's pipeline placed", n, err, len(p.Steps)) } - // A-336: a package published heard before its branch's half is held starts nothing, heard again once it is: its - // dependents are the half's edges (§747). - held = false - app := entries["app"] - app.Edges = []catalog.Edge{{From: "mesh-sdk:go", To: "app:program", Kind: 1}} - entries["app"] = app - published := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "next", Source: 7, Module: "mesh-sdk", Version: "0.1.17"} - if n, err := s.Published(published, late); n != 0 || err == nil { - t.Fatalf("the publish before the half started %d (%v), want none and an error, heard again", n, err) - } - held = true - if n, err := s.Published(published, late); n != 1 || err != nil { - t.Fatalf("the publish with the half held started %d (%v)", n, err) - } - p, _, _ = store.Get(PublishedKey("next", 7, "mesh-sdk:go", "0.1.17")) - if len(p.Steps) == 0 { - t.Errorf("the publish's pipeline placed %d steps, want its steps", len(p.Steps)) - } - // A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running, // no step placed) is placed and asked as a newly started one when the half is held; no hold is read any more. var old seat.Pipeline -- 2.54.0 From 4af1ba99cfcbf2ccdf8b7afa291d9727551914f6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:12:36 +0100 Subject: [PATCH 25/30] Design 47 H1 (5a): the runner's integration tests key a pipeline by its catalog revision The queue's and the test runs' integration tests took several sources in at one catalog revision, which now are one key. Each take-in has its own revision; the test runs' test also checks that a test ask names revision and source. A replaced reader of another pipeline's run stays the unit tests' (one revision is one pipeline). Builds: design 47 H H-A-213 (4), H-A-219 --- .../queue_integration_test.go | 4 +- .../testruns_integration_test.go | 52 +++++++------------ 2 files changed, 20 insertions(+), 36 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index 6b319931..8e9fb2d3 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -101,7 +101,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { } } answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs}) - answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries}) + answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 100, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries}) answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) // The router of the operator's asks holds the seat operator-channel; its record of each ask is in its state asks. const router = "operator-router" @@ -218,7 +218,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { } take := func(source int64, movers ...string) { t.Helper() - ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: 1, Call: fmt.Sprintf("%016x", source)} + ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: source, Call: fmt.Sprintf("%016x", source)} for _, m := range movers { ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index a0aa6696..e39fabf9 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -37,7 +37,7 @@ import ( // ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step // that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a // run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed; -// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended, +// a run that keeps saying its progress past its bound is not ended, // and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the // same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its // other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and @@ -327,7 +327,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { // One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its // ask is taken (A-469b), its holder present and nothing failing it (A-466y). - const p3 = "main.3" + // A pipeline is keyed by its catalog revision (H-A-213 (4)): source 3's take-in of revision 1 is main.1. + const p3 = "main.1" takeIn("app", "c1", "src2:aa", 3, 1) placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747) passBuild(p3, "app", "c1", "src2:aa") @@ -340,8 +341,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { } var testAsk nodebuildgo.StepAsk if err := json.Unmarshal(last.Data, &testAsk); err != nil || testAsk.Step == nil || - testAsk.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25" { - t.Fatalf("the last test ask = %s (%v), want its fetch to carry the entry's pin", last.Data, err) + testAsk.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25" || testAsk.Step.Revision != 1 || testAsk.Step.Source != 3 { + t.Fatalf("the last test ask = %s (%v), want its fetch to carry the entry's pin, revision 1 and source 3 (H-A-219)", last.Data, err) } if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 { t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers) @@ -483,39 +484,20 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { // A second revision with the same digests asks its own runs, so a failed test runs again (§724). revision.Store(2) publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4}) - const p4, p5 = "main.4", "main.5" + const p4 = "main.2" takeIn("app", "c1", "src2:aa", 4, 2) placed(p4) run4, r4 := runOfStep(p4, "app:program", "integration") if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask { t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask) } - // A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads - // the run, the first is cancelled and leaves it, and the run's ask is not dropped. - takeIn("app", "c1", "src2:aa", 5, 2) - placed(p5) - waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 }) - if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" { - t.Fatalf("cancel: %s", e) - } - waitFor(t, ctx, logs, func() bool { - r, _ := runOf(run4) - return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5 - }) - if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask { - t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r) - } - mu.Lock() - dropped := append([]string(nil), drops...) - mu.Unlock() - if len(dropped) != 0 { - t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String()) - } + // A replaced reader leaving the ask to the other readers of another pipeline (§726) is the unit tests': one revision is + // one pipeline (H-A-213 (4)). // A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720). taken(r4.Ask) waitFor(t, ctx, logs, func() bool { - return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun && - step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun + return step(read(p4), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun && + step(read(p4), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun }) if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") { t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r) @@ -528,8 +510,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { // A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not // asked (§535, §647; A-470g, A-470h). - const web = "main.7" - takeIn("web", "w1", "src2:ww", 7, 2) + revision.Store(3) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 7}) + const web = "main.3" + takeIn("web", "w1", "src2:ww", 7, 3) waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 }) if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" { t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e) @@ -580,10 +564,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { t.Fatal(err) } } - revision.Store(3) - publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5}) - const p6 = "main.6" - takeIn("app", "c1", "src2:aa", 6, 3) + revision.Store(4) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 4, Kind: "trunk", Branch: "main", Source: 6}) + const p6 = "main.4" + takeIn("app", "c1", "src2:aa", 6, 4) waitFor(t, ctx, logs, func() bool { s := step(read(p6), "app:program", "unit") return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib") -- 2.54.0 From 3f62410c22e8ec5c7e72f4269bd38ffb1bbd7945 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:16:53 +0100 Subject: [PATCH 26/30] Design 47 H1 (5c): a package-published names the catalog revision its build was made from One source now has several catalog revisions, so a publish naming branch and source alone does not say which pipeline published it. The seat mesh-builds' package-published gains an optional revision (seatgen regenerated). The version reserved for a build keeps the catalog revision number: from the picture's half when the picture starts the build (the same revision its source comes from), from the step ask's run when the ask reaches the build first. Its package-published says it. A reservation from before the field says none. Builds: design 47 H H-A-218, H-A-220 --- .../cmd/mesh-delivery-builder/catalog.go | 14 +++++++++--- .../cmd/mesh-delivery-builder/catalog_test.go | 17 ++++++++++++++ .../internal/build/numbers.go | 2 +- .../internal/build/revisionwait.go | 2 +- .../internal/build/run.go | 4 +++- .../internal/build/step.go | 22 +++++++++++++++++++ .../internal/build/step_test.go | 12 +++++----- .../internal/versions/reserve.go | 14 ++++++++---- .../internal/versions/reserve_test.go | 8 +++---- .../go/seats/meshbuilds/contract.go | 2 ++ .../seats/mesh-builds/seat.json | 4 ++++ .../ts/seats/meshbuilds/index.d.ts | 2 ++ 12 files changed, 84 insertions(+), 19 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go index 1d935875..5ff5f0bd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go @@ -101,9 +101,7 @@ func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, e } origin := build.Origin{Branch: branch, First: first} if cat, ok := c.branches.Catalog(branch); ok { - if half, ok := cat.View(); ok { - origin.Source = half.Revision.Source - } + origin = pictureOrigin(origin, cat.View) } var e meshcatalog.CatalogEntry if err := json.Unmarshal(entry, &e); err != nil || e.Module != module { @@ -132,6 +130,16 @@ func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, e } } +// pictureOrigin is the origin of an entry the picture takes in: the source revision and the catalog revision number of the +// half the view holds, one revision for both (design 47 G §371, H-A-218, H-A-220), so a build the picture starts before any +// step ask reserves a version that names its revision; none while the view holds no half. +func pictureOrigin(o build.Origin, view func() (meshcatalogclient.Half, bool)) build.Origin { + if half, ok := view(); ok { + o.Source, o.Revision = half.Revision.Source, half.Revision.Revision + } + return o +} + // picture is build.Builder.Picture: the source revision the view of branch holds (design 47 H H-A-155), false while the branch // is not followed or its view holds none. func (c *catalogHalf) picture(branch string) (int64, bool) { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go index 8d9c45b1..26bdeabd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go @@ -4,7 +4,9 @@ import ( "context" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "mesh-delivery-builder/internal/build" ) // TestTheManifestOfAnEntryIsReadByItsDigest: the builder reads a manifest from the entry that holds it, by the entry's manifest @@ -25,3 +27,18 @@ func TestTheManifestOfAnEntryIsReadByItsDigest(t *testing.T) { t.Errorf("an entry with no manifest kept one: %v, %v", err, m.byFp) } } + +// H-A-220: an entry the picture takes in has the source revision and the catalog revision number of the half its view holds, +// one revision for both; a view holding no half names neither. +func TestAnEntryThePictureTakesInNamesItsHalfsRevision(t *testing.T) { + o := build.Origin{Branch: "main", First: true} + got := pictureOrigin(o, func() (meshcatalogclient.Half, bool) { + return meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Revision: 47, Source: 45}}, true + }) + if got != (build.Origin{Branch: "main", First: true, Source: 45, Revision: 47}) { + t.Errorf("origin = %+v, want source 45 at revision 47", got) + } + if got := pictureOrigin(o, func() (meshcatalogclient.Half, bool) { return meshcatalogclient.Half{}, false }); got != o { + t.Errorf("origin with no half = %+v, want %+v", got, o) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 2aeefc0e..867a24e6 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -194,7 +194,7 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu for _, t := range plan.Tasks { refs = append(refs, module+":"+t.Artifact) } - src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, Rebuild: rebuild} + src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild} for _, t := range plan.Tasks { src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go") } diff --git a/modules/mesh-delivery-builder/internal/build/revisionwait.go b/modules/mesh-delivery-builder/internal/build/revisionwait.go index 274ec20c..0c4ec3d4 100644 --- a/modules/mesh-delivery-builder/internal/build/revisionwait.go +++ b/modules/mesh-delivery-builder/internal/build/revisionwait.go @@ -103,7 +103,7 @@ func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, } first := b.FirstBranch == nil || b.FirstBranch(run.Branch, e.Module) b.trunkFP[key] = true - b.origin[key] = Origin{Branch: run.Branch, Source: run.Source, First: first} + b.origin[key] = Origin{Branch: run.Branch, Source: run.Source, Revision: run.Revision, First: first} } b.mu.Unlock() b.logf("%s: %s; the catalog keeps revision %d with its entries: the build goes on", run.Pipeline, below, run.Revision) diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index b87cfd1a..f4662d49 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -237,7 +237,9 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) { type Origin struct { Branch string Source int64 - First bool + // Revision is the catalog revision number a step ask of the build named (design 47 H H-A-218), 0 until one did. + Revision int64 + First bool } // Entry takes the catalog graph entry of one module on one watched branch in (design 47 G §348, §614) and builds its diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index 09687285..b19cbb25 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -39,6 +39,7 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui if outcome, said := b.awaitRevision(ctx, run); outcome != "" { return finish(outcome, said) } + b.askedRevision(run) var failed []string for _, e := range run.Entries { outcome, said := b.buildEntry(ctx, e) @@ -55,6 +56,27 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui return finish(meshbuilds.StepEndedOutcomePassed, "") } +// askedRevision records on the origin of each entry of run the catalog revision the ask names (design 47 H H-A-218), where +// the origin is of the ask's branch and source revision: the version reserved for its build names it, and so does its +// package-published, so the runner finds the publishing pipeline by branch and revision. +func (b *Builder) askedRevision(run meshbuilds.StepRun) { + if run.Revision <= 0 { + return + } + b.mu.Lock() + defer b.mu.Unlock() + for _, e := range run.Entries { + key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint) + if err != nil { + continue + } + if o, ok := b.origin[key]; ok && o.Branch == run.Branch && o.Source == run.Source { + o.Revision = run.Revision + b.origin[key] = o + } + } +} + // buildEntry builds one entry at its fingerprint and waits for its end. func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outcome, said string) { entry, err := b.StepEntry(ctx, e.Fingerprint) diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 2a0e23dd..ef35acf7 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -362,6 +362,7 @@ func aStepAskWaitsForItsRevision(t *testing.T) { if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 || keptReads.Load() != 0 { t.Errorf("end = %+v, reserved from %+v, kept revisions read %d; want passed with one version reserved from revision 41, none read", end, res.sources, keptReads.Load()) } + } // H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, whose entry of m moved @@ -378,16 +379,17 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) { read = append(read, fmt.Sprintf("%s %d", branch, revision)) return 45, map[string]string{"m": fp1}, nil } - run := meshbuilds.StepRun{Pipeline: "main.45.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 45, Source: 45, + run := meshbuilds.StepRun{Pipeline: "main.47.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 47, Source: 45, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}} end := r.b.BuildStep(t.Context(), run) r.b.Wait() - if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 45"}) { - t.Errorf("end = %+v, kept revisions read %v; want passed, revision 45 of main read once", end, read) + if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 47"}) { + t.Errorf("end = %+v, kept revisions read %v; want passed, revision 47 of main read once", end, read) } // Its fingerprint is recorded as the branch's: its build reserves its version from source revision 45 (H-A-155). - if len(res.sources) != 1 || res.sources[0].Revision != 45 { - t.Errorf("reserved from %+v; want one version reserved from revision 45", res.sources) + // The version reserved names the catalog revision the ask named, which its package-published says (H-A-218). + if len(res.sources) != 1 || res.sources[0].Revision != 45 || res.sources[0].CatalogRevision != 47 { + t.Errorf("reserved from %+v; want one version reserved from source revision 45 at catalog revision 47 (H-A-218)", res.sources) } r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1)) r2.b.RevisionWait = time.Nanosecond diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 0bd46450..635805af 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -65,6 +65,9 @@ type ReservedVersion struct { // (design 47 G §371, A-57): what `package-published` says of the publication. Branch string `json:"branch,omitempty"` Source int64 `json:"source,omitempty"` + // Revision is the catalog revision number the build's step ask named, the indexer's count (design 47 H H-A-218): what + // `package-published` names beside the source, so the runner finds the publishing pipeline; 0 when no step ask named one. + Revision int64 `json:"revision,omitempty"` // Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with // (kind 3), kept for `package-published` (design 47 G §466, §673, A-57). Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"` @@ -91,6 +94,8 @@ type Source struct { // that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication. Branch string Revision int64 + // CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did. + CatalogRevision int64 // Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced // and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b). Rebuild bool @@ -177,7 +182,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts } } out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(), - Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision} + Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision} if src.GoPackage { out.Tag = GoTagOf(out.Version) } @@ -443,11 +448,12 @@ type Marked struct { } // PackagePublished is the seat mesh-builds' package-published of one artifact the holder said published at the reservation's -// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch and -// the source revision the build was made from, and the dependencies its step result named. +// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch, +// the source revision the build was made from and the catalog revision its step ask named (H-A-218), and the dependencies its +// step result named. func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.PackagePublished { at = at.UTC() - return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Module: m.Reserved.Module, + return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Revision: m.Reserved.Revision, Module: m.Reserved.Module, Version: m.Reserved.Version, Digest: digest, Dependencies: m.Reserved.Dependencies[m.Artifact]} } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index 3b68a10b..3c1bed16 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -125,7 +125,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { ctx := context.Background() store := &reserveMem{mem: newMem()} r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"1.4.2"}, nil }} - v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{Branch: "main", Revision: 41}) + v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{Branch: "main", Revision: 41, CatalogRevision: 44}) if err != nil || v.Version != "1.4.3" { t.Fatalf("reserved %q (%v), want 1.4.3", v.Version, err) } @@ -154,12 +154,12 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { if err != nil || len(marked) != 1 { t.Fatalf("a published of 1.4.3 with a built digest marked %v (%v)", marked, err) } - // Design 47 G §52, §673, A-57: the package-published of that artifact carries the digest published, the branch and the source - // revision the reservation was made for, and the dependencies its step result named. + // Design 47 G §52, §673, A-57: the package-published of that artifact carries the digest published, the branch, the source + // revision and the catalog revision (H-A-218) the reservation was made for, and the dependencies its step result named. at := time.Unix(100, 0) said := marked[0].PackagePublished("sha256:aa", at) atUTC := at.UTC() - if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) { + if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Revision: 44, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) { t.Errorf("package-published = %+v, want %+v", said, wantSaid) } if marked, _, _ := r.Published(ctx, "m-a", "1.4.3", "sha256:other"); len(marked) != 0 { diff --git a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go index 836f14e0..dc45a781 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go @@ -95,6 +95,8 @@ type PackagePublished struct { Digest string `json:"digest"` // the module Module string `json:"module"` + // the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field + Revision int64 `json:"revision,omitempty"` // the source revision of that branch the build was made from Source int64 `json:"source"` V int64 `json:"v"` diff --git a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json index d10eec4f..2caddbdf 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json @@ -180,6 +180,10 @@ "type": "integer", "description": "the source revision of that branch the build was made from" }, + "revision": { + "type": "integer", + "description": "the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field" + }, "dependencies": { "type": "array", "items": { diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts index d142d6d9..68e2c00e 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts @@ -69,6 +69,8 @@ export interface PackagePublished { "digest": string; /** the module */ "module": string; + /** the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field */ + "revision"?: number; /** the source revision of that branch the build was made from */ "source": number; "v": number; -- 2.54.0 From 671dc5a9630ec84f0abb5bbe09390c9f9a50c1c7 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:17:55 +0100 Subject: [PATCH 27/30] Design 47 H1 (5c): the runner takes a publish into the pipeline of the revision it names One source now has several catalog revisions, so the publishing pipeline is found by the branch and revision the package-published names. A publish naming no revision, from a reservation made before the field, goes to its source's lowest-numbered revision, the take-in's. The seat mesh-builds' package-published gains its optional revision here too, identical to the builder branch's, so this branch compiles alone. Builds: design 47 H H-A-218, H-A-220 --- .../go/seats/meshbuilds/contract.go | 2 ++ .../seats/mesh-builds/seat.json | 4 ++++ .../ts/seats/meshbuilds/index.d.ts | 2 ++ .../internal/pipeline/start.go | 12 ++++++---- .../internal/pipeline/start_test.go | 24 +++++++++++++++++++ 5 files changed, 40 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go index 4cc3285b..179a6890 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go @@ -95,6 +95,8 @@ type PackagePublished struct { Digest string `json:"digest"` // the module Module string `json:"module"` + // the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field + Revision int64 `json:"revision,omitempty"` // the source revision of that branch the build was made from Source int64 `json:"source"` V int64 `json:"v"` diff --git a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json index fc3d717d..e100e48c 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json @@ -180,6 +180,10 @@ "type": "integer", "description": "the source revision of that branch the build was made from" }, + "revision": { + "type": "integer", + "description": "the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field" + }, "dependencies": { "type": "array", "items": { diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts index 947f2e52..bf8e68c7 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts @@ -69,6 +69,8 @@ export interface PackagePublished { "digest": string; /** the module */ "module": string; + /** the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field */ + "revision"?: number; /** the source revision of that branch the build was made from */ "source": number; "v": number; diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 7d1a50ea..41f92321 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -893,14 +893,18 @@ func (s *Starter) Release(store Updater, key string, view func(branch string) (H }, s.Registration, view, "the publishes its movers waited for") } -// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the source -// revision that published it, and places the movers that waited for it (design 47 H §746, H-A-84). It answers that -// source's pipeline, read, and whether one is held. +// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the catalog +// revision that published it, found by the branch and revision the publish names (H-A-218), and places the movers that +// waited for it (design 47 H §746, H-A-84). It answers that pipeline, read, and whether one is held. func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch string) (Half, bool)) (seat.Pipeline, bool, error) { if s.Store == nil { return seat.Pipeline{}, false, nil } - key, ok := s.sourcePipeline(ev.Branch, ev.Source) + key, ok := Key(ev.Branch, ev.Revision), true + if ev.Revision <= 0 { + // A publish of an older builder names no revision: its source's lowest-numbered revision, the take-in's (H-A-218). + key, ok = s.sourcePipeline(ev.Branch, ev.Source) + } if !ok { return seat.Pipeline{}, false, nil } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 7baa680c..3c5cbfe0 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -339,3 +339,27 @@ func TestATakeInOfTheOldFormIsRefused(t *testing.T) { t.Errorf("the new form = %+v (%v)", ev, err) } } + +// H-A-218: a package-published is taken into the pipeline of the branch and revision it names; one naming no revision (an +// older builder) into its source's lowest-numbered revision, the take-in's. +func TestAPublishIsTakenIntoThePipelineOfItsRevision(t *testing.T) { + mem := memStore{} + for _, r := range []int64{7, 8} { + mem[Key("main", r)] = struct { + p seat.Pipeline + rev uint64 + }{seat.Pipeline{Pipeline: Key("main", r), Branch: "main", Source: 4, Revision: r, Outcome: seat.PipelineOutcomeRunning, + Movers: []seat.Mover{{Module: "mesh-sdk"}}}, 1} + } + s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(string, any) error { return nil }} + view := func(string) (Half, bool) { return Half{Revision: 8}, true } + published := func(key string) []string { m, _ := MoverOf(mem[key].p, "mesh-sdk"); return m.Published } + ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Revision: 8, Version: "0.1.18"} + if err := s.Published(ev, view); err != nil || len(published("main.8")) != 1 || len(published("main.7")) != 0 { + t.Fatalf("a publish of revision 8 (%v): main.8 %v, main.7 %v; want main.8's mover alone", err, published("main.8"), published("main.7")) + } + ev.Revision, ev.Artifact = 0, "mesh-sdk:ts" + if err := s.Published(ev, view); err != nil || len(published("main.7")) != 1 || len(published("main.8")) != 1 { + t.Errorf("a publish naming no revision (%v): main.7 %v, main.8 %v; want main.7's, the source's lowest", err, published("main.7"), published("main.8")) + } +} -- 2.54.0 From 626a6fb3586545a8d7ec0143660ce39b457cb91f Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:25:08 +0100 Subject: [PATCH 28/30] Design 47 H1: the pipeline-runner seat names a pipeline's key and its subjects by revision A pipeline is keyed by its catalog revision's number, so the seat's key form is . (the generated PipelinesByBranchRevisionKey and its siblings; the fixtures follow), and pipeline-started, pipeline-ended, wait-held and wait-ended are said on ., carrying the same number as the key; every reader takes the body. The client's SourceSubject is RevisionSubject. The published key form, under which pipelines were kept before, stays. Builds: design 47 H H-A-213 (4), H-A-219, H-A-221 --- .../meshpipelinerunnerclient/client.go | 34 +++++------ .../meshpipelinerunnerclient/client_test.go | 6 +- .../go/seats/meshpipelinerunner/contract.go | 56 +++++++++---------- .../seats/meshpipelinerunner/contract_test.go | 6 +- .../mesh-pipeline-runner/fixtures/states.json | 6 +- .../seats/mesh-pipeline-runner/seat.json | 26 ++++----- .../ts/seats/meshpipelinerunner/index.d.ts | 34 +++++------ .../ts/seats/meshpipelinerunner/index.js | 30 +++++----- .../internal/index/catalog_revisions.go | 2 +- .../internal/pipeline/queue.go | 4 +- .../internal/pipeline/read.go | 2 +- .../internal/pipeline/start.go | 12 +++- .../internal/pipeline/start_test.go | 6 +- .../internal/pipeline/steps.go | 6 +- 14 files changed, 119 insertions(+), 111 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client.go b/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client.go index eb466b0b..e1702ce1 100644 --- a/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client.go +++ b/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client.go @@ -17,8 +17,9 @@ import ( // eventPrefix starts every event subject of the seat. const eventPrefix = "mesh.seat." + seat.Seat + ".event." -// perSource are the events said per source revision, on .. (design 47 H §746, H-A-72 (3)). -var perSource = []string{seat.EventPipelineStarted, seat.EventPipelineEnded, seat.EventWaitHeld, seat.EventWaitEnded} +// perRevision are the events said per pipeline, on .., its catalog revision's number as its key +// names it (design 47 H §746, H-A-72 (3), H-A-221). +var perRevision = []string{seat.EventPipelineStarted, seat.EventPipelineEnded, seat.EventWaitHeld, seat.EventWaitEnded} // Subject is where the runner says a step's event or a run's start: mesh.seat.mesh-pipeline-runner.event.. // .., each name escaped as a key's name is, so a branch holding a dot is one token (design 47 H @@ -27,41 +28,42 @@ func Subject(event, module, branch, commit string) string { return eventPrefix + event + "." + keys.EscapeName(module) + "." + keys.EscapeName(branch) + "." + keys.EscapeName(commit) } -// SourceSubject is where the runner says a pipeline's start and end and its waits: mesh.seat.mesh-pipeline-runner. -// event..., the branch escaped as a key's name is (design 47 H §746, H-A-72 (3)). -func SourceSubject(event, branch string, source int64) string { - return eventPrefix + event + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10) +// RevisionSubject is where the runner says a pipeline's start and end and its waits: mesh.seat.mesh-pipeline-runner. +// event..., the pipeline's catalog revision number, the branch escaped as a key's name is (design +// 47 H §746, H-A-72 (3), H-A-221). +func RevisionSubject(event, branch string, revision int64) string { + return eventPrefix + event + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10) } // Of is what an event's subject names: a step's event or a run's start names the module, the branch and the commit; an -// event per source revision names the branch and the source. +// event per pipeline names the branch and the pipeline's catalog revision (H-A-221). type Of struct { Event, Module, Branch, Commit string - Source int64 + Revision int64 } -// Parse reads a subject Subject or SourceSubject built: an event of the seat's, and its names unescaped. +// Parse reads a subject Subject or RevisionSubject built: an event of the seat's, and its names unescaped. func Parse(subject string) (Of, error) { rest, ok := strings.CutPrefix(subject, eventPrefix) toks := strings.Split(rest, ".") - bySource := len(toks) > 0 && slices.Contains(perSource, toks[0]) + byRevision := len(toks) > 0 && slices.Contains(perRevision, toks[0]) want := 4 - if bySource { + if byRevision { want = 3 } if !ok || len(toks) != want || !slices.Contains(seat.Events(), toks[0]) { return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q is no event subject of the seat %s", subject, seat.Seat) } - if bySource { + if byRevision { branch, err := keys.UnescapeName(toks[1]) if err != nil || branch == "" { return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: branch: %v", subject, err) } - source, err := strconv.ParseInt(toks[2], 10, 64) - if err != nil || source < 1 { - return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: no source revision", subject) + revision, err := strconv.ParseInt(toks[2], 10, 64) + if err != nil || revision < 1 { + return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: no revision", subject) } - return Of{Event: toks[0], Branch: branch, Source: source}, nil + return Of{Event: toks[0], Branch: branch, Revision: revision}, nil } names := make([]string, 0, 3) for i, t := range toks[1:] { diff --git a/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client_test.go b/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client_test.go index 355017f2..f76ef511 100644 --- a/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client_test.go +++ b/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client_test.go @@ -17,14 +17,14 @@ func TestSubjectsAndBodies(t *testing.T) { if of, err := Parse(s); err != nil || of != (Of{Event: seat.EventStepEnded, Module: "nats", Branch: "release.1", Commit: "c1"}) { t.Errorf("parse %s: %+v, %v", s, of, err) } - ps := SourceSubject(seat.EventPipelineEnded, "release.1", 3589) + ps := RevisionSubject(seat.EventPipelineEnded, "release.1", 3589) if ps != "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.release=2e1.3589" { t.Errorf("pipeline-ended subject %s", ps) } - if of, err := Parse(ps); err != nil || of != (Of{Event: seat.EventPipelineEnded, Branch: "release.1", Source: 3589}) { + if of, err := Parse(ps); err != nil || of != (Of{Event: seat.EventPipelineEnded, Branch: "release.1", Revision: 3589}) { t.Errorf("parse %s: %+v, %v", ps, of, err) } - if of, err := Parse(SourceSubject(seat.EventWaitHeld, "main", 7)); err != nil || of.Source != 7 || of.Module != "" { + if of, err := Parse(RevisionSubject(seat.EventWaitHeld, "main", 7)); err != nil || of.Revision != 7 || of.Module != "" { t.Errorf("wait-held: %+v, %v", of, err) } for _, bad := range []string{"not.an.event.subject", diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go index 058625fb..8b5f340e 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go @@ -65,9 +65,9 @@ const ( // StreamPipelineStarted is the stream pipeline-started is kept in. const StreamPipelineStarted = "SEAT_mesh-pipeline-runner_pipeline-started" -// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started... -func SubjectPipelineStarted(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source +// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started... +func SubjectPipelineStarted(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision } // SubjectPipelineStartedAll hears every pipeline-started. @@ -76,9 +76,9 @@ const SubjectPipelineStartedAll = "mesh.seat.mesh-pipeline-runner.event.pipeline // StreamPipelineEnded is the stream pipeline-ended is kept in. const StreamPipelineEnded = "SEAT_mesh-pipeline-runner_pipeline-ended" -// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended... -func SubjectPipelineEnded(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source +// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended... +func SubjectPipelineEnded(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision } // SubjectPipelineEndedAll hears every pipeline-ended. @@ -120,9 +120,9 @@ const SubjectStepEndedAll = "mesh.seat.mesh-pipeline-runner.event.step-ended.*.* // StreamWaitHeld is the stream wait-held is kept in. const StreamWaitHeld = "SEAT_mesh-pipeline-runner_wait-held" -// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held... -func SubjectWaitHeld(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source +// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held... +func SubjectWaitHeld(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision } // SubjectWaitHeldAll hears every wait-held. @@ -131,9 +131,9 @@ const SubjectWaitHeldAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*" // StreamWaitEnded is the stream wait-ended is kept in. const StreamWaitEnded = "SEAT_mesh-pipeline-runner_wait-ended" -// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended... -func SubjectWaitEnded(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source +// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended... +func SubjectWaitEnded(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision } // SubjectWaitEndedAll hears every wait-ended. @@ -156,11 +156,11 @@ const ( StateStepOutputs = "step-outputs" ) -// PipelinesByBranchSourceKey is the key of one value of the state pipelines under the form ".". -func PipelinesByBranchSourceKey(branch, source string) string { return branch + "." + source } +// PipelinesByBranchRevisionKey is the key of one value of the state pipelines under the form ".". +func PipelinesByBranchRevisionKey(branch, revision string) string { return branch + "." + revision } -// ParsePipelinesByBranchSourceKey reads the parts of a key of the form ".", in the order of the form; the last result is false for a key of another form. -func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) { +// ParsePipelinesByBranchRevisionKey reads the parts of a key of the form ".", in the order of the form; the last result is false for a key of another form. +func ParsePipelinesByBranchRevisionKey(key string) (string, string, bool) { rest := key v0, after0, found0 := strings.Cut(rest, ".") if !found0 || v0 == "" { @@ -178,9 +178,9 @@ func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) { return v0, v2, true } -// DecodePipelinesByBranchSourceValue reads one value of the state pipelines under the form ".", refusing a field PipelinesByBranchSourceValue does not know and any text after the value. -func DecodePipelinesByBranchSourceValue(raw []byte) (PipelinesByBranchSourceValue, error) { - var v PipelinesByBranchSourceValue +// DecodePipelinesByBranchRevisionValue reads one value of the state pipelines under the form ".", refusing a field PipelinesByBranchRevisionValue does not know and any text after the value. +func DecodePipelinesByBranchRevisionValue(raw []byte) (PipelinesByBranchRevisionValue, error) { + var v PipelinesByBranchRevisionValue dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&v); err != nil { @@ -670,7 +670,7 @@ type Pipeline struct { // the modules the pipeline moves, its take-in's entries (design 47 H §746) Movers []Mover `json:"movers"` Outcome string `json:"outcome"` - // the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) + // the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) Pipeline string `json:"pipeline"` // for a pipeline a package's publish started, that publish (A-202) Published *Published `json:"published,omitempty"` @@ -720,7 +720,7 @@ type PipelineEnded struct { // per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2) Movers []MoverEnded `json:"movers"` Outcome string `json:"outcome"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // the catalog graph revision of the branch the pipeline runs for Revision int64 `json:"revision,omitempty"` @@ -753,7 +753,7 @@ type PipelineStarted struct { Call string `json:"call,omitempty"` // the pipeline's movers Movers []Mover `json:"movers"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // for a pipeline a package's publish started, that publish (A-202) Published *Published `json:"published,omitempty"` @@ -814,7 +814,7 @@ type StepEvent struct { // the module's build-input fingerprint at the commit Fingerprint string `json:"fingerprint,omitempty"` Module string `json:"module"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` Result StepResult `json:"result"` // the catalog graph revision of the branch the pipeline runs for @@ -966,7 +966,7 @@ type ToolchainUsed struct { Version string `json:"version"` } -// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72). +// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221). type Wait struct { Artifact string `json:"artifact,omitempty"` // the step's own ask, where it has one (design 47 G §731) @@ -983,7 +983,7 @@ type Wait struct { Module string `json:"module,omitempty"` // what the step waits on (design 47 *Health and going back*) On string `json:"on"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // the catalog graph revision of the branch the pipeline runs for Revision int64 `json:"revision,omitempty"` @@ -1024,7 +1024,7 @@ const ( // Waiting is a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72). type Waiting struct { - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` } @@ -1355,8 +1355,8 @@ type FullEvent = Fill // RoomEvent is the body of the event room. type RoomEvent = Fill -// PipelinesByBranchSourceValue is one value of the state pipelines under the key .. -type PipelinesByBranchSourceValue = Pipeline +// PipelinesByBranchRevisionValue is one value of the state pipelines under the key .. +type PipelinesByBranchRevisionValue = Pipeline // PipelinesByBranchSourcePublishedPackageVersionValue is one value of the state pipelines under the key ..published... type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go index 23b59e9c..4aaa6d53 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go @@ -45,7 +45,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")}, {name: "a take-in's pipeline kept, its steps naming their mover", subject: "", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, @@ -73,13 +73,13 @@ func TestFixtures(t *testing.T) { }, subjectIs: nil}, {name: "a pipeline waiting its turn, its seq given", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, {name: "a waiting pipeline names the movers a newer take-in took out of it", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json index cb8363f2..9229229f 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json @@ -1,7 +1,7 @@ [ { "name": "a take-in's pipeline kept, its steps naming their mover", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3589", "branch": "main", @@ -200,7 +200,7 @@ }, { "name": "a pipeline waiting its turn, its seq given", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3590", "branch": "main", @@ -224,7 +224,7 @@ }, { "name": "a waiting pipeline names the movers a newer take-in took out of it", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3590", "branch": "main", diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json index 5bf14a52..fd84b289 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json @@ -672,7 +672,7 @@ "events": [ { "name": "pipeline-started", - "subject": "pipeline-started..", + "subject": "pipeline-started..", "body": { "$ref": "#/defs/PipelineStarted" }, @@ -683,7 +683,7 @@ }, { "name": "pipeline-ended", - "subject": "pipeline-ended..", + "subject": "pipeline-ended..", "body": { "$ref": "#/defs/PipelineEnded" }, @@ -727,7 +727,7 @@ }, { "name": "wait-held", - "subject": "wait-held..", + "subject": "wait-held..", "body": { "$ref": "#/defs/Wait" }, @@ -738,7 +738,7 @@ }, { "name": "wait-ended", - "subject": "wait-ended..", + "subject": "wait-ended..", "body": { "$ref": "#/defs/Wait" }, @@ -778,11 +778,11 @@ "description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747), and one key per mover of a pipeline holding that mover's steps (H-A-187)", "key-forms": [ { - "key": ".", + "key": ".", "value": { "$ref": "#/defs/Pipeline" }, - "description": "a take-in's pipeline" + "description": "a pipeline of one catalog revision, keyed by the revision's number (design 47 H-A-213 (4), H-A-219); one kept from before under its source has the same form" }, { "key": "..published..", @@ -1199,7 +1199,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)" + "description": "the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)" }, "branch": { "type": "string", @@ -1313,7 +1313,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "branch": { "type": "string", @@ -1388,7 +1388,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "branch": { "type": "string", @@ -1495,7 +1495,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "module": { "type": "string" @@ -1550,7 +1550,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "module": { "type": "string", @@ -1635,7 +1635,7 @@ "description": "the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)" } }, - "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72)", + "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221)", "required": [ "v", "pipeline", @@ -1978,7 +1978,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" } }, "description": "a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72)", diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts index 748152b9..5c11adad 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -40,13 +40,13 @@ export declare const family: { readonly name: "mesh-pipeline-runner.events"; rea export declare const pipelineStartedStream: "SEAT_mesh-pipeline-runner_pipeline-started"; -export declare function pipelineStartedSubject(branch: string, source: string): string; +export declare function pipelineStartedSubject(branch: string, revision: string): string; export declare const pipelineStartedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*"; export declare const pipelineEndedStream: "SEAT_mesh-pipeline-runner_pipeline-ended"; -export declare function pipelineEndedSubject(branch: string, source: string): string; +export declare function pipelineEndedSubject(branch: string, revision: string): string; export declare const pipelineEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"; @@ -70,13 +70,13 @@ export declare const stepEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event. export declare const waitHeldStream: "SEAT_mesh-pipeline-runner_wait-held"; -export declare function waitHeldSubject(branch: string, source: string): string; +export declare function waitHeldSubject(branch: string, revision: string): string; export declare const waitHeldSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*"; export declare const waitEndedStream: "SEAT_mesh-pipeline-runner_wait-ended"; -export declare function waitEndedSubject(branch: string, source: string): string; +export declare function waitEndedSubject(branch: string, revision: string): string; export declare const waitEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*"; @@ -94,11 +94,11 @@ export declare const states: { readonly stepOutputs: "step-outputs"; }; -/** The key of one value of the state pipelines under the form ".". */ -export declare function pipelinesByBranchSourceKey(branch: string, source: string): string; +/** The key of one value of the state pipelines under the form ".". */ +export declare function pipelinesByBranchRevisionKey(branch: string, revision: string): string; -/** The parts of a key of the form ".", or undefined for a key of another form. */ -export declare function parsePipelinesByBranchSourceKey(_key: string): { branch: string; source: string } | undefined; +/** The parts of a key of the form ".", or undefined for a key of another form. */ +export declare function parsePipelinesByBranchRevisionKey(_key: string): { branch: string; revision: string } | undefined; /** The key of one value of the state pipelines under the form "..published..". */ export declare function pipelinesByBranchSourcePublishedPackageVersionKey(branch: string, source: string, packagePart: string, version: string): string; @@ -261,7 +261,7 @@ export interface Pipeline { /** the modules the pipeline moves, its take-in's entries (design 47 H §746) */ "movers": Mover[]; "outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled"; - /** the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */ + /** the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */ "pipeline": string; /** for a pipeline a package's publish started, that publish (A-202) */ "published"?: Published; @@ -300,7 +300,7 @@ export interface PipelineEnded { /** per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2) */ "movers": MoverEnded[]; "outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled"; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** the catalog graph revision of the branch the pipeline runs for */ "revision"?: number; @@ -322,7 +322,7 @@ export interface PipelineStarted { "call"?: string; /** the pipeline's movers */ "movers": Mover[]; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** for a pipeline a package's publish started, that publish (A-202) */ "published"?: Published; @@ -383,7 +383,7 @@ export interface StepEvent { /** the module's build-input fingerprint at the commit */ "fingerprint"?: string; "module": string; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; "result": StepResult; /** the catalog graph revision of the branch the pipeline runs for */ @@ -490,7 +490,7 @@ export interface ToolchainUsed { "version": string; } -/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72) */ +/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221) */ export interface Wait { "artifact"?: string; /** the step's own ask, where it has one (design 47 G §731) */ @@ -507,7 +507,7 @@ export interface Wait { "module"?: string; /** what the step waits on (design 47 *Health and going back*) */ "on": "person" | "build node" | "revision view" | "pipelines-at-once"; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** the catalog graph revision of the branch the pipeline runs for */ "revision"?: number; @@ -527,7 +527,7 @@ export interface Wait { /** a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72) */ export interface Waiting { - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; } @@ -800,8 +800,8 @@ export type FullEvent = Fill; /** is the body of the event room. */ export type RoomEvent = Fill; -/** is one value of the state pipelines under the key .. */ -export type PipelinesByBranchSourceValue = Pipeline; +/** is one value of the state pipelines under the key .. */ +export type PipelinesByBranchRevisionValue = Pipeline; /** is one value of the state pipelines under the key ..published... */ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline; diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js index 213a3df3..d8ff25e6 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js @@ -40,16 +40,16 @@ export const family = { name: "mesh-pipeline-runner.events", maxAgeDays: 7 }; export const pipelineStartedStream = "SEAT_mesh-pipeline-runner_pipeline-started"; -export function pipelineStartedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source; +export function pipelineStartedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision; } export const pipelineStartedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*"; export const pipelineEndedStream = "SEAT_mesh-pipeline-runner_pipeline-ended"; -export function pipelineEndedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source; +export function pipelineEndedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision; } export const pipelineEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"; @@ -80,16 +80,16 @@ export const stepEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.step-en export const waitHeldStream = "SEAT_mesh-pipeline-runner_wait-held"; -export function waitHeldSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source; +export function waitHeldSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision; } export const waitHeldSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*"; export const waitEndedStream = "SEAT_mesh-pipeline-runner_wait-ended"; -export function waitEndedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source; +export function waitEndedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision; } export const waitEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*"; @@ -108,23 +108,23 @@ export const states = { stepOutputs: "step-outputs", }; -/** The key of one value of the state pipelines under the form ".". */ -export function pipelinesByBranchSourceKey(branch, source) { - return branch + "." + source; +/** The key of one value of the state pipelines under the form ".". */ +export function pipelinesByBranchRevisionKey(branch, revision) { + return branch + "." + revision; } -/** The parts of a key of the form ".", or undefined for a key of another form. */ -export function parsePipelinesByBranchSourceKey(_key) { +/** The parts of a key of the form ".", or undefined for a key of another form. */ +export function parsePipelinesByBranchRevisionKey(_key) { let _rest = _key; const _at0 = _rest.indexOf("."); if (_at0 <= 0) return undefined; const branch = _rest.slice(0, _at0); _rest = _rest.slice(_at0 + 1); if (_rest === "") return undefined; - const source = _rest; + const revision = _rest; _rest = ""; if (_rest !== "") return undefined; - return { branch, source }; + return { branch, revision }; } /** The key of one value of the state pipelines under the form "..published..". */ diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index ec1e7717..4fc72a07 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -331,7 +331,7 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int } // The take-in's revision pinned for its one pipeline before any judge can remove it (§743, §749, A-449a). if len(movers) > 0 && ix.Pins != nil { - record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(src, 10))) + record := PipelineRecord(runner.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(src, 10))) if err := ix.pin(ctx, PinKey(branch, src), record); err != nil { ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err) return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go index 855191db..cecc84a8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go @@ -164,7 +164,7 @@ func (q *Queue) leave(key string) error { return nil } -// queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on . with step pipeline and +// queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on . with step pipeline and // on pipelines-at-once (design 47 H §748, H-A-52). func queueWait(p seat.Pipeline, at time.Time, how string) seat.Wait { return seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Revision: p.Revision, Run: p.Run, @@ -175,7 +175,7 @@ func (s *Starter) sayQueueWait(event string, p seat.Pipeline, at time.Time, how if s.Say == nil { return } - if err := s.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), queueWait(p, at, how)); err != nil { + if err := s.Say(runnerclient.RevisionSubject(event, p.Branch, subjectRevision(p)), queueWait(p, at, how)); err != nil { s.logf("%s of %s is not said: %v", event, p.Pipeline, err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index e6a0762b..668d2ed8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -206,7 +206,7 @@ func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) if raw == nil { return seat.Pipeline{}, false, nil } - p, err := seat.DecodePipelinesByBranchSourceValue(raw) + p, err := seat.DecodePipelinesByBranchRevisionValue(raw) if err != nil { return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 41f92321..1b2e0911 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -48,7 +48,13 @@ type Begin struct { // the branch escaped as a key's name is (design 47 H §746, H-A-213 (4)); a take-in's number is unchanged where it equalled // its source. A pipeline kept under its source before (`.`) has the same form and is read as it is. func Key(branch string, revision int64) string { - return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10)) + return seat.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10)) +} + +// subjectRevision is the number p's pipeline-started, pipeline-ended and waits are said under, .: its +// key's number (H-A-221). +func subjectRevision(p seat.Pipeline) int64 { + return keyRevision(Begin{Revision: p.Revision, Source: p.Source}) } // keyRevision is the number b's pipeline is keyed by (H-A-213 (4)): its revision, or its source where the start names no @@ -300,12 +306,12 @@ func started(p seat.Pipeline, at time.Time) seat.PipelineStarted { Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run} } -// sayStartedPipeline says p's pipeline-started on . (§746). +// sayStartedPipeline says p's pipeline-started on . (§746). func (s *Starter) sayStartedPipeline(p seat.Pipeline, at time.Time) { if s.Say == nil { return } - if err := s.Say(runnerclient.SourceSubject(seat.EventPipelineStarted, p.Branch, p.Source), started(p, at)); err != nil { + if err := s.Say(runnerclient.RevisionSubject(seat.EventPipelineStarted, p.Branch, subjectRevision(p)), started(p, at)); err != nil { s.logf("pipeline-started of %s is not said: %v", p.Pipeline, err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 3c5cbfe0..d3f5700b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -41,7 +41,7 @@ func TestStarts(t *testing.T) { if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Movers: []seat.Mover{{Module: "app"}}}); err != nil { t.Fatal(err) } - if b, r, ok := seat.ParsePipelinesByBranchSourceKey(Key("release.1", 3)); !ok || r != "3" || b == "release" { + if b, r, ok := seat.ParsePipelinesByBranchRevisionKey(Key("release.1", 3)); !ok || r != "3" || b == "release" { t.Errorf("the key of release.1 is %q", Key("release.1", 3)) } if _, err := s.Start(Begin{Branch: "main", Movers: []seat.Mover{{Module: "app"}}}); err == nil { @@ -151,7 +151,7 @@ func (failing) Create(string, []byte) error { return errors.New("bus away") } // R4/AC1 (design 47 H §746, acceptance test 112): a take-in.taken naming mesh-sdk and one importer starts one pipeline // keyed . by compare-and-set, its movers the event's entries, at the take-in's revision, and says one -// pipeline-started on . naming the call and the movers; R4/AC2: heard twice, the second writes nothing. A +// pipeline-started on . naming the call and the movers; R4/AC2: heard twice, the second writes nothing. A // take-in with no entries and one naming no source revision start nothing. func TestATakeInIsOnePipeline(t *testing.T) { store := created{} @@ -182,7 +182,7 @@ func TestATakeInIsOnePipeline(t *testing.T) { if m, _ := MoverOf(p, "mesh-sdk"); m.Commit != "c1" || m.Fingerprint != "src2:aa" || m.Repository != "o/sdk" || m.Tier != 1 { t.Errorf("the mover mesh-sdk = %+v", m) } - if subjects[0] != "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.7" || said[0].Call != ev.Call || len(said[0].Movers) != 2 { + if subjects[0] != "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.12" || said[0].Call != ev.Call || len(said[0].Movers) != 2 { t.Errorf("pipeline-started %+v on %s", said[0], subjects[0]) } if _, err := s.TakenIn(catalog.TakeInTaken{Branch: "main", Entries: ev.Entries}, view); err == nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 5ac10c50..40524451 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -441,7 +441,7 @@ func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) erro } // said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall; -// an end is said once, on . (design 47 H §746). The runner says no group-ended (§764). +// an end is said once, on . (design 47 H §746). The runner says no group-ended (§764). func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error { for _, i := range asked { if err := h.say(seat.EventWaitHeld, p, &p.Steps[i]); err != nil { @@ -489,7 +489,7 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error subject, ev := stepEvent(event, p, *step, at) return h.Say(subject, ev) case seat.EventPipelineEnded: - return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), pipelineEnded(p)) + return h.Say(runnerclient.RevisionSubject(event, p.Branch, subjectRevision(p)), pipelineEnded(p)) default: w := seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Revision: p.Revision, Source: p.Source, Run: p.Run, Step: seat.WaitStepApproval, On: seat.WaitOnPerson, Since: at} @@ -497,7 +497,7 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error m, _ := MoverOf(p, step.Module) w.Module, w.Commit, w.Fingerprint, w.Step, w.Ask = m.Module, m.Commit, m.Fingerprint, step.Step, step.Ask } - return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), w) + return h.Say(runnerclient.RevisionSubject(event, p.Branch, subjectRevision(p)), w) } } -- 2.54.0 From 396893dcacca1604ee639a88550f4a662399b1dd Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:25:58 +0100 Subject: [PATCH 29/30] Design 47 H1: the pipeline-runner seat names a pipeline's key and its subjects by revision The seat's key form of a pipeline is ., and its four per-pipeline subjects are said on . (the definition and generated contracts the same as the runner branch's). The indexer pins a revision for the pipeline keyed PipelinesByBranchRevisionKey, and this branch's runner copy follows the renamed generated functions; the wire form is unchanged. Builds: design 47 H H-A-213 (4), H-A-219, H-A-221 --- .../go/seats/meshpipelinerunner/contract.go | 56 +++++++++---------- .../seats/meshpipelinerunner/contract_test.go | 6 +- .../mesh-pipeline-runner/fixtures/states.json | 6 +- .../seats/mesh-pipeline-runner/seat.json | 26 ++++----- .../ts/seats/meshpipelinerunner/index.d.ts | 34 +++++------ .../ts/seats/meshpipelinerunner/index.js | 30 +++++----- .../internal/index/catalog_revisions.go | 2 +- .../internal/pipeline/read.go | 2 +- .../internal/pipeline/start.go | 2 +- .../internal/pipeline/start_test.go | 2 +- 10 files changed, 83 insertions(+), 83 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go index 058625fb..8b5f340e 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go @@ -65,9 +65,9 @@ const ( // StreamPipelineStarted is the stream pipeline-started is kept in. const StreamPipelineStarted = "SEAT_mesh-pipeline-runner_pipeline-started" -// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started... -func SubjectPipelineStarted(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source +// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started... +func SubjectPipelineStarted(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision } // SubjectPipelineStartedAll hears every pipeline-started. @@ -76,9 +76,9 @@ const SubjectPipelineStartedAll = "mesh.seat.mesh-pipeline-runner.event.pipeline // StreamPipelineEnded is the stream pipeline-ended is kept in. const StreamPipelineEnded = "SEAT_mesh-pipeline-runner_pipeline-ended" -// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended... -func SubjectPipelineEnded(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source +// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended... +func SubjectPipelineEnded(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision } // SubjectPipelineEndedAll hears every pipeline-ended. @@ -120,9 +120,9 @@ const SubjectStepEndedAll = "mesh.seat.mesh-pipeline-runner.event.step-ended.*.* // StreamWaitHeld is the stream wait-held is kept in. const StreamWaitHeld = "SEAT_mesh-pipeline-runner_wait-held" -// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held... -func SubjectWaitHeld(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source +// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held... +func SubjectWaitHeld(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision } // SubjectWaitHeldAll hears every wait-held. @@ -131,9 +131,9 @@ const SubjectWaitHeldAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*" // StreamWaitEnded is the stream wait-ended is kept in. const StreamWaitEnded = "SEAT_mesh-pipeline-runner_wait-ended" -// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended... -func SubjectWaitEnded(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source +// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended... +func SubjectWaitEnded(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision } // SubjectWaitEndedAll hears every wait-ended. @@ -156,11 +156,11 @@ const ( StateStepOutputs = "step-outputs" ) -// PipelinesByBranchSourceKey is the key of one value of the state pipelines under the form ".". -func PipelinesByBranchSourceKey(branch, source string) string { return branch + "." + source } +// PipelinesByBranchRevisionKey is the key of one value of the state pipelines under the form ".". +func PipelinesByBranchRevisionKey(branch, revision string) string { return branch + "." + revision } -// ParsePipelinesByBranchSourceKey reads the parts of a key of the form ".", in the order of the form; the last result is false for a key of another form. -func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) { +// ParsePipelinesByBranchRevisionKey reads the parts of a key of the form ".", in the order of the form; the last result is false for a key of another form. +func ParsePipelinesByBranchRevisionKey(key string) (string, string, bool) { rest := key v0, after0, found0 := strings.Cut(rest, ".") if !found0 || v0 == "" { @@ -178,9 +178,9 @@ func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) { return v0, v2, true } -// DecodePipelinesByBranchSourceValue reads one value of the state pipelines under the form ".", refusing a field PipelinesByBranchSourceValue does not know and any text after the value. -func DecodePipelinesByBranchSourceValue(raw []byte) (PipelinesByBranchSourceValue, error) { - var v PipelinesByBranchSourceValue +// DecodePipelinesByBranchRevisionValue reads one value of the state pipelines under the form ".", refusing a field PipelinesByBranchRevisionValue does not know and any text after the value. +func DecodePipelinesByBranchRevisionValue(raw []byte) (PipelinesByBranchRevisionValue, error) { + var v PipelinesByBranchRevisionValue dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&v); err != nil { @@ -670,7 +670,7 @@ type Pipeline struct { // the modules the pipeline moves, its take-in's entries (design 47 H §746) Movers []Mover `json:"movers"` Outcome string `json:"outcome"` - // the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) + // the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) Pipeline string `json:"pipeline"` // for a pipeline a package's publish started, that publish (A-202) Published *Published `json:"published,omitempty"` @@ -720,7 +720,7 @@ type PipelineEnded struct { // per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2) Movers []MoverEnded `json:"movers"` Outcome string `json:"outcome"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // the catalog graph revision of the branch the pipeline runs for Revision int64 `json:"revision,omitempty"` @@ -753,7 +753,7 @@ type PipelineStarted struct { Call string `json:"call,omitempty"` // the pipeline's movers Movers []Mover `json:"movers"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // for a pipeline a package's publish started, that publish (A-202) Published *Published `json:"published,omitempty"` @@ -814,7 +814,7 @@ type StepEvent struct { // the module's build-input fingerprint at the commit Fingerprint string `json:"fingerprint,omitempty"` Module string `json:"module"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` Result StepResult `json:"result"` // the catalog graph revision of the branch the pipeline runs for @@ -966,7 +966,7 @@ type ToolchainUsed struct { Version string `json:"version"` } -// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72). +// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221). type Wait struct { Artifact string `json:"artifact,omitempty"` // the step's own ask, where it has one (design 47 G §731) @@ -983,7 +983,7 @@ type Wait struct { Module string `json:"module,omitempty"` // what the step waits on (design 47 *Health and going back*) On string `json:"on"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // the catalog graph revision of the branch the pipeline runs for Revision int64 `json:"revision,omitempty"` @@ -1024,7 +1024,7 @@ const ( // Waiting is a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72). type Waiting struct { - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` } @@ -1355,8 +1355,8 @@ type FullEvent = Fill // RoomEvent is the body of the event room. type RoomEvent = Fill -// PipelinesByBranchSourceValue is one value of the state pipelines under the key .. -type PipelinesByBranchSourceValue = Pipeline +// PipelinesByBranchRevisionValue is one value of the state pipelines under the key .. +type PipelinesByBranchRevisionValue = Pipeline // PipelinesByBranchSourcePublishedPackageVersionValue is one value of the state pipelines under the key ..published... type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go index 23b59e9c..4aaa6d53 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go @@ -45,7 +45,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")}, {name: "a take-in's pipeline kept, its steps naming their mover", subject: "", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, @@ -73,13 +73,13 @@ func TestFixtures(t *testing.T) { }, subjectIs: nil}, {name: "a pipeline waiting its turn, its seq given", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, {name: "a waiting pipeline names the movers a newer take-in took out of it", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json index cb8363f2..9229229f 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json @@ -1,7 +1,7 @@ [ { "name": "a take-in's pipeline kept, its steps naming their mover", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3589", "branch": "main", @@ -200,7 +200,7 @@ }, { "name": "a pipeline waiting its turn, its seq given", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3590", "branch": "main", @@ -224,7 +224,7 @@ }, { "name": "a waiting pipeline names the movers a newer take-in took out of it", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3590", "branch": "main", diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json index 5bf14a52..fd84b289 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json @@ -672,7 +672,7 @@ "events": [ { "name": "pipeline-started", - "subject": "pipeline-started..", + "subject": "pipeline-started..", "body": { "$ref": "#/defs/PipelineStarted" }, @@ -683,7 +683,7 @@ }, { "name": "pipeline-ended", - "subject": "pipeline-ended..", + "subject": "pipeline-ended..", "body": { "$ref": "#/defs/PipelineEnded" }, @@ -727,7 +727,7 @@ }, { "name": "wait-held", - "subject": "wait-held..", + "subject": "wait-held..", "body": { "$ref": "#/defs/Wait" }, @@ -738,7 +738,7 @@ }, { "name": "wait-ended", - "subject": "wait-ended..", + "subject": "wait-ended..", "body": { "$ref": "#/defs/Wait" }, @@ -778,11 +778,11 @@ "description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747), and one key per mover of a pipeline holding that mover's steps (H-A-187)", "key-forms": [ { - "key": ".", + "key": ".", "value": { "$ref": "#/defs/Pipeline" }, - "description": "a take-in's pipeline" + "description": "a pipeline of one catalog revision, keyed by the revision's number (design 47 H-A-213 (4), H-A-219); one kept from before under its source has the same form" }, { "key": "..published..", @@ -1199,7 +1199,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)" + "description": "the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)" }, "branch": { "type": "string", @@ -1313,7 +1313,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "branch": { "type": "string", @@ -1388,7 +1388,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "branch": { "type": "string", @@ -1495,7 +1495,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "module": { "type": "string" @@ -1550,7 +1550,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "module": { "type": "string", @@ -1635,7 +1635,7 @@ "description": "the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)" } }, - "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72)", + "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221)", "required": [ "v", "pipeline", @@ -1978,7 +1978,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" } }, "description": "a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72)", diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts index 748152b9..5c11adad 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -40,13 +40,13 @@ export declare const family: { readonly name: "mesh-pipeline-runner.events"; rea export declare const pipelineStartedStream: "SEAT_mesh-pipeline-runner_pipeline-started"; -export declare function pipelineStartedSubject(branch: string, source: string): string; +export declare function pipelineStartedSubject(branch: string, revision: string): string; export declare const pipelineStartedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*"; export declare const pipelineEndedStream: "SEAT_mesh-pipeline-runner_pipeline-ended"; -export declare function pipelineEndedSubject(branch: string, source: string): string; +export declare function pipelineEndedSubject(branch: string, revision: string): string; export declare const pipelineEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"; @@ -70,13 +70,13 @@ export declare const stepEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event. export declare const waitHeldStream: "SEAT_mesh-pipeline-runner_wait-held"; -export declare function waitHeldSubject(branch: string, source: string): string; +export declare function waitHeldSubject(branch: string, revision: string): string; export declare const waitHeldSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*"; export declare const waitEndedStream: "SEAT_mesh-pipeline-runner_wait-ended"; -export declare function waitEndedSubject(branch: string, source: string): string; +export declare function waitEndedSubject(branch: string, revision: string): string; export declare const waitEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*"; @@ -94,11 +94,11 @@ export declare const states: { readonly stepOutputs: "step-outputs"; }; -/** The key of one value of the state pipelines under the form ".". */ -export declare function pipelinesByBranchSourceKey(branch: string, source: string): string; +/** The key of one value of the state pipelines under the form ".". */ +export declare function pipelinesByBranchRevisionKey(branch: string, revision: string): string; -/** The parts of a key of the form ".", or undefined for a key of another form. */ -export declare function parsePipelinesByBranchSourceKey(_key: string): { branch: string; source: string } | undefined; +/** The parts of a key of the form ".", or undefined for a key of another form. */ +export declare function parsePipelinesByBranchRevisionKey(_key: string): { branch: string; revision: string } | undefined; /** The key of one value of the state pipelines under the form "..published..". */ export declare function pipelinesByBranchSourcePublishedPackageVersionKey(branch: string, source: string, packagePart: string, version: string): string; @@ -261,7 +261,7 @@ export interface Pipeline { /** the modules the pipeline moves, its take-in's entries (design 47 H §746) */ "movers": Mover[]; "outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled"; - /** the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */ + /** the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */ "pipeline": string; /** for a pipeline a package's publish started, that publish (A-202) */ "published"?: Published; @@ -300,7 +300,7 @@ export interface PipelineEnded { /** per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2) */ "movers": MoverEnded[]; "outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled"; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** the catalog graph revision of the branch the pipeline runs for */ "revision"?: number; @@ -322,7 +322,7 @@ export interface PipelineStarted { "call"?: string; /** the pipeline's movers */ "movers": Mover[]; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** for a pipeline a package's publish started, that publish (A-202) */ "published"?: Published; @@ -383,7 +383,7 @@ export interface StepEvent { /** the module's build-input fingerprint at the commit */ "fingerprint"?: string; "module": string; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; "result": StepResult; /** the catalog graph revision of the branch the pipeline runs for */ @@ -490,7 +490,7 @@ export interface ToolchainUsed { "version": string; } -/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72) */ +/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221) */ export interface Wait { "artifact"?: string; /** the step's own ask, where it has one (design 47 G §731) */ @@ -507,7 +507,7 @@ export interface Wait { "module"?: string; /** what the step waits on (design 47 *Health and going back*) */ "on": "person" | "build node" | "revision view" | "pipelines-at-once"; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** the catalog graph revision of the branch the pipeline runs for */ "revision"?: number; @@ -527,7 +527,7 @@ export interface Wait { /** a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72) */ export interface Waiting { - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; } @@ -800,8 +800,8 @@ export type FullEvent = Fill; /** is the body of the event room. */ export type RoomEvent = Fill; -/** is one value of the state pipelines under the key .. */ -export type PipelinesByBranchSourceValue = Pipeline; +/** is one value of the state pipelines under the key .. */ +export type PipelinesByBranchRevisionValue = Pipeline; /** is one value of the state pipelines under the key ..published... */ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline; diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js index 213a3df3..d8ff25e6 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js @@ -40,16 +40,16 @@ export const family = { name: "mesh-pipeline-runner.events", maxAgeDays: 7 }; export const pipelineStartedStream = "SEAT_mesh-pipeline-runner_pipeline-started"; -export function pipelineStartedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source; +export function pipelineStartedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision; } export const pipelineStartedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*"; export const pipelineEndedStream = "SEAT_mesh-pipeline-runner_pipeline-ended"; -export function pipelineEndedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source; +export function pipelineEndedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision; } export const pipelineEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"; @@ -80,16 +80,16 @@ export const stepEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.step-en export const waitHeldStream = "SEAT_mesh-pipeline-runner_wait-held"; -export function waitHeldSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source; +export function waitHeldSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision; } export const waitHeldSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*"; export const waitEndedStream = "SEAT_mesh-pipeline-runner_wait-ended"; -export function waitEndedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source; +export function waitEndedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision; } export const waitEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*"; @@ -108,23 +108,23 @@ export const states = { stepOutputs: "step-outputs", }; -/** The key of one value of the state pipelines under the form ".". */ -export function pipelinesByBranchSourceKey(branch, source) { - return branch + "." + source; +/** The key of one value of the state pipelines under the form ".". */ +export function pipelinesByBranchRevisionKey(branch, revision) { + return branch + "." + revision; } -/** The parts of a key of the form ".", or undefined for a key of another form. */ -export function parsePipelinesByBranchSourceKey(_key) { +/** The parts of a key of the form ".", or undefined for a key of another form. */ +export function parsePipelinesByBranchRevisionKey(_key) { let _rest = _key; const _at0 = _rest.indexOf("."); if (_at0 <= 0) return undefined; const branch = _rest.slice(0, _at0); _rest = _rest.slice(_at0 + 1); if (_rest === "") return undefined; - const source = _rest; + const revision = _rest; _rest = ""; if (_rest !== "") return undefined; - return { branch, source }; + return { branch, revision }; } /** The key of one value of the state pipelines under the form "..published..". */ diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 548e8c93..38a96ead 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -412,7 +412,7 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int } // The take-in's revision pinned for its one pipeline before any judge can remove it (§743, §749, A-449a). if len(movers) > 0 && ix.Pins != nil { - record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(number, 10))) + record := PipelineRecord(runner.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(number, 10))) if err := ix.pin(ctx, PinKey(branch, number), record); err != nil { ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, number), record, err) return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, number), err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index e6a0762b..668d2ed8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -206,7 +206,7 @@ func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) if raw == nil { return seat.Pipeline{}, false, nil } - p, err := seat.DecodePipelinesByBranchSourceValue(raw) + p, err := seat.DecodePipelinesByBranchRevisionValue(raw) if err != nil { return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index cfb8cc48..4c9c49cc 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -50,7 +50,7 @@ type Begin struct { // Key is a take-in's pipeline key in the state pipelines, `.`, the branch escaped as a key's name is // (design 47 H §746). func Key(branch string, source int64) string { - return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10)) + return seat.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(source, 10)) } // PublishedKey is the key of a pipeline a package's publish starts, `..published..`, diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 019127c7..07198145 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -40,7 +40,7 @@ func TestStarts(t *testing.T) { if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Movers: []seat.Mover{{Module: "app"}}}); err != nil { t.Fatal(err) } - if b, r, ok := seat.ParsePipelinesByBranchSourceKey(Key("release.1", 3)); !ok || r != "3" || b == "release" { + if b, r, ok := seat.ParsePipelinesByBranchRevisionKey(Key("release.1", 3)); !ok || r != "3" || b == "release" { t.Errorf("the key of release.1 is %q", Key("release.1", 3)) } if _, err := s.Start(Begin{Branch: "main", Movers: []seat.Mover{{Module: "app"}}}); err == nil { -- 2.54.0 From 6087306dfd64272903bef98774c1ac9abec6e89d Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:43:16 +0100 Subject: [PATCH 30/30] Design 47 H1: the pipelines store guards its shared maps, so the runner's start no longer crashes (H-A-187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner crashed at every start on a live bus with "concurrent map writes" in MoverStore.Get: the waiting pipelines of each branch are placed from goroutines of their own, and the store's last reads and inline marks were written under each pipeline's own lock alone. The store's mutex now guards the lock table, the last reads and the inline marks; behaviour is unchanged. The run test starts the runner over six kept pipelines in the old form, one of 437 steps, so the start path runs under the race detector. Builds: design 47 H H-A-187, H §774 --- .../run_integration_test.go | 36 +++++++++++++ .../internal/pipeline/moverkeys.go | 54 +++++++++++++++---- 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index b5bf1b95..a7b2e3cb 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -72,6 +72,9 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes t.Fatal(err) } } + // Kept pipelines in the old form, their steps inline in the record, as the runner's start finds them on a live bus + // (main.41-46, one of 437 steps): each start path reads them at once from its own goroutine (H-A-187, H §774). + keptPipelines(t, ctx, js, module) // The step log, the runner's old logs section (§470): the stream the bus makes for it. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { t.Fatal(err) @@ -340,3 +343,36 @@ func waitFor(t *testing.T, ctx context.Context, logs *syncBuffer, cond func() bo } t.Fatalf("it did not come about in time; the runner said:\n%s", logs.String()) } + +// keptPipelines writes six ended pipelines of main, main.41 to main.46, in the record form before H-A-187: every step inline, +// main.46 with 437 of them. +func keptPipelines(t *testing.T, ctx context.Context, js jetstream.JetStream, module string) { + t.Helper() + kv, err := js.KeyValue(ctx, module+"_pipelines") + if err != nil { + t.Fatal(err) + } + at := time.Date(2026, 11, 20, 23, 0, 0, 0, time.UTC) + for source := int64(41); source <= 46; source++ { + movers, count := 3, 12 + if source == 46 { + movers, count = 73, 437 + } + p := seat.Pipeline{V: 1, Pipeline: fmt.Sprintf("main.%d", source), Branch: "main", Source: source, Revision: source, + Outcome: seat.PipelineOutcomePassed, Started: &at, Ended: &at, EndCount: 1} + for i := range movers { + p.Movers = append(p.Movers, seat.Mover{Module: fmt.Sprintf("kept-%02d", i), Commit: "c0", Fingerprint: fmt.Sprintf("src2:k%02d", i)}) + } + for i := range count { + p.Steps = append(p.Steps, seat.StepResult{Module: p.Movers[i%movers].Module, Step: seat.StepResultStepBuild, + Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomePassed, Started: &at, Ended: &at}) + } + raw, err := json.Marshal(p) + if err != nil { + t.Fatal(err) + } + if _, err := kv.Put(ctx, p.Pipeline, raw); err != nil { + t.Fatal(err) + } + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go index 9b7f1e07..bc21891e 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go @@ -66,6 +66,8 @@ type MoverStore struct { Raw RawState Logf func(string, ...any) + // mu guards the three maps, which every pipeline's reads and writes share across goroutines (H §774): the lock table, + // the last reads and the records said inline. A pipeline's own lock is held across its read or write. mu sync.Mutex locks map[string]*sync.Mutex read map[string]readAt @@ -93,6 +95,42 @@ func (s *MoverStore) lock(key string) func() { return l.Unlock } +// seen is the last read of pipeline key, under mu. +func (s *MoverStore) seen(key string) (readAt, bool) { + s.mu.Lock() + defer s.mu.Unlock() + r, ok := s.read[key] + return r, ok +} + +// setSeen keeps the last read of pipeline key, under mu. +func (s *MoverStore) setSeen(key string, r readAt) { + s.mu.Lock() + defer s.mu.Unlock() + s.read[key] = r +} + +// markInline says whether pipeline key is said inline for the first time, marking it, under mu. +func (s *MoverStore) markInline(key string) bool { + s.mu.Lock() + defer s.mu.Unlock() + first := !s.inline[key] + s.inline[key] = true + return first +} + +// forget drops pipeline key's last read, and its inline mark when inline is true, under mu. +func (s *MoverStore) forget(key string, read, inline bool) { + s.mu.Lock() + defer s.mu.Unlock() + if read { + delete(s.read, key) + } + if inline { + delete(s.inline, key) + } +} + func (s *MoverStore) logf(f string, a ...any) { if s.Logf != nil { s.Logf(f, a...) @@ -233,10 +271,9 @@ func (s *MoverStore) Get(key string) (seat.Pipeline, uint64, error) { if err != nil { return seat.Pipeline{}, 0, err } - s.read[key] = readAt{rev: rev, movers: revs} + s.setSeen(key, readAt{rev: rev, movers: revs}) if len(p.Steps) > 0 { - if !s.inline[key] { - s.inline[key] = true + if s.markInline(key) { s.logf("the pipeline %s carries its %d steps inline in its record: they are read whole and split into one key per mover at its next write (H-A-187)", key, len(p.Steps)) } return p, rev, nil @@ -271,7 +308,7 @@ func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint6 return 0, err } // A mover key written since the read is another write between: the read named its revision. - if seen, ok := s.read[key]; ok && seen.rev == revision { + if seen, ok := s.seen(key); ok && seen.rev == revision { for k, r := range revs { if seen.movers[k] != r { return 0, ErrChanged @@ -301,9 +338,9 @@ func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint6 } } else { s.logf("the pipeline %s's steps are split into %d mover keys and its record no longer carries them (H-A-187)", key, len(want)) - delete(s.inline, key) + s.forget(key, false, true) } - s.read[key] = readAt{rev: rev, movers: after} + s.setSeen(key, readAt{rev: rev, movers: after}) return rev, nil } @@ -367,7 +404,7 @@ func (s *MoverStore) Create(key string, value []byte) error { if err != nil { return err } - s.read[key] = readAt{rev: rev, movers: after} + s.setSeen(key, readAt{rev: rev, movers: after}) return nil } @@ -386,8 +423,7 @@ func (s *MoverStore) Delete(key string) error { if err == nil { err = s.Raw.Delete(key) } - delete(s.read, key) - delete(s.inline, key) + s.forget(key, true, true) unlock() return err } -- 2.54.0