From 0fcc614e25b95128880fd96e8ffd0fac7b89f332 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 22:59:31 +0100 Subject: [PATCH 1/3] Build an ask of an older kept revision by reading it by number, never waiting (H-A-205) Two pipelines of consecutive revisions ran at once (main.45, 22:5x CET): the picture held source revision 46, whose entries had moved on, so the older pipeline's builds waited for a picture that only moves forward and failed "revision 45 of main not in the builder's picture within 2m0s". An ask of a source revision below the newest the picture holds now reads its catalog revision by number through the catalog seat's verb revision (kept while its pipeline runs, H-A-147) and is built when that revision is of its source and names every entry it asks; only a revision above the newest is waited for (H-A-155). Builds: design 47 H H-A-205, H-A-147, H-A-155 --- .../cmd/mesh-delivery-builder/keptrevision.go | 47 +++++++++++++++++++ .../cmd/mesh-delivery-builder/main.go | 2 + .../internal/build/revisionwait.go | 32 +++++++++++++ .../internal/build/run.go | 4 ++ .../internal/build/step_test.go | 38 ++++++++++++++- 5 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/keptrevision.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/keptrevision.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/keptrevision.go new file mode 100644 index 00000000..f9d6e55e --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/keptrevision.go @@ -0,0 +1,47 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// keptRevisionVia reads one kept revision of a branch by its number through the catalog seat's verb revision, page by page +// (design 47 H H-A-205, H-A-147): the source revision it was taken in from and the fingerprint of each module it names. +func keptRevisionVia(call toolCaller) func(context.Context, string, int64) (int64, map[string]string, error) { + return func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) { + fps := map[string]string{} + var source int64 + after := "" + for { + raw, err := call(meshcatalogclient.RevisionVerb, meshcatalog.RevisionInput{Branch: keys.EscapeName(branch), Revision: revision, After: after}) + if err != nil { + return 0, nil, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return 0, nil, fmt.Errorf("%s answered: %s", meshcatalogclient.RevisionVerb, said) + } + var a meshcatalog.RevisionAnswer + if err := json.Unmarshal(answer, &a); err != nil { + return 0, nil, fmt.Errorf("%s answered no revision: %w", meshcatalogclient.RevisionVerb, err) + } + if a.NoneYet || a.Revision == nil || a.Revision.Revision != revision { + return 0, nil, fmt.Errorf("%s answered no revision %d of %s", meshcatalogclient.RevisionVerb, revision, branch) + } + source = a.Revision.Source + for _, e := range a.Entries { + fps[e.Module] = e.Fp + } + if a.Next == "" { + return source, fps, nil + } + after = a.Next + } + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index a5167640..0faf4eaf 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -469,6 +469,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // a holder of the seat that is down holds no other part of it). half.b = b b.Picture = half.picture + // An ask of a revision below the newest the picture holds is read by number from the catalog seat (H-A-205). + b.KeptRevision = keptRevisionVia(calls.Call) branches.Held = func(string, meshcatalog.CatalogRevision) { b.PictureHeld() } // a revision held wakes a step ask's wait (H-A-162) half.ready = func() { if err := b.Sweep(ctx, half.pictured()...); err != nil { diff --git a/modules/mesh-delivery-builder/internal/build/revisionwait.go b/modules/mesh-delivery-builder/internal/build/revisionwait.go index 2a052bbc..2ae3c745 100644 --- a/modules/mesh-delivery-builder/internal/build/revisionwait.go +++ b/modules/mesh-delivery-builder/internal/build/revisionwait.go @@ -20,6 +20,11 @@ import ( // revision its view holds (PictureHeld, H-A-162), never a // timer or a poll, up to RevisionWait (the setting revision-wait-minutes); past it the ask fails, naming the revision and the // branch, and nothing is built. A step ask with no source (a run on request) does not wait. +// +// **The picture holds every kept revision, not only the newest** (design 47 H H-A-205, H-A-147): an ask of a source revision +// below the newest the picture holds, whose entries the picture no longer names, asks the catalog seat's verb revision for +// its catalog revision by number, once, and is built when that kept revision is of its source and names every entry it +// asks; it never waits, since the picture only moves on. Only a revision above the newest is waited for. // DefaultRevisionWait is the bound of a step ask's wait for its revision, its setting revision-wait-minutes' default (H-A-156). const DefaultRevisionWait = 2 * time.Minute @@ -68,12 +73,39 @@ func (b *Builder) holdsRevision(branch string, source int64, entries []meshbuild return true } +// keptRevisionHolds answers no outcome when the catalog keeps the ask's revision (its catalog revision by number, of its +// source revision) and that revision names every entry of the ask; else failed or could not run, with its words. +func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, held int64) (outcome, said string) { + below := fmt.Sprintf("revision %d of %s is below the %d the builder's picture holds", run.Source, run.Branch, held) + if b.KeptRevision == nil || run.Revision <= 0 { + return meshbuilds.StepEndedOutcomeFailed, below + ", and the ask names no catalog revision to read it by" + } + source, fps, err := b.KeptRevision(ctx, run.Branch, run.Revision) + if err != nil { + b.logf("%s: %s, and the catalog's revision %d cannot be read: %v", run.Pipeline, below, run.Revision, err) + return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s, and the catalog's revision %d cannot be read: %v", below, run.Revision, err) + } + if source != run.Source { + return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d is of source revision %d", below, run.Revision, source) + } + for _, e := range run.Entries { + if fps[e.Module] != e.Fingerprint { + return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d names %s at %q, not %s", below, run.Revision, e.Module, fps[e.Module], e.Fingerprint) + } + } + b.logf("%s: %s; the catalog keeps revision %d with its entries: the build goes on", run.Pipeline, below, run.Revision) + return "", "" +} + // awaitRevision returns no outcome once the builder's picture of the ask's branch holds its source revision; past the bound the // step's outcome failed, at the builder's stop could not run, each with its words. func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (outcome, said string) { if b.Picture == nil || run.Source <= 0 || run.Branch == "" { return "", "" } + if held, ok := b.Picture(run.Branch); ok && run.Source < held && !b.holdsRevision(run.Branch, run.Source, run.Entries) { + return b.keptRevisionHolds(ctx, run, held) + } bound := b.revisionWait() notHeld := fmt.Sprintf("revision %d of %s not in the builder's picture", run.Source, run.Branch) wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8) diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ba6f3e3c..4bfb6df4 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -110,6 +110,10 @@ type Builder struct { // Picture answers the source revision the builder's picture of a watched branch holds, false while it holds none (design 47 // H H-A-155, H-A-141): a step ask naming a newer one waits for it. Nil waits for nothing. Picture func(branch string) (source int64, held bool) + // KeptRevision asks the catalog seat's verb revision for one kept revision of a branch by its number (design 47 H + // H-A-205, H-A-147): the source revision it was taken in from and the fingerprint of each module it names. A step ask of + // a source revision below the newest the picture holds is built when that kept revision names its entries. Nil reads none. + KeptRevision func(ctx context.Context, branch string, revision int64) (source int64, fingerprints map[string]string, err error) // RevisionWait bounds a step ask's wait for its revision, its setting revision-wait-minutes (H-A-156); DefaultRevisionWait // when 0. RevisionWait time.Duration diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 6a777cf2..2bf0847a 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -119,6 +119,7 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T) } t.Run("H-A-155: an ask waits for the picture to take its revision in, then builds with its version reserved", aStepAskWaitsForItsRevision) t.Run("H-A-155: past the bound the ask fails naming its revision, and nothing is built", aStepAskPastItsRevisionBoundFails) + t.Run("H-A-205: an ask of a revision below the picture's newest is read by number and built", anOlderKeptRevisionIsReadByNumber) t.Run("H-A-162: a revision held that changes no entry wakes the ask", aRevisionHeldWakesTheAsk) t.Run("H-A-165: a package with no pin waits for its release and is built with it", aPackageWithNoPinWaitsForItsRelease) t.Run("H-A-165: past its bound a package with no release fails the step", aPackageWithNoReleaseFailsPastItsBound) @@ -329,6 +330,11 @@ func aStepAskWaitsForItsRevision(t *testing.T) { held.Store(40) asked := make(chan struct{}, 1) r, res, entry := revisionRig(t, &held, asked) + var keptReads atomic.Int64 // H-A-205: a revision above the newest is waited for, never read by number + r.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { + keptReads.Add(1) + return 0, nil, errors.New("not asked") + } ended := make(chan meshbuilds.StepEnded, 1) go func() { ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41, @@ -353,8 +359,36 @@ func aStepAskWaitsForItsRevision(t *testing.T) { t.Fatal("the ask was not woken by the entry the picture took in") } r.b.Wait() - if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 { - t.Errorf("end = %+v, reserved from %+v; want passed with one version reserved from revision 41", end, res.sources) + if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 || keptReads.Load() != 0 { + t.Errorf("end = %+v, reserved from %+v, kept revisions read %d; want passed with one version reserved from revision 41, none read", end, res.sources, keptReads.Load()) + } +} + +// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, whose entry of m moved +// on, while the older pipeline asks the build of revision 45. Its kept catalog revision is read by number and the build runs; +// a kept revision naming another fingerprint fails the step, naming it. +func anOlderKeptRevisionIsReadByNumber(t *testing.T) { + var held atomic.Int64 + held.Store(46) + r, _, _ := revisionRig(t, &held, make(chan struct{}, 1)) + r.b.RevisionWait = time.Nanosecond // a wait would fail at once: the older revision is never waited for + var read []string + r.b.KeptRevision = func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) { + read = append(read, fmt.Sprintf("%s %d", branch, revision)) + return 45, map[string]string{"m": fp1}, nil + } + run := meshbuilds.StepRun{Pipeline: "main.45.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 45, Source: 45, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}} + end := r.b.BuildStep(t.Context(), run) + r.b.Wait() + if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 45"}) { + t.Errorf("end = %+v, kept revisions read %v; want passed, revision 45 of main read once", end, read) + } + r.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { + return 45, map[string]string{"m": "src2:other"}, nil + } + if end := r.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") { + t.Errorf("end = %+v; want failed naming the fingerprint the kept revision names", end) } } From 014f7b7c2f39ee72d639f3a9e8f766685f626ae1 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 23:03:45 +0100 Subject: [PATCH 2/3] Reserve the version of an ask admitted by its kept revision (H-A-205) An ask of an older kept revision was admitted, but after a builder restart its fingerprint had never been heard on the branch, so its build neither reserved a version nor was checked against one and passed silently. Each asked fingerprint is now recorded as the branch's (its source revision, and first as the module's registration says), without moving what the picture heard, so the newer fingerprint's build is not superseded. Builds: design 47 H H-A-205, H-A-155 --- .../cmd/mesh-delivery-builder/main.go | 1 + .../internal/build/revisionwait.go | 13 +++++++++++++ .../mesh-delivery-builder/internal/build/run.go | 4 ++++ .../internal/build/step_test.go | 15 +++++++++++---- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 0faf4eaf..60097089 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -471,6 +471,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro b.Picture = half.picture // An ask of a revision below the newest the picture holds is read by number from the catalog seat (H-A-205). b.KeptRevision = keptRevisionVia(calls.Call) + b.FirstBranch = half.first branches.Held = func(string, meshcatalog.CatalogRevision) { b.PictureHeld() } // a revision held wakes a step ask's wait (H-A-162) half.ready = func() { if err := b.Sweep(ctx, half.pictured()...); err != nil { diff --git a/modules/mesh-delivery-builder/internal/build/revisionwait.go b/modules/mesh-delivery-builder/internal/build/revisionwait.go index 2ae3c745..274ec20c 100644 --- a/modules/mesh-delivery-builder/internal/build/revisionwait.go +++ b/modules/mesh-delivery-builder/internal/build/revisionwait.go @@ -93,6 +93,19 @@ func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d names %s at %q, not %s", below, run.Revision, e.Module, fps[e.Module], e.Fingerprint) } } + // Each asked fingerprint is recorded as the branch's, so its build reserves its version (H-A-155) as one the picture + // took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded. + b.mu.Lock() + for _, e := range run.Entries { + key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint) + if err != nil || b.origin[key].Branch != "" { + continue + } + first := b.FirstBranch == nil || b.FirstBranch(run.Branch, e.Module) + b.trunkFP[key] = true + b.origin[key] = Origin{Branch: run.Branch, Source: run.Source, First: first} + } + b.mu.Unlock() b.logf("%s: %s; the catalog keeps revision %d with its entries: the build goes on", run.Pipeline, below, run.Revision) return "", "" } diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 4bfb6df4..b87cfd1a 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -114,6 +114,10 @@ type Builder struct { // H-A-205, H-A-147): the source revision it was taken in from and the fingerprint of each module it names. A step ask of // a source revision below the newest the picture holds is built when that kept revision names its entries. Nil reads none. KeptRevision func(ctx context.Context, branch string, revision int64) (source int64, fingerprints map[string]string, err error) + // FirstBranch says whether branch is the first watched branch of the module's registration, the one whose builds reserve + // a version (design 47 G §371): an ask admitted by its kept revision is recorded as that branch's (H-A-205, H-A-155). Nil + // takes every branch as the first, as an origin never heard is. + FirstBranch func(branch, module string) bool // RevisionWait bounds a step ask's wait for its revision, its setting revision-wait-minutes (H-A-156); DefaultRevisionWait // when 0. RevisionWait time.Duration diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 2bf0847a..2a0e23dd 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -370,8 +370,9 @@ func aStepAskWaitsForItsRevision(t *testing.T) { func anOlderKeptRevisionIsReadByNumber(t *testing.T) { var held atomic.Int64 held.Store(46) - r, _, _ := revisionRig(t, &held, make(chan struct{}, 1)) - r.b.RevisionWait = time.Nanosecond // a wait would fail at once: the older revision is never waited for + r, res, _ := revisionRig(t, &held, make(chan struct{}, 1)) // a picture at 46 that never heard 45 (a restart) + r.b.RevisionWait = time.Nanosecond // a wait would fail at once: the older revision is never waited for + r.b.FirstBranch = func(branch, module string) bool { return branch == "main" && module == "m" } var read []string r.b.KeptRevision = func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) { read = append(read, fmt.Sprintf("%s %d", branch, revision)) @@ -384,10 +385,16 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) { if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 45"}) { t.Errorf("end = %+v, kept revisions read %v; want passed, revision 45 of main read once", end, read) } - r.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { + // Its fingerprint is recorded as the branch's: its build reserves its version from source revision 45 (H-A-155). + if len(res.sources) != 1 || res.sources[0].Revision != 45 { + t.Errorf("reserved from %+v; want one version reserved from revision 45", res.sources) + } + r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1)) + r2.b.RevisionWait = time.Nanosecond + r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { return 45, map[string]string{"m": "src2:other"}, nil } - if end := r.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") { + if end := r2.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") { t.Errorf("end = %+v; want failed naming the fingerprint the kept revision names", end) } } From 3f62410c22e8ec5c7e72f4269bd38ffb1bbd7945 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 00:16:53 +0100 Subject: [PATCH 3/3] Design 47 H1 (5c): a package-published names the catalog revision its build was made from One source now has several catalog revisions, so a publish naming branch and source alone does not say which pipeline published it. The seat mesh-builds' package-published gains an optional revision (seatgen regenerated). The version reserved for a build keeps the catalog revision number: from the picture's half when the picture starts the build (the same revision its source comes from), from the step ask's run when the ask reaches the build first. Its package-published says it. A reservation from before the field says none. Builds: design 47 H H-A-218, H-A-220 --- .../cmd/mesh-delivery-builder/catalog.go | 14 +++++++++--- .../cmd/mesh-delivery-builder/catalog_test.go | 17 ++++++++++++++ .../internal/build/numbers.go | 2 +- .../internal/build/revisionwait.go | 2 +- .../internal/build/run.go | 4 +++- .../internal/build/step.go | 22 +++++++++++++++++++ .../internal/build/step_test.go | 12 +++++----- .../internal/versions/reserve.go | 14 ++++++++---- .../internal/versions/reserve_test.go | 8 +++---- .../go/seats/meshbuilds/contract.go | 2 ++ .../seats/mesh-builds/seat.json | 4 ++++ .../ts/seats/meshbuilds/index.d.ts | 2 ++ 12 files changed, 84 insertions(+), 19 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go index 1d935875..5ff5f0bd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go @@ -101,9 +101,7 @@ func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, e } origin := build.Origin{Branch: branch, First: first} if cat, ok := c.branches.Catalog(branch); ok { - if half, ok := cat.View(); ok { - origin.Source = half.Revision.Source - } + origin = pictureOrigin(origin, cat.View) } var e meshcatalog.CatalogEntry if err := json.Unmarshal(entry, &e); err != nil || e.Module != module { @@ -132,6 +130,16 @@ func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, e } } +// pictureOrigin is the origin of an entry the picture takes in: the source revision and the catalog revision number of the +// half the view holds, one revision for both (design 47 G §371, H-A-218, H-A-220), so a build the picture starts before any +// step ask reserves a version that names its revision; none while the view holds no half. +func pictureOrigin(o build.Origin, view func() (meshcatalogclient.Half, bool)) build.Origin { + if half, ok := view(); ok { + o.Source, o.Revision = half.Revision.Source, half.Revision.Revision + } + return o +} + // picture is build.Builder.Picture: the source revision the view of branch holds (design 47 H H-A-155), false while the branch // is not followed or its view holds none. func (c *catalogHalf) picture(branch string) (int64, bool) { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go index 8d9c45b1..26bdeabd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go @@ -4,7 +4,9 @@ import ( "context" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "mesh-delivery-builder/internal/build" ) // TestTheManifestOfAnEntryIsReadByItsDigest: the builder reads a manifest from the entry that holds it, by the entry's manifest @@ -25,3 +27,18 @@ func TestTheManifestOfAnEntryIsReadByItsDigest(t *testing.T) { t.Errorf("an entry with no manifest kept one: %v, %v", err, m.byFp) } } + +// H-A-220: an entry the picture takes in has the source revision and the catalog revision number of the half its view holds, +// one revision for both; a view holding no half names neither. +func TestAnEntryThePictureTakesInNamesItsHalfsRevision(t *testing.T) { + o := build.Origin{Branch: "main", First: true} + got := pictureOrigin(o, func() (meshcatalogclient.Half, bool) { + return meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Revision: 47, Source: 45}}, true + }) + if got != (build.Origin{Branch: "main", First: true, Source: 45, Revision: 47}) { + t.Errorf("origin = %+v, want source 45 at revision 47", got) + } + if got := pictureOrigin(o, func() (meshcatalogclient.Half, bool) { return meshcatalogclient.Half{}, false }); got != o { + t.Errorf("origin with no half = %+v, want %+v", got, o) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 2aeefc0e..867a24e6 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -194,7 +194,7 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu for _, t := range plan.Tasks { refs = append(refs, module+":"+t.Artifact) } - src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, Rebuild: rebuild} + src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild} for _, t := range plan.Tasks { src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go") } diff --git a/modules/mesh-delivery-builder/internal/build/revisionwait.go b/modules/mesh-delivery-builder/internal/build/revisionwait.go index 274ec20c..0c4ec3d4 100644 --- a/modules/mesh-delivery-builder/internal/build/revisionwait.go +++ b/modules/mesh-delivery-builder/internal/build/revisionwait.go @@ -103,7 +103,7 @@ func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, } first := b.FirstBranch == nil || b.FirstBranch(run.Branch, e.Module) b.trunkFP[key] = true - b.origin[key] = Origin{Branch: run.Branch, Source: run.Source, First: first} + b.origin[key] = Origin{Branch: run.Branch, Source: run.Source, Revision: run.Revision, First: first} } b.mu.Unlock() b.logf("%s: %s; the catalog keeps revision %d with its entries: the build goes on", run.Pipeline, below, run.Revision) diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index b87cfd1a..f4662d49 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -237,7 +237,9 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) { type Origin struct { Branch string Source int64 - First bool + // Revision is the catalog revision number a step ask of the build named (design 47 H H-A-218), 0 until one did. + Revision int64 + First bool } // Entry takes the catalog graph entry of one module on one watched branch in (design 47 G §348, §614) and builds its diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index 09687285..b19cbb25 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -39,6 +39,7 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui if outcome, said := b.awaitRevision(ctx, run); outcome != "" { return finish(outcome, said) } + b.askedRevision(run) var failed []string for _, e := range run.Entries { outcome, said := b.buildEntry(ctx, e) @@ -55,6 +56,27 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui return finish(meshbuilds.StepEndedOutcomePassed, "") } +// askedRevision records on the origin of each entry of run the catalog revision the ask names (design 47 H H-A-218), where +// the origin is of the ask's branch and source revision: the version reserved for its build names it, and so does its +// package-published, so the runner finds the publishing pipeline by branch and revision. +func (b *Builder) askedRevision(run meshbuilds.StepRun) { + if run.Revision <= 0 { + return + } + b.mu.Lock() + defer b.mu.Unlock() + for _, e := range run.Entries { + key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint) + if err != nil { + continue + } + if o, ok := b.origin[key]; ok && o.Branch == run.Branch && o.Source == run.Source { + o.Revision = run.Revision + b.origin[key] = o + } + } +} + // buildEntry builds one entry at its fingerprint and waits for its end. func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outcome, said string) { entry, err := b.StepEntry(ctx, e.Fingerprint) diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 2a0e23dd..ef35acf7 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -362,6 +362,7 @@ func aStepAskWaitsForItsRevision(t *testing.T) { if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 || keptReads.Load() != 0 { t.Errorf("end = %+v, reserved from %+v, kept revisions read %d; want passed with one version reserved from revision 41, none read", end, res.sources, keptReads.Load()) } + } // H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, whose entry of m moved @@ -378,16 +379,17 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) { read = append(read, fmt.Sprintf("%s %d", branch, revision)) return 45, map[string]string{"m": fp1}, nil } - run := meshbuilds.StepRun{Pipeline: "main.45.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 45, Source: 45, + run := meshbuilds.StepRun{Pipeline: "main.47.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 47, Source: 45, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}} end := r.b.BuildStep(t.Context(), run) r.b.Wait() - if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 45"}) { - t.Errorf("end = %+v, kept revisions read %v; want passed, revision 45 of main read once", end, read) + if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 47"}) { + t.Errorf("end = %+v, kept revisions read %v; want passed, revision 47 of main read once", end, read) } // Its fingerprint is recorded as the branch's: its build reserves its version from source revision 45 (H-A-155). - if len(res.sources) != 1 || res.sources[0].Revision != 45 { - t.Errorf("reserved from %+v; want one version reserved from revision 45", res.sources) + // The version reserved names the catalog revision the ask named, which its package-published says (H-A-218). + if len(res.sources) != 1 || res.sources[0].Revision != 45 || res.sources[0].CatalogRevision != 47 { + t.Errorf("reserved from %+v; want one version reserved from source revision 45 at catalog revision 47 (H-A-218)", res.sources) } r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1)) r2.b.RevisionWait = time.Nanosecond diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 0bd46450..635805af 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -65,6 +65,9 @@ type ReservedVersion struct { // (design 47 G §371, A-57): what `package-published` says of the publication. Branch string `json:"branch,omitempty"` Source int64 `json:"source,omitempty"` + // Revision is the catalog revision number the build's step ask named, the indexer's count (design 47 H H-A-218): what + // `package-published` names beside the source, so the runner finds the publishing pipeline; 0 when no step ask named one. + Revision int64 `json:"revision,omitempty"` // Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with // (kind 3), kept for `package-published` (design 47 G §466, §673, A-57). Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"` @@ -91,6 +94,8 @@ type Source struct { // that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication. Branch string Revision int64 + // CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did. + CatalogRevision int64 // Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced // and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b). Rebuild bool @@ -177,7 +182,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts } } out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(), - Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision} + Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision} if src.GoPackage { out.Tag = GoTagOf(out.Version) } @@ -443,11 +448,12 @@ type Marked struct { } // PackagePublished is the seat mesh-builds' package-published of one artifact the holder said published at the reservation's -// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch and -// the source revision the build was made from, and the dependencies its step result named. +// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch, +// the source revision the build was made from and the catalog revision its step ask named (H-A-218), and the dependencies its +// step result named. func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.PackagePublished { at = at.UTC() - return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Module: m.Reserved.Module, + return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Revision: m.Reserved.Revision, Module: m.Reserved.Module, Version: m.Reserved.Version, Digest: digest, Dependencies: m.Reserved.Dependencies[m.Artifact]} } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index 3b68a10b..3c1bed16 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -125,7 +125,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { ctx := context.Background() store := &reserveMem{mem: newMem()} r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"1.4.2"}, nil }} - v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{Branch: "main", Revision: 41}) + v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{Branch: "main", Revision: 41, CatalogRevision: 44}) if err != nil || v.Version != "1.4.3" { t.Fatalf("reserved %q (%v), want 1.4.3", v.Version, err) } @@ -154,12 +154,12 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { if err != nil || len(marked) != 1 { t.Fatalf("a published of 1.4.3 with a built digest marked %v (%v)", marked, err) } - // Design 47 G §52, §673, A-57: the package-published of that artifact carries the digest published, the branch and the source - // revision the reservation was made for, and the dependencies its step result named. + // Design 47 G §52, §673, A-57: the package-published of that artifact carries the digest published, the branch, the source + // revision and the catalog revision (H-A-218) the reservation was made for, and the dependencies its step result named. at := time.Unix(100, 0) said := marked[0].PackagePublished("sha256:aa", at) atUTC := at.UTC() - if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) { + if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Revision: 44, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) { t.Errorf("package-published = %+v, want %+v", said, wantSaid) } if marked, _, _ := r.Published(ctx, "m-a", "1.4.3", "sha256:other"); len(marked) != 0 { diff --git a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go index 836f14e0..dc45a781 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go @@ -95,6 +95,8 @@ type PackagePublished struct { Digest string `json:"digest"` // the module Module string `json:"module"` + // the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field + Revision int64 `json:"revision,omitempty"` // the source revision of that branch the build was made from Source int64 `json:"source"` V int64 `json:"v"` diff --git a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json index d10eec4f..2caddbdf 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json @@ -180,6 +180,10 @@ "type": "integer", "description": "the source revision of that branch the build was made from" }, + "revision": { + "type": "integer", + "description": "the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field" + }, "dependencies": { "type": "array", "items": { diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts index d142d6d9..68e2c00e 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts @@ -69,6 +69,8 @@ export interface PackagePublished { "digest": string; /** the module */ "module": string; + /** the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field */ + "revision"?: number; /** the source revision of that branch the build was made from */ "source": number; "v": number;