Join fix/47-H1-builder-revision-picture

This commit is contained in:
2026-11-21 00:27:10 +01:00
11 changed files with 216 additions and 15 deletions
@@ -101,9 +101,7 @@ func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, e
}
origin := build.Origin{Branch: branch, First: first}
if cat, ok := c.branches.Catalog(branch); ok {
if half, ok := cat.View(); ok {
origin.Source = half.Revision.Source
}
origin = pictureOrigin(origin, cat.View)
}
var e meshcatalog.CatalogEntry
if err := json.Unmarshal(entry, &e); err != nil || e.Module != module {
@@ -132,6 +130,16 @@ func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, e
}
}
// pictureOrigin is the origin of an entry the picture takes in: the source revision and the catalog revision number of the
// half the view holds, one revision for both (design 47 G §371, H-A-218, H-A-220), so a build the picture starts before any
// step ask reserves a version that names its revision; none while the view holds no half.
func pictureOrigin(o build.Origin, view func() (meshcatalogclient.Half, bool)) build.Origin {
if half, ok := view(); ok {
o.Source, o.Revision = half.Revision.Source, half.Revision.Revision
}
return o
}
// picture is build.Builder.Picture: the source revision the view of branch holds (design 47 H H-A-155), false while the branch
// is not followed or its view holds none.
func (c *catalogHalf) picture(branch string) (int64, bool) {
@@ -4,7 +4,9 @@ import (
"context"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-builder/internal/build"
)
// TestTheManifestOfAnEntryIsReadByItsDigest: the builder reads a manifest from the entry that holds it, by the entry's manifest
@@ -25,3 +27,18 @@ func TestTheManifestOfAnEntryIsReadByItsDigest(t *testing.T) {
t.Errorf("an entry with no manifest kept one: %v, %v", err, m.byFp)
}
}
// H-A-220: an entry the picture takes in has the source revision and the catalog revision number of the half its view holds,
// one revision for both; a view holding no half names neither.
func TestAnEntryThePictureTakesInNamesItsHalfsRevision(t *testing.T) {
o := build.Origin{Branch: "main", First: true}
got := pictureOrigin(o, func() (meshcatalogclient.Half, bool) {
return meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Revision: 47, Source: 45}}, true
})
if got != (build.Origin{Branch: "main", First: true, Source: 45, Revision: 47}) {
t.Errorf("origin = %+v, want source 45 at revision 47", got)
}
if got := pictureOrigin(o, func() (meshcatalogclient.Half, bool) { return meshcatalogclient.Half{}, false }); got != o {
t.Errorf("origin with no half = %+v, want %+v", got, o)
}
}
@@ -0,0 +1,47 @@
package main
import (
"context"
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// keptRevisionVia reads one kept revision of a branch by its number through the catalog seat's verb revision, page by page
// (design 47 H H-A-205, H-A-147): the source revision it was taken in from and the fingerprint of each module it names.
func keptRevisionVia(call toolCaller) func(context.Context, string, int64) (int64, map[string]string, error) {
return func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) {
fps := map[string]string{}
var source int64
after := ""
for {
raw, err := call(meshcatalogclient.RevisionVerb, meshcatalog.RevisionInput{Branch: keys.EscapeName(branch), Revision: revision, After: after})
if err != nil {
return 0, nil, err
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return 0, nil, fmt.Errorf("%s answered: %s", meshcatalogclient.RevisionVerb, said)
}
var a meshcatalog.RevisionAnswer
if err := json.Unmarshal(answer, &a); err != nil {
return 0, nil, fmt.Errorf("%s answered no revision: %w", meshcatalogclient.RevisionVerb, err)
}
if a.NoneYet || a.Revision == nil || a.Revision.Revision != revision {
return 0, nil, fmt.Errorf("%s answered no revision %d of %s", meshcatalogclient.RevisionVerb, revision, branch)
}
source = a.Revision.Source
for _, e := range a.Entries {
fps[e.Module] = e.Fp
}
if a.Next == "" {
return source, fps, nil
}
after = a.Next
}
}
}
@@ -469,6 +469,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// a holder of the seat that is down holds no other part of it).
half.b = b
b.Picture = half.picture
// An ask of a revision below the newest the picture holds is read by number from the catalog seat (H-A-205).
b.KeptRevision = keptRevisionVia(calls.Call)
b.FirstBranch = half.first
branches.Held = func(string, meshcatalog.CatalogRevision) { b.PictureHeld() } // a revision held wakes a step ask's wait (H-A-162)
half.ready = func() {
if err := b.Sweep(ctx, half.pictured()...); err != nil {
@@ -194,7 +194,7 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu
for _, t := range plan.Tasks {
refs = append(refs, module+":"+t.Artifact)
}
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, Rebuild: rebuild}
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild}
for _, t := range plan.Tasks {
src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go")
}
@@ -20,6 +20,11 @@ import (
// revision its view holds (PictureHeld, H-A-162), never a
// timer or a poll, up to RevisionWait (the setting revision-wait-minutes); past it the ask fails, naming the revision and the
// branch, and nothing is built. A step ask with no source (a run on request) does not wait.
//
// **The picture holds every kept revision, not only the newest** (design 47 H H-A-205, H-A-147): an ask of a source revision
// below the newest the picture holds, whose entries the picture no longer names, asks the catalog seat's verb revision for
// its catalog revision by number, once, and is built when that kept revision is of its source and names every entry it
// asks; it never waits, since the picture only moves on. Only a revision above the newest is waited for.
// DefaultRevisionWait is the bound of a step ask's wait for its revision, its setting revision-wait-minutes' default (H-A-156).
const DefaultRevisionWait = 2 * time.Minute
@@ -68,12 +73,52 @@ func (b *Builder) holdsRevision(branch string, source int64, entries []meshbuild
return true
}
// keptRevisionHolds answers no outcome when the catalog keeps the ask's revision (its catalog revision by number, of its
// source revision) and that revision names every entry of the ask; else failed or could not run, with its words.
func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, held int64) (outcome, said string) {
below := fmt.Sprintf("revision %d of %s is below the %d the builder's picture holds", run.Source, run.Branch, held)
if b.KeptRevision == nil || run.Revision <= 0 {
return meshbuilds.StepEndedOutcomeFailed, below + ", and the ask names no catalog revision to read it by"
}
source, fps, err := b.KeptRevision(ctx, run.Branch, run.Revision)
if err != nil {
b.logf("%s: %s, and the catalog's revision %d cannot be read: %v", run.Pipeline, below, run.Revision, err)
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s, and the catalog's revision %d cannot be read: %v", below, run.Revision, err)
}
if source != run.Source {
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d is of source revision %d", below, run.Revision, source)
}
for _, e := range run.Entries {
if fps[e.Module] != e.Fingerprint {
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d names %s at %q, not %s", below, run.Revision, e.Module, fps[e.Module], e.Fingerprint)
}
}
// Each asked fingerprint is recorded as the branch's, so its build reserves its version (H-A-155) as one the picture
// took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded.
b.mu.Lock()
for _, e := range run.Entries {
key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint)
if err != nil || b.origin[key].Branch != "" {
continue
}
first := b.FirstBranch == nil || b.FirstBranch(run.Branch, e.Module)
b.trunkFP[key] = true
b.origin[key] = Origin{Branch: run.Branch, Source: run.Source, Revision: run.Revision, First: first}
}
b.mu.Unlock()
b.logf("%s: %s; the catalog keeps revision %d with its entries: the build goes on", run.Pipeline, below, run.Revision)
return "", ""
}
// awaitRevision returns no outcome once the builder's picture of the ask's branch holds its source revision; past the bound the
// step's outcome failed, at the builder's stop could not run, each with its words.
func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (outcome, said string) {
if b.Picture == nil || run.Source <= 0 || run.Branch == "" {
return "", ""
}
if held, ok := b.Picture(run.Branch); ok && run.Source < held && !b.holdsRevision(run.Branch, run.Source, run.Entries) {
return b.keptRevisionHolds(ctx, run, held)
}
bound := b.revisionWait()
notHeld := fmt.Sprintf("revision %d of %s not in the builder's picture", run.Source, run.Branch)
wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8)
@@ -110,6 +110,14 @@ type Builder struct {
// Picture answers the source revision the builder's picture of a watched branch holds, false while it holds none (design 47
// H H-A-155, H-A-141): a step ask naming a newer one waits for it. Nil waits for nothing.
Picture func(branch string) (source int64, held bool)
// KeptRevision asks the catalog seat's verb revision for one kept revision of a branch by its number (design 47 H
// H-A-205, H-A-147): the source revision it was taken in from and the fingerprint of each module it names. A step ask of
// a source revision below the newest the picture holds is built when that kept revision names its entries. Nil reads none.
KeptRevision func(ctx context.Context, branch string, revision int64) (source int64, fingerprints map[string]string, err error)
// FirstBranch says whether branch is the first watched branch of the module's registration, the one whose builds reserve
// a version (design 47 G §371): an ask admitted by its kept revision is recorded as that branch's (H-A-205, H-A-155). Nil
// takes every branch as the first, as an origin never heard is.
FirstBranch func(branch, module string) bool
// RevisionWait bounds a step ask's wait for its revision, its setting revision-wait-minutes (H-A-156); DefaultRevisionWait
// when 0.
RevisionWait time.Duration
@@ -229,7 +237,9 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) {
type Origin struct {
Branch string
Source int64
First bool
// Revision is the catalog revision number a step ask of the build named (design 47 H H-A-218), 0 until one did.
Revision int64
First bool
}
// Entry takes the catalog graph entry of one module on one watched branch in (design 47 G §348, §614) and builds its
@@ -39,6 +39,7 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui
if outcome, said := b.awaitRevision(ctx, run); outcome != "" {
return finish(outcome, said)
}
b.askedRevision(run)
var failed []string
for _, e := range run.Entries {
var pins map[string]string
@@ -66,6 +67,27 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui
return finish(meshbuilds.StepEndedOutcomePassed, "")
}
// askedRevision records on the origin of each entry of run the catalog revision the ask names (design 47 H H-A-218), where
// the origin is of the ask's branch and source revision: the version reserved for its build names it, and so does its
// package-published, so the runner finds the publishing pipeline by branch and revision.
func (b *Builder) askedRevision(run meshbuilds.StepRun) {
if run.Revision <= 0 {
return
}
b.mu.Lock()
defer b.mu.Unlock()
for _, e := range run.Entries {
key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint)
if err != nil {
continue
}
if o, ok := b.origin[key]; ok && o.Branch == run.Branch && o.Source == run.Source {
o.Revision = run.Revision
b.origin[key] = o
}
}
}
// buildEntry builds one entry at its fingerprint and waits for its end; pins is set, when it is done, to the pins its
// done key keeps it was built with, the registry's included (H-A-207).
func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) {
@@ -119,6 +119,7 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T)
}
t.Run("H-A-155: an ask waits for the picture to take its revision in, then builds with its version reserved", aStepAskWaitsForItsRevision)
t.Run("H-A-155: past the bound the ask fails naming its revision, and nothing is built", aStepAskPastItsRevisionBoundFails)
t.Run("H-A-205: an ask of a revision below the picture's newest is read by number and built", anOlderKeptRevisionIsReadByNumber)
t.Run("H-A-162: a revision held that changes no entry wakes the ask", aRevisionHeldWakesTheAsk)
t.Run("H-A-165: a package with no pin waits for its release and is built with it", aPackageWithNoPinWaitsForItsRelease)
t.Run("H-A-165: past its bound a package with no release fails the step", aPackageWithNoReleaseFailsPastItsBound)
@@ -355,6 +356,11 @@ func aStepAskWaitsForItsRevision(t *testing.T) {
held.Store(40)
asked := make(chan struct{}, 1)
r, res, entry := revisionRig(t, &held, asked)
var keptReads atomic.Int64 // H-A-205: a revision above the newest is waited for, never read by number
r.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
keptReads.Add(1)
return 0, nil, errors.New("not asked")
}
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
@@ -379,8 +385,45 @@ func aStepAskWaitsForItsRevision(t *testing.T) {
t.Fatal("the ask was not woken by the entry the picture took in")
}
r.b.Wait()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 {
t.Errorf("end = %+v, reserved from %+v; want passed with one version reserved from revision 41", end, res.sources)
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 || keptReads.Load() != 0 {
t.Errorf("end = %+v, reserved from %+v, kept revisions read %d; want passed with one version reserved from revision 41, none read", end, res.sources, keptReads.Load())
}
}
// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, whose entry of m moved
// on, while the older pipeline asks the build of revision 45. Its kept catalog revision is read by number and the build runs;
// a kept revision naming another fingerprint fails the step, naming it.
func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
var held atomic.Int64
held.Store(46)
r, res, _ := revisionRig(t, &held, make(chan struct{}, 1)) // a picture at 46 that never heard 45 (a restart)
r.b.RevisionWait = time.Nanosecond // a wait would fail at once: the older revision is never waited for
r.b.FirstBranch = func(branch, module string) bool { return branch == "main" && module == "m" }
var read []string
r.b.KeptRevision = func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) {
read = append(read, fmt.Sprintf("%s %d", branch, revision))
return 45, map[string]string{"m": fp1}, nil
}
run := meshbuilds.StepRun{Pipeline: "main.47.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 47, Source: 45,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}
end := r.b.BuildStep(t.Context(), run)
r.b.Wait()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 47"}) {
t.Errorf("end = %+v, kept revisions read %v; want passed, revision 47 of main read once", end, read)
}
// Its fingerprint is recorded as the branch's: its build reserves its version from source revision 45 (H-A-155).
// The version reserved names the catalog revision the ask named, which its package-published says (H-A-218).
if len(res.sources) != 1 || res.sources[0].Revision != 45 || res.sources[0].CatalogRevision != 47 {
t.Errorf("reserved from %+v; want one version reserved from source revision 45 at catalog revision 47 (H-A-218)", res.sources)
}
r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1))
r2.b.RevisionWait = time.Nanosecond
r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
return 45, map[string]string{"m": "src2:other"}, nil
}
if end := r2.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") {
t.Errorf("end = %+v; want failed naming the fingerprint the kept revision names", end)
}
}
@@ -65,6 +65,9 @@ type ReservedVersion struct {
// (design 47 G §371, A-57): what `package-published` says of the publication.
Branch string `json:"branch,omitempty"`
Source int64 `json:"source,omitempty"`
// Revision is the catalog revision number the build's step ask named, the indexer's count (design 47 H H-A-218): what
// `package-published` names beside the source, so the runner finds the publishing pipeline; 0 when no step ask named one.
Revision int64 `json:"revision,omitempty"`
// Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with
// (kind 3), kept for `package-published` (design 47 G §466, §673, A-57).
Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"`
@@ -91,6 +94,8 @@ type Source struct {
// that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication.
Branch string
Revision int64
// CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did.
CatalogRevision int64
// Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced
// and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b).
Rebuild bool
@@ -177,7 +182,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
}
}
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(),
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision}
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision}
if src.GoPackage {
out.Tag = GoTagOf(out.Version)
}
@@ -443,11 +448,12 @@ type Marked struct {
}
// PackagePublished is the seat mesh-builds' package-published of one artifact the holder said published at the reservation's
// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch and
// the source revision the build was made from, and the dependencies its step result named.
// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch,
// the source revision the build was made from and the catalog revision its step ask named (H-A-218), and the dependencies its
// step result named.
func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.PackagePublished {
at = at.UTC()
return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Module: m.Reserved.Module,
return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Revision: m.Reserved.Revision, Module: m.Reserved.Module,
Version: m.Reserved.Version, Digest: digest, Dependencies: m.Reserved.Dependencies[m.Artifact]}
}
@@ -125,7 +125,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
ctx := context.Background()
store := &reserveMem{mem: newMem()}
r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"1.4.2"}, nil }}
v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{Branch: "main", Revision: 41})
v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{Branch: "main", Revision: 41, CatalogRevision: 44})
if err != nil || v.Version != "1.4.3" {
t.Fatalf("reserved %q (%v), want 1.4.3", v.Version, err)
}
@@ -154,12 +154,12 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
if err != nil || len(marked) != 1 {
t.Fatalf("a published of 1.4.3 with a built digest marked %v (%v)", marked, err)
}
// Design 47 G §52, §673, A-57: the package-published of that artifact carries the digest published, the branch and the source
// revision the reservation was made for, and the dependencies its step result named.
// Design 47 G §52, §673, A-57: the package-published of that artifact carries the digest published, the branch, the source
// revision and the catalog revision (H-A-218) the reservation was made for, and the dependencies its step result named.
at := time.Unix(100, 0)
said := marked[0].PackagePublished("sha256:aa", at)
atUTC := at.UTC()
if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) {
if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Revision: 44, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) {
t.Errorf("package-published = %+v, want %+v", said, wantSaid)
}
if marked, _, _ := r.Published(ctx, "m-a", "1.4.3", "sha256:other"); len(marked) != 0 {