diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index 0ca00286..2d160479 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -169,7 +169,7 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) { changed := false var kept []tierMover defer func() { - if err := e.startTiers(p.Pipeline, kept); err != nil { + if err := e.startTiers(p.Pipeline, p.Source, kept); err != nil { e.logf("the tier progress of %s is not written: %v", p.Pipeline, err) } }() diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go index 5211d564..5c45c24f 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers.go @@ -42,11 +42,14 @@ type tierMover struct { Tier int64 `json:"tier"` } -// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, and the movers. +// tierProgress is a pipeline's tier progress (H-A-56): the tier whose wants are written, since when, the movers, and the source +// revision its pipeline-ended names, said in pipeline-wanted beside the key's revision (design 47 H-A-214, H-A-216); zero in a +// value kept before the field, read as the key's number. type tierProgress struct { Tier int64 `json:"tier"` Since time.Time `json:"since"` Movers []tierMover `json:"movers"` + Source int64 `json:"source,omitempty"` } // tierKey is the key of pipeline's tier progress; pipeline is the runner's key `.`, its branch escaped. @@ -60,7 +63,7 @@ func isTierKey(key string) bool { // startTiers writes the tier progress of a passed pipeline whose movers' versions are now kept, its first tier the lowest of // theirs, since now. A pipeline heard again writes nothing. -func (e *pipelineEnds) startTiers(pipeline string, movers []tierMover) error { +func (e *pipelineEnds) startTiers(pipeline string, source int64, movers []tierMover) error { if len(movers) == 0 { return nil } @@ -68,7 +71,7 @@ func (e *pipelineEnds) startTiers(pipeline string, movers []tierMover) error { for _, m := range movers { first = min(first, m.Tier) } - value, err := json.Marshal(tierProgress{Tier: first, Since: e.clock().UTC(), Movers: movers}) + value, err := json.Marshal(tierProgress{Tier: first, Since: e.clock().UTC(), Movers: movers, Source: source}) if err != nil { return err } @@ -131,7 +134,7 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st continue } pipeline := strings.TrimPrefix(key, tierPrefix) - branchEsc, source, _ := strings.Cut(pipeline, ".") + branchEsc, number, _ := strings.Cut(pipeline, ".") branch, err := keys.UnescapeName(branchEsc) if err != nil { continue @@ -144,7 +147,7 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st notRunning := p.notRunning(n, placedOn, pipeline, branch, t, t.Tier) late := !now.Before(t.Since.Add(bound)) if late { - p.tierLate(pipeline, branch, source, t, notRunning) + p.tierLate(pipeline, branch, number, t, notRunning) break } if len(notRunning) > 0 { @@ -165,10 +168,10 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st p.logf("the tier progress %s is not deleted: %v", key, err) break } - p.sayPipelineWanted(pipeline, branch, source, meshwants.PipelineWantedOutcomeRunning, nil) + p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeRunning, nil) break } - value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers}) + value, err := json.Marshal(tierProgress{Tier: later, Since: now, Movers: t.Movers, Source: t.Source}) if err == nil { err = p.ends.st.swap(key, value, rev) } @@ -217,7 +220,7 @@ func (p *passer) notRunning(n assign.Nodes, placedOn func(module, branch string) // tierLate ends a pipeline's delivery at a tier past tier-bound: the finding tier-late is raised and pipeline-wanted says tier-late, // once each; the finding clears once the late movers run or a later pipeline moved them. The progress stays, so no later tier is // written, until no later tier's mover is kept at this pipeline any more. -func (p *passer) tierLate(pipeline, branch, source string, t tierProgress, notRunning []meshwants.NotRunning) { +func (p *passer) tierLate(pipeline, branch, number string, t tierProgress, notRunning []meshwants.NotRunning) { o := tierLateFinding(pipeline, t.Tier, notRunning) key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind) if len(notRunning) > 0 { @@ -231,7 +234,7 @@ func (p *passer) tierLate(pipeline, branch, source string, t tierProgress, notRu p.said[key+".raised"] = true delete(p.said, key+".cleared") p.logf("pipeline %s: tier %d is late (tier-bound %s): %s; no later tier is written", pipeline, t.Tier, p.tierBoundOf(), o.Summary) - p.sayPipelineWanted(pipeline, branch, source, meshwants.PipelineWantedOutcomeTierLate, notRunning) + p.sayPipelineWanted(pipeline, branch, number, t.Source, meshwants.PipelineWantedOutcomeTierLate, notRunning) } } else if !p.said[key+".cleared"] { if p.findings.Delete != nil { @@ -271,11 +274,16 @@ func tierLateFinding(pipeline string, tier int64, notRunning []meshwants.NotRunn Source: "assigner." + kindTierLate} } -// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome). -func (p *passer) sayPipelineWanted(pipeline, branch, source, outcome string, notRunning []meshwants.NotRunning) { - var src int64 - _, _ = fmt.Sscan(source, &src) - body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: src, Outcome: outcome, NotRunning: notRunning} +// sayPipelineWanted says the seat mesh-wants' pipeline-wanted of pipeline, once (its id the pipeline and the outcome): its +// revision the number of the pipeline's key `.`, its source the one the pipeline-ended named (design 47 +// H-A-214, H-A-216), the key's number for a progress kept before it held one. +func (p *passer) sayPipelineWanted(pipeline, branch, number string, source int64, outcome string, notRunning []meshwants.NotRunning) { + var revision int64 + _, _ = fmt.Sscan(number, &revision) + if source <= 0 { + source = revision + } + body := meshwants.PipelineWantedEvent{V: keys.Version, Pipeline: pipeline, Branch: branch, Source: source, Revision: revision, Outcome: outcome, NotRunning: notRunning} event := meshwants.EventPipelineWanted + "." + pipeline if err := p.wantsSay(event, body, "pipeline-wanted."+pipeline+"."+outcome); err != nil { p.logf("pipeline-wanted of %s is not said: %v", pipeline, err) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go index 39d9bb39..9937f775 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tiers_test.go @@ -62,8 +62,11 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { p.w.mu.Unlock() p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) p.ends.trunk, p.ends.now = p.w.trunkName, func() time.Time { return now } - if _, err := p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed), - mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))); err != nil { + // The pipeline of revision 5 (its key's number) of source 4: a re-pin's revision (H-A-213, H-A-216). + e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed), + mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed)) + e.Source = 4 + if _, err := p.ends.take(e); err != nil { t.Fatal(err) } wanted := func(node, module string) bool { @@ -88,7 +91,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web}) _ = p.w.hearRunning("running.g14.web", raw) p.wants(context.Background()) - if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || len(findings) != 0 { + if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || said[0].Revision != 5 || said[0].Source != 4 || len(findings) != 0 { t.Errorf("every tier runs: said %+v, findings %v", said, findings) } if _, _, ok := p.ends.tiersOf(trunkBranch + ".5"); ok { @@ -108,8 +111,14 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) { if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") { t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings) } - if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" { - t.Errorf("pipeline-wanted tier-late is not said once: %+v", said) + if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeTierLate || len(said[0].NotRunning) != 1 || said[0].NotRunning[0].Module != "db" || + said[0].Revision != 5 || said[0].Source != 4 { + t.Errorf("pipeline-wanted tier-late is not said once, naming revision 5 of source 4: %+v", said) + } + // A tier progress kept before it held its source says the key's number as both (H-A-216). + p.sayPipelineWanted(trunkBranch+".5", trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil) + if len(said) != 2 || said[1].Revision != 5 || said[1].Source != 5 { + t.Errorf("a progress with no source: said %+v, want revision and source 5", said) } if wanted("g14", "web") || wanted("ace", "web") { t.Error("a consumer's want is written after its provider's tier was late") diff --git a/modules/mesh-delivery-builder/builds/builds.go b/modules/mesh-delivery-builder/builds/builds.go index 8981c7dc..04c3c6cd 100644 --- a/modules/mesh-delivery-builder/builds/builds.go +++ b/modules/mesh-delivery-builder/builds/builds.go @@ -242,6 +242,11 @@ type BuilderBuild struct { // Imports are, on a done key, the mesh packages the built programs import, as the build module answered them // (BuildAnswer.Imports; novox/hq ADR 0593 §2): the builder carries them onto the build's end (BuildLogEntry.Imports). Imports []string `json:"imports,omitempty"` + // Packages are, on a done key only, the mesh packages' versions its Go build was built with, by module path, the ones + // the registry gave a package the entry named with none included (H-A-207): the build step's end reports them, so the + // runner asks the steps that waited for the pins with what the build was built against. A reader that does not know + // the field reads the build as reporting none. + Packages map[string]string `json:"packages,omitempty"` // Needs are, on a done key only, the reader capabilities (`form:`) the build's reference forms need of a // node-engine, beside the ones its manifest needs (novox/hq ADR 0618 §1). A reader that does not know the field // reads the build as needing none. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go index 1d935875..5ff5f0bd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go @@ -101,9 +101,7 @@ func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, e } origin := build.Origin{Branch: branch, First: first} if cat, ok := c.branches.Catalog(branch); ok { - if half, ok := cat.View(); ok { - origin.Source = half.Revision.Source - } + origin = pictureOrigin(origin, cat.View) } var e meshcatalog.CatalogEntry if err := json.Unmarshal(entry, &e); err != nil || e.Module != module { @@ -132,6 +130,16 @@ func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, e } } +// pictureOrigin is the origin of an entry the picture takes in: the source revision and the catalog revision number of the +// half the view holds, one revision for both (design 47 G §371, H-A-218, H-A-220), so a build the picture starts before any +// step ask reserves a version that names its revision; none while the view holds no half. +func pictureOrigin(o build.Origin, view func() (meshcatalogclient.Half, bool)) build.Origin { + if half, ok := view(); ok { + o.Source, o.Revision = half.Revision.Source, half.Revision.Revision + } + return o +} + // picture is build.Builder.Picture: the source revision the view of branch holds (design 47 H H-A-155), false while the branch // is not followed or its view holds none. func (c *catalogHalf) picture(branch string) (int64, bool) { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go index 8d9c45b1..26bdeabd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go @@ -4,7 +4,9 @@ import ( "context" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "mesh-delivery-builder/internal/build" ) // TestTheManifestOfAnEntryIsReadByItsDigest: the builder reads a manifest from the entry that holds it, by the entry's manifest @@ -25,3 +27,18 @@ func TestTheManifestOfAnEntryIsReadByItsDigest(t *testing.T) { t.Errorf("an entry with no manifest kept one: %v, %v", err, m.byFp) } } + +// H-A-220: an entry the picture takes in has the source revision and the catalog revision number of the half its view holds, +// one revision for both; a view holding no half names neither. +func TestAnEntryThePictureTakesInNamesItsHalfsRevision(t *testing.T) { + o := build.Origin{Branch: "main", First: true} + got := pictureOrigin(o, func() (meshcatalogclient.Half, bool) { + return meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Revision: 47, Source: 45}}, true + }) + if got != (build.Origin{Branch: "main", First: true, Source: 45, Revision: 47}) { + t.Errorf("origin = %+v, want source 45 at revision 47", got) + } + if got := pictureOrigin(o, func() (meshcatalogclient.Half, bool) { return meshcatalogclient.Half{}, false }); got != o { + t.Errorf("origin with no half = %+v, want %+v", got, o) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/keptrevision.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/keptrevision.go new file mode 100644 index 00000000..f9d6e55e --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/keptrevision.go @@ -0,0 +1,47 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// keptRevisionVia reads one kept revision of a branch by its number through the catalog seat's verb revision, page by page +// (design 47 H H-A-205, H-A-147): the source revision it was taken in from and the fingerprint of each module it names. +func keptRevisionVia(call toolCaller) func(context.Context, string, int64) (int64, map[string]string, error) { + return func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) { + fps := map[string]string{} + var source int64 + after := "" + for { + raw, err := call(meshcatalogclient.RevisionVerb, meshcatalog.RevisionInput{Branch: keys.EscapeName(branch), Revision: revision, After: after}) + if err != nil { + return 0, nil, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return 0, nil, fmt.Errorf("%s answered: %s", meshcatalogclient.RevisionVerb, said) + } + var a meshcatalog.RevisionAnswer + if err := json.Unmarshal(answer, &a); err != nil { + return 0, nil, fmt.Errorf("%s answered no revision: %w", meshcatalogclient.RevisionVerb, err) + } + if a.NoneYet || a.Revision == nil || a.Revision.Revision != revision { + return 0, nil, fmt.Errorf("%s answered no revision %d of %s", meshcatalogclient.RevisionVerb, revision, branch) + } + source = a.Revision.Source + for _, e := range a.Entries { + fps[e.Module] = e.Fp + } + if a.Next == "" { + return source, fps, nil + } + after = a.Next + } + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index a5167640..60097089 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -469,6 +469,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // a holder of the seat that is down holds no other part of it). half.b = b b.Picture = half.picture + // An ask of a revision below the newest the picture holds is read by number from the catalog seat (H-A-205). + b.KeptRevision = keptRevisionVia(calls.Call) + b.FirstBranch = half.first branches.Held = func(string, meshcatalog.CatalogRevision) { b.PictureHeld() } // a revision held wakes a step ask's wait (H-A-162) half.ready = func() { if err := b.Sweep(ctx, half.pictured()...); err != nil { diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 2aeefc0e..867a24e6 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -194,7 +194,7 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu for _, t := range plan.Tasks { refs = append(refs, module+":"+t.Artifact) } - src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, Rebuild: rebuild} + src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild} for _, t := range plan.Tasks { src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go") } diff --git a/modules/mesh-delivery-builder/internal/build/revisionwait.go b/modules/mesh-delivery-builder/internal/build/revisionwait.go index 2a052bbc..0c4ec3d4 100644 --- a/modules/mesh-delivery-builder/internal/build/revisionwait.go +++ b/modules/mesh-delivery-builder/internal/build/revisionwait.go @@ -20,6 +20,11 @@ import ( // revision its view holds (PictureHeld, H-A-162), never a // timer or a poll, up to RevisionWait (the setting revision-wait-minutes); past it the ask fails, naming the revision and the // branch, and nothing is built. A step ask with no source (a run on request) does not wait. +// +// **The picture holds every kept revision, not only the newest** (design 47 H H-A-205, H-A-147): an ask of a source revision +// below the newest the picture holds, whose entries the picture no longer names, asks the catalog seat's verb revision for +// its catalog revision by number, once, and is built when that kept revision is of its source and names every entry it +// asks; it never waits, since the picture only moves on. Only a revision above the newest is waited for. // DefaultRevisionWait is the bound of a step ask's wait for its revision, its setting revision-wait-minutes' default (H-A-156). const DefaultRevisionWait = 2 * time.Minute @@ -68,12 +73,52 @@ func (b *Builder) holdsRevision(branch string, source int64, entries []meshbuild return true } +// keptRevisionHolds answers no outcome when the catalog keeps the ask's revision (its catalog revision by number, of its +// source revision) and that revision names every entry of the ask; else failed or could not run, with its words. +func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, held int64) (outcome, said string) { + below := fmt.Sprintf("revision %d of %s is below the %d the builder's picture holds", run.Source, run.Branch, held) + if b.KeptRevision == nil || run.Revision <= 0 { + return meshbuilds.StepEndedOutcomeFailed, below + ", and the ask names no catalog revision to read it by" + } + source, fps, err := b.KeptRevision(ctx, run.Branch, run.Revision) + if err != nil { + b.logf("%s: %s, and the catalog's revision %d cannot be read: %v", run.Pipeline, below, run.Revision, err) + return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s, and the catalog's revision %d cannot be read: %v", below, run.Revision, err) + } + if source != run.Source { + return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d is of source revision %d", below, run.Revision, source) + } + for _, e := range run.Entries { + if fps[e.Module] != e.Fingerprint { + return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d names %s at %q, not %s", below, run.Revision, e.Module, fps[e.Module], e.Fingerprint) + } + } + // Each asked fingerprint is recorded as the branch's, so its build reserves its version (H-A-155) as one the picture + // took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded. + b.mu.Lock() + for _, e := range run.Entries { + key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint) + if err != nil || b.origin[key].Branch != "" { + continue + } + first := b.FirstBranch == nil || b.FirstBranch(run.Branch, e.Module) + b.trunkFP[key] = true + b.origin[key] = Origin{Branch: run.Branch, Source: run.Source, Revision: run.Revision, First: first} + } + b.mu.Unlock() + b.logf("%s: %s; the catalog keeps revision %d with its entries: the build goes on", run.Pipeline, below, run.Revision) + return "", "" +} + // awaitRevision returns no outcome once the builder's picture of the ask's branch holds its source revision; past the bound the // step's outcome failed, at the builder's stop could not run, each with its words. func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (outcome, said string) { if b.Picture == nil || run.Source <= 0 || run.Branch == "" { return "", "" } + if held, ok := b.Picture(run.Branch); ok && run.Source < held && !b.holdsRevision(run.Branch, run.Source, run.Entries) { + return b.keptRevisionHolds(ctx, run, held) + } bound := b.revisionWait() notHeld := fmt.Sprintf("revision %d of %s not in the builder's picture", run.Source, run.Branch) wait, stop := context.WithTimeout(ctx, bound) // a context's bound, no timer (ADR 0437 §8) diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ba6f3e3c..86d65f6c 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -110,6 +110,14 @@ type Builder struct { // Picture answers the source revision the builder's picture of a watched branch holds, false while it holds none (design 47 // H H-A-155, H-A-141): a step ask naming a newer one waits for it. Nil waits for nothing. Picture func(branch string) (source int64, held bool) + // KeptRevision asks the catalog seat's verb revision for one kept revision of a branch by its number (design 47 H + // H-A-205, H-A-147): the source revision it was taken in from and the fingerprint of each module it names. A step ask of + // a source revision below the newest the picture holds is built when that kept revision names its entries. Nil reads none. + KeptRevision func(ctx context.Context, branch string, revision int64) (source int64, fingerprints map[string]string, err error) + // FirstBranch says whether branch is the first watched branch of the module's registration, the one whose builds reserve + // a version (design 47 G §371): an ask admitted by its kept revision is recorded as that branch's (H-A-205, H-A-155). Nil + // takes every branch as the first, as an origin never heard is. + FirstBranch func(branch, module string) bool // RevisionWait bounds a step ask's wait for its revision, its setting revision-wait-minutes (H-A-156); DefaultRevisionWait // when 0. RevisionWait time.Duration @@ -229,7 +237,9 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) { type Origin struct { Branch string Source int64 - First bool + // Revision is the catalog revision number a step ask of the build named (design 47 H H-A-218), 0 until one did. + Revision int64 + First bool } // Entry takes the catalog graph entry of one module on one watched branch in (design 47 G §348, §614) and builds its @@ -833,6 +843,7 @@ type lane struct { tcVersion string // and the version it ran kept string // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty outOfStep []string // the generated files its build module's compare found out of step, passed or failed + pins map[string]string // the mesh packages' versions its Go ask was built with (H-A-207) err error } @@ -982,7 +993,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module used = nil } done := builds.BuilderBuild{State: builds.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used, - Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts)} // the forms it is written in (ADR 0618 §1) + Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts), Packages: pinsOf(lanes)} // the forms it is written in (ADR 0618 §1) for _, l := range lanes { if l.kept != "" { done.KeptOn = l.node // design 47 G §671: the build node that keeps the build @@ -1144,6 +1155,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes for _, fb := range req.Builds { delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465) } + l.pins = maps.Clone(ask.Packages) // kept with the done key, so its step's end reports them (H-A-207) } if id := asked.resume(language); id != "" { ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it @@ -1326,6 +1338,20 @@ func (b *Builder) takeOutOfStep(key string) []string { // importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593 // §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that // kept one reports only what it built. +// pinsOf is the mesh packages' versions the lanes were built with (H-A-207), nil when none was pinned. +func pinsOf(lanes []*lane) map[string]string { + var out map[string]string + for _, l := range lanes { + for p, v := range l.pins { + if out == nil { + out = map[string]string{} + } + out[p] = v + } + } + return out +} + func importsOf(lanes []*lane) []string { seen := map[string]bool{} for _, l := range lanes { diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index 09687285..efb91d8f 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -39,9 +39,21 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui if outcome, said := b.awaitRevision(ctx, run); outcome != "" { return finish(outcome, said) } + b.askedRevision(run) var failed []string for _, e := range run.Entries { - outcome, said := b.buildEntry(ctx, e) + var pins map[string]string + outcome, said := b.buildEntry(ctx, e, &pins) + // The end reports the pins each entry was built with, so the runner asks its held test steps with them (H-A-207). + for path, version := range pins { + if version == "" { + continue + } + if end.Packages == nil { + end.Packages = map[string]string{} + } + end.Packages[path] = version + } switch outcome { case meshbuilds.StepEndedOutcomeCouldNotRun: return finish(outcome, said) @@ -55,8 +67,30 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui return finish(meshbuilds.StepEndedOutcomePassed, "") } -// buildEntry builds one entry at its fingerprint and waits for its end. -func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outcome, said string) { +// askedRevision records on the origin of each entry of run the catalog revision the ask names (design 47 H H-A-218), where +// the origin is of the ask's branch and source revision: the version reserved for its build names it, and so does its +// package-published, so the runner finds the publishing pipeline by branch and revision. +func (b *Builder) askedRevision(run meshbuilds.StepRun) { + if run.Revision <= 0 { + return + } + b.mu.Lock() + defer b.mu.Unlock() + for _, e := range run.Entries { + key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint) + if err != nil { + continue + } + if o, ok := b.origin[key]; ok && o.Branch == run.Branch && o.Source == run.Source { + o.Revision = run.Revision + b.origin[key] = o + } + } +} + +// buildEntry builds one entry at its fingerprint and waits for its end; pins is set, when it is done, to the pins its +// done key keeps it was built with, the registry's included (H-A-207). +func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) { entry, err := b.StepEntry(ctx, e.Fingerprint) if err != nil { return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry %s at %s cannot be read: %v", e.Module, e.Fingerprint, err) @@ -78,6 +112,8 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc defer b.dropEnd(key, ended) if done, failedWords, found := b.keyEnd(ctx, key); found { if done { + // A key done before reports the pins it was built with, kept with it, never the ones a publish gives now. + *pins = b.keptPins(ctx, key) return b.passedWithItsVersion(ctx, entry) } return meshbuilds.StepEndedOutcomeFailed, failedWords @@ -88,6 +124,7 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc select { case log := <-ended: if log.Outcome == builds.BuildDone { + *pins = b.keptPins(ctx, key) return b.passedWithItsVersion(ctx, entry) } return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error) @@ -136,6 +173,20 @@ func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.Ca return meshbuilds.StepEndedOutcomePassed, "" } +// keptPins are the pins a done key keeps it was built with (H-A-207); none when it keeps none or cannot be read, so the +// runner ends the steps that waited for them could not run, loud. +func (b *Builder) keptPins(ctx context.Context, key string) map[string]string { + raw, _, ok, err := b.Claimer.Store.Get(ctx, key) + if err != nil || !ok { + return nil + } + v, err := builds.DecodeBuilderBuild(raw) + if err != nil || v.State != builds.BuildDone { + return nil + } + return v.Packages +} + // keyEnd reads the build key: whether it is done, and its error when it failed for good; found only for a key that has ended. func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed string, found bool) { raw, _, ok, err := b.Claimer.Store.Get(ctx, key) diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 6a777cf2..4b63b05a 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -119,6 +119,7 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T) } t.Run("H-A-155: an ask waits for the picture to take its revision in, then builds with its version reserved", aStepAskWaitsForItsRevision) t.Run("H-A-155: past the bound the ask fails naming its revision, and nothing is built", aStepAskPastItsRevisionBoundFails) + t.Run("H-A-205: an ask of a revision below the picture's newest is read by number and built", anOlderKeptRevisionIsReadByNumber) t.Run("H-A-162: a revision held that changes no entry wakes the ask", aRevisionHeldWakesTheAsk) t.Run("H-A-165: a package with no pin waits for its release and is built with it", aPackageWithNoPinWaitsForItsRelease) t.Run("H-A-165: past its bound a package with no release fails the step", aPackageWithNoReleaseFailsPastItsBound) @@ -189,6 +190,32 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) { !reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) { t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks) } + // The end reports the pins the build was built with, the one the publish gave included (H-A-207). + if !reflect.DeepEqual(end.Packages, map[string]string{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) { + t.Errorf("the end reports the pins %v; want the ones it was built with", end.Packages) + } + // A done key reports the pins it keeps it was built with, never the newer one a publish gives now (H-A-207). + r.b.PackagePublished(unpinnedX, "v0.1.5") + again := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}) + if again.Outcome != meshbuilds.StepEndedOutcomePassed || again.Packages[unpinnedX] != "v0.1.3" { + t.Errorf("the done key's end = %+v; want passed reporting x at v0.1.3, as built", again) + } + // A done key that keeps no pins reports none, so the runner ends the waiting steps loud. + v, err := builds.DecodeBuilderBuild(r.st.vals[r.key]) + if err != nil { + t.Fatal(err) + } + v.Packages = nil + raw, err := builds.EncodeBuilderBuild(v) + if err != nil { + t.Fatal(err) + } + r.st.vals[r.key] = raw + if bare := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); len(bare.Packages) != 0 { + t.Errorf("a done key keeping no pins reported %v; want none", bare.Packages) + } } func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) { @@ -329,6 +356,11 @@ func aStepAskWaitsForItsRevision(t *testing.T) { held.Store(40) asked := make(chan struct{}, 1) r, res, entry := revisionRig(t, &held, asked) + var keptReads atomic.Int64 // H-A-205: a revision above the newest is waited for, never read by number + r.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { + keptReads.Add(1) + return 0, nil, errors.New("not asked") + } ended := make(chan meshbuilds.StepEnded, 1) go func() { ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41, @@ -353,8 +385,45 @@ func aStepAskWaitsForItsRevision(t *testing.T) { t.Fatal("the ask was not woken by the entry the picture took in") } r.b.Wait() - if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 { - t.Errorf("end = %+v, reserved from %+v; want passed with one version reserved from revision 41", end, res.sources) + if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 || keptReads.Load() != 0 { + t.Errorf("end = %+v, reserved from %+v, kept revisions read %d; want passed with one version reserved from revision 41, none read", end, res.sources, keptReads.Load()) + } + +} + +// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, whose entry of m moved +// on, while the older pipeline asks the build of revision 45. Its kept catalog revision is read by number and the build runs; +// a kept revision naming another fingerprint fails the step, naming it. +func anOlderKeptRevisionIsReadByNumber(t *testing.T) { + var held atomic.Int64 + held.Store(46) + r, res, _ := revisionRig(t, &held, make(chan struct{}, 1)) // a picture at 46 that never heard 45 (a restart) + r.b.RevisionWait = time.Nanosecond // a wait would fail at once: the older revision is never waited for + r.b.FirstBranch = func(branch, module string) bool { return branch == "main" && module == "m" } + var read []string + r.b.KeptRevision = func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) { + read = append(read, fmt.Sprintf("%s %d", branch, revision)) + return 45, map[string]string{"m": fp1}, nil + } + run := meshbuilds.StepRun{Pipeline: "main.47.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 47, Source: 45, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}} + end := r.b.BuildStep(t.Context(), run) + r.b.Wait() + if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 47"}) { + t.Errorf("end = %+v, kept revisions read %v; want passed, revision 47 of main read once", end, read) + } + // Its fingerprint is recorded as the branch's: its build reserves its version from source revision 45 (H-A-155). + // The version reserved names the catalog revision the ask named, which its package-published says (H-A-218). + if len(res.sources) != 1 || res.sources[0].Revision != 45 || res.sources[0].CatalogRevision != 47 { + t.Errorf("reserved from %+v; want one version reserved from source revision 45 at catalog revision 47 (H-A-218)", res.sources) + } + r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1)) + r2.b.RevisionWait = time.Nanosecond + r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { + return 45, map[string]string{"m": "src2:other"}, nil + } + if end := r2.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") { + t.Errorf("end = %+v; want failed naming the fingerprint the kept revision names", end) } } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 0bd46450..635805af 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -65,6 +65,9 @@ type ReservedVersion struct { // (design 47 G §371, A-57): what `package-published` says of the publication. Branch string `json:"branch,omitempty"` Source int64 `json:"source,omitempty"` + // Revision is the catalog revision number the build's step ask named, the indexer's count (design 47 H H-A-218): what + // `package-published` names beside the source, so the runner finds the publishing pipeline; 0 when no step ask named one. + Revision int64 `json:"revision,omitempty"` // Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with // (kind 3), kept for `package-published` (design 47 G §466, §673, A-57). Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"` @@ -91,6 +94,8 @@ type Source struct { // that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication. Branch string Revision int64 + // CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did. + CatalogRevision int64 // Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced // and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b). Rebuild bool @@ -177,7 +182,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts } } out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(), - Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision} + Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision} if src.GoPackage { out.Tag = GoTagOf(out.Version) } @@ -443,11 +448,12 @@ type Marked struct { } // PackagePublished is the seat mesh-builds' package-published of one artifact the holder said published at the reservation's -// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch and -// the source revision the build was made from, and the dependencies its step result named. +// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch, +// the source revision the build was made from and the catalog revision its step ask named (H-A-218), and the dependencies its +// step result named. func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.PackagePublished { at = at.UTC() - return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Module: m.Reserved.Module, + return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Revision: m.Reserved.Revision, Module: m.Reserved.Module, Version: m.Reserved.Version, Digest: digest, Dependencies: m.Reserved.Dependencies[m.Artifact]} } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index 3b68a10b..3c1bed16 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -125,7 +125,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { ctx := context.Background() store := &reserveMem{mem: newMem()} r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"1.4.2"}, nil }} - v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{Branch: "main", Revision: 41}) + v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{Branch: "main", Revision: 41, CatalogRevision: 44}) if err != nil || v.Version != "1.4.3" { t.Fatalf("reserved %q (%v), want 1.4.3", v.Version, err) } @@ -154,12 +154,12 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { if err != nil || len(marked) != 1 { t.Fatalf("a published of 1.4.3 with a built digest marked %v (%v)", marked, err) } - // Design 47 G §52, §673, A-57: the package-published of that artifact carries the digest published, the branch and the source - // revision the reservation was made for, and the dependencies its step result named. + // Design 47 G §52, §673, A-57: the package-published of that artifact carries the digest published, the branch, the source + // revision and the catalog revision (H-A-218) the reservation was made for, and the dependencies its step result named. at := time.Unix(100, 0) said := marked[0].PackagePublished("sha256:aa", at) atUTC := at.UTC() - if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) { + if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Revision: 44, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) { t.Errorf("package-published = %+v, want %+v", said, wantSaid) } if marked, _, _ := r.Published(ctx, "m-a", "1.4.3", "sha256:other"); len(marked) != 0 { diff --git a/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client.go b/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client.go index eb466b0b..e1702ce1 100644 --- a/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client.go +++ b/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client.go @@ -17,8 +17,9 @@ import ( // eventPrefix starts every event subject of the seat. const eventPrefix = "mesh.seat." + seat.Seat + ".event." -// perSource are the events said per source revision, on .. (design 47 H §746, H-A-72 (3)). -var perSource = []string{seat.EventPipelineStarted, seat.EventPipelineEnded, seat.EventWaitHeld, seat.EventWaitEnded} +// perRevision are the events said per pipeline, on .., its catalog revision's number as its key +// names it (design 47 H §746, H-A-72 (3), H-A-221). +var perRevision = []string{seat.EventPipelineStarted, seat.EventPipelineEnded, seat.EventWaitHeld, seat.EventWaitEnded} // Subject is where the runner says a step's event or a run's start: mesh.seat.mesh-pipeline-runner.event.. // .., each name escaped as a key's name is, so a branch holding a dot is one token (design 47 H @@ -27,41 +28,42 @@ func Subject(event, module, branch, commit string) string { return eventPrefix + event + "." + keys.EscapeName(module) + "." + keys.EscapeName(branch) + "." + keys.EscapeName(commit) } -// SourceSubject is where the runner says a pipeline's start and end and its waits: mesh.seat.mesh-pipeline-runner. -// event..., the branch escaped as a key's name is (design 47 H §746, H-A-72 (3)). -func SourceSubject(event, branch string, source int64) string { - return eventPrefix + event + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10) +// RevisionSubject is where the runner says a pipeline's start and end and its waits: mesh.seat.mesh-pipeline-runner. +// event..., the pipeline's catalog revision number, the branch escaped as a key's name is (design +// 47 H §746, H-A-72 (3), H-A-221). +func RevisionSubject(event, branch string, revision int64) string { + return eventPrefix + event + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10) } // Of is what an event's subject names: a step's event or a run's start names the module, the branch and the commit; an -// event per source revision names the branch and the source. +// event per pipeline names the branch and the pipeline's catalog revision (H-A-221). type Of struct { Event, Module, Branch, Commit string - Source int64 + Revision int64 } -// Parse reads a subject Subject or SourceSubject built: an event of the seat's, and its names unescaped. +// Parse reads a subject Subject or RevisionSubject built: an event of the seat's, and its names unescaped. func Parse(subject string) (Of, error) { rest, ok := strings.CutPrefix(subject, eventPrefix) toks := strings.Split(rest, ".") - bySource := len(toks) > 0 && slices.Contains(perSource, toks[0]) + byRevision := len(toks) > 0 && slices.Contains(perRevision, toks[0]) want := 4 - if bySource { + if byRevision { want = 3 } if !ok || len(toks) != want || !slices.Contains(seat.Events(), toks[0]) { return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q is no event subject of the seat %s", subject, seat.Seat) } - if bySource { + if byRevision { branch, err := keys.UnescapeName(toks[1]) if err != nil || branch == "" { return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: branch: %v", subject, err) } - source, err := strconv.ParseInt(toks[2], 10, 64) - if err != nil || source < 1 { - return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: no source revision", subject) + revision, err := strconv.ParseInt(toks[2], 10, 64) + if err != nil || revision < 1 { + return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: no revision", subject) } - return Of{Event: toks[0], Branch: branch, Source: source}, nil + return Of{Event: toks[0], Branch: branch, Revision: revision}, nil } names := make([]string, 0, 3) for i, t := range toks[1:] { diff --git a/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client_test.go b/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client_test.go index 355017f2..f76ef511 100644 --- a/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client_test.go +++ b/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient/client_test.go @@ -17,14 +17,14 @@ func TestSubjectsAndBodies(t *testing.T) { if of, err := Parse(s); err != nil || of != (Of{Event: seat.EventStepEnded, Module: "nats", Branch: "release.1", Commit: "c1"}) { t.Errorf("parse %s: %+v, %v", s, of, err) } - ps := SourceSubject(seat.EventPipelineEnded, "release.1", 3589) + ps := RevisionSubject(seat.EventPipelineEnded, "release.1", 3589) if ps != "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.release=2e1.3589" { t.Errorf("pipeline-ended subject %s", ps) } - if of, err := Parse(ps); err != nil || of != (Of{Event: seat.EventPipelineEnded, Branch: "release.1", Source: 3589}) { + if of, err := Parse(ps); err != nil || of != (Of{Event: seat.EventPipelineEnded, Branch: "release.1", Revision: 3589}) { t.Errorf("parse %s: %+v, %v", ps, of, err) } - if of, err := Parse(SourceSubject(seat.EventWaitHeld, "main", 7)); err != nil || of.Source != 7 || of.Module != "" { + if of, err := Parse(RevisionSubject(seat.EventWaitHeld, "main", 7)); err != nil || of.Revision != 7 || of.Module != "" { t.Errorf("wait-held: %+v, %v", of, err) } for _, bad := range []string{"not.an.event.subject", diff --git a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go index 836f14e0..179a6890 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go @@ -95,6 +95,8 @@ type PackagePublished struct { Digest string `json:"digest"` // the module Module string `json:"module"` + // the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field + Revision int64 `json:"revision,omitempty"` // the source revision of that branch the build was made from Source int64 `json:"source"` V int64 `json:"v"` @@ -114,9 +116,11 @@ type StepEnded struct { // the node the builder ran it on, as in the subject Node string `json:"node"` // never passed for a build that built nothing - Outcome string `json:"outcome"` - Pipeline string `json:"pipeline"` - Rerun bool `json:"rerun,omitempty"` + Outcome string `json:"outcome"` + // the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207) + Packages map[string]string `json:"packages,omitempty"` + Pipeline string `json:"pipeline"` + Rerun bool `json:"rerun,omitempty"` // why it failed or could not run: the entry not built, the part not heard Said string `json:"said,omitempty"` Started *time.Time `json:"started,omitempty"` diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go index 261c6f66..1ad5e63d 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go @@ -715,6 +715,10 @@ type CatalogRevision struct { Base int64 `json:"base,omitempty"` // the watched branch the revision is of Branch string `json:"branch,omitempty"` + // what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's + Call string `json:"call,omitempty"` + // the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's + CallKind string `json:"call-kind,omitempty"` // the seat definitions it was judged with, each by its digest, read whole through seat-definitions Definitions []CatalogRevisionDefinitionsItem `json:"definitions,omitempty"` // one per module, by name @@ -725,8 +729,12 @@ type CatalogRevision struct { Heads string `json:"heads,omitempty"` // trunk or candidate Kind string `json:"kind"` + // for call-kind release, the module released (H-A-215) + Module string `json:"module,omitempty"` // the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover Movers []TakeInEntry `json:"movers,omitempty"` + // for call-kind take-in-again, the repository taken in again (H-A-215) + Repository string `json:"repository,omitempty"` // the catalog graph revision's number, counted by its holder Revision int64 `json:"revision"` // the source revision it was taken in from; for a candidate, the source revision its heads are merged onto @@ -735,8 +743,18 @@ type CatalogRevision struct { Trunk []Head `json:"trunk,omitempty"` // the layout version V int64 `json:"v,omitempty"` + // for call-kind release, the version released (H-A-215) + Version string `json:"version,omitempty"` } +// The values of CatalogRevision's call-kind. +const ( + CatalogRevisionCallKindTakeIn = "take-in" + CatalogRevisionCallKindRelease = "release" + CatalogRevisionCallKindTakeInAgain = "take-in-again" + CatalogRevisionCallKindRecompute = "recompute" +) + // Edge is one edge of the catalog graph between artifact references, B before A (ADR 0666 section 14): kind 1 B published before A builds (a package A imports), 2 B published before A delivers (fetched by reference at its start), 3 B delivered before A builds (its build module and toolchain), 4 B delivered before A delivers (a provision, bus state, a format's reader), 5 B has acted for A's new version (an acting rule), 6 B runs a version that serves A (a verb or a manifest word A uses). type Edge struct { // declared in the artifact entry's needs of A's manifest, where nothing else says it @@ -1301,6 +1319,10 @@ type CatalogRevisionEvent struct { At *time.Time `json:"at,omitempty"` // the watched branch the revision is of, escaped as a key name is Branch string `json:"branch,omitempty"` + // what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's + Call string `json:"call,omitempty"` + // the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's + CallKind string `json:"call-kind,omitempty"` // the modules whose entries differ from the revision before it of the same kind Changed []string `json:"changed,omitempty"` // how many entries it lists @@ -1308,12 +1330,26 @@ type CatalogRevisionEvent struct { Group string `json:"group,omitempty"` Heads string `json:"heads,omitempty"` // trunk or candidate: a candidate's moves no running part - Kind string `json:"kind"` - Revision int64 `json:"revision"` - Source int64 `json:"source"` - V int64 `json:"v"` + Kind string `json:"kind"` + // for call-kind release, the module released (H-A-215) + Module string `json:"module,omitempty"` + // for call-kind take-in-again, the repository taken in again (H-A-215) + Repository string `json:"repository,omitempty"` + Revision int64 `json:"revision"` + Source int64 `json:"source"` + V int64 `json:"v"` + // for call-kind release, the version released (H-A-215) + Version string `json:"version,omitempty"` } +// The values of CatalogRevisionEvent's call-kind. +const ( + CatalogRevisionEventCallKindTakeIn = "take-in" + CatalogRevisionEventCallKindRelease = "release" + CatalogRevisionEventCallKindTakeInAgain = "take-in-again" + CatalogRevisionEventCallKindRecompute = "recompute" +) + // FillingEvent is the body of the event filling. type FillingEvent = Fill diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go index e8f85fa8..5258e5ea 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go @@ -17,7 +17,7 @@ func TestFixtures(t *testing.T) { decode func([]byte) (any, error) subjectIs *regexp.Regexp }{ - {name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", + {name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"revision\":12,\"source\":3589,\"v\":1,\"version\":\"0.1.27\"}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"revision\":12,\"source\":3589,\"v\":1,\"version\":\"0.1.27\"}", decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")}, {name: "a candidate's revision kept", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}", decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")}, @@ -59,7 +59,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v ModulesAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "the module verb (captured: xclip)", subject: "", given: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", wire: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", decode: func(b []byte) (any, error) { var v ModuleAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, - {name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", + {name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1,\"version\":\"0.1.27\"}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"call-kind\":\"release\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"module\":\"mesh-sdk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1,\"version\":\"0.1.27\"}", decode: func(b []byte) (any, error) { var v CatalogRevisionsByBranchRevisionValue err := json.Unmarshal(b, &v) diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go index 058625fb..8b5f340e 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go @@ -65,9 +65,9 @@ const ( // StreamPipelineStarted is the stream pipeline-started is kept in. const StreamPipelineStarted = "SEAT_mesh-pipeline-runner_pipeline-started" -// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started... -func SubjectPipelineStarted(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source +// SubjectPipelineStarted is where pipeline-started is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-started... +func SubjectPipelineStarted(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision } // SubjectPipelineStartedAll hears every pipeline-started. @@ -76,9 +76,9 @@ const SubjectPipelineStartedAll = "mesh.seat.mesh-pipeline-runner.event.pipeline // StreamPipelineEnded is the stream pipeline-ended is kept in. const StreamPipelineEnded = "SEAT_mesh-pipeline-runner_pipeline-ended" -// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended... -func SubjectPipelineEnded(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source +// SubjectPipelineEnded is where pipeline-ended is published, on mesh.seat.mesh-pipeline-runner.event.pipeline-ended... +func SubjectPipelineEnded(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision } // SubjectPipelineEndedAll hears every pipeline-ended. @@ -120,9 +120,9 @@ const SubjectStepEndedAll = "mesh.seat.mesh-pipeline-runner.event.step-ended.*.* // StreamWaitHeld is the stream wait-held is kept in. const StreamWaitHeld = "SEAT_mesh-pipeline-runner_wait-held" -// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held... -func SubjectWaitHeld(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source +// SubjectWaitHeld is where wait-held is published, on mesh.seat.mesh-pipeline-runner.event.wait-held... +func SubjectWaitHeld(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision } // SubjectWaitHeldAll hears every wait-held. @@ -131,9 +131,9 @@ const SubjectWaitHeldAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*" // StreamWaitEnded is the stream wait-ended is kept in. const StreamWaitEnded = "SEAT_mesh-pipeline-runner_wait-ended" -// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended... -func SubjectWaitEnded(branch, source string) string { - return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source +// SubjectWaitEnded is where wait-ended is published, on mesh.seat.mesh-pipeline-runner.event.wait-ended... +func SubjectWaitEnded(branch, revision string) string { + return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision } // SubjectWaitEndedAll hears every wait-ended. @@ -156,11 +156,11 @@ const ( StateStepOutputs = "step-outputs" ) -// PipelinesByBranchSourceKey is the key of one value of the state pipelines under the form ".". -func PipelinesByBranchSourceKey(branch, source string) string { return branch + "." + source } +// PipelinesByBranchRevisionKey is the key of one value of the state pipelines under the form ".". +func PipelinesByBranchRevisionKey(branch, revision string) string { return branch + "." + revision } -// ParsePipelinesByBranchSourceKey reads the parts of a key of the form ".", in the order of the form; the last result is false for a key of another form. -func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) { +// ParsePipelinesByBranchRevisionKey reads the parts of a key of the form ".", in the order of the form; the last result is false for a key of another form. +func ParsePipelinesByBranchRevisionKey(key string) (string, string, bool) { rest := key v0, after0, found0 := strings.Cut(rest, ".") if !found0 || v0 == "" { @@ -178,9 +178,9 @@ func ParsePipelinesByBranchSourceKey(key string) (string, string, bool) { return v0, v2, true } -// DecodePipelinesByBranchSourceValue reads one value of the state pipelines under the form ".", refusing a field PipelinesByBranchSourceValue does not know and any text after the value. -func DecodePipelinesByBranchSourceValue(raw []byte) (PipelinesByBranchSourceValue, error) { - var v PipelinesByBranchSourceValue +// DecodePipelinesByBranchRevisionValue reads one value of the state pipelines under the form ".", refusing a field PipelinesByBranchRevisionValue does not know and any text after the value. +func DecodePipelinesByBranchRevisionValue(raw []byte) (PipelinesByBranchRevisionValue, error) { + var v PipelinesByBranchRevisionValue dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&v); err != nil { @@ -670,7 +670,7 @@ type Pipeline struct { // the modules the pipeline moves, its take-in's entries (design 47 H §746) Movers []Mover `json:"movers"` Outcome string `json:"outcome"` - // the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) + // the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) Pipeline string `json:"pipeline"` // for a pipeline a package's publish started, that publish (A-202) Published *Published `json:"published,omitempty"` @@ -720,7 +720,7 @@ type PipelineEnded struct { // per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2) Movers []MoverEnded `json:"movers"` Outcome string `json:"outcome"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // the catalog graph revision of the branch the pipeline runs for Revision int64 `json:"revision,omitempty"` @@ -753,7 +753,7 @@ type PipelineStarted struct { Call string `json:"call,omitempty"` // the pipeline's movers Movers []Mover `json:"movers"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // for a pipeline a package's publish started, that publish (A-202) Published *Published `json:"published,omitempty"` @@ -814,7 +814,7 @@ type StepEvent struct { // the module's build-input fingerprint at the commit Fingerprint string `json:"fingerprint,omitempty"` Module string `json:"module"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` Result StepResult `json:"result"` // the catalog graph revision of the branch the pipeline runs for @@ -966,7 +966,7 @@ type ToolchainUsed struct { Version string `json:"version"` } -// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72). +// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221). type Wait struct { Artifact string `json:"artifact,omitempty"` // the step's own ask, where it has one (design 47 G §731) @@ -983,7 +983,7 @@ type Wait struct { Module string `json:"module,omitempty"` // what the step waits on (design 47 *Health and going back*) On string `json:"on"` - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` // the catalog graph revision of the branch the pipeline runs for Revision int64 `json:"revision,omitempty"` @@ -1024,7 +1024,7 @@ const ( // Waiting is a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72). type Waiting struct { - // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) + // the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) Pipeline string `json:"pipeline"` } @@ -1355,8 +1355,8 @@ type FullEvent = Fill // RoomEvent is the body of the event room. type RoomEvent = Fill -// PipelinesByBranchSourceValue is one value of the state pipelines under the key .. -type PipelinesByBranchSourceValue = Pipeline +// PipelinesByBranchRevisionValue is one value of the state pipelines under the key .. +type PipelinesByBranchRevisionValue = Pipeline // PipelinesByBranchSourcePublishedPackageVersionValue is one value of the state pipelines under the key ..published... type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go index 23b59e9c..4aaa6d53 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract_test.go @@ -45,7 +45,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")}, {name: "a take-in's pipeline kept, its steps naming their mover", subject: "", given: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"0123456789abcdef\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"passed\",\"pipeline\":\"main.3589\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.3588\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"module\":\"nats\",\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"module\":\"nats\",\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, @@ -73,13 +73,13 @@ func TestFixtures(t *testing.T) { }, subjectIs: nil}, {name: "a pipeline waiting its turn, its seq given", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, {name: "a waiting pipeline names the movers a newer take-in took out of it", subject: "", given: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}", wire: "{\"branch\":\"main\",\"call\":\"fedcba9876543210\",\"movers\":[{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0124\",\"module\":\"nats\",\"repository\":\"mesh-networking\",\"tier\":1}],\"outcome\":\"waiting\",\"pipeline\":\"main.3590\",\"revision\":13,\"seq\":2,\"source\":3590,\"steps\":[],\"superseded\":[\"mesh-config\"],\"v\":1}", decode: func(b []byte) (any, error) { - var v PipelinesByBranchSourceValue + var v PipelinesByBranchRevisionValue err := json.Unmarshal(b, &v) return v, err }, subjectIs: nil}, diff --git a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go index a4dc8aa9..0df94fed 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go @@ -83,9 +83,9 @@ const SubjectWantedAll = "mesh.seat.mesh-wants.event.wanted.*.*" // StreamPipelineWanted is the stream pipeline-wanted is kept in. const StreamPipelineWanted = "SEAT_mesh-wants_pipeline-wanted" -// SubjectPipelineWanted is where pipeline-wanted is published, on mesh.seat.mesh-wants.event.pipeline-wanted... -func SubjectPipelineWanted(branch, source string) string { - return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + source +// SubjectPipelineWanted is where pipeline-wanted is published, on mesh.seat.mesh-wants.event.pipeline-wanted... +func SubjectPipelineWanted(branch, revision string) string { + return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + revision } // SubjectPipelineWantedAll hears every pipeline-wanted. @@ -1001,10 +1001,12 @@ type PipelineWanted struct { // for tier-late, the late tier's movers not running and the machines they do not run on NotRunning []NotRunning `json:"not-running,omitempty"` Outcome string `json:"outcome"` - // the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746) + // the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746, H-A-213 (4)) Pipeline string `json:"pipeline"` - Source int64 `json:"source"` - V int64 `json:"v"` + // the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event + Revision int64 `json:"revision,omitempty"` + Source int64 `json:"source"` + V int64 `json:"v"` } // The values of PipelineWanted's outcome. diff --git a/modules/mesh-delivery-contracts/go/seats/meshwants/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshwants/contract_test.go index daeed530..d623d162 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshwants/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshwants/contract_test.go @@ -29,7 +29,7 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v WantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wanted\\.[^.*> ]+\\.[^.*> ]+$")}, {name: "a want removed, its node left or its module unplaced", subject: "mesh.seat.mesh-wants.event.wanted.n1.xclip", given: "{\"at\":\"2026-11-09T19:05:00Z\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"module\":\"xclip\",\"node\":\"n1\",\"removed\":true,\"revision\":41,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:05:00Z\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"module\":\"xclip\",\"node\":\"n1\",\"removed\":true,\"revision\":41,\"v\":1}", decode: func(b []byte) (any, error) { var v WantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wanted\\.[^.*> ]+\\.[^.*> ]+$")}, - {name: "a pipeline's wants all run", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3589", given: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3589\",\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3589\",\"source\":3589,\"v\":1}", + {name: "a pipeline's wants all run", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", given: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3590\",\"revision\":3590,\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"outcome\":\"running\",\"pipeline\":\"main.3590\",\"revision\":3590,\"source\":3589,\"v\":1}", decode: func(b []byte) (any, error) { var v PipelineWantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.pipeline-wanted\\.[^.*> ]+\\.[^.*> ]+$")}, {name: "a pipeline's tier late, naming the movers and machines not running", subject: "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", given: "{\"branch\":\"main\",\"not-running\":[{\"machines\":[\"anchor\",\"laptop\"],\"module\":\"nats\"}],\"outcome\":\"tier-late\",\"pipeline\":\"main.3590\",\"source\":3590,\"v\":1}", wire: "{\"branch\":\"main\",\"not-running\":[{\"machines\":[\"anchor\",\"laptop\"],\"module\":\"nats\"}],\"outcome\":\"tier-late\",\"pipeline\":\"main.3590\",\"source\":3590,\"v\":1}", decode: func(b []byte) (any, error) { var v PipelineWantedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.pipeline-wanted\\.[^.*> ]+\\.[^.*> ]+$")}, diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go index 7ed4f8fe..213116c3 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildgo/contract.go @@ -905,8 +905,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go index d2136e78..391f8658 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildimage/contract.go @@ -903,8 +903,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go index 47ca8f1d..bd89858a 100644 --- a/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/nodebuildnode/contract.go @@ -903,8 +903,10 @@ type StepRun struct { Pipeline string `json:"pipeline"` // the step is asked again by the runner's rerun Rerun bool `json:"rerun,omitempty"` - // the source revision of the watched branch the pipeline runs for; absent for a run on request + // the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request Revision int64 `json:"revision,omitempty"` + // the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request + Source int64 `json:"source,omitempty"` // the pipeline step Step string `json:"step"` Toolchain *ToolchainUsed `json:"toolchain,omitempty"` diff --git a/modules/mesh-delivery-contracts/parts/build-step.json b/modules/mesh-delivery-contracts/parts/build-step.json index fcaebd62..0390aecb 100644 --- a/modules/mesh-delivery-contracts/parts/build-step.json +++ b/modules/mesh-delivery-contracts/parts/build-step.json @@ -1521,7 +1521,11 @@ }, "revision": { "type": "integer", - "description": "the source revision of the watched branch the pipeline runs for; absent for a run on request" + "description": "the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request" + }, + "source": { + "type": "integer", + "description": "the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request" }, "fingerprint": { "type": "string", diff --git a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json index d10eec4f..e100e48c 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json @@ -180,6 +180,10 @@ "type": "integer", "description": "the source revision of that branch the build was made from" }, + "revision": { + "type": "integer", + "description": "the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field" + }, "dependencies": { "type": "array", "items": { @@ -363,6 +367,13 @@ "$ref": "#/defs/BuildEntry" }, "description": "each entry the step built, by module and fingerprint" + }, + "packages": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207)" } }, "required": [ diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json index fd3678df..1c0b4dd0 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/events.json @@ -8,6 +8,10 @@ "revision": 12, "kind": "trunk", "source": 3589, + "call": "release of mesh-sdk 0.1.27", + "call-kind": "release", + "module": "mesh-sdk", + "version": "0.1.27", "entries": 2, "changed": [ "xclip" diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json index e537ca82..965b8ff6 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/states.json @@ -209,7 +209,11 @@ "repository": "novox/mesh-catalog" } ], - "base": 9 + "base": 9, + "call": "release of mesh-sdk 0.1.27", + "call-kind": "release", + "module": "mesh-sdk", + "version": "0.1.27" } }, { diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json index e9db48fc..164a7576 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json @@ -657,6 +657,32 @@ "source": { "type": "integer" }, + "call": { + "type": "string", + "description": "what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's" + }, + "call-kind": { + "type": "string", + "enum": [ + "take-in", + "release", + "take-in-again", + "recompute" + ], + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's" + }, + "module": { + "type": "string", + "description": "for call-kind release, the module released (H-A-215)" + }, + "version": { + "type": "string", + "description": "for call-kind release, the version released (H-A-215)" + }, + "repository": { + "type": "string", + "description": "for call-kind take-in-again, the repository taken in again (H-A-215)" + }, "group": { "type": "string" }, @@ -1871,6 +1897,32 @@ "type": "integer", "description": "the source revision it was taken in from; for a candidate, the source revision its heads are merged onto" }, + "call": { + "type": "string", + "description": "what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's" + }, + "call-kind": { + "type": "string", + "enum": [ + "take-in", + "release", + "take-in-again", + "recompute" + ], + "description": "the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's" + }, + "module": { + "type": "string", + "description": "for call-kind release, the module released (H-A-215)" + }, + "version": { + "type": "string", + "description": "for call-kind release, the version released (H-A-215)" + }, + "repository": { + "type": "string", + "description": "for call-kind take-in-again, the repository taken in again (H-A-215)" + }, "group": { "type": "string", "description": "a candidate's group" diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json index cb8363f2..9229229f 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/fixtures/states.json @@ -1,7 +1,7 @@ [ { "name": "a take-in's pipeline kept, its steps naming their mover", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3589", "branch": "main", @@ -200,7 +200,7 @@ }, { "name": "a pipeline waiting its turn, its seq given", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3590", "branch": "main", @@ -224,7 +224,7 @@ }, { "name": "a waiting pipeline names the movers a newer take-in took out of it", - "of": "state:pipelines/.", + "of": "state:pipelines/.", "given": { "pipeline": "main.3590", "branch": "main", diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json index 5bf14a52..fd84b289 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json @@ -672,7 +672,7 @@ "events": [ { "name": "pipeline-started", - "subject": "pipeline-started..", + "subject": "pipeline-started..", "body": { "$ref": "#/defs/PipelineStarted" }, @@ -683,7 +683,7 @@ }, { "name": "pipeline-ended", - "subject": "pipeline-ended..", + "subject": "pipeline-ended..", "body": { "$ref": "#/defs/PipelineEnded" }, @@ -727,7 +727,7 @@ }, { "name": "wait-held", - "subject": "wait-held..", + "subject": "wait-held..", "body": { "$ref": "#/defs/Wait" }, @@ -738,7 +738,7 @@ }, { "name": "wait-ended", - "subject": "wait-ended..", + "subject": "wait-ended..", "body": { "$ref": "#/defs/Wait" }, @@ -778,11 +778,11 @@ "description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747), and one key per mover of a pipeline holding that mover's steps (H-A-187)", "key-forms": [ { - "key": ".", + "key": ".", "value": { "$ref": "#/defs/Pipeline" }, - "description": "a take-in's pipeline" + "description": "a pipeline of one catalog revision, keyed by the revision's number (design 47 H-A-213 (4), H-A-219); one kept from before under its source has the same form" }, { "key": "..published..", @@ -1199,7 +1199,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)" + "description": "the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83)" }, "branch": { "type": "string", @@ -1313,7 +1313,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "branch": { "type": "string", @@ -1388,7 +1388,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "branch": { "type": "string", @@ -1495,7 +1495,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "module": { "type": "string" @@ -1550,7 +1550,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" }, "module": { "type": "string", @@ -1635,7 +1635,7 @@ "description": "the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)" } }, - "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72)", + "description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221)", "required": [ "v", "pipeline", @@ -1978,7 +1978,7 @@ "properties": { "pipeline": { "type": "string", - "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" + "description": "the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747)" } }, "description": "a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72)", diff --git a/modules/mesh-delivery-contracts/seats/mesh-wants/fixtures/events.json b/modules/mesh-delivery-contracts/seats/mesh-wants/fixtures/events.json index feeace4e..3381500a 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-wants/fixtures/events.json +++ b/modules/mesh-delivery-contracts/seats/mesh-wants/fixtures/events.json @@ -65,12 +65,13 @@ { "name": "a pipeline's wants all run", "of": "event:pipeline-wanted", - "subject": "mesh.seat.mesh-wants.event.pipeline-wanted.main.3589", + "subject": "mesh.seat.mesh-wants.event.pipeline-wanted.main.3590", "given": { "v": 1, - "pipeline": "main.3589", + "pipeline": "main.3590", "branch": "main", "source": 3589, + "revision": 3590, "outcome": "running" } }, diff --git a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json index f09ae1f9..b827436e 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json @@ -434,7 +434,7 @@ }, { "name": "pipeline-wanted", - "subject": "pipeline-wanted..", + "subject": "pipeline-wanted..", "keep": { "max-age-days": 30, "max-mib": 64 @@ -1725,7 +1725,7 @@ }, "pipeline": { "type": "string", - "description": "the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746)" + "description": "the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746, H-A-213 (4))" }, "branch": { "type": "string" @@ -1733,6 +1733,10 @@ "source": { "type": "integer" }, + "revision": { + "type": "integer", + "description": "the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event" + }, "outcome": { "type": "string", "enum": [ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts index d142d6d9..bf8e68c7 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts @@ -69,6 +69,8 @@ export interface PackagePublished { "digest": string; /** the module */ "module": string; + /** the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field */ + "revision"?: number; /** the source revision of that branch the build was made from */ "source": number; "v": number; @@ -89,6 +91,8 @@ export interface StepEnded { "node": string; /** never passed for a build that built nothing */ "outcome": "passed" | "failed" | "nothing to run" | "could not run"; + /** the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207) */ + "packages"?: Record; "pipeline": string; "rerun"?: boolean; /** why it failed or could not run: the entry not built, the part not heard */ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts index b3d0c697..1bef6feb 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshcatalog/index.d.ts @@ -277,6 +277,10 @@ export interface CatalogRevision { "base"?: number; /** the watched branch the revision is of */ "branch"?: string; + /** what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of , a take-in again's take-in again of ; absent for a candidate's */ + "call"?: string; + /** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217): a take-in's, a re-pin's on a module's release, a take-in again's, a recompute's that moved a fingerprint (its call naming the reason); absent for a candidate's */ + "call-kind"?: "take-in" | "release" | "take-in-again" | "recompute"; /** the seat definitions it was judged with, each by its digest, read whole through seat-definitions */ "definitions"?: CatalogRevisionDefinitionsItem[]; /** one per module, by name */ @@ -287,8 +291,12 @@ export interface CatalogRevision { "heads"?: string; /** trunk or candidate */ "kind": string; + /** for call-kind release, the module released (H-A-215) */ + "module"?: string; /** the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover */ "movers"?: TakeInEntry[]; + /** for call-kind take-in-again, the repository taken in again (H-A-215) */ + "repository"?: string; /** the catalog graph revision's number, counted by its holder */ "revision": number; /** the source revision it was taken in from; for a candidate, the source revision its heads are merged onto */ @@ -297,6 +305,8 @@ export interface CatalogRevision { "trunk"?: Head[]; /** the layout version */ "v"?: number; + /** for call-kind release, the version released (H-A-215) */ + "version"?: string; } /** one edge of the catalog graph between artifact references, B before A (ADR 0666 section 14): kind 1 B published before A builds (a package A imports), 2 B published before A delivers (fetched by reference at its start), 3 B delivered before A builds (its build module and toolchain), 4 B delivered before A delivers (a provision, bus state, a format's reader), 5 B has acted for A's new version (an acting rule), 6 B runs a version that serves A (a verb or a manifest word A uses) */ @@ -796,6 +806,10 @@ export interface CatalogRevisionEvent { "at"?: string; /** the watched branch the revision is of, escaped as a key name is */ "branch"?: string; + /** what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's */ + "call"?: string; + /** the kind of what made the revision, which a reader reads instead of call's words (design 47 H-A-213 (3), H-A-214, H-A-215, H-A-217); absent for a candidate's */ + "call-kind"?: "take-in" | "release" | "take-in-again" | "recompute"; /** the modules whose entries differ from the revision before it of the same kind */ "changed"?: string[]; /** how many entries it lists */ @@ -804,9 +818,15 @@ export interface CatalogRevisionEvent { "heads"?: string; /** trunk or candidate: a candidate's moves no running part */ "kind": string; + /** for call-kind release, the module released (H-A-215) */ + "module"?: string; + /** for call-kind take-in-again, the repository taken in again (H-A-215) */ + "repository"?: string; "revision": number; "source": number; "v": number; + /** for call-kind release, the version released (H-A-215) */ + "version"?: string; } /** is the body of the event filling. */ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts index 748152b9..5c11adad 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -40,13 +40,13 @@ export declare const family: { readonly name: "mesh-pipeline-runner.events"; rea export declare const pipelineStartedStream: "SEAT_mesh-pipeline-runner_pipeline-started"; -export declare function pipelineStartedSubject(branch: string, source: string): string; +export declare function pipelineStartedSubject(branch: string, revision: string): string; export declare const pipelineStartedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*"; export declare const pipelineEndedStream: "SEAT_mesh-pipeline-runner_pipeline-ended"; -export declare function pipelineEndedSubject(branch: string, source: string): string; +export declare function pipelineEndedSubject(branch: string, revision: string): string; export declare const pipelineEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"; @@ -70,13 +70,13 @@ export declare const stepEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event. export declare const waitHeldStream: "SEAT_mesh-pipeline-runner_wait-held"; -export declare function waitHeldSubject(branch: string, source: string): string; +export declare function waitHeldSubject(branch: string, revision: string): string; export declare const waitHeldSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*"; export declare const waitEndedStream: "SEAT_mesh-pipeline-runner_wait-ended"; -export declare function waitEndedSubject(branch: string, source: string): string; +export declare function waitEndedSubject(branch: string, revision: string): string; export declare const waitEndedSubjectAll: "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*"; @@ -94,11 +94,11 @@ export declare const states: { readonly stepOutputs: "step-outputs"; }; -/** The key of one value of the state pipelines under the form ".". */ -export declare function pipelinesByBranchSourceKey(branch: string, source: string): string; +/** The key of one value of the state pipelines under the form ".". */ +export declare function pipelinesByBranchRevisionKey(branch: string, revision: string): string; -/** The parts of a key of the form ".", or undefined for a key of another form. */ -export declare function parsePipelinesByBranchSourceKey(_key: string): { branch: string; source: string } | undefined; +/** The parts of a key of the form ".", or undefined for a key of another form. */ +export declare function parsePipelinesByBranchRevisionKey(_key: string): { branch: string; revision: string } | undefined; /** The key of one value of the state pipelines under the form "..published..". */ export declare function pipelinesByBranchSourcePublishedPackageVersionKey(branch: string, source: string, packagePart: string, version: string): string; @@ -261,7 +261,7 @@ export interface Pipeline { /** the modules the pipeline moves, its take-in's entries (design 47 H §746) */ "movers": Mover[]; "outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled"; - /** the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */ + /** the pipeline's key, ., ..published.. for a pipeline a publish started, or .run. for a run on request; a branch escaped as a key's name is (design 47 H §746, §747; H-A-83) */ "pipeline": string; /** for a pipeline a package's publish started, that publish (A-202) */ "published"?: Published; @@ -300,7 +300,7 @@ export interface PipelineEnded { /** per mover its fingerprint, released version, step results and tier, in the delivery order (design 47 H §746, H-A-2) */ "movers": MoverEnded[]; "outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled"; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** the catalog graph revision of the branch the pipeline runs for */ "revision"?: number; @@ -322,7 +322,7 @@ export interface PipelineStarted { "call"?: string; /** the pipeline's movers */ "movers": Mover[]; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** for a pipeline a package's publish started, that publish (A-202) */ "published"?: Published; @@ -383,7 +383,7 @@ export interface StepEvent { /** the module's build-input fingerprint at the commit */ "fingerprint"?: string; "module": string; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; "result": StepResult; /** the catalog graph revision of the branch the pipeline runs for */ @@ -490,7 +490,7 @@ export interface ToolchainUsed { "version": string; } -/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72) */ +/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283); said on . (design 47 H-A-72, H-A-221) */ export interface Wait { "artifact"?: string; /** the step's own ask, where it has one (design 47 G §731) */ @@ -507,7 +507,7 @@ export interface Wait { "module"?: string; /** what the step waits on (design 47 *Health and going back*) */ "on": "person" | "build node" | "revision view" | "pipelines-at-once"; - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; /** the catalog graph revision of the branch the pipeline runs for */ "revision"?: number; @@ -527,7 +527,7 @@ export interface Wait { /** a pipeline waiting its turn under pipelines-at-once (design 47 H §748, H-A-72) */ export interface Waiting { - /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ + /** the pipeline's key, ., or ..published.. for a pipeline a publish started; a branch escaped as a key's name is (design 47 H §746, §747) */ "pipeline": string; } @@ -800,8 +800,8 @@ export type FullEvent = Fill; /** is the body of the event room. */ export type RoomEvent = Fill; -/** is one value of the state pipelines under the key .. */ -export type PipelinesByBranchSourceValue = Pipeline; +/** is one value of the state pipelines under the key .. */ +export type PipelinesByBranchRevisionValue = Pipeline; /** is one value of the state pipelines under the key ..published... */ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline; diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js index 213a3df3..d8ff25e6 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js @@ -40,16 +40,16 @@ export const family = { name: "mesh-pipeline-runner.events", maxAgeDays: 7 }; export const pipelineStartedStream = "SEAT_mesh-pipeline-runner_pipeline-started"; -export function pipelineStartedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + source; +export function pipelineStartedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-started." + branch + "." + revision; } export const pipelineStartedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-started.*.*"; export const pipelineEndedStream = "SEAT_mesh-pipeline-runner_pipeline-ended"; -export function pipelineEndedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + source; +export function pipelineEndedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.pipeline-ended." + branch + "." + revision; } export const pipelineEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"; @@ -80,16 +80,16 @@ export const stepEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.step-en export const waitHeldStream = "SEAT_mesh-pipeline-runner_wait-held"; -export function waitHeldSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + source; +export function waitHeldSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.wait-held." + branch + "." + revision; } export const waitHeldSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-held.*.*"; export const waitEndedStream = "SEAT_mesh-pipeline-runner_wait-ended"; -export function waitEndedSubject(branch, source) { - return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + source; +export function waitEndedSubject(branch, revision) { + return "mesh.seat.mesh-pipeline-runner.event.wait-ended." + branch + "." + revision; } export const waitEndedSubjectAll = "mesh.seat.mesh-pipeline-runner.event.wait-ended.*.*"; @@ -108,23 +108,23 @@ export const states = { stepOutputs: "step-outputs", }; -/** The key of one value of the state pipelines under the form ".". */ -export function pipelinesByBranchSourceKey(branch, source) { - return branch + "." + source; +/** The key of one value of the state pipelines under the form ".". */ +export function pipelinesByBranchRevisionKey(branch, revision) { + return branch + "." + revision; } -/** The parts of a key of the form ".", or undefined for a key of another form. */ -export function parsePipelinesByBranchSourceKey(_key) { +/** The parts of a key of the form ".", or undefined for a key of another form. */ +export function parsePipelinesByBranchRevisionKey(_key) { let _rest = _key; const _at0 = _rest.indexOf("."); if (_at0 <= 0) return undefined; const branch = _rest.slice(0, _at0); _rest = _rest.slice(_at0 + 1); if (_rest === "") return undefined; - const source = _rest; + const revision = _rest; _rest = ""; if (_rest !== "") return undefined; - return { branch, source }; + return { branch, revision }; } /** The key of one value of the state pipelines under the form "..published..". */ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts index 0b53c871..d4ef255a 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts @@ -50,7 +50,7 @@ export declare const wantedSubjectAll: "mesh.seat.mesh-wants.event.wanted.*.*"; export declare const pipelineWantedStream: "SEAT_mesh-wants_pipeline-wanted"; -export declare function pipelineWantedSubject(branch: string, source: string): string; +export declare function pipelineWantedSubject(branch: string, revision: string): string; export declare const pipelineWantedSubjectAll: "mesh.seat.mesh-wants.event.pipeline-wanted.*.*"; @@ -412,8 +412,10 @@ export interface PipelineWanted { /** for tier-late, the late tier's movers not running and the machines they do not run on */ "not-running"?: NotRunning[]; "outcome": "running" | "tier-late"; - /** the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746) */ + /** the runner's pipeline key, ., a branch escaped as a key's name is (design 47 H §746, H-A-213 (4)) */ "pipeline": string; + /** the pipeline's catalog revision number, the number of its key; the indexer frees the pins of that revision, not of another revision of the same source (design 47 H-A-214); absent from an older event */ + "revision"?: number; "source": number; "v": number; } diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js index a544df9a..b0c21d4b 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js @@ -52,8 +52,8 @@ export const wantedSubjectAll = "mesh.seat.mesh-wants.event.wanted.*.*"; export const pipelineWantedStream = "SEAT_mesh-wants_pipeline-wanted"; -export function pipelineWantedSubject(branch, source) { - return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + source; +export function pipelineWantedSubject(branch, revision) { + return "mesh.seat.mesh-wants.event.pipeline-wanted." + branch + "." + revision; } export const pipelineWantedSubjectAll = "mesh.seat.mesh-wants.event.pipeline-wanted.*.*"; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildgo/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildimage/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts index 72bbb451..93c0f352 100644 --- a/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/nodebuildnode/parts/build-step.d.ts @@ -510,8 +510,10 @@ export interface StepRun { "pipeline": string; /** the step is asked again by the runner's rerun */ "rerun"?: boolean; - /** the source revision of the watched branch the pipeline runs for; absent for a run on request */ + /** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */ "revision"?: number; + /** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */ + "source"?: number; /** the pipeline step */ "step": "lint" | "unit" | "integration"; "toolchain"?: ToolchainUsed; diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go new file mode 100644 index 00000000..8eb969dd --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "fmt" + "strconv" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-indexer/internal/index" +) + +// entryAnswer is what indexer_entry answers: the trunk revision read, the entry's packages (the pins: each other-repository +// package with the version it builds against, empty while none is published), its named paths by artifact, its fingerprint +// and its manifest's digest. +type entryAnswer struct { + Module string `json:"module"` + Revision int64 `json:"revision"` + Packages map[string]string `json:"packages"` + Paths map[string][]string `json:"paths"` + Fp string `json:"fp,omitempty"` + ManifestDigest string `json:"manifest-digest,omitempty"` +} + +// entryReadTool is indexer_entry (H-A-204; tools-are-how-a-module-is-spoken-to) over the indexer's catalog-revisions objects: +// one module's entry pins and paths at a trunk revision, small, so a pin can be read without the revision whole (1.3 MB, +// more than the bus carries). +func entryReadTool(o index.Objects) stdio.Tool { + return stdio.Tool{Name: index.EntryPinsTool, + Description: "Read one module's catalog graph entry pins: the packages it builds against with the version each is pinned to " + + "(an empty version while none is published), the paths each artifact is built from, its fingerprint and its manifest " + + "digest, at the newest trunk revision or at a trunk revision given; a small answer, for when the revision whole is too " + + "large for the bus (H-A-204).", + Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module whose entry is read"}, + "revision": map[string]any{"type": "integer", "description": "the trunk revision; none reads the newest"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + module = strings.TrimSpace(module) + if module == "" { + return nil, fmt.Errorf("%s names the module", index.EntryPinsTool) + } + revision, err := revisionArg(args["revision"]) + if err != nil { + return nil, err + } + rev, e, err := index.TrunkEntryAt(context.Background(), o, module, revision) + if err != nil { + return nil, err + } + out := entryAnswer{Module: e.Module, Revision: rev.Revision, Packages: e.Packages, Paths: map[string][]string{}, + Fp: e.Fp, ManifestDigest: e.ManifestDigest} + if out.Packages == nil { + out.Packages = map[string]string{} + } + if e.Inputs != nil && e.Inputs.Paths != nil { + out.Paths = e.Inputs.Paths + } + return out, nil + }} +} + +// revisionArg is the revision argument as a number: absent is 0 (the newest); a number or its digits as a string. +func revisionArg(v any) (int64, error) { + switch r := v.(type) { + case nil: + return 0, nil + case float64: + if r < 0 || r != float64(int64(r)) { + return 0, fmt.Errorf("revision %v is not a trunk revision's number", r) + } + return int64(r), nil + case string: + if strings.TrimSpace(r) == "" { + return 0, nil + } + n, err := strconv.ParseInt(strings.TrimSpace(r), 10, 64) + if err != nil || n < 0 { + return 0, fmt.Errorf("revision %q is not a trunk revision's number", r) + } + return n, nil + } + return 0, fmt.Errorf("revision %v is not a trunk revision's number", v) +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go new file mode 100644 index 00000000..aefb9942 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/entry_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + + "mesh-delivery-indexer/internal/index" +) + +// memObjects is catalog-revisions in memory. +type memObjects map[string][]byte + +func (m memObjects) Put(_ context.Context, name string, data []byte) error { + m[name] = data + return nil +} +func (m memObjects) Get(_ context.Context, name string) ([]byte, error) { + d, ok := m[name] + if !ok { + return nil, index.ErrNoObject + } + return d, nil +} +func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil } +func (m memObjects) Names(context.Context) ([]string, error) { + var out []string + for n := range m { + out = append(out, n) + } + return out, nil +} +func (memObjects) Fill(context.Context) (int64, int64, error) { return 0, 0, nil } + +// TestEntryTool: indexer_entry answers one entry's pins, paths, fingerprint and manifest digest, small, at the newest trunk +// revision and at a given one read from its kept object; an unknown module or revision is an error (H-A-204). +func TestEntryTool(t *testing.T) { + o := memObjects{} + keep := func(revision int64, entries ...meshcatalog.CatalogEntry) { + rev := meshcatalog.CatalogRevision{Branch: index.TrunkBranch, Revision: revision} + for _, e := range entries { + e.Manifest = map[string]any{"module": e.Module, "big": strings.Repeat("x", 64<<10)} // the manifest whole is never answered + data, _ := json.Marshal(e) + digest := e.Module + "-" + e.Fp + o[index.EntryObject(digest)] = data + rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: e.Module, Digest: digest}) + } + data, _ := json.Marshal(rev) + o[index.BranchObject(index.TrunkBranch, revision)] = data + } + paths := &meshcatalog.EntryInputs{Paths: map[string][]string{"mesh-grants:program": {"modules/mesh-grants", "sdk"}}} + keep(7, meshcatalog.CatalogEntry{Module: "mesh-grants", Fp: "src2:old", ManifestDigest: "sha256:a", Inputs: paths, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26"}}, meshcatalog.CatalogEntry{Module: "other", Fp: "src2:o"}) + keep(9, meshcatalog.CatalogEntry{Module: "mesh-grants", Fp: "src2:new", ManifestDigest: "sha256:b", Inputs: paths, + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.27", "busgrants": ""}}) + tool := entryReadTool(o) + if tool.Name != "indexer_entry" { + t.Fatalf("name %s", tool.Name) + } + for _, c := range []struct { + name string + args map[string]any + want entryAnswer + err string + }{ + {"newest", map[string]any{"module": "mesh-grants"}, entryAnswer{Module: "mesh-grants", Revision: 9, Fp: "src2:new", ManifestDigest: "sha256:b", + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.27", "busgrants": ""}, Paths: paths.Paths}, ""}, + {"given", map[string]any{"module": "mesh-grants", "revision": float64(7)}, entryAnswer{Module: "mesh-grants", Revision: 7, Fp: "src2:old", ManifestDigest: "sha256:a", + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26"}, Paths: paths.Paths}, ""}, + {"given as digits, no paths", map[string]any{"module": "other", "revision": "7"}, entryAnswer{Module: "other", Revision: 7, Fp: "src2:o", + Packages: map[string]string{}, Paths: map[string][]string{}}, ""}, + {"not in the newest", map[string]any{"module": "other"}, entryAnswer{}, "holds no module other"}, + {"revision not kept", map[string]any{"module": "mesh-grants", "revision": float64(8)}, entryAnswer{}, "revision 8 is not kept"}, + {"no module", map[string]any{}, entryAnswer{}, "names the module"}, + } { + t.Run(c.name, func(t *testing.T) { + got, err := tool.Run(c.args) + if c.err != "" { + if err == nil || !strings.Contains(err.Error(), c.err) { + t.Fatalf("error %v, want %q", err, c.err) + } + return + } + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, c.want) { + t.Fatalf("answer %+v, want %+v", got, c.want) + } + data, _ := json.Marshal(got) + if len(data) >= 8<<10 { + t.Fatalf("answer of %d bytes, want under 8 KB", len(data)) + } + }) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index c4311fa4..a2eba94a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -193,7 +193,7 @@ func main() { tools = append(tools, catalogTools(trunk.State, kept, watched)...) // The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly // the parts whose entry declares it (novox/hq ADR 0663). - tools = append(tools, index.RevisionTool(revisions, kept), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), + tools = append(tools, index.RevisionTool(revisions, kept), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), entryReadTool(revisions), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}), index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}), index.InputsReadTool(keptRead{stdio.State(index.InputsState)}, stateobjects.ForState(index.InputsState).Objects)) tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index bcf46813..76a24231 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -433,7 +433,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro done() done = step("following the assignments") followAssignments(ctx, reader, seatHolders.holders(), process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views reads nothing and indexer_first_view_again is not heard", logger.Printf), cred, js, gate, func(ctx context.Context) { - if err := ix.Recompute(ctx); err != nil { + if err := ix.RecomputeKept(ctx, "the assignments of a build module"); err != nil { logger.Printf("writing the modules again after the assignments of a build module changed: %v", err) } }, logger.Printf) @@ -449,7 +449,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // the toolchain. return ix.ToolchainReported(ctx) }, - recompute: ix.Recompute}, func() []string { return buildModuleNames(ix.BuildModules()) }, logger) + recompute: func(ctx context.Context) error { return ix.RecomputeKept(ctx, "the toolchains read") }}, func() []string { return buildModuleNames(ix.BuildModules()) }, logger) if err := tools.start(); err != nil { return err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index c57388a2..cc42bf67 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -90,7 +90,7 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 }{ {name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"}, {name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"}, - {name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.12","branch":"main","source":12,"outcome":"tier-late"}`, wanted: true, acked: 1}, + {name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.13","branch":"main","source":12,"revision":13,"outcome":"tier-late"}`, wanted: true, acked: 1}, {name: "not json", body: `nope`, termd: 1}, {name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"}, {name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"}, @@ -100,8 +100,8 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 switch { case row.wanted: heardPipelineWanted(context.Background(), ix, m, t.Logf) - if ix.wanted != "main.12 tier-late" { - t.Errorf("%s: was handed %q, want main.12 tier-late", row.name, ix.wanted) + if ix.wanted != "main.12.13 tier-late" { // its revision handed beside its source (H-A-214) + t.Errorf("%s: was handed %q, want main.12.13 tier-late", row.name, ix.wanted) } case row.ended: heardPipelineEnded(context.Background(), ix, m, t.Logf) @@ -169,8 +169,11 @@ func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarte return nil } -func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64, outcome string) error { +func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source, revision int64, outcome string) error { f.wanted = fmt.Sprintf("%s.%d %s", branch, source, outcome) + if revision > 0 { + f.wanted = fmt.Sprintf("%s.%d.%d %s", branch, source, revision, outcome) + } return f.err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go index fdbf67aa..1c095858 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go @@ -21,7 +21,7 @@ type runTaker interface { StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error - HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error + HeardPipelineWanted(ctx context.Context, branch string, source, revision int64, outcome string) error JudgePins(ctx context.Context) } @@ -99,7 +99,7 @@ func heardPipelineWanted(ctx context.Context, ix runTaker, m delivered, logf fun _ = m.Term() return } - err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Outcome) + err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Revision, e.Outcome) settleRun(m, err, "pipeline-wanted of "+e.Pipeline, logf) if err == nil { ix.JudgePins(ctx) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go index ff1b77f2..bdfc2a5d 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series.go @@ -22,8 +22,9 @@ var ( ) // heardVersion reads one message of the consumer as a version, false for any other subject. A package's is kept in its -// series; one that does not decode is said and acknowledged; a write that fails is left unacknowledged, so the bus gives -// it again. +// series; one that does not decode is said and acknowledged; a write that fails is left unacknowledged, so the bus gives it +// again. It recomputes no module: a fingerprint moves only with a kept catalog revision, a release's by the re-pin its +// package-published starts, one revision per module release (design 47 H-A-211, H-A-213 (2)). func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) bool { var h index.HeardVersion switch seat, ok := publishedSeats[m.Subject()]; { @@ -51,12 +52,6 @@ func heardVersion(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf if kept { logf("packages: %s %s kept in its series (published %v)", h.Package, h.Version, h.Published) } - if kept && h.Published { - // A series' newest moved: its users' fingerprints take it (DLV-I6), in the same turn as the publish is heard. - if err := ix.Recompute(ctx); err != nil { - logf("writing the modules again after %s %s was published: %v", h.Package, h.Version, err) - } - } _ = m.Ack() return true } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go new file mode 100644 index 00000000..c58fb27f --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/series_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry" + + "mesh-delivery-indexer/internal/index" +) + +// ackedMsg is a message that counts its acknowledgements. +type ackedMsg struct { + jetstream.Msg + subject string + data []byte + acked int +} + +func (a *ackedMsg) Subject() string { return a.subject } +func (a *ackedMsg) Data() []byte { return a.data } +func (a *ackedMsg) Ack() error { a.acked++; return nil } + +// unlistedStore is a store whose listing fails: a recompute over it fails. +type unlistedStore struct{ memStore } + +func (unlistedStore) Keys(context.Context) ([]string, error) { + return nil, errors.New("the bus does not answer") +} + +// TestHeardVersionMovesNoFingerprint: a registry's published is kept in its series and acknowledged, and recomputes no +// module: a fingerprint moves only with a kept revision, a release's by its re-pin on the package-published (design 47 +// H-A-211, H-A-213 (2); the pin specification's leftover L3). The modules state here fails any recompute. +func TestHeardVersionMovesNoFingerprint(t *testing.T) { + data, err := json.Marshal(gopackageregistry.PublishedEvent{Kind: "package", Name: "git.example/novox/lib", Version: "0.1.4"}) + if err != nil { + t.Fatal(err) + } + ix := &index.Indexer{Packages: &memStore{vals: map[string][]byte{}}, Modules: &unlistedStore{memStore{vals: map[string][]byte{}}}, + Manifests: &memStore{vals: map[string][]byte{}}} + m := &ackedMsg{subject: gopackageregistry.SubjectPublished, data: data} + if !heardVersion(context.Background(), ix, m, t.Logf) { + t.Fatal("the registry's published is not taken as a version") + } + if m.acked != 1 { + t.Errorf("acknowledged %d times, want once: a publish recomputes nothing that could fail", m.acked) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go index cde7d759..4a42bcdd 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/sourcemoved.go @@ -286,6 +286,14 @@ func (a actor) HeldAgain(ctx context.Context, branch string) index.TakenIn { return r } +// TakeInAgain takes a repository in again at its kept merge commit (indexer_take_in_again, H-A-208, H-A-210), one take-in +// again at a time beside the held ones. +func (a actor) TakeInAgain(ctx context.Context, repository string) index.TakenIn { + a.s.again.Lock() + defer a.s.again.Unlock() + return a.Indexer.TakeInAgain(ctx, repository) +} + // HeldSkip ends a branch's held source revision unwritten (indexer_take_in_skip): take-in.taken says it skipped. func (a actor) HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn { r := index.TakenIn{Module: branch} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 14625a36..31c7f4a8 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -24,6 +24,8 @@ type takerIn interface { // HeldAgain and HeldSkip act on a watched branch's held source-moved (design 47 H §744). HeldAgain(ctx context.Context, branch string) index.TakenIn HeldSkip(ctx context.Context, branch string, source int64, by, why string) index.TakenIn + // TakeInAgain takes a repository in again at its kept merge commit (H-A-208, H-A-210). + TakeInAgain(ctx context.Context, repository string) index.TakenIn } // takenIn answers one value of the takeins state (a hand act, issue 1945): a value not yet answered is acted on now and @@ -53,6 +55,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream } else if a.Held != "" { asked, r = "take-in again of a held take-in", ix.HeldAgain(ctx, a.Held) a.Module = a.Held + } else if a.Again != "" { + asked, r = "take-in again of a repository", ix.TakeInAgain(ctx, a.Again) + a.Module = a.Again } else if a.Registry != nil { asked, r = "registrations' act "+a.Registry.Verb, ix.RegistryAct(ctx, a) } else if a.Recompute { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go index c42a2a37..3055a5ac 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein_integration_test.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "os" "slices" "strconv" @@ -447,37 +448,42 @@ func TestIndexerWhole(t *testing.T) { // The first revision: no mover, no pin; its entries are the base. sayMoved(t, ctx, js, "main", 10, c10, true) - if first := heard.next(t, "main", 10); len(first.Entries) != 0 || first.Revision != 10 { - t.Fatalf("the first revision said %+v, want revision 10 with no entries", first) + // A revision's number is the indexer's own count, the branch's newest + 1 (design 47 H-A-213 (1)): the later take-ins of + // sources 11 to 13 are counted on from the number of source 10's. + first := heard.next(t, "main", 10) + r := func(source int64) int64 { return first.Revision + source - 10 } + rn := func(source int64) string { return fmt.Sprintf("main.%d", r(source)) } + if len(first.Entries) != 0 || first.Revision != 2 || first.Source != 10 { + t.Fatalf("the first revision said %+v, want revision 2 (the bed's seed is main.1, of source 0) of source 10 with no entries", first) } - untilKey(t, ctx, pins, index.PinKey("main", 10), absent) + untilKey(t, ctx, pins, index.PinKey("main", r(10)), absent) // One take-in, one revision carrying its movers, pinned for its pipeline; said again from the kept movers. sayMoved(t, ctx, js, "main", 11, c11, false) taken := heard.next(t, "main", 11) if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Commit != c11 || taken.Entries[0].Fingerprint == "" || - taken.Entries[0].Repository != "novox/mesh-catalog" || taken.Revision != 11 || taken.Call == "" { + taken.Entries[0].Repository != "novox/mesh-catalog" || taken.Revision != r(11) || taken.Call == "" { t.Fatalf("take-in.taken of main 11 = %+v, want avahi at %s as its one mover", taken, c11) } - rev, _, err := index.ReadRevision(ctx, objects, "main.11") + rev, _, err := index.ReadRevision(ctx, objects, rn(11)) if err != nil || len(rev.Movers) != 1 || rev.Movers[0] != taken.Entries[0] { t.Fatalf("main.11 = %+v (%v), want it kept carrying the movers said", rev, err) } - untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.11")) - untilKey(t, ctx, pins, index.PinKey("main", 10), pinned("pipeline.main.11")) // validate's base, H-A-147 + untilKey(t, ctx, pins, index.PinKey("main", r(11)), pinned("pipeline."+rn(11))) + untilKey(t, ctx, pins, index.PinKey("main", r(10)), pinned("pipeline."+rn(11))) // validate's base, H-A-147 info, err := objects.open(ctx) if err != nil { t.Fatal(err) } - before, err := info.GetInfo(ctx, "main.11") + before, err := info.GetInfo(ctx, rn(11)) if err != nil { t.Fatal(err) } sayMoved(t, ctx, js, "main", 11, c11, false) - if again := heard.next(t, "main", 11); len(again.Entries) != 1 || again.Entries[0] != taken.Entries[0] || again.Revision != 11 { + if again := heard.next(t, "main", 11); len(again.Entries) != 1 || again.Entries[0] != taken.Entries[0] || again.Revision != r(11) { t.Errorf("the source-moved given again said %+v, want the kept movers %+v", again, taken.Entries) } - if after, err := info.GetInfo(ctx, "main.11"); err != nil || after.NUID != before.NUID { + if after, err := info.GetInfo(ctx, rn(11)); err != nil || after.NUID != before.NUID { t.Errorf("main.11 was written again (%v): a source-moved given again keeps no second revision", err) } @@ -485,11 +491,11 @@ func TestIndexerWhole(t *testing.T) { // (H-A-147), which the take-in pinned for it. sayMoved(t, ctx, js, "main", 12, c12, false) heard.next(t, "main", 12) - if err := pins.Delete(ctx, index.PinKey("main", 12)); err != nil { // the take-in's own pin, so pipeline-started alone writes it below + if err := pins.Delete(ctx, index.PinKey("main", r(12))); err != nil { // the take-in's own pin, so pipeline-started alone writes it below t.Fatal(err) } - sayPipeline(t, ctx, js, false, "main", 12) - untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12")) + sayPipeline(t, ctx, js, false, "main", r(12)) + untilKey(t, ctx, pins, index.PinKey("main", r(12)), pinned("pipeline."+rn(12))) // A run on request: its candidate and its key written on run-started, from avahi's files at the run's commit (A-160, A-174). manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") if err != nil { @@ -515,7 +521,7 @@ func TestIndexerWhole(t *testing.T) { heard.acked(t, "main") } restart() - for _, name := range []string{"main.10", "main.11"} { + for _, name := range []string{rn(10), rn(11)} { if _, _, err := index.ReadRevision(ctx, objects, name); err != nil { t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err) } @@ -553,27 +559,27 @@ func TestIndexerWhole(t *testing.T) { t.Errorf("the held source-moved's consumer %+v (%v), want nothing unacknowledged", info, err) } setCap(t, ctx, js, 1<<30) // a larger cap alone wakes nothing: no timer takes a held take-in in - sayPipeline(t, ctx, js, true, "main", 11) + sayPipeline(t, ctx, js, true, "main", r(11)) if got := heard.next(t, "main", 13); len(got.Entries) != 1 || got.Entries[0].Commit != c13 { t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13) } untilKey(t, ctx, held, keys.EscapeName("main"), absent) // main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170). - untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi"))) - untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins + untilKey(t, ctx, pins, index.PinKey("main", r(11)), pinned("pipeline."+rn(12), index.ReleaseRecord("main", "avahi"))) + untilRevisionGone(t, ctx, js, rn(10)) // the end freed both of main.11's pins // main.12's passed pipeline releases avahi again; its pipeline-wanted, the seat mesh-wants' (its wants written to the end), // frees main.11's older release, and main.11 goes (H-A-168, H-A-170, H-A-173). - sayPipeline(t, ctx, js, true, "main", 12) - untilKey(t, ctx, pins, index.PinKey("main", 12), func(v []byte) bool { + sayPipeline(t, ctx, js, true, "main", r(12)) + untilKey(t, ctx, pins, index.PinKey("main", r(12)), func(v []byte) bool { var p index.PinValue return v != nil && json.Unmarshal(v, &p) == nil && slices.Contains(p.Records, index.ReleaseRecord("main", "avahi")) }) - wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: "main.12", Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning}) + wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: rn(12), Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning}) if _, err := js.Publish(ctx, meshwants.SubjectPipelineWanted("main", "12"), wantedRaw); err != nil { t.Fatal(err) } - untilKey(t, ctx, pins, index.PinKey("main", 11), absent) - untilRevisionGone(t, ctx, js, "main.11") + untilKey(t, ctx, pins, index.PinKey("main", r(11)), absent) + untilRevisionGone(t, ctx, js, rn(11)) if n := len(heard.pending); n != 0 { t.Errorf("take-in.pending said %d more times for one held source revision, want once", n) } diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_values.go b/modules/mesh-delivery-indexer/internal/index/candidate_values.go index 62cc8b33..f443f807 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidate_values.go +++ b/modules/mesh-delivery-indexer/internal/index/candidate_values.go @@ -39,7 +39,9 @@ func checkCandidateModule(m meshcatalog.CandidatesByGroupValuePullsItemFpsValue) return fmt.Errorf("index: the named path %q names no git object", p) } } - if err := sdkstep.PackageVersions(m.Packages).Check(); err != nil { + // An entry names a package with no published version yet with an empty version (design 47 H S8, H-A-204): it is checked + // as an entry's packages, not as an ask's. + if err := sdkstep.PackageVersions(m.Packages).CheckEntry(); err != nil { return err } if m.Fp2 != "" { diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_values_test.go b/modules/mesh-delivery-indexer/internal/index/candidate_values_test.go new file mode 100644 index 00000000..fa25038a --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/candidate_values_test.go @@ -0,0 +1,30 @@ +package index + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" +) + +// TestCheckCandidateModulePins: an entry's package with no published version (an empty pin) passes a run on request's +// candidate; a pin that is no version is refused (design 47 H S8, the pin specification's item 4). +func TestCheckCandidateModulePins(t *testing.T) { + for _, c := range []struct { + name string + packages map[string]string + ok bool + }{ + {"an empty pin", map[string]string{"git.example/novox/lib/go": ""}, true}, + {"a version", map[string]string{"git.example/novox/lib/go": "v0.1.4"}, true}, + {"no version", map[string]string{"git.example/novox/lib/go": "latest"}, false}, + } { + t.Run(c.name, func(t *testing.T) { + m := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Fp2: "src2:" + strings.Repeat("ab", 32), Packages: c.packages} + err := checkCandidateModule(m) + if (err == nil) != c.ok { + t.Errorf("checkCandidateModule(%v) = %v, want ok %v", c.packages, err, c.ok) + } + }) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go index 5e239931..66319e48 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_branches_test.go @@ -36,13 +36,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { for _, n := range names { have[n] = true } - for _, want := range []string{"main.5", "feature/x=2ey.7", "feature/x=2ey.8"} { + for _, want := range []string{"main.1", "feature/x=2ey.1", "feature/x=2ey.2"} { if !have[want] { t.Errorf("the objects hold %v, want %s", names, want) } } - if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 8 { - t.Errorf("said %+v, want main 5, then the escaped branch 7 and 8", said) + if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 2 || said[2].Source != 8 { + t.Errorf("said %+v, want main 1 of source 5, then the escaped branch 1 and 2 of sources 7 and 8", said) } answer := func(branch string, revision int64) meshcatalog.RevisionAnswer { a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Revision: revision}) @@ -51,14 +51,14 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { } return a } - if a := answer("main", 0); a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" { + if a := answer("main", 0); a.Revision.Revision != 1 || a.Revision.Source != 5 || a.Entries[0].Commit != "c1" { t.Errorf("main answered %+v", a) } - if a := answer("feature/x=2ey", 0); a.Revision.Revision != 8 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" { - t.Errorf("the branch's newest answered %+v, want its revision 8 at c3", a.Revision) + if a := answer("feature/x=2ey", 0); a.Revision.Revision != 2 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" { + t.Errorf("the branch's newest answered %+v, want its revision 2 at c3", a.Revision) } - if a := answer("feature/x=2ey", 7); a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" { - t.Errorf("the branch's revision 7 answered %+v", a.Revision) + if a := answer("feature/x=2ey", 1); a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" { + t.Errorf("the branch's revision 1 answered %+v", a.Revision) } if a := answer("beta", 0); !a.NoneYet { t.Errorf("a branch with no revision answered %+v, want none-yet", a) @@ -68,15 +68,22 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) { bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) { return Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Source: source}) } - if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" { + if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Revision.Source != 5 || a.Entries[0].Commit != "c1" { t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err) } - if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" { - t.Errorf("the branch's source 7 answered %+v (%v), want its revision 7 at c2", a.Revision, err) + if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" { + t.Errorf("the branch's source 7 answered %+v (%v), want its revision 1 at c2", a.Revision, err) } - if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 8 { + if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 { t.Errorf("the branch's source 8 answered %+v (%v)", a.Revision, err) } + // H-A-213: a re-pin's revision of source 8 answers below the take-in's: source 8 answers revision 2. + if _, err := ix.keepBranchEntries(ctx, "feature/x.y", 8, releaseCall("mesh-sdk", "0.1.27"), tip("c4"), nil); err != nil { + t.Fatal(err) + } + if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 || a.Entries[0].Commit != "c3" { + t.Errorf("the branch's source 8 after a re-pin answered %+v (%v), want the take-in's revision 2", a.Revision, err) + } if _, err := bySource("feature/x=2ey", 9); err == nil { t.Error("a source no revision was taken in from was answered") } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index ec1e7717..38a96ead 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "slices" "strconv" "strings" @@ -87,6 +88,37 @@ func BranchRevisionOf(branch, name string) (int64, bool) { return r, err == nil && r > 0 } +// watchedBranches are the watched branches the bucket keeps a revision of, the first included when it keeps one, sorted: +// each name . but an entry's, a candidate's and the earlier forms' (earlierForm). +func (ix *Indexer) watchedBranches(ctx context.Context) ([]string, error) { + if ix.Revisions == nil { + return nil, nil + } + names, err := ix.Revisions.Names(ctx) + if err != nil { + return nil, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err) + } + seen := map[string]bool{} + for _, n := range names { + if strings.HasPrefix(n, "entry.") || strings.HasPrefix(n, "candidate.") || earlierForm(n) { + continue + } + i := strings.LastIndexByte(n, '.') + if i <= 0 { + continue + } + if r, err := strconv.ParseInt(n[i+1:], 10, 64); err != nil || r <= 0 { + continue + } + branch, err := keys.UnescapeName(n[:i]) + if err != nil { + continue + } + seen[branch] = true + } + return slices.Sorted(maps.Keys(seen)), nil +} + // TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one. func TrunkRevisionOf(name string) (int64, bool) { return BranchRevisionOf(TrunkBranch, name) } @@ -210,6 +242,54 @@ func readNewestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Ca } } +// TrunkEntryAt is a module's entry in a trunk revision, with that revision's list (H-A-204): for revision 0 the newest trunk +// revision read as entryOf reads it (readNewestOf); else the kept list of that trunk revision and only the module's entry +// object, so one entry is read without the revision whole. +func TrunkEntryAt(ctx context.Context, o Objects, module string, revision int64) (meshcatalog.CatalogRevision, meshcatalog.CatalogEntry, error) { + if revision == 0 { + rev, entries, found, err := readNewestOf(ctx, o, TrunkBranch) + if err != nil { + return rev, meshcatalog.CatalogEntry{}, err + } + if !found { + return rev, meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet") + } + for _, e := range entries { + if e.Module == module { + return rev, e, nil + } + } + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d holds no module %s", rev.Revision, module) + } + name := BranchObject(TrunkBranch, revision) + var rev meshcatalog.CatalogRevision + data, err := o.Get(ctx, name) + if errors.Is(err, ErrNoObject) { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d is not kept", revision) + } + if err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s: %w", name, err) + } + if err := json.Unmarshal(data, &rev); err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s: %w", name, err) + } + for _, ref := range rev.Entries { + if ref.Module != module { + continue + } + data, err := o.Get(ctx, EntryObject(ref.Digest)) + if err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s, the entry of %s (%s): %w", name, module, ref.Digest, err) + } + var e meshcatalog.CatalogEntry + if err := json.Unmarshal(data, &e); err != nil { + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("revision %s, the entry of %s: %w", name, module, err) + } + return rev, e, nil + } + return rev, meshcatalog.CatalogEntry{}, fmt.Errorf("the trunk revision %d holds no module %s", revision, module) +} + // changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name. func changedEntries(before, after []meshcatalog.EntryRef) []string { was := map[string]string{} @@ -266,17 +346,22 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata } // KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design -// 47 H §743, H-A-32): ., the trunk's included, numbered by the source, and says catalog-revision naming the -// branch. movers are the take-in's: kept on the revision (H-A-92), and the revision pinned for the take-in's pipeline, -// pipeline.., in the same write when there is one (A-449a), with the branch's previous revision (H-A-147). A revision already kept under that name is not written -// again (read before write, under the branch's one consumer); ErrObjectsFull is a store too full to keep it. +// 47 H §743, H-A-32), as keepBranchEntries does with no call. func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error { - return ix.keepBranchEntries(ctx, branch, src, entries, movers) + _, err := ix.keepBranchEntries(ctx, branch, src, revisionCall{}, entries, movers) + return err } -func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error { +// keepBranchEntries keeps entries as the next catalog revision of a watched branch, ., the trunk's included: n is +// the indexer's own count, the branch's newest revision + 1, and src the gate's source revision it was read at, its field +// source (design 47 H-A-213 (1)); call is what made it (H-A-212, H-A-213 (3)): a take-in's the gate's call, a re-pin's +// "release of ", a take-in again's "take-in again of ". It says catalog-revision naming the +// branch, its source and its call. movers are kept on the revision (H-A-92), and the revision pinned for its one pipeline, +// pipeline.., in the same write when there is one (A-449a, H-A-213 (4)), with the branch's previous revision +// (H-A-147). It answers the number it kept; ErrObjectsFull is a store too full to keep it. +func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, call revisionCall, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) (int64, error) { if ix.Revisions == nil { - return nil + return 0, nil } if branch == "" { branch = TrunkBranch @@ -286,28 +371,24 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int newest, _, err := newestOf(ctx, ix.Revisions, branch) if errors.Is(err, process.ErrNotYet) { ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects) - return err + return 0, err } if err != nil { ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err) - return fmt.Errorf("the newest catalog graph revision: %w", err) - } - if _, err := ix.Revisions.Get(ctx, BranchObject(branch, src)); err == nil { - ix.logf("catalog graph: revision %s is kept already: not written again", BranchObject(branch, src)) - return nil - } else if !errors.Is(err, ErrNoObject) { - return fmt.Errorf("revision %s: %w", BranchObject(branch, src), err) + return 0, fmt.Errorf("the newest catalog graph revision: %w", err) } + number := newest.Revision + 1 entries = WithEdges(ix.withActs(ctx, entries)) // the acting rules and the edges between artifact references are written here, once, for every reader (A-208) at := ix.now().UTC() - rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: src, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries) + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: number, Kind: "trunk", Branch: branch, Source: src, Call: call.Call, + CallKind: call.Kind, Module: call.Module, Version: call.Version, Repository: call.Repository, At: &at}, entries) if err != nil { - return err + return 0, err } rev.Movers = movers // The base of the validate step's case before: the newest revision kept before this write, which the take-in pins for its // pipeline below, named on the revision in this same write (H-A-147, H-A-151, H-A-154); none on the branch's first take-in. - pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0 && newest.Revision != src + pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0 if pinsBase { rev.Base = newest.Revision } @@ -319,32 +400,32 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull) ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity) ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused}) - return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull) + return 0, fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull) } if err := ix.indexEntries(ctx, rev, entries); err != nil { ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) - return err + return 0, err } if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) - return err + return 0, err } // The take-in's revision pinned for its one pipeline before any judge can remove it (§743, §749, A-449a). if len(movers) > 0 && ix.Pins != nil { - record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(src, 10))) - if err := ix.pin(ctx, PinKey(branch, src), record); err != nil { - ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err) - return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err) + record := PipelineRecord(runner.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(number, 10))) + if err := ix.pin(ctx, PinKey(branch, number), record); err != nil { + ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, number), record, err) + return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, number), err) } // The branch's previous revision, the newest kept before this write, pinned for the same pipeline: it is the base of // the validate step's case before (H-A-108 (3), H-A-147), kept as long as the pipeline that needs it. if pinsBase { if err := ix.pin(ctx, PinKey(branch, rev.Base), record); err != nil { ix.logf("pins: the previous revision %s is not pinned for %s: %v", BranchObject(branch, newest.Revision), record, err) - return fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err) + return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err) } } else { - ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, src), record) + ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, number), record) } } // The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it. @@ -364,16 +445,48 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 { ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity}) } - ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed)) + ix.logf("catalog graph: revision %d of source revision %d kept (%s), %d entries, %d changed", rev.Revision, src, call.Call, len(rev.Entries), len(changed)) if ix.SayCatalogRevision != nil { if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind, - Branch: keys.EscapeName(branch), Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil { + Branch: keys.EscapeName(branch), Source: src, Call: call.Call, CallKind: call.Kind, Module: call.Module, Version: call.Version, + Repository: call.Repository, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil { ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err) } } - return nil + return number, nil } +// revisionCall is what made a catalog revision (design 47 H-A-213 (3), H-A-215): call, the words for a person; Kind, one of the +// seat mesh-catalog's three call kinds, which a reader reads instead of the words; Module and Version for a release, Repository +// for a take-in again. The zero value names no call (a revision kept by hand or a test). +type revisionCall struct { + Call, Kind, Module, Version, Repository string +} + +// takeInCall is a take-in's: the gate's call its source revision names. +func takeInCall(call string) revisionCall { + return revisionCall{Call: call, Kind: meshcatalog.CatalogRevisionCallKindTakeIn} +} + +// releaseCall is a re-pin's on the release of module at version (H-A-213 (2)). +func releaseCall(module, version string) revisionCall { + return revisionCall{Call: ReleaseCall(module, version), Kind: meshcatalog.CatalogRevisionCallKindRelease, Module: module, Version: version} +} + +// takeInAgainCall is a take-in again's of repository (H-A-212). +func takeInAgainCall(repository string) revisionCall { + return revisionCall{Call: TakeInAgainCall(repository), Kind: meshcatalog.CatalogRevisionCallKindTakeInAgain, Repository: repository} +} + +// recomputeCall is a recompute's that moved a fingerprint, its call naming the reason (H-A-217). +func recomputeCall(reason string) revisionCall { + return revisionCall{Call: RecomputeCall(reason), Kind: meshcatalog.CatalogRevisionCallKindRecompute} +} + +// RecomputeCall is the call of the catalog revision a recompute keeps: its reason, such as the assignments of a build module, +// a toolchain or a hand act (design 47 H-A-217). +func RecomputeCall(reason string) string { return "recompute on " + reason } + // HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held). const HeldObjectsFull = "objects full" diff --git a/modules/mesh-delivery-indexer/internal/index/entry_verb.go b/modules/mesh-delivery-indexer/internal/index/entry_verb.go index 96aee4e8..e310ac84 100644 --- a/modules/mesh-delivery-indexer/internal/index/entry_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/entry_verb.go @@ -21,6 +21,8 @@ const ( EntryIndexState = "entry-index" // EntryIndexTool is the tool that reads a fingerprint's key and its object's digest (class read). EntryIndexTool = "indexer_entries" + // EntryPinsTool is the tool that reads one module's entry pins and paths at a trunk revision (class read; H-A-204). + EntryPinsTool = "indexer_entry" ) // EntryIndexValue is one key of entry-index: the digest of the entry object of the fingerprint the key names. diff --git a/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go b/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go index 5de12ff7..de25475f 100644 --- a/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go +++ b/modules/mesh-delivery-indexer/internal/index/entry_verb_test.go @@ -49,7 +49,7 @@ func TestAKeptRevisionIndexesItsEntries(t *testing.T) { ctx := context.Background() fp := "src2:" + strings.Repeat("d", 64) ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }} - if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { + if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } if got, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: fp}); err != nil || got.Entry.Module != "avahi" { diff --git a/modules/mesh-delivery-indexer/internal/index/held.go b/modules/mesh-delivery-indexer/internal/index/held.go index c947f68e..5e8774ad 100644 --- a/modules/mesh-delivery-indexer/internal/index/held.go +++ b/modules/mesh-delivery-indexer/internal/index/held.go @@ -260,6 +260,32 @@ func HeldTools(held Store, ask func(key string, value []byte) (TakeInAsk, error) } return a.Result, nil }}, + {Name: TakeInAgainTool, + Description: "Take a repository in again now at the merge commit the indexer keeps of it (design 47 H-A-208, H-A-210): its " + + "module graph is read again (each module's requirements, transitively) and the modules state recomputed as a " + + "package's publish re-pins it, so a changed take-in rule reaches a repository with no merge since; pins and " + + "fingerprints move, no catalog revision is kept and the gate is not asked. An unknown repository is refused.", + Input: map[string]any{"type": "object", "required": []string{"repository"}, "properties": map[string]any{ + "repository": map[string]any{"type": "string", "description": "the repository as owner/name, such as novox/mesh-host"}, + "why": map[string]any{"type": "string", "description": "why it is taken in again now"}}}, + Run: func(args map[string]any) (any, error) { + repository := str(args, "repository") + if repository == "" { + return nil, errors.New("name the repository (repository) as owner/name") + } + data, err := json.Marshal(TakeInAsk{V: 1, Again: repository, Why: str(args, "why"), By: stdio.CallerOf(args), At: now().UTC()}) + if err != nil { + return nil, err + } + a, err := ask("again."+keys.EscapeName(repository), data) + if err != nil { + return nil, err + } + if a.Result != nil && !a.Result.Stored { + return a.Result, fmt.Errorf("%s is not taken in again: %s", repository, a.Result.Answer) + } + return a.Result, nil + }}, {Name: TakeInSkipTool, Description: "End a watched branch's held source-moved unwritten (design 47 H §744): it is acknowledged, take-in.taken is said " + "with no entries and skipped, and the hand act is recorded through mesh-hand-acts before it is asked. indexer_pending " + diff --git a/modules/mesh-delivery-indexer/internal/index/package_trees_test.go b/modules/mesh-delivery-indexer/internal/index/package_trees_test.go index 999bb220..06c1fc06 100644 --- a/modules/mesh-delivery-indexer/internal/index/package_trees_test.go +++ b/modules/mesh-delivery-indexer/internal/index/package_trees_test.go @@ -137,7 +137,7 @@ func TestTheEntryPinsEveryRequirementOfTheModuleGraph(t *testing.T) { t.Errorf("packages = %+v, want the client's requirement and state's mesh-sdk, and no module of the repository", m.Inputs2) } builds := "git.novox.be/novox/mesh-delivery/modules/b/builds" - pins := releasedPins(sdkstep.PackageVersions{builds: "v0.1.106", conditions: "v0.1.4"}, nil, catalogue.Manifest{}, "novox/mesh-delivery") + pins := releasedPins(sdkstep.PackageVersions{builds: "v0.1.106", conditions: "v0.1.4"}, nil, catalogue.Manifest{}, "novox/mesh-delivery", nil) if _, pinned := pins[builds]; pinned || len(pins) != 1 { t.Errorf("pins = %v, want %s only, never a module of the repository", pins, conditions) } diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index 33816fa8..92a543d2 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -119,8 +119,9 @@ func ReleaseRecord(branch, module string) string { // every revision of the branch below it; a mover the revision holds no release of (superseded, no version released) keeps its // older release's revision. Outcome tier-late frees nothing: its later tiers' wants are not written, though their release // records are held, and the same pipeline's later running frees them. A pipeline whose revision is not kept frees nothing and -// says so; one applied twice changes nothing. -func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error { +// says so; one applied twice changes nothing. The pipeline's revision is the one the event names (design 47 H-A-214); an event +// naming none (revision zero, said before the field) is of the source's lowest-numbered revision, the take-in's. +func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source, revision int64, outcome string) error { if ix.Pins == nil || ix.Revisions == nil { return errors.New("the indexer keeps no state " + PinsState) } @@ -128,12 +129,21 @@ func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, sourc ix.logf("pins: pipeline-wanted of %s %d is %s: it frees no release until its wants are written everywhere (H-A-174)", branch, source, outcome) return nil } - name, err := revisionOfSource(ctx, ix.Revisions, branch, source) - if err != nil { - ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err) - return nil + var name string + if revision > 0 { + name = RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: revision}) + } else { + var err error + if name, err = revisionOfSource(ctx, ix.Revisions, branch, source); err != nil { + ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err) + return nil + } } rev, _, err := ReadRevision(ctx, ix.Revisions, name) + if revision > 0 && errors.Is(err, ErrNoObject) { + ix.logf("pins: pipeline-wanted of %s revision %d frees no release: %s is not kept", branch, revision, name) + return nil + } if err != nil { return fmt.Errorf("%s: %w", name, err) } diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index 4a255ebd..c6de5147 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -106,42 +106,54 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te t.Fatal(err) } } - end := func(source int64, outcome, version string) { + endOf := func(source, revision int64, outcome, version string) { t.Helper() - e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome, + e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", revision), Revision: revision, Source: source, Outcome: outcome, Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: version}}} if err := ix.HeardPins(ctx, nil, &e); err != nil { t.Fatal(err) } ix.JudgePins(ctx) } + end := func(source int64, outcome, version string) { t.Helper(); endOf(source, source, outcome, version) } + // An event naming no revision (said before the field) is of the source's lowest-numbered revision, the take-in's (H-A-214). wanted := func(source int64, outcome string) { t.Helper() - if err := ix.HeardPipelineWanted(ctx, "main", source, outcome); err != nil { + if err := ix.HeardPipelineWanted(ctx, "main", source, 0, outcome); err != nil { t.Fatal(err) } ix.JudgePins(ctx) } - take(10) - end(10, runner.PipelineEndedOutcomePassed, "0.1.10") - take(11) - end(11, runner.PipelineEndedOutcomeFailed, "") - want("a passed release past a later failed pipeline", "main.10", "main.11") - take(12) - end(12, runner.PipelineEndedOutcomePassed, "") - wanted(12, meshwants.PipelineWantedOutcomeRunning) - want("a passed pipeline releasing no version frees no older release", "main.10", "main.12") - take(13) - end(13, runner.PipelineEndedOutcomePassed, "0.1.13") - want("two releases, the unreleased one between them gone", "main.10", "main.13") - wanted(13, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing - want("a tier-late pipeline-wanted", "main.10", "main.13") - wanted(13, meshwants.PipelineWantedOutcomeRunning) - wanted(13, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more - want("the newer release written everywhere", "main.13") - if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { - t.Errorf("the pin of main.13 holds %s, want its release alone", v) + take(4) + end(4, runner.PipelineEndedOutcomePassed, "0.1.10") + take(5) + end(5, runner.PipelineEndedOutcomeFailed, "") + want("a passed release past a later failed pipeline", "main.4", "main.5") + take(6) + end(6, runner.PipelineEndedOutcomePassed, "") + wanted(6, meshwants.PipelineWantedOutcomeRunning) + want("a passed pipeline releasing no version frees no older release", "main.4", "main.6") + take(7) + end(7, runner.PipelineEndedOutcomePassed, "0.1.13") + want("two releases, the unreleased one between them gone", "main.4", "main.7") + wanted(7, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing + want("a tier-late pipeline-wanted", "main.4", "main.7") + wanted(7, meshwants.PipelineWantedOutcomeRunning) + wanted(7, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more + want("the newer release written everywhere", "main.7") + if v, _, _, _ := ix.Pins.Get(ctx, "main.7"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) { + t.Errorf("the pin of main.7 holds %s, want its release alone", v) } + // H-A-214: a re-pin's revision 8 of the same source 7 releases 0.1.14; its pipeline-wanted names revision 8, so the release + // on revision 8 frees the older one on revision 7, which the source's lowest-numbered revision would not. + take(7) + endOf(7, 8, runner.PipelineEndedOutcomePassed, "0.1.14") + want("a re-pin's release beside the take-in's", "main.7", "main.8") + if err := ix.HeardPipelineWanted(ctx, "main", 7, 8, meshwants.PipelineWantedOutcomeRunning); err != nil { + t.Fatal(err) + } + ix.JudgePins(ctx) + want("the re-pin's release written everywhere", "main.8") } func mustRecords(t *testing.T, v []byte) []string { @@ -168,21 +180,22 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { before []int64 // sources taken in with no mover before the take-in of source 5 whileRuns []string afterTheEnd []string - base int64 // the base main.5 names (H-A-154) + at int64 // the number the take-in of source 5 keeps: the branch's newest + 1 (H-A-213 (1)) + base int64 // the base that revision names (H-A-154) passed bool // main.5's pipeline passed: its release pins main.5 unreleased bool // its mover released no version there, so nothing pins main.5 (H-A-173) then []int64 // sources taken in with no mover after the end released []string }{ - {name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4}, - {name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2}, - {name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}}, - {name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, - base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}}, - {name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, - base: 4, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.6"}}, - {name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"}, - base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}}, + {name: "the previous revision kept and pinned", before: []int64{3, 4}, at: 3, whileRuns: []string{"main.2", "main.3"}, afterTheEnd: []string{"main.3"}, base: 2}, + {name: "the newest kept before, whatever its source", before: []int64{2}, at: 2, whileRuns: []string{"main.1", "main.2"}, afterTheEnd: []string{"main.2"}, base: 1}, + {name: "no revision before it pins only its own", at: 1, whileRuns: []string{"main.1"}, afterTheEnd: []string{"main.1"}}, + {name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"}, + base: 1, passed: true, then: []int64{6}, afterTheEnd: []string{"main.2", "main.3"}, released: []string{"main.2"}}, + {name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"}, + base: 1, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.3"}}, + {name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"}, + base: 1, then: []int64{6}, afterTheEnd: []string{"main.3"}}, } { t.Run(tc.name, func(t *testing.T) { ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(), @@ -206,18 +219,19 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) { keep(s, nil) } keep(5, mover) - if rev, _, err := ReadRevision(ctx, ix.Revisions, "main.5"); err != nil || rev.Base != tc.base { - t.Errorf("main.5 names the base %d (%v), want %d, the revision its take-in pinned (H-A-154)", rev.Base, err, tc.base) + at := fmt.Sprintf("main.%d", tc.at) + if rev, _, err := ReadRevision(ctx, ix.Revisions, at); err != nil || rev.Base != tc.base || rev.Source != 5 { + t.Errorf("%s names the source %d and the base %d (%v), want 5 and %d, the revision its take-in pinned (H-A-154)", at, rev.Source, rev.Base, err, tc.base) } if got := kept(); !slices.Equal(got, tc.whileRuns) { t.Errorf("while the pipeline runs: kept %v, want %v", got, tc.whileRuns) } for _, k := range tc.whileRuns { - if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5"}) { + if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline." + at}) { t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v) } } - end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed, + end := runner.PipelineEnded{Branch: "main", Pipeline: at, Revision: tc.at, Outcome: runner.PipelineEndedOutcomeFailed, Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: "0.1.5"}}} if tc.unreleased { end.Movers[0].Version = "" diff --git a/modules/mesh-delivery-indexer/internal/index/repin_test.go b/modules/mesh-delivery-indexer/internal/index/repin_test.go new file mode 100644 index 00000000..584c3708 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/repin_test.go @@ -0,0 +1,100 @@ +package index + +import ( + "context" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Design 47 H-A-204, H-A-201, H-A-165: an entry's packages name every requirement of its module graph of another +// repository: one the form's versions leave out (an import form naming only what the build imported) is pinned by its +// series' newest published version, and one none publishes yet is kept with no version, never left out. +func TestAnEntryNamesEveryRequirementPinnedOrWaiting(t *testing.T) { + const ( + wireguard = "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go" + busgrants = "git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants" + sdk = "git.novox.be/novox/mesh-sdk/go" + ) + series := map[string]SeriesPackage{"wg": {Name: "mesh-wireguard/go", GoModule: wireguard, Series: catalogue.DefaultSeries, + Newest: &meshcatalog.PackagesByPackageSeriesValueNewest{Version: "0.1.47"}}} + pins := releasedPins(sdkstep.PackageVersions{sdk: "c0ffee"}, series, catalogue.Manifest{}, "novox/mesh-catalog", []string{busgrants, sdk, wireguard}) + want := sdkstep.PackageVersions{sdk: "", wireguard: "v0.1.47", busgrants: ""} + if len(pins) != len(want) { + t.Fatalf("pins = %v, want %v", pins, want) + } + for p, v := range want { + if got, named := pins[p]; !named || got != v { + t.Errorf("the pin of %s is %q (named %v), want %q", p, got, named, v) + } + } +} + +// Design 47 H-A-204, H-A-165: a release re-pins every kept entry naming one of its packages, of any repository and +// changed or not, by its pins or by its kept requirements alone: here g, of another repository, holds no pin of mesh-sdk. +// H-A-211, H-A-212, H-A-213: on every watched branch holding g, the re-pin keeps the branch's next revision, of the same +// source as the one before it, with the call "release of mesh-sdk 0.1.27" and g as its mover at its new fingerprint, and +// says catalog-revision with that call. +func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) { + ctx := context.Background() + objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)} + ix, mods, _ := ptpIndexer(t, objects) + ix.Revisions, ix.Packages = newMemObjects(1<<20), newMem() + reportToolchain(t, ix, "mesh-build-go", "n1", "go", "1.26.0") + var said []meshcatalog.CatalogRevisionEvent + ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { said = append(said, e); return nil } + f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, nil) + if err := ix.Apply(ctx, f); err != nil { + t.Fatal(err) + } + g, _ := moduleOf(t, mods, "g") + if g.Inputs2 == nil || !slices.Contains(g.Inputs2.Packages, "git.novox.be/novox/mesh-sdk/go") { + t.Fatalf("g = %+v, want mesh-sdk among its kept requirements", g.Inputs2) + } + g.Packages = nil + mods.put("g", mustEncode(t, g)) + mods.put("mesh-sdk", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: strings.Repeat("1", 40), + Path: meshcatalogclient.RootPath, None2: None2Pending})) + entries, err := ix.viewEntries(ctx) + if err != nil { + t.Fatal(err) + } + sources := map[string]int64{"main": 5, "feature/x": 9} + for branch, src := range sources { + if err := ix.KeepBranchEntries(ctx, branch, src, entries, nil); err != nil { + t.Fatal(err) + } + } + said = nil + if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: "git.novox.be/novox/mesh-sdk/go", Version: "0.1.27", Published: true}); err != nil { + t.Fatal(err) + } + if n, err := ix.Repin(ctx, "mesh-sdk", "0.1.27"); err != nil || n != 2 { + t.Errorf("a release of mesh-sdk re-pinned %d modules (%v), want g on each of the two branches, which requires it with no pin", n, err) + } + now, _ := moduleOf(t, mods, "g") + if now.Fp2 == "" || now.Fp2 == g.Fp2 || now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.27" { + t.Fatalf("g after the re-pin = %s pinned %v, want a new fingerprint pinned to v0.1.27 (was %s)", now.Fp2, now.Packages, g.Fp2) + } + for branch, src := range sources { + rev, kept, found, err := readNewestOf(ctx, ix.Revisions, branch) + if err != nil || !found || rev.Revision != 2 || rev.Source != src || rev.Call != "release of mesh-sdk 0.1.27" || + rev.CallKind != meshcatalog.CatalogRevisionCallKindRelease || rev.Module != "mesh-sdk" || rev.Version != "0.1.27" || // H-A-215 + len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != now.Fp2 { + t.Errorf("%s's newest = %+v (%v), want revision 2 of source %d, its call the release, g its mover at %s", branch, rev, err, src, now.Fp2) + continue + } + if i := slices.IndexFunc(kept, func(e meshcatalog.CatalogEntry) bool { return e.Module == "g" }); i < 0 || kept[i].Fp != now.Fp2 { + t.Errorf("%s's newest holds no entry of g at %s", branch, now.Fp2) + } + } + if len(said) != 2 || said[0].Call != "release of mesh-sdk 0.1.27" || said[1].Call != said[0].Call || + said[0].CallKind != meshcatalog.CatalogRevisionEventCallKindRelease || said[1].Module != "mesh-sdk" || said[1].Version != "0.1.27" { + t.Errorf("said %+v, want catalog-revision of each branch with the release's call", said) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/revision_tools.go b/modules/mesh-delivery-indexer/internal/index/revision_tools.go index 279e41e8..cd98aba6 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_tools.go @@ -272,7 +272,7 @@ func (ix *Indexer) RecomputeByHand(ctx context.Context, by, why string) TakenIn r.Answer = "the newest catalog graph revision cannot be read: " + err.Error() return r } - if err := ix.Recompute(ctx); err != nil { + if err := ix.RecomputeKept(ctx, fmt.Sprintf("a hand act by %s (%s)", by, why)); err != nil { r.Answer = "the recompute failed: " + err.Error() return r } diff --git a/modules/mesh-delivery-indexer/internal/index/revision_verb.go b/modules/mesh-delivery-indexer/internal/index/revision_verb.go index 20402883..2667fbb4 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_verb.go @@ -167,18 +167,24 @@ func revisionOfSource(ctx context.Context, o Objects, branch string, source int6 } } slices.SortFunc(found, func(a, b int64) int { return int(b - a) }) + // A re-pin or a take-in again keeps a later revision of the same source (H-A-211, H-A-213): the source's own revision is + // the lowest numbered of it, the take-in's. + lowest := "" for _, r := range found { name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: r}) rev, _, err := ReadRevision(ctx, o, name) if err != nil { return "", err } - if rev.Source == source { - return name, nil - } if rev.Source < source { break } + if rev.Source == source { + lowest = name + } + } + if lowest != "" { + return lowest, nil } return "", fmt.Errorf("no revision of %q was taken in from the source revision %d", branch, source) } diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again.go b/modules/mesh-delivery-indexer/internal/index/take_in_again.go new file mode 100644 index 00000000..e27a23f1 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again.go @@ -0,0 +1,78 @@ +package index + +import ( + "context" + "errors" + "fmt" + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// TakeInAgainTool takes a repository in again at its kept merge commit (design 47 H-A-208, H-A-210): a rule change of the +// take-in (such as the walk of the module graph, H-A-201) reaches a repository with no merge since. +const TakeInAgainTool = "indexer_take_in_again" + +// TakeInAgainCall is the call of the catalog revision a take-in again keeps (design 47 H-A-212, H-A-213 (3)). +func TakeInAgainCall(repository string) string { return "take-in again of " + repository } + +// TakeInAgain re-reads the repository's module graph at the merge commit its taken key keeps (wholeAt, the reading of every +// take-in: the kept requirements of each of its modules, H-A-201, H-A-157) and applies it as a take-in does, whose recompute +// of the modules state is the one a publish re-pin runs (recomputeLocked): pins and fingerprints move as they would there. +// It keeps the result as the trunk's next catalog revision of the same source, its call TakeInAgainCall (H-A-212, H-A-213), +// and asks no gate. A repository with no taken key is refused, said in the answer. +func (ix *Indexer) TakeInAgain(ctx context.Context, repository string) TakenIn { + repository = strings.TrimSpace(repository) + r := TakenIn{Module: repository, Repository: repository} + refuse := func(format string, a ...any) TakenIn { + r.Answer = "refused: " + fmt.Sprintf(format, a...) + ix.logf("take-in again of %s: %s", repository, r.Answer) + return r + } + if ix.Taken == nil || ix.Modules == nil { + return refuse("the indexer keeps no taken merges or no modules, so nothing is taken in again") + } + key, err := trunk.RepositoryKey(repository) + if err != nil { + return refuse("%q names no repository as owner/name", repository) + } + value, _, found, err := ix.Taken.Get(ctx, key) + if errors.Is(err, process.ErrNotYet) { + return refuse("the state taken is not on the bus yet") + } + if err != nil { + return refuse("taken.%s cannot be read: %v", key, err) + } + if !found { + return refuse("the indexer keeps no merge of %s (no key taken.%s): an unknown repository, or one never taken in", repository, key) + } + t, err := takein.DecodeTaken(value) + if err != nil || t.Merge == "" { + return refuse("taken.%s names no merge commit", key) + } + ix.pendingMu.Lock() // one take-in step at a time (pending.go) + defer ix.pendingMu.Unlock() + f, err := ix.wholeAt(ctx, repository, t.Merge) + if err != nil { + return refuse("%v", err) + } + f.Branch = TrunkBranch + applied, err := ix.ApplyMessage(ctx, f) + if err != nil { + return refuse("taking %s at %.12s in again: %v", repository, t.Merge, err) + } + // The take-in again keeps its revision as a re-pin does: the branch's next, of the same source, its call naming the + // repository, its movers every module whose fingerprint moved (design 47 H-A-212, H-A-213). + if err := ix.keepRepinned(ctx, ix, TrunkBranch, takeInAgainCall(repository), nil); err != nil { + return refuse("taken in again at %.12s, but its catalog revision is not kept: %v", t.Merge, err) + } + r.Stored, r.Commit = true, t.Merge + r.Answer = fmt.Sprintf("taken in again at %.12s: %d modules written", t.Merge, len(applied.Written)) + if len(applied.Written) > 0 { + r.Answer += " (" + strings.Join(applied.Written, ", ") + ")" + } + ix.logf("take-in again of %s: %s", repository, r.Answer) + return r +} diff --git a/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go new file mode 100644 index 00000000..1fe65e2d --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/take_in_again_test.go @@ -0,0 +1,113 @@ +package index + +import ( + "context" + "maps" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" +) + +// Design 47 H-A-208, H-A-210: a repository taken in before the walk of the module graph (H-A-201) keeps its modules' direct +// requirements only; indexer_take_in_again re-reads it at its kept merge commit and recomputes the modules state, so the +// module's kept requirements and pins gain the transitive package, and keeps the trunk's next catalog revision of the same +// source with the call "take-in again of " (H-A-212, H-A-213). An unknown repository is refused. +func TestATakeInAgainReachesARepositoryWithNoMergeSince(t *testing.T) { + const conditions = "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)} + ix, mods, _ := ptpIndexer(t, objects) + ix.Taken, ix.Revisions, ix.Packages = newMem(), newMemObjects(1<<20), newMem() + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + direct := map[string]string{"modules/g": "module g\n\nrequire git.novox.be/novox/mesh-delivery/modules/b/builds" + ptpZero + "\n"} + walked := map[string]string{"modules/c/client": "module git.novox.be/novox/mesh-delivery/modules/c/client\n\nrequire " + conditions + ptpZero + "\n"} + goMods := direct // before the rule: the client's go.mod reads as requiring nothing + ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) { + if c, ok := goMods[dir]; ok { + return []byte(c), nil + } + if dir == "modules/c/client" { + return []byte("module git.novox.be/novox/mesh-delivery/modules/c/client\n"), nil + } + return []byte(ptpGoMods[dir]), nil + } + f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, + map[string]map[string]string{"modules/g": {"go.sum": avahiGoSum}}) + var readAt []string + ix.Read = func(_ context.Context, repository, sha, _ string) (RepoRead, error) { + readAt = append(readAt, repository+"@"+sha) + return RepoRead{Root: f.Root, Trees: f.Trees, Manifests: f.Manifests, Locks: f.Locks, Packages: f.Packages}, nil + } + ctx := context.Background() + for p, v := range map[string]string{conditions: "0.1.4", "git.novox.be/novox/mesh-sdk/go": "0.1.27"} { + if _, err := ix.HearVersion(ctx, HeardVersion{Package: p, GoModule: p, Version: v, Published: true}); err != nil { + t.Fatal(err) + } + } + // the modules that hold the two packages, so their published versions pin them (fingerprint.HolderOf) + mods.put("mesh-sdk", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: strings.Repeat("1", 40), + Path: meshcatalogclient.RootPath, None2: None2Pending})) + mods.put("meshconditionsclient", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-contracts", Commit: strings.Repeat("2", 40), + Path: "health-and-repair/go/clients/meshconditionsclient", None2: None2Pending})) + if err := ix.Apply(ctx, f); err != nil { + t.Fatal(err) + } + before, _ := moduleOf(t, mods, "g") + if before.Inputs2 == nil || slices.Contains(before.Inputs2.Packages, conditions) || strings.Contains(before.None2, conditions) { + t.Fatalf("before: packages = %+v (%s), want no transitive requirement yet", before.Inputs2, before.None2) + } + entries, err := ix.viewEntries(ctx) + if err != nil { + t.Fatal(err) + } + if err := ix.KeepBranchEntries(ctx, TrunkBranch, 4, entries, nil); err != nil { + t.Fatal(err) + } + if r := ix.TakeInAgain(ctx, "novox/unknown"); r.Stored || !strings.HasPrefix(r.Answer, "refused:") { + t.Errorf("unknown repository: %+v, want refused", r) + } + goMods = maps.Clone(direct) + maps.Copy(goMods, walked) + r := ix.TakeInAgain(ctx, "novox/mesh-delivery") + if !r.Stored || r.Commit != merge { + t.Fatalf("take-in again = %+v, want stored at the kept merge %s", r, merge) + } + if len(readAt) == 0 || readAt[0] != "novox/mesh-delivery@"+merge { + t.Errorf("read at %v, want the kept merge commit", readAt) + } + m, _ := moduleOf(t, mods, "g") + if m.Inputs2 == nil || !slices.Contains(m.Inputs2.Packages, conditions) { + t.Errorf("after: kept requirements = %+v, want %s", m.Inputs2, conditions) + } + // The recompute pins it at its published version, which moves the fingerprint (H-A-167, H-A-204), and the take-in again + // keeps the trunk's next revision of the same source, its call naming the repository, g its mover (H-A-212, H-A-213). + if m.Fp2 == "" || m.Fp2 == before.Fp2 || m.Packages[conditions] != "v0.1.4" { + t.Errorf("after: g = %s (%s) pinned %v, want a new fingerprint pinned to v0.1.4 (was %s)", m.Fp2, m.None2, m.Packages, before.Fp2) + } + rev, _, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) + if err != nil || !found || rev.Revision != 2 || rev.Source != 4 || rev.Call != "take-in again of novox/mesh-delivery" || + rev.CallKind != meshcatalog.CatalogRevisionCallKindTakeInAgain || rev.Repository != "novox/mesh-delivery" || // H-A-215 + len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != m.Fp2 { + t.Errorf("the newest trunk revision = %+v (%v), want revision 2 of source 4 with the take-in again's call and g its mover", rev, err) + } + // H-A-217: a recompute that moves no fingerprint keeps nothing; one that moves one (a toolchain reported) keeps the trunk's + // next revision of the same source, its call-kind recompute, its call naming the reason, the moved modules its movers. + if err := ix.RecomputeKept(ctx, "the assignments of a build module"); err != nil { + t.Fatal(err) + } + if rev, _, _, _ := readNewestOf(ctx, ix.Revisions, TrunkBranch); rev.Revision != 2 { + t.Errorf("a recompute that moved nothing kept revision %d (%s), want none", rev.Revision, rev.Call) + } + reportToolchainRecipe(t, ix, "mesh-build-go", "novox", "go", "go1.27.1", "2") // a recipe change moves a Go module at once + moved, _ := moduleOf(t, mods, "g") + rev, _, found, err = readNewestOf(ctx, ix.Revisions, TrunkBranch) + if err != nil || !found || rev.Revision != 3 || rev.Source != 4 || rev.CallKind != meshcatalog.CatalogRevisionCallKindRecompute || + rev.Call != RecomputeCall("a toolchain reported") || !slices.ContainsFunc(rev.Movers, func(e meshcatalog.TakeInEntry) bool { + return e.Module == "g" && e.Fingerprint == moved.Fp2 && moved.Fp2 != m.Fp2 + }) { + t.Errorf("after a toolchain reported: the newest trunk revision = %+v (%v), want revision 3 of source 4, kind recompute, g its mover at %s (was %s)", + rev, err, moved.Fp2, m.Fp2) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index 934e19c9..3cba48d0 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -46,6 +46,8 @@ type TakeInAsk struct { HeldSource int64 `json:"held-source,omitempty"` Skip bool `json:"skip,omitempty"` NamedMember string `json:"named-member,omitempty"` + // Again names the repository taken in again at its kept merge commit (indexer_take_in_again, take_in_again.go; H-A-208, H-A-210). + Again string `json:"again,omitempty"` // RunDrop asks one run on request, by its group run-, to be dropped with its candidate (indexer_run_drop, runs_tools.go, A-194). RunDrop string `json:"run-drop,omitempty"` // Registry asks one of the registrations' acts (registrations.go): a trusted change of one, the verb registrations, diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source.go b/modules/mesh-delivery-indexer/internal/index/takein_source.go index 0b4d4947..667eb11a 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source.go @@ -2,9 +2,11 @@ package index import ( "context" + "encoding/json" "errors" "fmt" "maps" + "slices" "strconv" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -25,9 +27,10 @@ func TakenID(branch string, source int64) string { // TakeInSource takes one source revision of a watched branch in, as the gate's source-moved names it (design 47 H §743): for each // repository it moves, the trees at the named commit through the seat git (wholeAt) and the fingerprints of the modules // registered for it; its movers (moversOf) against the branch's newest catalog revision; one catalog revision of the branch, -// ., carrying the movers and pinned for the take-in's pipeline when it has one (KeepBranchEntries); and +// the branch's next, . of source , carrying the movers and pinned for the take-in's pipeline when it has one +// (keepBranchEntries, H-A-213); and // take-in.taken once, with branch, source, revision, call and one entry per mover; a branch's first revision (start) has no -// mover (§768). A revision already kept under that name (a +// mover (§768). A revision already kept of that source (a // restart before the acknowledgement) is said again from its kept movers, recomputing nothing (H-A-92). The first watched // branch is taken in on the indexer's own view; another on its newest revision's, computed and kept only as that revision. // An error leaves the source-moved held, written into the state held behind its branch's earlier ones (ErrHeldUnwritten when @@ -57,13 +60,13 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE } ix.pendingMu.Lock() defer ix.pendingMu.Unlock() - name := BranchObject(e.Branch, e.Source) - if kept, _, err := ReadRevision(ctx, ix.Revisions, name); err == nil { - taken.Revision, taken.Entries = kept.Revision, kept.Movers - ix.logf("take-in: %s is kept already: take-in.taken said again from its %d movers", name, len(kept.Movers)) - return taken, ix.sayTaken(ctx, taken) - } else if !errors.Is(err, ErrNoObject) { + // A source revision kept already is found by the revision's field source, never by its number (H-A-213 (1)). + if kept, found, err := takenOfSource(ctx, ix.Revisions, e.Branch, e.Source); err != nil { return taken, err + } else if found { + taken.Revision, taken.Entries = kept.Revision, kept.Movers + ix.logf("take-in: source revision %d of %s is kept already as %s: take-in.taken said again from its %d movers", e.Source, e.Branch, RevisionObject(kept), len(kept.Movers)) + return taken, ix.sayTaken(ctx, taken) } _, before, _, err := readNewestOf(ctx, ix.Revisions, e.Branch) if err != nil { @@ -106,14 +109,15 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE if e.Start { movers = nil // a branch's first revision moves nothing: its entries are the base later take-ins are held against (§768, H-A-19) } - if err := ix.KeepBranchEntries(ctx, e.Branch, e.Source, entries, movers); err != nil { + number, err := ix.keepBranchEntries(ctx, e.Branch, e.Source, takeInCall(e.Call), entries, movers) + if err != nil { if errors.Is(err, ErrObjectsFull) { ix.heldFull.Store(true) // room is said once the store has room again (RoomAgain), and wakes the take-in (§744) } return taken, err } - taken.Revision, taken.Entries = e.Source, movers - ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, name, len(e.Heads), len(movers)) + taken.Revision, taken.Entries = number, movers + ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, BranchObject(e.Branch, number), len(e.Heads), len(movers)) return taken, ix.sayTaken(ctx, taken) } @@ -157,22 +161,57 @@ func (ix *Indexer) SourceToTake(ctx context.Context, branch string, a meshdelive if ix.Revisions == nil { return e, false, errors.New("the indexer keeps no catalog revisions") } - names, err := ix.Revisions.Names(ctx) + // The gate's source a branch is taken in at is its newest revision's field source, never the revision's number (H-A-213 (1)). + newest, found, err := newestOf(ctx, ix.Revisions, branch) if err != nil { - return e, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err) + return e, false, fmt.Errorf("the newest catalog revision of %s: %w", branch, err) } - var kept int64 - for _, n := range names { - if r, ok := BranchRevisionOf(branch, n); ok && r > kept { - kept = r - } - } - if kept >= a.Revision { + if found && newest.Source >= a.Revision { return e, false, nil } - e.Start = kept == 0 + e.Start = !found for _, r := range a.Repositories { e.Heads = append(e.Heads, meshdelivery.SourceMovedEventHeadsItem{Repository: r.Repository, Commit: r.Head}) } return e, true, nil } + +// takenOfSource is the revision a take-in of source src kept on branch: of the branch's revisions whose field source is src, +// the lowest numbered (a re-pin or a take-in again keeps the same source on a later number, H-A-211, H-A-213). The revisions +// are read from the newest down while their source is not below src; none is false. +func takenOfSource(ctx context.Context, o Objects, branch string, src int64) (meshcatalog.CatalogRevision, bool, error) { + names, err := o.Names(ctx) + if err != nil { + return meshcatalog.CatalogRevision{}, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err) + } + var numbers []int64 + for _, n := range names { + if r, ok := BranchRevisionOf(branch, n); ok { + numbers = append(numbers, r) + } + } + slices.Sort(numbers) + var kept meshcatalog.CatalogRevision + found := false + for i := len(numbers) - 1; i >= 0; i-- { + name := BranchObject(branch, numbers[i]) + data, err := o.Get(ctx, name) + if errors.Is(err, ErrNoObject) { + continue // removed under the read by the pin judge + } + if err != nil { + return kept, false, fmt.Errorf("revision %s: %w", name, err) + } + var rev meshcatalog.CatalogRevision + if err := json.Unmarshal(data, &rev); err != nil { + return kept, false, fmt.Errorf("%s: %w", name, err) + } + if rev.Source < src { + break + } + if rev.Source == src { + kept, found = rev, true + } + } + return kept, found, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go index d36933e0..2e439e85 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -43,7 +43,8 @@ func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTa return ix, objects, said, f } -// Design 47 H §743, H-A-32, H-A-92: a take-in writes one catalog revision . carrying its movers, pins it for the +// Design 47 H §743, H-A-32, H-A-92, H-A-213 (1): a take-in writes one catalog revision ., n the branch's newest + 1 +// and its source the gate's, carrying its movers, pins it for the // take-in's pipeline and says take-in.taken once; a redelivery writes nothing again and says the same; a revision with no mover // pins nothing and says no entries; the old form is refused. func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { @@ -56,19 +57,20 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { t.Fatal(err) } if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Fingerprint == "" || taken.Entries[0].Commit != f.Merge || - taken.Revision != 7 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call { - t.Fatalf("taken %+v, want revision 7 of main with avahi as its one mover", taken) + taken.Revision != 1 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call { + t.Fatalf("taken %+v, want revision 1 (the branch's first) of source 7 of main with avahi as its one mover", taken) } - rev, _, err := ReadRevision(ctx, objects, "main.7") - if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 { - t.Fatalf("main.7 = %+v (%v), want it kept carrying the movers", rev, err) + rev, _, err := ReadRevision(ctx, objects, "main.1") + if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 || + rev.Call != moved.Call || rev.CallKind != meshcatalog.CatalogRevisionCallKindTakeIn { // H-A-215 + t.Fatalf("main.1 = %+v (%v), want it kept carrying the movers", rev, err) } - if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 7)); !found { - t.Error("main.7 is not pinned for its pipeline") + if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 1)); !found { + t.Error("main.1 is not pinned for its pipeline") } else { var pv PinValue - if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.7"}) { - t.Errorf("main.7 is pinned by %s, want pipeline.main.7", v) + if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.1"}) { + t.Errorf("main.1 is pinned by %s, want pipeline.main.1", v) } } puts := objects.puts @@ -79,14 +81,14 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { same := moved same.Source, same.Call = 8, "fedcba9876543210" none, err := ix.TakeInSource(ctx, same) - if err != nil || len(none.Entries) != 0 || none.Revision != 8 { - t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 8 with no entries", none, err) + if err != nil || len(none.Entries) != 0 || none.Revision != 2 || none.Source != 8 { + t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 2 of source 8 with no entries", none, err) } - if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 8)); found { + if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 2)); found { t.Error("a take-in with no mover pinned its revision") } - if _, _, err := ReadRevision(ctx, objects, "main.8"); err != nil { - t.Errorf("main.8 is not kept: %v", err) + if _, _, err := ReadRevision(ctx, objects, "main.2"); err != nil { + t.Errorf("main.2 is not kept: %v", err) } if len(*said) != 3 { t.Errorf("said %d take-in.taken, want 3", len(*said)) diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index f8c1eb67..dfba512a 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -79,9 +79,22 @@ func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value return ix.ToolchainReported(ctx) } -// ToolchainReported is what a changed toolchain moves: the modules written again (Recompute). +// ToolchainReported is what a changed toolchain moves: the modules written again, kept as a revision (RecomputeKept). func (ix *Indexer) ToolchainReported(ctx context.Context) error { - return ix.Recompute(ctx) + return ix.RecomputeKept(ctx, "a toolchain reported") +} + +// RecomputeKept is Recompute kept as a re-pin is (design 47 H-A-217): when it moved at least one fingerprint against the +// trunk's newest revision, it keeps the trunk's next revision of the same source, its movers the modules that moved, its +// call-kind recompute and its call naming reason, pinned for its pipeline, and says catalog-revision; one that moves none +// keeps nothing. Recompute writes the trunk's modules state, so only the trunk's revision is kept. +func (ix *Indexer) RecomputeKept(ctx context.Context, reason string) error { + ix.pendingMu.Lock() // one take-in step at a time, so the trunk's newest stays the one compared against (pending.go) + defer ix.pendingMu.Unlock() + if err := ix.Recompute(ctx); err != nil { + return err + } + return ix.keepRepinned(ctx, ix, TrunkBranch, recomputeCall(reason), nil) } // agreed is the toolchain of each language every reporting node agrees on, or why there is none. @@ -298,7 +311,7 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc r.fp, r.none = f.fp, f.none if r.fp != "" { // a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167) - r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository) + r.fp, r.pkgs = pinnedOf(f, series, k.manifest, k.cur.Repository, kept.Packages) r.tcs = toolchainsUsed(toolchains) } } @@ -517,7 +530,7 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, if f.none != "" || f.fp == "" { return "", nil, false } - fp, pkgs := pinnedOf(f, series, k.manifest, k.held.Repository) + fp, pkgs := pinnedOf(f, series, k.manifest, k.held.Repository, kept.Packages) return fp, pkgs, fp == k.held.Fp2 } @@ -544,7 +557,7 @@ func (ix *Indexer) RecomputeAtStart(ctx context.Context) error { ix.logf("no toolchain is read yet: the new-form fingerprints are computed when one is") return nil } - return ix.Recompute(ctx) + return ix.RecomputeKept(ctx, "the indexer's start") } // fillPaths gives every module whose key names no directory (a key written before the indexer learned paths) the one its @@ -598,29 +611,128 @@ func importsOf(pkgs sdkstep.PackageVersions, repository string) bool { return false } +// ReleaseCall is the call of the catalog revision a re-pin keeps: the release of a module at a version (design 47 H-A-213 (2)). +func ReleaseCall(module, version string) string { return "release of " + module + " " + version } + // Repin moves the pins of every module whose packages hold a package of module (a held module's directory holds it) to -// module's newest released version, the pass of H-A-161 for those modules alone: a version-released of module starts it -// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It answers how many it named. -func (ix *Indexer) Repin(ctx context.Context, module string) (int, error) { +// module's newest released version, the pass of H-A-161 for those modules alone: a release of module at version starts it +// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It runs on every watched branch +// holding such a module: the first on the indexer's own modules state, another on its newest revision's picture (as a +// take-in does), and keeps on each whose re-pinned fingerprints moved the branch's next catalog revision, of the same +// source as its newest, with the call ReleaseCall and those modules as its movers, pinned for the release's pipeline, and +// says catalog-revision; no take-in.taken (H-A-211, H-A-212, H-A-213). It answers how many modules it named. +func (ix *Indexer) Repin(ctx context.Context, module, version string) (int, error) { if ix.Modules == nil || ix.Manifests == nil { return 0, nil } - held, err := ix.heldModules(ctx) + ix.pendingMu.Lock() // one take-in step at a time, so the branch's newest stays the one re-pinned (pending.go) + defer ix.pendingMu.Unlock() + branches, err := ix.watchedBranches(ctx) if err != nil { return 0, err } + if !slices.Contains(branches, TrunkBranch) { + branches = append([]string{TrunkBranch}, branches...) // the modules state is re-pinned whether a revision is kept or not + } + named := 0 + for _, branch := range branches { + x := ix + if !isTrunkBranch(branch) { + view, nodes, _, err := BranchView(ctx, ix.Revisions, branch) + if err != nil { + return named, err + } + d := ix.dry() + d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) + d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) } + x = d.ix + } + only, err := x.repinned(ctx, module) + if err != nil { + return named, err + } + if len(only) == 0 { + continue + } + named += len(only) + x.recompute.Lock() + err = x.recomputeLocked(ctx, nil, nil, "", only) + x.recompute.Unlock() + if err != nil { + return named, err + } + if err := ix.keepRepinned(ctx, x, branch, releaseCall(module, version), only); err != nil { + return named, err + } + } + return named, nil +} + +// repinned are the modules the indexer's modules state holds whose packages name a package of module: by their pins, and +// by their kept requirements, which name a package they hold no pin of yet (design 47 H-A-204, H-A-165, H-A-167). +func (ix *Indexer) repinned(ctx context.Context, module string) (map[string]bool, error) { + held, err := ix.heldModules(ctx) + if err != nil { + return nil, err + } only := map[string]bool{} for key, m := range held { - for p := range m.Packages { + names := slices.Collect(maps.Keys(m.Packages)) + if m.Inputs2 != nil { + names = append(names, m.Inputs2.Packages...) + } + for _, p := range names { if moduleOfKey(fingerprint.HolderOf(p, held)) == module { only[key] = true } } } - if len(only) == 0 { - return 0, nil + return only, nil +} + +// keepRepinned keeps x's view, re-pinned for the modules only names (every module for nil), as branch's next catalog revision +// of the source of its newest, with call, when a fingerprint of them moved against that newest; its movers are those that +// moved (H-A-211, H-A-212, H-A-213). A branch with no revision kept yet keeps none: there is no source to keep it of. +func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch string, call revisionCall, only map[string]bool) error { + if ix.Revisions == nil { + return nil } - ix.recompute.Lock() - defer ix.recompute.Unlock() - return len(only), ix.recomputeLocked(ctx, nil, nil, "", only) + newest, before, found, err := readNewestOf(ctx, ix.Revisions, branch) + if err != nil { + return fmt.Errorf("the newest catalog revision of %s: %w", branch, err) + } + if !found { + ix.logf("re-pin (%s): %s keeps no revision yet, so none is kept for it", call.Call, branch) + return nil + } + entries, err := x.viewEntries(ctx) + if err != nil { + return err + } + was := make(map[string]string, len(before)) + for _, e := range before { + was[e.Module] = e.Fp + } + var movers []meshcatalog.TakeInEntry + moved := slices.Sorted(maps.Keys(only)) + if only == nil { // a take-in again or a recompute: every module whose fingerprint moved (H-A-212, H-A-217) + moved = slices.Sorted(maps.Keys(entries)) + } + for _, key := range moved { + e, ok := entries[key] + if !ok || e.Fp == "" || was[e.Module] == e.Fp { + continue + } + movers = append(movers, meshcatalog.TakeInEntry{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository}) + } + if len(movers) == 0 { + ix.logf("re-pin (%s): no fingerprint of %s moved, so no revision is kept", call.Call, branch) + return nil + } + number, err := ix.keepBranchEntries(ctx, branch, newest.Source, call, entries, movers) + if err != nil { + return err + } + ix.logf("re-pin (%s): %s kept as revision %d of source %d, %d movers", call.Call, branch, number, newest.Source, len(movers)) + return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 561b544a..059aa977 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -504,7 +504,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // } return first.KeepBranchEntries(context.Background(), TrunkBranch, source, entries, nil) } - if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 6 || + if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 || firstSaid[0].Entries != int64(len(builderModules(t))) { t.Fatalf("the take-in said %+v (%v), want trunk revision 1 with every module", firstSaid, err) } @@ -519,7 +519,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if err := takeIn(7); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) { t.Errorf("the next take-in after a write said %+v (%v), want revision 7 naming avahi", firstSaid, err) } - if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 7 { + if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 2 { t.Errorf("the view at a restart is read at %d (%v), want 7", at, err) } else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") { t.Errorf("the view at a restart lost the verdict: %s", raw) @@ -536,21 +536,21 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Errorf("the first-view tools %+v serve no modules", FirstViewTools) } if a, err := views.FirstView(testSelf, ModulesState, trunkStates.State(ModulesState), trunkStates.State(views.MarksState), "", time.Now()); err != nil || - a.Revision != 7 || len(a.Values) != len(builderModules(t)) { - t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 7", len(a.Values), a.Revision, err) + a.Revision != 2 || len(a.Values) != len(builderModules(t)) { + t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 2", len(a.Values), a.Revision, err) } else if m, err := meshcatalogclient.DecodeModule(a.Values[viewKey]); err != nil || m.Manifest == "" { t.Errorf("the first view's avahi = %s (%v), want the module's value naming its manifest", a.Values[viewKey], err) } - if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":7`) { - t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 7", e, ok, err) + if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) { + t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err) } // A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never // one revision read per key (about 11 s a page live, 2026-11-14). reads := 0 keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}} if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil || - a.Revision != 7 || len(a.Values) != len(builderModules(t)) || a.Next != "" { - t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 7 in one page", len(a.Values), a.Revision, a.Next, err) + a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" { + t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err) } if _, _, err := keptView.State("graph").Get("avahi"); err != nil || reads != 1 { t.Errorf("the kept view read the revision %d times (%v), want once", reads, err) @@ -822,7 +822,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { if _, err := ix.HearVersion(context.Background(), HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: "0.2.0", Commit: strings.Repeat("1", 40), GoModule: sdk, Published: true}); err != nil { t.Fatal(err) } - if _, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil { // the pass for avahi alone: mesh-tools is held stale here + if _, err := ix.Repin(context.Background(), "mesh-sdk", "0.1.27"); err != nil { // the pass for avahi alone: mesh-tools is held stale here t.Fatal(err) } if b, _ := moduleOf(t, mods, "avahi"); b.Fp2 != a.Fp2 || b.Packages[sdk] != "v0.1.18" { diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index 9500d302..e764bdad 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -376,12 +376,20 @@ func isFloor(raw []byte) bool { // release number is never a pin. A package with none published in that series is kept with no version, so its kind-1 // edge stands (H §746) and its build waits for the publish (H-A-165). A module of the module's own repository is never // pinned, whatever a build reported: it is built from the tree (design 47 H-A-198, H-A-201, H-A-202). nil for none. -func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest, repository string) sdkstep.PackageVersions { - if len(versions) == 0 { +// +// required are the mesh packages the module's go.mod graph requires (its kept packages, H-A-201): each of another +// repository is named whether or not the form's versions name it, with no version while none is published, never left +// out, so the builder waits for its publish instead of the build module refusing (design 47 H-A-204, H-A-165). +func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest, repository string, required []string) sdkstep.PackageVersions { + names := slices.Collect(maps.Keys(versions)) + if repository != "" { + names = append(names, fingerprint.OtherPackages(required, repository)...) + } + if len(names) == 0 { return nil } out := sdkstep.PackageVersions{} - for p := range versions { + for _, p := range names { if repo, _, ok := trunk.PackageRepository(p); ok && repository != "" && trunk.SameRepository(repo, repository) { continue } @@ -403,8 +411,8 @@ func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPack // form's fingerprint takes them (design 47 H-A-167, H-A-172; ADR 0573 §4) and a re-pin moves it, the module a mover // built again against the release; a fingerprint with no pin is the form's as it is. The held fingerprint of a build that // stands is kept while its pins are the ones it was built with, else its import form takes the new pins. -func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, repository string) (string, sdkstep.PackageVersions) { - pins := releasedPins(f.versions, series, m, repository) +func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest, repository string, required []string) (string, sdkstep.PackageVersions) { + pins := releasedPins(f.versions, series, m, repository, required) if f.stands { if maps.Equal(f.built, map[string]string(pins)) { return f.fp, pins @@ -443,7 +451,7 @@ func PinsOf(packages map[string]string) sdkstep.PackageVersions { // repin is Repin of a release of module at version, said in the journal; its failure is the caller's error. func (ix *Indexer) repin(ctx context.Context, module, version string) error { - n, err := ix.Repin(ctx, module) + n, err := ix.Repin(ctx, module, version) if err != nil { return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err) } diff --git a/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go b/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go index cdc1fc3f..d7a136cd 100644 --- a/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go +++ b/modules/mesh-delivery-indexer/internal/index/wanted_names_test.go @@ -102,7 +102,7 @@ func TestAnEntryNoRevisionListsStaysWhileAWantNamesItAndGoesAfterTheWantMovesOn( n := NewNamed(newMem(), nil) ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }} n.Changed = ix.RemoveUnlisted - if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { + if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { // the revision that listed it is removed @@ -213,7 +213,7 @@ func TestNoEntryIsRemovedWhileAStartReadWasEmptyOrAnEventWasNotApplied(t *testin fp := namedFp("c") setup := func(n *Named) (*Indexer, func() bool) { ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }} - if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { + if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil { t.Fatal(err) } if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 8a24e6af..81ef37ba 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -205,9 +205,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build index.FingerprintsAgainTool, // issue 1966 // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, - index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 - index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 42 || !slices.Contains(m.Tools, "modules") { + index.EntryIndexTool, index.EntryPinsTool, index.NamedDropTool, // design 47 G §675, H-A-204 + index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.PinDropTool, index.PendingTool, index.TakeInPendingTool, index.TakeInAgainTool, index.TakeInSkipTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) + if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 0eeaa80c..bc75c22c 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -57,6 +57,7 @@ "indexer_recompute", "indexer_drop_candidate_graph", "indexer_entries", + "indexer_entry", "indexer_named_drop", "indexer_proposals", "indexer_proposal_drop", @@ -66,6 +67,7 @@ "indexer_pin_drop", "indexer_pending", "indexer_take_in_pending", + "indexer_take_in_again", "indexer_take_in_skip", "indexer_runs", "indexer_run_drop" diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go index 26a92e9c..f0d87ef5 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/candidate_test.go @@ -71,3 +71,28 @@ func TestPlaceLostRunsReadsEachCandidateNewest(t *testing.T) { t.Errorf("asked %+v, want the candidate's group alone", asked[1]) } } + +// H-A-213 (2), (4): a catalog-revision heard is handed whole to the start of a revision's pipeline; a start that fails is +// an error, so the bus delivers the event again, and a body that is no event is ended. +func TestARevisionHeardIsHandedToItsStart(t *testing.T) { + var got []catalog.CatalogRevisionEvent + fail := false + hear := revisedHear(func(ev catalog.CatalogRevisionEvent) (int, error) { + got = append(got, ev) + if fail { + return 0, errors.New("not read") + } + return 1, nil + }, func(string, ...any) {}) + data := []byte(`{"v":1,"kind":"trunk","branch":"main","revision":10,"source":41,"call-kind":"release","call":"release of mesh-sdk 0.1.17"}`) + if err := hear(t.Context(), data); err != nil || len(got) != 1 || got[0].Revision != 10 || got[0].CallKind != "release" || got[0].Source != 41 { + t.Fatalf("heard %+v (%v), want the release revision 10", got, err) + } + fail = true + if err := hear(t.Context(), data); err == nil { + t.Error("a start that failed is not an error, so the event is never heard again") + } + if err := hear(t.Context(), []byte("no event")); err != nil { + t.Errorf("a body that is no event = %v, want it ended", err) + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 9488c743..85891217 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -98,6 +98,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro var advancer atomic.Pointer[func(string)] var placer atomic.Pointer[func(catalog.CatalogRevisionEvent) error] var registrar atomic.Pointer[func(catalog.Registration)] + var reviser atomic.Pointer[func(catalog.CatalogRevisionEvent) (int, error)] rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) { if f := resumer.Load(); f != nil { (*f)(branch) @@ -115,6 +116,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if f := registrar.Load(); f != nil { (*f)(reg) } + }, func(ev catalog.CatalogRevisionEvent) (int, error) { + if f := reviser.Load(); f != nil { + return (*f)(ev) + } + return 0, fmt.Errorf("the revision %d of %s is heard before the runner can start its pipeline", ev.Revision, ev.Branch) }, logger.Printf) if err != nil { return err @@ -142,7 +148,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } logger.Printf("widens: each read bound %s, %d catalog revisions kept", readBound, revisionsKept) - hearer.Widens = widenerOf(calls.Call, readBound, revisionsKept, logger.Printf).Widens + widener := widenerOf(calls.Call, readBound, revisionsKept, logger.Printf) + hearer.Widens = widener.Widens hearer.LogLast = stepLog{ctx: ctx, js: js, module: cred.Module}.Last hearer.Where = routerRecordOf(cred.Module, calls.Call, calls.Ask) hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf} @@ -154,6 +161,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} hearer.AskSteps = starter.AskSteps hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) } + hearer.AskPinHeld = func(key, module string, pins map[string]string) { + if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil { + logger.Printf("the steps of %s's %s that waited for its pins are not asked: %v", key, module, err) + } + } hearer.Release = func(key string) { if _, err := starter.Release(hearer.Store, key, rv.Catalog.View); err != nil { logger.Printf("the movers of %s waiting for a build are not placed: %v", key, err) @@ -197,6 +209,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } registrar.Store(®ister) + // A release's, a take-in again's or a recompute's revision starts one pipeline, its revision read by its number + // (H-A-213 (2), (4)). + revise := func(ev catalog.CatalogRevisionEvent) (int, error) { return starter.Revised(ev, widener.Revision) } + reviser.Store(&revise) starter.Store = store // The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in // the state queue, placed at the start from it and on every end. diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index 6b319931..8e9fb2d3 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -101,7 +101,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { } } answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs}) - answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries}) + answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 100, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries}) answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) // The router of the operator's asks holds the seat operator-channel; its record of each ask is in its state asks. const router = "operator-router" @@ -218,7 +218,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) { } take := func(source int64, movers ...string) { t.Helper() - ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: 1, Call: fmt.Sprintf("%016x", source)} + ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: source, Call: fmt.Sprintf("%016x", source)} for _, m := range movers { ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index b5bf1b95..a7b2e3cb 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -72,6 +72,9 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes t.Fatal(err) } } + // Kept pipelines in the old form, their steps inline in the record, as the runner's start finds them on a live bus + // (main.41-46, one of 437 steps): each start path reads them at once from its own goroutine (H-A-187, H §774). + keptPipelines(t, ctx, js, module) // The step log, the runner's old logs section (§470): the stream the bus makes for it. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil { t.Fatal(err) @@ -340,3 +343,36 @@ func waitFor(t *testing.T, ctx context.Context, logs *syncBuffer, cond func() bo } t.Fatalf("it did not come about in time; the runner said:\n%s", logs.String()) } + +// keptPipelines writes six ended pipelines of main, main.41 to main.46, in the record form before H-A-187: every step inline, +// main.46 with 437 of them. +func keptPipelines(t *testing.T, ctx context.Context, js jetstream.JetStream, module string) { + t.Helper() + kv, err := js.KeyValue(ctx, module+"_pipelines") + if err != nil { + t.Fatal(err) + } + at := time.Date(2026, 11, 20, 23, 0, 0, 0, time.UTC) + for source := int64(41); source <= 46; source++ { + movers, count := 3, 12 + if source == 46 { + movers, count = 73, 437 + } + p := seat.Pipeline{V: 1, Pipeline: fmt.Sprintf("main.%d", source), Branch: "main", Source: source, Revision: source, + Outcome: seat.PipelineOutcomePassed, Started: &at, Ended: &at, EndCount: 1} + for i := range movers { + p.Movers = append(p.Movers, seat.Mover{Module: fmt.Sprintf("kept-%02d", i), Commit: "c0", Fingerprint: fmt.Sprintf("src2:k%02d", i)}) + } + for i := range count { + p.Steps = append(p.Steps, seat.StepResult{Module: p.Movers[i%movers].Module, Step: seat.StepResultStepBuild, + Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomePassed, Started: &at, Ended: &at}) + } + raw, err := json.Marshal(p) + if err != nil { + t.Fatal(err) + } + if _, err := kv.Put(ctx, p.Pipeline, raw); err != nil { + t.Fatal(err) + } + } +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go index 5be6db24..1f9365ce 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "github.com/nats-io/nats.go/jetstream" @@ -11,10 +12,10 @@ import ( "mesh-delivery-pipeline/internal/pipeline" ) -// hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one -// pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is -// written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard -// after a stop. +// hearPublished hears each package-published (design 47 H §746, §747): in the running pipeline of the revision that +// published it, the movers waiting for it are placed; it starts no pipeline (a release's revision does, H-A-213 (2)). The +// message is acknowledged once that is written, on a goroutine of its own, so a slow write holds no other event; the +// durable consumer delivers a publish heard after a stop. func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error { return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) { var ev builds.PackagePublished @@ -24,18 +25,38 @@ func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, l return } go func() { - n, err := starter.Published(ev, v.View) - if err != nil { - logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err) + if err := starter.Published(ev, v.View); err != nil { + logf("the publish of %s %s on %s: %v; heard again", ev.Artifact, ev.Version, ev.Branch, err) _ = m.Nak() return } - logf("the publish of %s %s on %s started %d pipelines", ev.Artifact, ev.Version, ev.Branch, n) + logf("the publish of %s %s on %s is heard", ev.Artifact, ev.Version, ev.Branch) _ = m.Ack() }() }) } +// revisedHear is the handler of catalog-revision that starts the pipeline of a revision whose call kind is not a +// take-in's (design 47 H-A-212, H-A-213 (2), H-A-217): every other revision is left to the views that hear it. An error +// has the event heard again; a second start of its key writes nothing. +func revisedHear(revised func(ev catalog.CatalogRevisionEvent) (int, error), logf func(string, ...any)) func(context.Context, []byte) error { + return func(_ context.Context, data []byte) error { + var ev catalog.CatalogRevisionEvent + if err := json.Unmarshal(data, &ev); err != nil { + logf("a catalog-revision is not read: %v", err) + return nil + } + n, err := revised(ev) + if err != nil { + return err + } + if n > 0 { + logf("the revision %d of %s (%s, %s) started %d pipeline", ev.Revision, ev.Branch, ev.CallKind, ev.Call, n) + } + return nil + } +} + // hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged // once it is written, and heard again when a write failed, so the key already written starts nothing twice. func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index d6915534..c4aa18c1 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -128,7 +128,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any)) if bs.Shared != nil { shared = bs.Shared(m.Data()) } - _, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared) + var pins map[string]string + if bs.Pins != nil { + pins = bs.Pins(m.Data()) + } + _, err = h.OnStepEndedPinned(bs, m.Subject(), ev, shared, pins) switch { case errors.Is(err, pipeline.ErrRefused): logf("%v", err) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index 95ed9670..e39fabf9 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -37,7 +37,7 @@ import ( // ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step // that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a // run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed; -// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended, +// a run that keeps saying its progress past its bound is not ended, // and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the // same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its // other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and @@ -88,7 +88,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil { + askStream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}) + if err != nil { t.Fatal(err) } @@ -116,6 +117,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{}) entries := []catalog.CatalogEntry{ {Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app", + Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25"}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, {Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}}, {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, @@ -325,12 +327,23 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { // One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its // ask is taken (A-469b), its holder present and nothing failing it (A-466y). - const p3 = "main.3" + // A pipeline is keyed by its catalog revision (H-A-213 (4)): source 3's take-in of revision 1 is main.1. + const p3 = "main.1" takeIn("app", "c1", "src2:aa", 3, 1) placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747) passBuild(p3, "app", "c1", "src2:aa") unitRun, unit := runOfStep(p3, "app:program", "unit") integRun, integ := runOfStep(p3, "app:program", "integration") + // A test ask fetches with the entry's pins, as the builder gives them the build (H-A-203). + last, err := askStream.GetLastMsgForSubject(ctx, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner") + if err != nil { + t.Fatal(err) + } + var testAsk nodebuildgo.StepAsk + if err := json.Unmarshal(last.Data, &testAsk); err != nil || testAsk.Step == nil || + testAsk.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25" || testAsk.Step.Revision != 1 || testAsk.Step.Source != 3 { + t.Fatalf("the last test ask = %s (%v), want its fetch to carry the entry's pin, revision 1 and source 3 (H-A-219)", last.Data, err) + } if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 { t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers) } @@ -471,39 +484,20 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { // A second revision with the same digests asks its own runs, so a failed test runs again (§724). revision.Store(2) publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4}) - const p4, p5 = "main.4", "main.5" + const p4 = "main.2" takeIn("app", "c1", "src2:aa", 4, 2) placed(p4) run4, r4 := runOfStep(p4, "app:program", "integration") if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask { t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask) } - // A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads - // the run, the first is cancelled and leaves it, and the run's ask is not dropped. - takeIn("app", "c1", "src2:aa", 5, 2) - placed(p5) - waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 }) - if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" { - t.Fatalf("cancel: %s", e) - } - waitFor(t, ctx, logs, func() bool { - r, _ := runOf(run4) - return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5 - }) - if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask { - t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r) - } - mu.Lock() - dropped := append([]string(nil), drops...) - mu.Unlock() - if len(dropped) != 0 { - t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String()) - } + // A replaced reader leaving the ask to the other readers of another pipeline (§726) is the unit tests': one revision is + // one pipeline (H-A-213 (4)). // A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720). taken(r4.Ask) waitFor(t, ctx, logs, func() bool { - return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun && - step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun + return step(read(p4), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun && + step(read(p4), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun }) if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") { t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r) @@ -516,8 +510,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { // A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not // asked (§535, §647; A-470g, A-470h). - const web = "main.7" - takeIn("web", "w1", "src2:ww", 7, 2) + revision.Store(3) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 7}) + const web = "main.3" + takeIn("web", "w1", "src2:ww", 7, 3) waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 }) if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" { t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e) @@ -568,10 +564,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { t.Fatal(err) } } - revision.Store(3) - publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5}) - const p6 = "main.6" - takeIn("app", "c1", "src2:aa", 6, 3) + revision.Store(4) + publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 4, Kind: "trunk", Branch: "main", Source: 6}) + const p6 = "main.4" + takeIn("app", "c1", "src2:aa", 6, 4) waitFor(t, ctx, logs, func() bool { s := step(read(p6), "app:program", "unit") return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib") diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go index 3a620f52..440e2c3d 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go @@ -98,7 +98,7 @@ func (h *holderView) Holders(seatName string) ([]string, bool) { // after, then start hub once. func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State, call meshcatalogclient.Call, onReady, onAdvance func(branch string), candidate func(ctx context.Context, ev catalog.CatalogRevisionEvent) error, - registered func(reg catalog.Registration), logf func(string, ...any)) (*runnerViews, error) { + registered func(reg catalog.Registration), revised func(ev catalog.CatalogRevisionEvent) (int, error), logf func(string, ...any)) (*runnerViews, error) { findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf) if err != nil { return nil, err @@ -153,6 +153,11 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr if err := meshcatalogclient.WireCandidatesProcess(reader, candidate); err != nil { return nil, err } + // A trunk revision of a release, a take-in again or a recompute starts one pipeline of its own, read by its number + // (H-A-212, H-A-213 (2), H-A-217), heard on the same event's one subscription. + if err := reader.AdoptEvent(catalog.Seat, catalog.EventCatalogRevision, revisedHear(revised, logf)); err != nil { + return nil, err + } reader.Start(ctx, js, cred, status) return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings, Holders: &holderView{reader: reader, catalog: cv, logf: logf}}, nil } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go index 211738a7..12fef5af 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens.go @@ -166,7 +166,8 @@ func catalogRevision(call func(string, any) (json.RawMessage, error), kept *revi if h, ok := kept.get(branch, a.Revision.Revision); ok { return h, nil } - half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{}} + half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{}, + Call: a.Revision.Call, Movers: a.Revision.Movers} in.Revision = a.Revision.Revision // the pages of one revision, though a newer one is said between them } for _, e := range a.Entries { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go index fd32e2f8..60a7d333 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged.go @@ -231,11 +231,11 @@ func (a *Ager) newest(p seat.Pipeline) (bool, error) { if err != nil { return false, fmt.Errorf("the pipelines of %s are not listed: %w", p.Branch, err) } - // Newest for one of its movers: no later pipeline of the branch moves it. + // Newest for one of its movers: no pipeline of a later revision of the branch moves it (H-A-213 (4)). for _, m := range p.Movers { newer := false for _, k := range pkeys { - if q, _, err := a.Pipelines.Get(k); err == nil && q.Source > p.Source && Moves(q, m.Module) { + if q, _, err := a.Pipelines.Get(k); err == nil && keyRevision(Begin{Revision: q.Revision, Source: q.Source}) > keyRevision(Begin{Revision: p.Revision, Source: p.Source}) && Moves(q, m.Module) { newer = true break } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go index 2270238e..47da9a57 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/aged_test.go @@ -142,3 +142,23 @@ func TestOneStepEndAgesAtMostTwentyEntries(t *testing.T) { t.Fatalf("%d outputs deleted, sequence %d; want 20 and 20", len(f.outputs), f.aged["7d"]) } } + +// H-A-213 (4): a branch's newest pipeline for a mover is the one of the highest revision number, never of the highest source: +// a release revision of the same source is newer than its take-in's. +func TestTheNewestPipelineIsOfTheHighestRevision(t *testing.T) { + f := newAgedFixture() + for key, p := range map[string]seat.Pipeline{ + "main.41": {Pipeline: "main.41", Branch: "main", Source: 41, Revision: 41, Movers: []seat.Mover{{Module: "app"}}}, + "main.42": {Pipeline: "main.42", Branch: "main", Source: 41, Revision: 42, Movers: []seat.Mover{{Module: "app"}}}, + } { + f.pipes.memStore[key] = struct { + p seat.Pipeline + rev uint64 + }{p, 1} + } + for key, want := range map[string]bool{"main.41": false, "main.42": true} { + if got, err := f.ager.newest(f.pipes.memStore[key].p); got != want || err != nil { + t.Errorf("%s newest = %v (%v), want %v", key, got, err, want) + } + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go index abf01a7c..ee636c4a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/ended_test.go @@ -113,9 +113,19 @@ func TestAReleasedVersionIsTheMoversVersion(t *testing.T) { other := store[pkey] other.p.Pipeline, other.p.Branch, other.p.Movers = "beta.5", "beta", append([]seat.Mover(nil), other.p.Movers...) store["beta.5"] = other + // A pipeline is keyed by its revision (H-A-213 (4)): main.8, a later revision of source 5, is written too; main.6, of + // source 6, is not. + for key, source := range map[string]int64{"main.8": 5, "main.6": 6} { + again := store[pkey] + again.p.Pipeline, again.p.Source, again.p.Movers = key, source, append([]seat.Mover(nil), again.p.Movers...) + store[key] = again + } ev := catalog.VersionReleasedEvent{Module: "app", Version: "0.4.1", Source: 5} - if n, err := h.Released(ev, "main"); n != 1 || err != nil { - t.Fatalf("released wrote %d (%v), want app's mover of main.5", n, err) + if n, err := h.Released(ev, "main"); n != 2 || err != nil { + t.Fatalf("released wrote %d (%v), want app's mover of main.5 and main.8", n, err) + } + if m, _ := MoverOf(store["main.6"].p, "app"); m.Version != "" { + t.Errorf("main.6's mover = %+v, want no version: another source", m) } if m, _ := MoverOf(store[pkey].p, "app"); m.Version != "0.4.1" { t.Errorf("app's mover = %+v, want version 0.4.1", m) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go index 9b7f1e07..bc21891e 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go @@ -66,6 +66,8 @@ type MoverStore struct { Raw RawState Logf func(string, ...any) + // mu guards the three maps, which every pipeline's reads and writes share across goroutines (H §774): the lock table, + // the last reads and the records said inline. A pipeline's own lock is held across its read or write. mu sync.Mutex locks map[string]*sync.Mutex read map[string]readAt @@ -93,6 +95,42 @@ func (s *MoverStore) lock(key string) func() { return l.Unlock } +// seen is the last read of pipeline key, under mu. +func (s *MoverStore) seen(key string) (readAt, bool) { + s.mu.Lock() + defer s.mu.Unlock() + r, ok := s.read[key] + return r, ok +} + +// setSeen keeps the last read of pipeline key, under mu. +func (s *MoverStore) setSeen(key string, r readAt) { + s.mu.Lock() + defer s.mu.Unlock() + s.read[key] = r +} + +// markInline says whether pipeline key is said inline for the first time, marking it, under mu. +func (s *MoverStore) markInline(key string) bool { + s.mu.Lock() + defer s.mu.Unlock() + first := !s.inline[key] + s.inline[key] = true + return first +} + +// forget drops pipeline key's last read, and its inline mark when inline is true, under mu. +func (s *MoverStore) forget(key string, read, inline bool) { + s.mu.Lock() + defer s.mu.Unlock() + if read { + delete(s.read, key) + } + if inline { + delete(s.inline, key) + } +} + func (s *MoverStore) logf(f string, a ...any) { if s.Logf != nil { s.Logf(f, a...) @@ -233,10 +271,9 @@ func (s *MoverStore) Get(key string) (seat.Pipeline, uint64, error) { if err != nil { return seat.Pipeline{}, 0, err } - s.read[key] = readAt{rev: rev, movers: revs} + s.setSeen(key, readAt{rev: rev, movers: revs}) if len(p.Steps) > 0 { - if !s.inline[key] { - s.inline[key] = true + if s.markInline(key) { s.logf("the pipeline %s carries its %d steps inline in its record: they are read whole and split into one key per mover at its next write (H-A-187)", key, len(p.Steps)) } return p, rev, nil @@ -271,7 +308,7 @@ func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint6 return 0, err } // A mover key written since the read is another write between: the read named its revision. - if seen, ok := s.read[key]; ok && seen.rev == revision { + if seen, ok := s.seen(key); ok && seen.rev == revision { for k, r := range revs { if seen.movers[k] != r { return 0, ErrChanged @@ -301,9 +338,9 @@ func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint6 } } else { s.logf("the pipeline %s's steps are split into %d mover keys and its record no longer carries them (H-A-187)", key, len(want)) - delete(s.inline, key) + s.forget(key, false, true) } - s.read[key] = readAt{rev: rev, movers: after} + s.setSeen(key, readAt{rev: rev, movers: after}) return rev, nil } @@ -367,7 +404,7 @@ func (s *MoverStore) Create(key string, value []byte) error { if err != nil { return err } - s.read[key] = readAt{rev: rev, movers: after} + s.setSeen(key, readAt{rev: rev, movers: after}) return nil } @@ -386,8 +423,7 @@ func (s *MoverStore) Delete(key string) error { if err == nil { err = s.Raw.Delete(key) } - delete(s.read, key) - delete(s.inline, key) + s.forget(key, true, true) unlock() return err } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go index 0065aabd..95f5536a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go @@ -70,14 +70,14 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if n, err := s.TakenIn(ev, view); n != 1 || err != nil { t.Fatalf("the take-in started %d (%v)", n, err) } - p, _, _ := mem.Get("main.4") + p, _, _ := mem.Get("main.7") if !placed(p, "mesh-sdk") || !placed(p, "builder") || placed(p, "app") || placed(p, "tool") { t.Fatalf("placed at the start: %+v, want mesh-sdk and builder only", p.Steps) } - if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); n != 0 || err != nil { - t.Fatalf("the publish started %d (%v), want none", n, err) + if err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); err != nil { + t.Fatalf("the publish is not heard: %v", err) } - p, _, _ = mem.Get("main.4") + p, _, _ = mem.Get("main.7") if sdk, _ := MoverOf(p, "mesh-sdk"); !placed(p, "app") || placed(p, "tool") || len(mem) != 1 || !slices.Contains(sdk.Published, "mesh-sdk:go") { t.Fatalf("after mesh-sdk's publish: %d pipelines, app placed %v, tool placed %v, mesh-sdk %+v", len(mem), placed(p, "app"), placed(p, "tool"), sdk) } @@ -87,22 +87,22 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { b := stepsOf(p, "builder")[0] bs := stepSeatOf(t, builds.Seat) end := func(outcome string) StepEnded { - e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.4","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`)) + e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.7","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`)) if err != nil { t.Fatal(err) } return e } - before := mem["main.4"] + before := mem["main.7"] if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("passed")); err != nil { t.Fatal(err) } - if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning { + if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning { t.Errorf("after builder's build passed, tool's steps = %+v, want placed and asked", stepsOf(p, "tool")) } // The same pipeline, builder's build failed: tool is placed with every step not asked, each saying its own step-ended // naming the dependency (H-A-101 (2)). - mem["main.4"] = before + mem["main.7"] = before var notAsked []seat.StepEvent s.Say = func(subject string, body any) error { if ev, ok := body.(seat.StepEvent); ok && strings.Contains(subject, ".step-ended.tool.") { @@ -113,7 +113,7 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("failed")); err != nil { t.Fatal(err) } - if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked { + if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked { t.Errorf("after builder's build failed, tool's steps = %+v, want placed not asked", stepsOf(p, "tool")) } if len(notAsked) == 0 || len(notAsked) != len(slices.DeleteFunc(stepsOf(p, "tool"), func(st seat.StepResult) bool { return st.Outcome != seat.StepResultOutcomeNotAsked })) { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go index 455f8fa5..bd99d2c8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/pins_test.go @@ -51,7 +51,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing. if n, err := s.TakenIn(taken, view); n != 1 || err != nil { t.Fatalf("the take-in started %d (%v)", n, err) } - p, _, _ := store.Get("main.947") + p, _, _ := store.Get("main.357") if p.Revision != 357 || len(p.Steps) != 0 || len(asks.asks) != 0 { t.Fatalf("a take-in of revision 357 with the view at 356 started %+v with %d asks, want revision 357 and no step", p, len(asks.asks)) } @@ -60,7 +60,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing. } viewRevision = 357 n, err := s.Waiting(store.memStore, "main", view) - p, _, _ = store.Get("main.947") + p, _, _ = store.Get("main.357") if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Revision != 357 { t.Errorf("the view reaching 357 placed %d (%v): %d steps, %d asks at revision %d", n, err, len(p.Steps), len(asks.asks), p.Revision) } @@ -87,7 +87,7 @@ func TestAViewThatReachesTheRevisionDuringTheStartsPlacesTheHeldBackPipelines(t if n, err := s.TakenIn(taken, view); n != 1 || err != nil { t.Fatalf("started %d (%v)", n, err) } - if p, _, _ := store.Get("main.947"); len(p.Steps) == 0 || len(asks.asks) == 0 { + if p, _, _ := store.Get("main.357"); len(p.Steps) == 0 || len(asks.asks) == 0 { t.Errorf("the pipeline has %d steps and %d asks after the view reached 357 during the start, want them placed", len(p.Steps), len(asks.asks)) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index af5758d6..a7dbf327 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -3,7 +3,10 @@ package pipeline import ( "crypto/sha256" "encoding/hex" + "errors" "fmt" + "maps" + "slices" "sort" "strconv" "strings" @@ -140,6 +143,13 @@ func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[ steps = append(steps, s) continue } + // A Go step is asked only once every pin of its entry has a version (H-A-206): until then it waits on the + // publish, as the build does (H-A-165), and the publish re-pins the entry and moves the mover (H-A-204). + if path := unpinnedPackage(entry); path != "" && language == sdkstep.AskGo { + s.Seat, s.Outcome, s.Started, s.Why = bs, seat.StepResultOutcomeWaiting, &now, PinWait+path + steps = append(steps, s) + continue + } s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now s.Ask = AskID(startOf(p, now), p.Pipeline, m.Module, a.Ref, step) body := stepAsk(p, m, entry, a, language, step, s.Ask, asker, now) @@ -227,6 +237,132 @@ type Asked struct { Commit string } +// PinWait opens the why of a Go step held because its entry names a package with no version yet (H-A-206), the build's +// own wait form (H-A-165); the package's path follows. +const PinWait = "waits on the publish of " + +// unpinnedPackage is the first package, in path order, the entry names with no version, empty when every one has one. +func unpinnedPackage(e catalog.CatalogEntry) string { + for _, path := range slices.Sorted(maps.Keys(e.Packages)) { + if e.Packages[path] == "" { + return path + } + } + return "" +} + +// pinHeld says whether a step waits for its entry's pins (H-A-206). +func pinHeld(s seat.StepResult) bool { + return s.Outcome == seat.StepResultOutcomeWaiting && strings.HasPrefix(s.Why, PinWait) +} + +// endPinHeld ends each of build's mover's steps waiting for its entry's pins (H-A-206, H-A-207): as the build ended when +// it did not pass, so no step waits on a publish the build no longer waits for; could not run, loud, when the build passed +// and its end names no pins to run them with. It answers the steps it ended. +func endPinHeld(p *seat.Pipeline, build seat.StepResult, pins map[string]string, now time.Time) []int { + outcome, why := build.Outcome, "the build ended "+build.Outcome+" while this step waited: %s; "+build.Why + if stepPassed(build) { + if len(pins) > 0 { + return nil + } + outcome, why = seat.StepResultOutcomeCouldNotRun, "the build passed, but its end names no pins to run this step with (H-A-207); it %s" + } + var ended []int + for i := range p.Steps { + st := &p.Steps[i] + if st.Module != build.Module || !pinHeld(*st) { + continue + } + st.Outcome, st.Why, st.Ended = outcome, fmt.Sprintf(why, st.Why), &now + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) + ended = append(ended, i) + } + return ended +} + +// readRevision is revision's read, an error when none is given. +func readRevision(revision func(branch string, revision int64) (Half, error), branch string, n int64) (Half, error) { + if revision == nil { + return Half{}, errors.New("the runner reads no catalog revision") + } + return revision(branch, n) +} + +// heldOn says whether module has a step of p waiting for its entry's pins. +func heldOn(p seat.Pipeline, module string) bool { + return slices.ContainsFunc(p.Steps, func(st seat.StepResult) bool { return st.Module == module && pinHeld(st) }) +} + +// AskPinHeld asks the steps of pipeline key's mover module that waited for its entry's pins (H-A-206) with the pins its +// build passed with (H-A-207): each running under its own ask, its fetch the entry's as Place builds it with those pins. +// The entry is read at the pipeline's own catalog revision through revision, which keeps what it read (H-A-205), never +// the branch's newest: a revision that cannot be read, or whose entry is of another fingerprint than the mover's, or that +// names no artifact of the step, ends the step could not run, saying so. It answers how many it asked. +func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]string, revision func(branch string, revision int64) (Half, error)) (int, error) { + for { + p, rev, err := store.Get(key) + if err != nil { + return 0, fmt.Errorf("the pipeline %s is not read: %w", key, err) + } + m, moves := MoverOf(p, module) + if !moves || !heldOn(p, module) { + return 0, nil + } + var entry catalog.CatalogEntry + unread := "" + switch half, err := readRevision(revision, p.Branch, p.Revision); { + case err != nil: + unread = fmt.Sprintf("revision %d of %s, the pipeline's, cannot be read for its entry: %v", p.Revision, p.Branch, err) + case half.Entries[module].Fp != m.Fingerprint: + unread = fmt.Sprintf("revision %d of %s holds %s at %q, not the mover's %s", p.Revision, p.Branch, module, half.Entries[module].Fp, m.Fingerprint) + default: + entry = half.Entries[module] + } + entry.Packages = pins + now := s.now() + var asks []Asked + var ended []int + for i := range p.Steps { + st := &p.Steps[i] + if st.Module != module || !pinHeld(*st) { + continue + } + j := slices.IndexFunc(entry.Artifacts, func(a catalog.Artifact) bool { return a.Ref == st.Artifact }) + language, err := sdkstep.AskLanguageOfSeat(st.Seat) + if unread != "" || j < 0 || err != nil { + why := unread + if why == "" { + why = fmt.Sprintf("revision %d of %s names no artifact %s of %s to ask it of", p.Revision, p.Branch, st.Artifact, module) + } + st.Outcome, st.Ended = seat.StepResultOutcomeCouldNotRun, &now + st.Why = "its build passed with its pins, but " + why + " (H-A-207, H-A-205)" + p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now)) + ended = append(ended, i) + continue + } + a := entry.Artifacts[j] + st.Outcome, st.Why, st.Started = seat.StepResultOutcomeRunning, "", &now + st.Ask = AskID(startOf(p, now), p.Pipeline, module, a.Ref, st.Step) + ask := Asked{Seat: st.Seat, ID: st.Ask, Ask: stepAsk(p, m, entry, a, language, st.Step, st.Ask, s.Name, now), Toolchain: a.Language, Commit: m.Commit} + if a.Package != nil { + ask.Package = a.Package.Name + } + asks = append(asks, ask) + } + if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return 0, fmt.Errorf("the pipeline %s is not written: %w", key, err) + } + s.logf("%s: %s's build passed with its pins: %d steps that waited for them are asked, %d could not run (H-A-207)", key, module, len(asks), len(ended)) + s.sayNotAsked(p, ended) + if s.Steps == nil || len(asks) == 0 { + return 0, nil + } + return len(asks), s.askSteps(key, asks) + } +} + func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk { // A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the // artifact's named paths in it when the entry has them. @@ -234,9 +370,21 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar if paths := artifactPaths(e, a.Ref); len(paths) > 0 { fetch.Paths = paths } + // The entry's pins, the same the builder gives the build (H-A-203, H-A-183): a test runs against what the build was + // built against, and the Go build module refuses a requirement of another repository with no pin (H-A-192). A + // package the entry names with no pin yet is left out, as the builder pins it only from its publish. + for path, version := range e.Packages { + if version == "" { + continue + } + if fetch.Packages == nil { + fetch.Packages = map[string]string{} + } + fetch.Packages[path] = version + } return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now, Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: m.Commit, Fetch: fetch, Fingerprint: m.Fingerprint, - Module: m.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}} + Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}} } // askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index d54f44a8..7f46c658 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -2,6 +2,8 @@ package pipeline import ( "encoding/json" + "fmt" + "reflect" "strconv" "strings" "testing" @@ -44,7 +46,7 @@ func placeFixture() (catalog.Registration, map[string]catalog.CatalogEntry) { Artifacts: map[string]catalog.StepStatuses{"tools": {Lint: "bypass"}}, }} return reg, map[string]catalog.CatalogEntry{ - "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, + "app": {Module: "app", Path: "modules/app", Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25"}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}, {Ref: "app:web", Language: "rust"}}, Inputs: &catalog.EntryInputs{Paths: map[string][]string{"app:program": {"cmd/app"}}}}, "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, } @@ -100,10 +102,131 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) { a := asks.asks[1].(nodebuildgo.StepAsk) if asks.at[1] != "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner" || a.Kind != "step" || a.Asker != "the-runner" || a.ID != p.Steps[1].Ask || asks.ids[1] != a.ID || a.Step.Commit != "c1" || a.Step.Fingerprint != "src2:aa" || a.Step.Pipeline != "main.4" || a.Step.Artifact != "app:program" || - a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Revision != 4 || + a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Source != 4 || len(a.Step.Fetch.Paths) != 1 || a.Step.Fetch.Paths[0] != "cmd/app" { t.Errorf("the ask = %+v at %s", a, asks.at[1]) } + // Every step ask fetches with the entry's pins, as the builder gives them the build (H-A-203, H-A-192). + for _, x := range asks.asks[1:] { + if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && (len(b.Step.Fetch.Packages) != 1 || b.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25") { + t.Errorf("the %s ask of %s fetches the pins %v, want the entry's", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages) + } + } + // An entry naming a package with no version yet asks no Go step: each waits on its publish, as the build does, said in + // the journal (H-A-206); the build is still asked. + waiting := map[string]catalog.CatalogEntry{} + for k, e := range entries { + waiting[k] = e + } + held := waiting["app"] + held.Packages = map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": ""} + held.Fp = "src2:ab" + waiting["app"] = held + var journal []string + heldAsks := &stepAsks{} + hs := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: heldAsks, Name: "the-runner", + Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }, + Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }} + if ok, err := hs.Start(Begin{Branch: "main", Source: 5, Revision: 3, Entries: waiting, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}}); !ok || err != nil { + t.Fatalf("start = %v, %v", ok, err) + } + for _, x := range heldAsks.asks { + if b, ok := x.(nodebuildgo.StepAsk); ok { + t.Errorf("the %s of %s is asked with a package unpinned", b.Step.Step, b.Step.Artifact) + } + } + var hp seat.Pipeline + if err := json.Unmarshal(store["main.3"], &hp); err != nil { + t.Fatal(err) + } + if st := hp.Steps[1]; st.Step != "lint" || st.Outcome != seat.StepResultOutcomeWaiting || st.Ask != "" || st.Why != "waits on the publish of git.novox.be/novox/x/go" { + t.Errorf("the held lint step = %+v; want it waiting on the publish", st) + } + // The build ending without passing past its bound ends each held step as it ended (H-A-206). + failed := hp.Steps[0] + failed.Outcome, failed.Why = seat.StepResultOutcomeFailed, "package git.novox.be/novox/x/go has no released version" + failedP := hp + failedP.Steps = append([]seat.StepResult(nil), hp.Steps...) + if ended := endPinHeld(&failedP, failed, nil, time.Now()); len(ended) == 0 || failedP.Steps[1].Outcome != seat.StepResultOutcomeFailed || failedP.Steps[1].Ended == nil { + t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", failedP.Steps[1], len(ended)) + } + // A build passed whose end names no pins ends them could not run, loud (H-A-207). + passed := hp.Steps[0] + passed.Outcome = seat.StepResultOutcomePassed + bare := hp + bare.Steps = append([]seat.StepResult(nil), hp.Steps...) + if ended := endPinHeld(&bare, passed, nil, time.Now()); len(ended) == 0 || bare.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || + !strings.Contains(bare.Steps[1].Why, "names no pins") { + t.Errorf("after a build passed naming no pins the held lint step = %+v; want could not run naming it", bare.Steps[1]) + } + // A build passed naming its pins asks the held steps with them: its end heard, the runner asks each with the pins the + // build was built with (H-A-207). + pinsBuilt := map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": "v0.1.4"} + ms := memStore{} + ms["main.5"] = struct { + p seat.Pipeline + rev uint64 + }{hp, 1} + // The entry is read at the pipeline's own revision (3), never the newer revision 4 that moved the module (H-A-205). + moved := held + moved.Fp, moved.Path = "src2:ac", "modules/moved" + revisions := func(branch string, n int64) (Half, error) { + switch n { + case 3: + return Half{Revision: 3, Entries: waiting}, nil + case 0, 4: + return Half{Revision: 4, Entries: map[string]catalog.CatalogEntry{"app": moved}}, nil + } + return Half{}, fmt.Errorf("no revision %d", n) + } + var askedWith map[string]string + h := &Hearer{Store: ms, Say: func(string, any) error { return nil }, Name: "the-runner", + AskPinHeld: func(key, module string, pins map[string]string) { + askedWith = pins + if _, err := hs.AskPinHeld(ms, key, module, pins, revisions); err != nil { + t.Errorf("ask held: %v", err) + } + }} + buildEnd := StepEnded{ID: hp.Steps[0].Ask, Pipeline: "main.5", Step: "build", Node: "n1", Outcome: seat.StepResultOutcomePassed, + Module: "app", Commit: "c1", Fingerprint: "src2:ab"} + if _, err := h.OnStepEndedPinned(StepSeats[3], StepSeats[3].Prefix+"n1", buildEnd, nil, pinsBuilt); err != nil { + t.Fatalf("the build's end: %v", err) + } + if !reflect.DeepEqual(askedWith, pinsBuilt) { + t.Errorf("the held steps were asked with %v; want the build's pins", askedWith) + } + var askedHeld int + for _, x := range heldAsks.asks { + if b, ok := x.(nodebuildgo.StepAsk); ok { + askedHeld++ + if !reflect.DeepEqual(b.Step.Fetch.Packages, pinsBuilt) { + t.Errorf("the %s of %s fetches %v; want the build's pins", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages) + } + if b.Step.Fetch.Dir != "modules/app" { + t.Errorf("the %s of %s fetches %s; want the pipeline's revision's directory", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Dir) + } + } + } + after, _, _ := ms.Get("main.5") + if askedHeld == 0 || after.Steps[1].Outcome == seat.StepResultOutcomeWaiting || after.Steps[1].Ask == "" { + t.Errorf("after the build passed %d held steps were asked, lint = %+v; want each asked", askedHeld, after.Steps[1]) + } + // A mover whose fingerprint the pipeline's revision does not hold ends its held steps could not run, naming it. + other := hp + other.Pipeline, other.Steps, other.Movers = "main.4", append([]seat.StepResult(nil), hp.Steps...), []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:zz", Repository: "o/app"}} + ms["main.4"] = struct { + p seat.Pipeline + rev uint64 + }{other, 1} + if n, err := hs.AskPinHeld(ms, "main.4", "app", pinsBuilt, revisions); n != 0 || err != nil { + t.Errorf("asked %d held steps of a mover its revision does not hold (%v); want none", n, err) + } + if o, _, _ := ms.Get("main.4"); o.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(o.Steps[1].Why, "not the mover's src2:zz") { + t.Errorf("the held lint step of a moved fingerprint = %+v; want could not run naming it", o.Steps[1]) + } + if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") { + t.Errorf("the journal says %q; want the held step named", journal) + } // An artifact that names no paths still fetches its module alone: the directory with no paths (principle 10, A-447). for _, x := range asks.asks[2:] { if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && b.Step.Artifact == "app:tools" && (b.Step.Fetch.Dir != "modules/app" || len(b.Step.Fetch.Paths) != 0) { @@ -409,7 +532,8 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }} start := func(source, revision int64) seat.Pipeline { t.Helper() - if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { + // Keyed by its source, so pipelines of one revision are several (one kept under its source before H-A-213 (4)). + if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { t.Fatalf("start %d = %v, %v", source, ok, err) } p, _, _ := store.Get(Key("main", source)) @@ -428,6 +552,12 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) { if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" { t.Fatalf("the test steps asked = %q, want the program's three alone", got) } + // Every step ask names the catalog revision in revision and the gate's source in source (H-A-219). + for _, a := range asks.asks { + if b, ok := a.(nodebuildgo.StepAsk); ok && (b.Step.Revision != 9 || b.Step.Source != 4) { + t.Errorf("the %s ask of %s names revision %d and source %d, want 9 and 4", b.Step.Step, b.Step.Artifact, b.Step.Revision, b.Step.Source) + } + } for _, level := range buildSteps { asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level) if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil { @@ -628,13 +758,13 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) { } } } - p, _, _ := mem.Get("main.4") + p, _, _ := mem.Get("main.7") if validates != 1 || compositions != 0 || stepNamed(p, "", seat.StepResultStepComposition).Outcome != seat.StepResultOutcomeWaiting { t.Fatalf("placed: %d validate and %d composition asks, steps %+v; want one validate, no composition yet", validates, compositions, p.Steps) } h := &Hearer{Store: mem, Name: "the-runner", Say: func(string, any) error { return nil }, AskSteps: s.AskSteps} bs := stepSeatOf(t, wants.Seat) - raw := `{"id":"` + validateAsk + `","pipeline":"main.4","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}` + raw := `{"id":"` + validateAsk + `","pipeline":"main.7","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}` ev, err := bs.Decode([]byte(raw)) if err != nil { t.Fatal(err) @@ -643,7 +773,7 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) { if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil { t.Fatal(err) } - p, _, _ = mem.Get("main.4") + p, _, _ = mem.Get("main.7") for _, st := range p.Steps { if st.Step == seat.StepResultStepValidate && st.Outcome != outcome { t.Errorf("%s: %s's validate is %s, want every mover's read from the one run", outcome, st.Module, st.Outcome) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go index 855191db..cecc84a8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go @@ -164,7 +164,7 @@ func (q *Queue) leave(key string) error { return nil } -// queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on . with step pipeline and +// queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on . with step pipeline and // on pipelines-at-once (design 47 H §748, H-A-52). func queueWait(p seat.Pipeline, at time.Time, how string) seat.Wait { return seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Revision: p.Revision, Run: p.Run, @@ -175,7 +175,7 @@ func (s *Starter) sayQueueWait(event string, p seat.Pipeline, at time.Time, how if s.Say == nil { return } - if err := s.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), queueWait(p, at, how)); err != nil { + if err := s.Say(runnerclient.RevisionSubject(event, p.Branch, subjectRevision(p)), queueWait(p, at, how)); err != nil { s.logf("%s of %s is not said: %v", event, p.Pipeline, err) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go index 586b656e..54249bca 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go @@ -96,7 +96,7 @@ func queueFixture() (*Starter, memStore, memQueue, *[]string, *[]seat.Wait, *ste func takeIn(t *testing.T, s *Starter, view func(string) (Half, bool)) func(source int64, modules ...string) { return func(source int64, modules ...string) { t.Helper() - ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7} + ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: source} for _, m := range modules { ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index e6a0762b..668d2ed8 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -206,7 +206,7 @@ func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) if raw == nil { return seat.Pipeline{}, false, nil } - p, err := seat.DecodePipelinesByBranchSourceValue(raw) + p, err := seat.DecodePipelinesByBranchRevisionValue(raw) if err != nil { return seat.Pipeline{}, false, fmt.Errorf("%s %s: %w", seat.StatePipelines, key, err) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go index 751fbf9a..1f0cc57a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/registrations.go @@ -99,6 +99,10 @@ type Half struct { Revision int64 Source int64 Entries map[string]catalog.CatalogEntry + // Call and Movers are the revision's own, as the revision verb answers them (H-A-213): what made it and its movers; + // a view's half names none. + Call string + Movers []catalog.TakeInEntry } // Views is the runner's views of the catalog half, one per watched branch the registrations name (design 47 G §21, diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/released.go b/modules/mesh-delivery-pipeline/internal/pipeline/released.go index 76028fce..840ca598 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/released.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/released.go @@ -3,7 +3,6 @@ package pipeline import ( "errors" "fmt" - "strconv" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/keys" @@ -21,12 +20,13 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int, if branch != "" { b = keys.EscapeName(branch) } - source := strconv.FormatInt(ev.Source, 10) + // A pipeline is keyed by its revision (H-A-213 (4)), so the pipelines of a source are those naming it, of either key + // form a source's pipelines were kept under. var pkeys []string - for _, f := range []string{b + "." + source, b + "." + source + ".published.*.*"} { + for _, f := range []string{b + ".*", b + ".*.published.*.*"} { k, err := h.Store.Keys(f) if err != nil { - return 0, fmt.Errorf("the pipelines of source %s are not listed: %w", source, err) + return 0, fmt.Errorf("the pipelines of source %d are not listed: %w", ev.Source, err) } pkeys = append(pkeys, k...) } @@ -37,6 +37,9 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int, if err != nil { return wrote, fmt.Errorf("the pipeline %s is not read: %w", key, err) } + if p.Source != ev.Source || p.Run != "" { + break + } i := -1 for j, m := range p.Movers { if m.Module == ev.Module { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index cfb8cc48..1b2e0911 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "maps" "slices" "sort" "strconv" @@ -28,7 +27,7 @@ type Created interface { Create(key string, value []byte) error } -// Begin is what a start names: the pipeline's key (watched branch and source revision, design 47 H §746), the take-in's +// Begin is what a start names: the pipeline's key (watched branch and catalog revision, design 47 H §746, H-A-213), the take-in's // call and catalog graph revision, its movers, and the catalog half of that branch. type Begin struct { Branch string @@ -39,28 +38,36 @@ type Begin struct { Movers []seat.Mover // Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them. Entries map[string]catalog.CatalogEntry - // Published is the package's publish that started the pipeline, nil for any other start (§747). - Published *seat.Published - // Key is the pipeline's key when it is not . (a publish's, §747); empty for a take-in's. + // Key is the pipeline's key when it is not .; empty for every start the runner makes. Key string // unheld says the branch's catalog half is not held yet: the pipeline starts with no steps, placed once it is (A-336). unheld bool } -// Key is a take-in's pipeline key in the state pipelines, `.`, the branch escaped as a key's name is -// (design 47 H §746). -func Key(branch string, source int64) string { - return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10)) +// Key is a pipeline's key in the state pipelines, `.`, the catalog revision's number the indexer counts, +// the branch escaped as a key's name is (design 47 H §746, H-A-213 (4)); a take-in's number is unchanged where it equalled +// its source. A pipeline kept under its source before (`.`) has the same form and is read as it is. +func Key(branch string, revision int64) string { + return seat.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10)) } -// PublishedKey is the key of a pipeline a package's publish starts, `..published..`, -// each part escaped as a key's name is (design 47 H §747). -func PublishedKey(branch string, source int64, pkg, version string) string { - return seat.PipelinesByBranchSourcePublishedPackageVersionKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), - keys.EscapeName(pkg), keys.EscapeName(version)) +// subjectRevision is the number p's pipeline-started, pipeline-ended and waits are said under, .: its +// key's number (H-A-221). +func subjectRevision(p seat.Pipeline) int64 { + return keyRevision(Begin{Revision: p.Revision, Source: p.Source}) } -// keyForms are the filters of every key form of the state pipelines: a take-in's, a run's and a publish's. +// keyRevision is the number b's pipeline is keyed by (H-A-213 (4)): its revision, or its source where the start names no +// revision (a take-in.taken of an indexer whose revision numbers were its sources, H-A-211). +func keyRevision(b Begin) int64 { + if b.Revision > 0 { + return b.Revision + } + return b.Source +} + +// keyForms are the filters of every key form of the state pipelines: a revision's, a run's, and a publish's kept from before +// a release revision started one pipeline (H-A-213 (2)). var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*"} // AllKeys are every pipeline's key, of each key form (§746, §747; H-A-83). @@ -189,7 +196,7 @@ func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]A } var none []string for _, st := range p.Steps { - if !open(st) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) { + if (!open(st) && !pinHeld(st)) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) { continue } if holders, read := s.Holders(st.Seat); read && len(holders) == 0 { @@ -282,7 +289,7 @@ func (s *Starter) sayStarted(key string, asks []Asked) { func newPipeline(b Begin, key string, now time.Time) seat.Pipeline { movers := append([]seat.Mover(nil), b.Movers...) return seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Call: b.Call, Revision: b.Revision, Movers: movers, - Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}, Published: b.Published} + Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}} } func (s *Starter) create(key string, p seat.Pipeline) error { @@ -299,12 +306,12 @@ func started(p seat.Pipeline, at time.Time) seat.PipelineStarted { Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run} } -// sayStartedPipeline says p's pipeline-started on . (§746). +// sayStartedPipeline says p's pipeline-started on . (§746). func (s *Starter) sayStartedPipeline(p seat.Pipeline, at time.Time) { if s.Say == nil { return } - if err := s.Say(runnerclient.SourceSubject(seat.EventPipelineStarted, p.Branch, p.Source), started(p, at)); err != nil { + if err := s.Say(runnerclient.RevisionSubject(seat.EventPipelineStarted, p.Branch, subjectRevision(p)), started(p, at)); err != nil { s.logf("pipeline-started of %s is not said: %v", p.Pipeline, err) } } @@ -337,6 +344,11 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog } steps, a := Place(*p, m, r, entries, now, s.Name) supersededWhy(*p, m.Module, entries, steps) + for _, st := range steps { + if pinHeld(st) { + s.logf("%s: %s's %s of %s is not asked: it %s, as its build does (H-A-206)", p.Pipeline, m.Module, st.Step, st.Artifact, st.Why) + } + } if failed != "" { why := "its dependency " + failed + " did not build" for k := range steps { @@ -490,7 +502,7 @@ func (s *Starter) Start(b Begin) (bool, error) { now := s.now() key := b.Key if key == "" { - key = Key(b.Branch, b.Source) + key = Key(b.Branch, keyRevision(b)) } if err := s.Fills.refuseStart(key); err != nil { return false, err @@ -887,14 +899,21 @@ func (s *Starter) Release(store Updater, key string, view func(branch string) (H }, s.Registration, view, "the publishes its movers waited for") } -// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the source -// revision that published it, and places the movers that waited for it (design 47 H §746, H-A-84). It answers that -// source's pipeline, read, and whether one is held. +// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the catalog +// revision that published it, found by the branch and revision the publish names (H-A-218), and places the movers that +// waited for it (design 47 H §746, H-A-84). It answers that pipeline, read, and whether one is held. func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch string) (Half, bool)) (seat.Pipeline, bool, error) { if s.Store == nil { return seat.Pipeline{}, false, nil } - key := Key(ev.Branch, ev.Source) + key, ok := Key(ev.Branch, ev.Revision), true + if ev.Revision <= 0 { + // A publish of an older builder names no revision: its source's lowest-numbered revision, the take-in's (H-A-218). + key, ok = s.sourcePipeline(ev.Branch, ev.Source) + } + if !ok { + return seat.Pipeline{}, false, nil + } for { p, rev, err := s.Store.Get(key) if err != nil { @@ -918,49 +937,81 @@ func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch st } } -// publishedKind is the edge a dependent builds on a published package by: B published before A builds, a package A -// imports (§53). -const publishedKind = 1 - -// Published hears a package-published (design 47 H §747, H-A-52): in the running pipeline of the source revision that -// published it, a mover that builds on it waits for it there (§746) and nothing starts; every module the catalog half -// holds a kind-1 edge from the package into that this pipeline does not move (a publish again of an older build, a -// dependent it does not move) is a mover of one pipeline keyed `..published..`, at the -// publishing source's catalog revision. The dependents come from the edges the runner holds, never from a comparison. -func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) (int, error) { - if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 { - return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev) - } - publishing, held, err := s.heardPublished(ev, view) +// sourcePipeline is the key of branch's pipeline of the lowest-numbered revision of source, the take-in's (H-A-213 (4), +// H-A-218): a pipeline is keyed by its revision, so one of a source is found by the source it names, never by its key. +func (s *Starter) sourcePipeline(branch string, source int64) (string, bool) { + pkeys, err := s.Store.Keys(keys.EscapeName(branch) + ".*") if err != nil { - return 0, err + s.logf("the pipelines of %s are not listed, so source %d's is not found: %v", branch, source, err) + return "", false } - half, ok := view(ev.Branch) - if !ok { - return 0, fmt.Errorf("the catalog half of %s is not held yet: the publish of %s %s is heard again once it is", ev.Branch, ev.Artifact, ev.Version) - } - var movers []seat.Mover - for _, module := range slices.Sorted(maps.Keys(half.Entries)) { - e := half.Entries[module] - if module == ev.Module || Moves(publishing, module) || !slices.ContainsFunc(e.Edges, func(x catalog.Edge) bool { - return x.Kind == publishedKind && x.From == ev.Artifact - }) { + found, low := "", int64(0) + for _, k := range pkeys { + p, _, err := s.Store.Get(k) + if err != nil || p.Branch != branch || p.Source != source || p.Run != "" { continue } - movers = append(movers, seat.Mover{Module: module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository}) + if r := keyRevision(Begin{Revision: p.Revision, Source: p.Source}); found == "" || r < low { + found, low = k, r + } } - if len(movers) == 0 { + return found, found != "" +} + +// Published hears a package-published (design 47 H §746, H-A-52): in the running pipeline of the revision that published +// it, a mover that builds on it waits for it there and is placed once it is heard. It starts no pipeline: a re-pin on a +// module's release is a catalog revision of its own, whose pipeline Revised starts (H-A-212, H-A-213 (2)). +func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) error { + if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 { + return fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev) + } + _, _, err := s.heardPublished(ev, view) + return err +} + +// revisedKinds are the call kinds of a catalog revision whose pipeline Revised starts: every kind but a take-in's, whose +// pipeline its take-in.taken starts (H-A-213 (2), H-A-217). +var revisedKinds = []string{catalog.CatalogRevisionEventCallKindRelease, catalog.CatalogRevisionEventCallKindTakeInAgain, + catalog.CatalogRevisionEventCallKindRecompute} + +// Revised starts, on a catalog-revision of the trunk kind whose call kind is release, take-in again or recompute, ONE +// pipeline keyed . by compare-and-set on its absence (design 47 H §746, H-A-212, H-A-213 (2) and (4), +// H-A-217): the revision is read by its number through the catalog seat's revision verb, never the newest half, and its +// movers are the revision's movers, its steps placed from the revision's entries. A revision of another kind, or one with +// no mover, starts nothing; a second hearing writes nothing. +func (s *Starter) Revised(ev catalog.CatalogRevisionEvent, read func(branch string, revision int64) (Half, error)) (int, error) { + if ev.Kind != "trunk" || !slices.Contains(revisedKinds, ev.CallKind) { return 0, nil } - revision := half.Revision - if held && publishing.Revision > 0 { - revision = publishing.Revision + branch, err := keys.UnescapeName(ev.Branch) + if err != nil || branch == "" || ev.Revision <= 0 { + return 0, fmt.Errorf("a catalog-revision names its watched branch and revision: branch %q revision %d", ev.Branch, ev.Revision) + } + if read == nil { + return 0, fmt.Errorf("the revision %d of %s cannot be read: the runner holds no revision reader", ev.Revision, branch) + } + half, err := read(branch, ev.Revision) + if err != nil { + return 0, fmt.Errorf("the revision %d of %s is not read, so its pipeline is not started: %w", ev.Revision, branch, err) + } + if half.Revision != ev.Revision { + return 0, fmt.Errorf("the revision verb answered revision %d of %s for %d", half.Revision, branch, ev.Revision) + } + if len(half.Movers) == 0 { + s.logf("the revision %d of %s (%s) names no mover: it starts no pipeline", ev.Revision, branch, ev.Call) + return 0, nil + } + movers := make([]seat.Mover, 0, len(half.Movers)) + for _, e := range half.Movers { + movers = append(movers, seat.Mover{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint, Repository: e.Repository}) } movers = Tiers(movers, half.Entries, s.logf) - started, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Call: publishing.Call, Revision: revision, Movers: movers, - Entries: half.Entries, Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version}, - Key: PublishedKey(ev.Branch, ev.Source, ev.Artifact, ev.Version)}) - if started { + call := half.Call + if call == "" { + call = ev.Call + } + ok, err := s.Start(Begin{Branch: branch, Source: half.Source, Call: call, Revision: half.Revision, Movers: movers, Entries: half.Entries}) + if ok { return 1, err } return 0, err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 019127c7..d3f5700b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -11,6 +11,7 @@ import ( builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" ) type created map[string][]byte @@ -40,7 +41,7 @@ func TestStarts(t *testing.T) { if _, err := s.Start(Begin{Branch: "release.1", Source: 3, Movers: []seat.Mover{{Module: "app"}}}); err != nil { t.Fatal(err) } - if b, r, ok := seat.ParsePipelinesByBranchSourceKey(Key("release.1", 3)); !ok || r != "3" || b == "release" { + if b, r, ok := seat.ParsePipelinesByBranchRevisionKey(Key("release.1", 3)); !ok || r != "3" || b == "release" { t.Errorf("the key of release.1 is %q", Key("release.1", 3)) } if _, err := s.Start(Begin{Branch: "main", Movers: []seat.Mover{{Module: "app"}}}); err == nil { @@ -55,12 +56,11 @@ func TestStarts(t *testing.T) { } } -// R10/AC1 and AC2 (design 47 H §747, H-A-52): a package-published of a mover of its source's running pipeline whose -// dependent that pipeline moves starts nothing: the dependent waits there. A publish again of an older build starts one -// pipeline keyed ..published.., its movers the package's kind-1 dependents from the -// edges the runner holds, at the publishing source's revision, and says pipeline-started with the publish; an ordering -// edge starts nothing; heard twice, the second writes nothing. -func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) { +// H-A-212, H-A-213 (2) and (4), H-A-217: a package-published starts no pipeline, the movers waiting for it in its +// pipeline placed there; a trunk catalog-revision of a release, a take-in again or a recompute starts ONE pipeline keyed +// ., the revision read by its number, its movers the revision's, its source and call the revision's; +// heard twice the second writes nothing; a take-in's revision, a candidate's and one with no mover start nothing. +func TestARevisionOfAReleaseStartsOnePipeline(t *testing.T) { mem := memStore{} var said []seat.PipelineStarted s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(_ string, body any) error { @@ -70,37 +70,60 @@ func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) { return nil }} edges := []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 1}} - view := func(string) (Half, bool) { - return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{ - "mesh-sdk": {Module: "mesh-sdk"}, - "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges}, - "tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges}, - "ordered": {Module: "ordered", Edges: []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 4}}}}}, true + entries := map[string]catalog.CatalogEntry{ + "mesh-sdk": {Module: "mesh-sdk"}, + "app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges}, + "tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges}} + view := func(string) (Half, bool) { return Half{Revision: 9, Entries: entries}, true } + ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 30, Module: "mesh-sdk", Version: "0.1.9"} + if err := s.Published(ev, view); err != nil || len(mem) != 0 { + t.Fatalf("a publish of an older build wrote %d pipelines (%v), want none", len(mem), err) } - mem[Key("main", 41)] = struct { - p seat.Pipeline - rev uint64 - }{seat.Pipeline{Pipeline: "main.41", Branch: "main", Source: 41, Revision: 8, Outcome: seat.PipelineOutcomeRunning, - Movers: []seat.Mover{{Module: "mesh-sdk"}, {Module: "app"}, {Module: "tool"}}}, 1} - ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"} - if n, err := s.Published(ev, view); n != 0 || err != nil || len(mem) != 1 { - t.Fatalf("a publish in a pipeline moving its dependents started %d (%v), %d pipelines", n, err, len(mem)) + var asked []int64 + movers := []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}, {Module: "tool", Commit: "c2", Fingerprint: "src2:bc"}} + read := func(branch string, revision int64) (Half, error) { + asked = append(asked, revision) + if branch != "release.1" { + return Half{}, fmt.Errorf("branch %q", branch) + } + r := Half{Revision: revision, Source: 41, Entries: entries, Call: "release of mesh-sdk 0.1.17"} + if revision != 12 { + r.Movers = movers + } + return r, nil } - // A publish again of an older build, at source 30: its pipeline is gone, so every kind-1 dependent is a mover. - ev.Source, ev.Version = 30, "0.1.9" + release := catalog.CatalogRevisionEvent{Kind: "trunk", Branch: keys.EscapeName("release.1"), Revision: 10, Source: 41, CallKind: "release", Call: "release of mesh-sdk 0.1.17"} for i := range 2 { - if n, err := s.Published(ev, view); n != 1-i || err != nil { + if n, err := s.Revised(release, read); n != 1-i || err != nil { t.Fatalf("hearing %d started %d (%v)", i+1, n, err) } } - key := PublishedKey("main", 30, "mesh-sdk:go", "0.1.9") - p, _, err := mem.Get(key) - if err != nil || len(p.Movers) != 2 || !Moves(p, "app") || !Moves(p, "tool") || Moves(p, "ordered") || p.Revision != 9 || - p.Published == nil || p.Published.Version != "0.1.9" || len(said) != 1 || said[0].Published == nil { - t.Fatalf("the pipeline %s = %+v (%v), said %+v", key, p, err, said) + p, _, err := mem.Get(Key("release.1", 10)) + if err != nil || len(mem) != 1 || len(p.Movers) != 2 || p.Revision != 10 || p.Source != 41 || p.Call != "release of mesh-sdk 0.1.17" || + len(said) != 1 || len(asked) != 2 || asked[0] != 10 { + t.Fatalf("the pipeline %s = %+v (%v), %d written, said %d, read %v", Key("release.1", 10), p, err, len(mem), len(said), asked) } - if app, _ := MoverOf(p, "app"); app.Commit != "c1" || app.Fingerprint != "src2:aa" || app.Repository != "o/app" { - t.Errorf("the mover app = %+v", app) + if app, _ := MoverOf(p, "app"); app.Fingerprint != "src2:ab" || app.Repository != "o/app" || app.Tier != 1 { + t.Errorf("the mover app = %+v, want the revision's", app) + } + for _, c := range []struct { + name string + ev catalog.CatalogRevisionEvent + want int + }{ + {"take-in again", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 11, CallKind: "take-in-again"}, 1}, + {"recompute with no mover", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 12, CallKind: "recompute"}, 0}, + {"recompute", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 13, CallKind: "recompute"}, 1}, + {"take-in", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 14, CallKind: "take-in"}, 0}, + {"candidate", catalog.CatalogRevisionEvent{Kind: "candidate", Branch: release.Branch, Revision: 15, CallKind: "release"}, 0}, + } { + if n, err := s.Revised(c.ev, read); n != c.want || err != nil { + t.Errorf("%s: started %d (%v), want %d", c.name, n, err, c.want) + } + } + if _, err := s.Revised(catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 16, CallKind: "release"}, + func(string, int64) (Half, error) { return Half{}, errors.New("no answer") }); err == nil { + t.Error("a revision that cannot be read is not an error, so it is never heard again") } } @@ -128,7 +151,7 @@ func (failing) Create(string, []byte) error { return errors.New("bus away") } // R4/AC1 (design 47 H §746, acceptance test 112): a take-in.taken naming mesh-sdk and one importer starts one pipeline // keyed . by compare-and-set, its movers the event's entries, at the take-in's revision, and says one -// pipeline-started on . naming the call and the movers; R4/AC2: heard twice, the second writes nothing. A +// pipeline-started on . naming the call and the movers; R4/AC2: heard twice, the second writes nothing. A // take-in with no entries and one naming no source revision start nothing. func TestATakeInIsOnePipeline(t *testing.T) { store := created{} @@ -153,13 +176,13 @@ func TestATakeInIsOnePipeline(t *testing.T) { t.Fatalf("written %d pipelines and said %d starts, want one each", len(store), len(said)) } var p seat.Pipeline - if err := json.Unmarshal(store["main.7"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 { - t.Fatalf("the pipeline main.7 = %+v (%v)", p, err) + if err := json.Unmarshal(store["main.12"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 { + t.Fatalf("the pipeline main.12 = %+v (%v)", p, err) } if m, _ := MoverOf(p, "mesh-sdk"); m.Commit != "c1" || m.Fingerprint != "src2:aa" || m.Repository != "o/sdk" || m.Tier != 1 { t.Errorf("the mover mesh-sdk = %+v", m) } - if subjects[0] != "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.7" || said[0].Call != ev.Call || len(said[0].Movers) != 2 { + if subjects[0] != "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.12" || said[0].Call != ev.Call || len(said[0].Movers) != 2 { t.Errorf("pipeline-started %+v on %s", said[0], subjects[0]) } if _, err := s.TakenIn(catalog.TakeInTaken{Branch: "main", Entries: ev.Entries}, view); err == nil { @@ -243,25 +266,6 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T if n != 1 || err != nil || len(p.Steps) == 0 { t.Errorf("the half of next placed %d (%v): %d steps, want the take-in's pipeline placed", n, err, len(p.Steps)) } - // A-336: a package published heard before its branch's half is held starts nothing, heard again once it is: its - // dependents are the half's edges (§747). - held = false - app := entries["app"] - app.Edges = []catalog.Edge{{From: "mesh-sdk:go", To: "app:program", Kind: 1}} - entries["app"] = app - published := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "next", Source: 7, Module: "mesh-sdk", Version: "0.1.17"} - if n, err := s.Published(published, late); n != 0 || err == nil { - t.Fatalf("the publish before the half started %d (%v), want none and an error, heard again", n, err) - } - held = true - if n, err := s.Published(published, late); n != 1 || err != nil { - t.Fatalf("the publish with the half held started %d (%v)", n, err) - } - p, _, _ = store.Get(PublishedKey("next", 7, "mesh-sdk:go", "0.1.17")) - if len(p.Steps) == 0 { - t.Errorf("the publish's pipeline placed %d steps, want its steps", len(p.Steps)) - } - // A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running, // no step placed) is placed and asked as a newly started one when the half is held; no hold is read any more. var old seat.Pipeline @@ -318,8 +322,8 @@ func TestTakenInStartsOnlyItsEntries(t *testing.T) { if err != nil || n != 1 { t.Fatalf("the take-in started %d (%v), want its one entry", n, err) } - if p, _, err := mem.Get("main.9"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") { - t.Errorf("main.9 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err) + if p, _, err := mem.Get("main.12"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") { + t.Errorf("main.12 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err) } } @@ -335,3 +339,27 @@ func TestATakeInOfTheOldFormIsRefused(t *testing.T) { t.Errorf("the new form = %+v (%v)", ev, err) } } + +// H-A-218: a package-published is taken into the pipeline of the branch and revision it names; one naming no revision (an +// older builder) into its source's lowest-numbered revision, the take-in's. +func TestAPublishIsTakenIntoThePipelineOfItsRevision(t *testing.T) { + mem := memStore{} + for _, r := range []int64{7, 8} { + mem[Key("main", r)] = struct { + p seat.Pipeline + rev uint64 + }{seat.Pipeline{Pipeline: Key("main", r), Branch: "main", Source: 4, Revision: r, Outcome: seat.PipelineOutcomeRunning, + Movers: []seat.Mover{{Module: "mesh-sdk"}}}, 1} + } + s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(string, any) error { return nil }} + view := func(string) (Half, bool) { return Half{Revision: 8}, true } + published := func(key string) []string { m, _ := MoverOf(mem[key].p, "mesh-sdk"); return m.Published } + ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Revision: 8, Version: "0.1.18"} + if err := s.Published(ev, view); err != nil || len(published("main.8")) != 1 || len(published("main.7")) != 0 { + t.Fatalf("a publish of revision 8 (%v): main.8 %v, main.7 %v; want main.8's mover alone", err, published("main.8"), published("main.7")) + } + ev.Revision, ev.Artifact = 0, "mesh-sdk:ts" + if err := s.Published(ev, view); err != nil || len(published("main.7")) != 1 || len(published("main.8")) != 1 { + t.Errorf("a publish naming no revision (%v): main.7 %v, main.8 %v; want main.7's, the source's lowest", err, published("main.7"), published("main.8")) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 9b39cf26..40524451 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -31,6 +31,8 @@ type StepSeat struct { // Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for // a seat whose ends name none. Shared func([]byte) []seat.Shared + // Pins reads the pins a build's end names it was built with (H-A-207); nil for a seat whose ends name none. + Pins func([]byte) map[string]string } // StepSeats are the three build seats, from their contracts. @@ -39,7 +41,7 @@ var StepSeats = []StepSeat{ {Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")}, {Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")}, // The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99). - {Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded}, + {Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded, Pins: buildPins}, // The seat mesh-wants answers the composition step (§22, §24). {Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared}, } @@ -77,6 +79,15 @@ func decodeWantsEnded(raw []byte) (StepEnded, error) { Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil } +// buildPins is the pins a build step's end of the seat mesh-builds names its entry was built with (H-A-207). +func buildPins(raw []byte) map[string]string { + var b builds.StepEnded + if json.Unmarshal(raw, &b) != nil { + return nil + } + return b.Packages +} + // decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one // entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is // refused (A-99). @@ -155,6 +166,9 @@ type Hearer struct { // Release places the movers of a pipeline whose awaited publishes are heard now, after a build step passed (a kind-3 // edge, §746; Starter.Release); nil places none. Release func(key string) + // AskPinHeld asks a mover's steps that waited for its entry's pins with the pins its build passed with (H-A-207; + // Starter.AskPinHeld); nil leaves them waiting. + AskPinHeld func(key, module string, pins map[string]string) // Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748; // Starter.Turn); nil takes none. Turn func(key string) @@ -197,6 +211,12 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi // of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726, // A-469m). func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) { + return h.OnStepEndedPinned(bs, subject, ev, shared, nil) +} + +// OnStepEndedPinned is OnStepEndedShared of a build's end naming the pins it was built with (H-A-207): a passed build's +// mover's steps that waited for its entry's pins are asked with them. +func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared, pins map[string]string) (seat.Pipeline, error) { node := strings.TrimPrefix(subject, bs.Prefix) if node == subject || node == "" || strings.Contains(node, ".") { return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat) @@ -272,6 +292,11 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh // Every mover's validate reads the one run (§745, H-A-51); its end asks each mover's composition, or ends it not // asked when validate did not pass. var due []Asked + askHeld := false + if s.Step == seat.StepResultStepBuild { + readers = append(readers, endPinHeld(&p, *s, pins, now)...) + askHeld = stepPassed(*s) && heldOn(p, s.Module) + } if s.Step == seat.StepResultStepValidate { readers = append(readers, sameValidate(&p, i, now)...) var notAsked []int @@ -324,6 +349,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh if release && h.Release != nil { h.Release(ev.Pipeline) } + if askHeld && h.AskPinHeld != nil { + h.AskPinHeld(ev.Pipeline, s.Module, pins) + } if h.AskSteps != nil && len(due) > 0 { if err := h.AskSteps(ev.Pipeline, due); err != nil { h.logf("the compositions of %s are written running and not asked, asked again at the next start: %v", ev.Pipeline, err) @@ -413,7 +441,7 @@ func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) erro } // said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall; -// an end is said once, on . (design 47 H §746). The runner says no group-ended (§764). +// an end is said once, on . (design 47 H §746). The runner says no group-ended (§764). func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error { for _, i := range asked { if err := h.say(seat.EventWaitHeld, p, &p.Steps[i]); err != nil { @@ -461,7 +489,7 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error subject, ev := stepEvent(event, p, *step, at) return h.Say(subject, ev) case seat.EventPipelineEnded: - return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), pipelineEnded(p)) + return h.Say(runnerclient.RevisionSubject(event, p.Branch, subjectRevision(p)), pipelineEnded(p)) default: w := seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Revision: p.Revision, Source: p.Source, Run: p.Run, Step: seat.WaitStepApproval, On: seat.WaitOnPerson, Since: at} @@ -469,7 +497,7 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error m, _ := MoverOf(p, step.Module) w.Module, w.Commit, w.Fingerprint, w.Step, w.Ask = m.Module, m.Commit, m.Fingerprint, step.Step, step.Ask } - return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), w) + return h.Say(runnerclient.RevisionSubject(event, p.Branch, subjectRevision(p)), w) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go index 780009af..ba0712e4 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/testruns_test.go @@ -113,7 +113,7 @@ func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline { "app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}}, "mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}, } - if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { + if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil { t.Fatalf("start = %v, %v", ok, err) } p, _, _ := f.store.Get(Key("main", source)) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go index 5264f76e..d86cec3b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs.go @@ -582,11 +582,11 @@ func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) { if !ok || half.Source <= 0 { return seat.StartAnswer{}, fmt.Errorf("the catalog half of %s holds no entry of %s at a source revision: nothing was started", in.Branch, in.Module) } - key := Key(in.Branch, half.Source) + key := Key(in.Branch, keyRevision(Begin{Revision: half.Revision, Source: half.Source})) if err := v.record(seat.VerbStart, "started the pipeline "+key+" by hand", in.Why); err != nil { return seat.StartAnswer{}, err } - // A hand start keys the newest revision's . with the module as its one mover (H-A-52). + // A hand start keys the newest revision's . with the module as its one mover (H-A-52, H-A-213 (4)). started, err := v.Starter.Start(Begin{Branch: in.Branch, Source: half.Source, Revision: half.Revision, Entries: half.Entries, Movers: []seat.Mover{{Module: in.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository, Tier: 1}}}) if err != nil { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go index d3a273a0..a2c8adae 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/verbs_test.go @@ -287,7 +287,7 @@ func TestStartAndAskAgain(t *testing.T) { _, entries := placeFixture() v.View = func(string) (Half, bool) { return Half{Source: 7, Revision: 3, Entries: entries}, true } a, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "no pipeline for the head"}) - if err != nil || a.Pipeline != "main.7" || made["main.7"] == nil { + if err != nil || a.Pipeline != "main.3" || made["main.3"] == nil { t.Fatalf("start = %+v, %v; written %v", a, err, len(made)) } if _, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "again"}); err == nil { diff --git a/modules/mesh-delivery-sdk/buildstep/ask_test.go b/modules/mesh-delivery-sdk/buildstep/ask_test.go index 5467bf23..e996d4c2 100644 --- a/modules/mesh-delivery-sdk/buildstep/ask_test.go +++ b/modules/mesh-delivery-sdk/buildstep/ask_test.go @@ -87,6 +87,7 @@ func TestAskCheckAndRoundTrip(t *testing.T) { {"no path", func(a *StepAsk) { a.Build.Path = "" }, "names no directory"}, {"short commit", func(a *StepAsk) { a.Build.Fetch.Commit = "abc" }, "full commit"}, {"bad package pin", func(a *StepAsk) { a.Packages = PackageVersions{"m": "main"} }, "neither a full commit"}, + {"empty package pin", func(a *StepAsk) { a.Packages = PackageVersions{"m/x": ""} }, "neither a full commit"}, } { a := goodBuild() c.edit(&a) @@ -94,6 +95,27 @@ func TestAskCheckAndRoundTrip(t *testing.T) { t.Errorf("%s: %v, want %q", c.name, err, c.want) } } + // Design 47 H, S8 (H-A-204, H-A-165): a catalog entry's packages name a package with no version published yet by an + // empty version, which waits for its publish; the entry form allows it, an ask's pins never do (above). + for _, c := range []struct { + name string + p PackageVersions + entry string // what CheckEntry's refusal holds, "" for none + pinsToo bool // Check (an ask's pins) passes it as well + }{ + {name: "an empty version", p: PackageVersions{"m/x": "", "m/y": "v0.1.3"}}, + {name: "versions only", p: PackageVersions{"m/y": "v0.1.3", "m/z": testCommit}, pinsToo: true}, + {name: "a version that is no version", p: PackageVersions{"m/x": "main"}, entry: "neither a full commit"}, + {name: "no module path", p: PackageVersions{"": ""}, entry: "no module path"}, + } { + err := c.p.CheckEntry() + if (err == nil) != (c.entry == "") || (err != nil && !strings.Contains(err.Error(), c.entry)) { + t.Errorf("%s: CheckEntry answered %v, want %q", c.name, err, c.entry) + } + if err := c.p.Check(); (err == nil) != c.pinsToo { + t.Errorf("%s: Check answered %v, want a pass %v", c.name, err, c.pinsToo) + } + } // 1921 (B0, G): a run ask goes only on the Go seat's check stream, alone, builds a clean package at a full commit // over heads named once at full commits, and names its input on a seat's verb by a digest within its bound. for _, c := range []struct { diff --git a/modules/mesh-delivery-sdk/buildstep/steps.go b/modules/mesh-delivery-sdk/buildstep/steps.go index 84b4e0db..33a4fa3a 100644 --- a/modules/mesh-delivery-sdk/buildstep/steps.go +++ b/modules/mesh-delivery-sdk/buildstep/steps.go @@ -184,16 +184,27 @@ var ( ) // PackageVersions are the versions of the mesh packages a build or a check builds against, by package (a Go module's -// path): each a full commit or a Go semantic version. +// path): each a full commit or a Go semantic version. A catalog entry's packages may also name a package with an empty +// version, one of another repository with no version published yet in its series, which waits for its publish (design +// 47 H, S8, S9; H-A-204, H-A-165): CheckEntry allows it, Check (an ask's pins) never. type PackageVersions map[string]string // Check refuses a package that is no module path or a version that is neither a full commit nor a Go semantic -// version, naming it. -func (p PackageVersions) Check() error { +// version, naming it: an ask's pins, an empty version among them. +func (p PackageVersions) Check() error { return p.check(false) } + +// CheckEntry is Check for a catalog entry's packages: an empty version is allowed, and stands for a package not +// published yet, whose build waits for its publish (design 47 H, S8; H-A-204, H-A-165). +func (p PackageVersions) CheckEntry() error { return p.check(true) } + +func (p PackageVersions) check(emptyAllowed bool) error { for pkg, version := range p { if pkg == "" || strings.ContainsAny(pkg, " \t\n@") || strings.HasPrefix(pkg, "/") { return fmt.Errorf("buildstep: the package %q is no module path", pkg) } + if version == "" && emptyAllowed { + continue + } if !askCommitOK.MatchString(version) && !PackageSemver.MatchString(version) { return fmt.Errorf("buildstep: the package %s names the version %q, which is neither a full commit nor a Go semantic version (v0.1.15-feat-x.2)", pkg, version) }