Join fix/47-H1-runner-published-revision
# Conflicts: # modules/mesh-delivery-indexer/internal/index/catalog_revisions.go # modules/mesh-delivery-pipeline/internal/pipeline/start.go
This commit is contained in:
@@ -242,6 +242,11 @@ type BuilderBuild struct {
|
||||
// Imports are, on a done key, the mesh packages the built programs import, as the build module answered them
|
||||
// (BuildAnswer.Imports; novox/hq ADR 0593 §2): the builder carries them onto the build's end (BuildLogEntry.Imports).
|
||||
Imports []string `json:"imports,omitempty"`
|
||||
// Packages are, on a done key only, the mesh packages' versions its Go build was built with, by module path, the ones
|
||||
// the registry gave a package the entry named with none included (H-A-207): the build step's end reports them, so the
|
||||
// runner asks the steps that waited for the pins with what the build was built against. A reader that does not know
|
||||
// the field reads the build as reporting none.
|
||||
Packages map[string]string `json:"packages,omitempty"`
|
||||
// Needs are, on a done key only, the reader capabilities (`form:<name>`) the build's reference forms need of a
|
||||
// node-engine, beside the ones its manifest needs (novox/hq ADR 0618 §1). A reader that does not know the field
|
||||
// reads the build as needing none.
|
||||
|
||||
@@ -833,6 +833,7 @@ type lane struct {
|
||||
tcVersion string // and the version it ran
|
||||
kept string // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty
|
||||
outOfStep []string // the generated files its build module's compare found out of step, passed or failed
|
||||
pins map[string]string // the mesh packages' versions its Go ask was built with (H-A-207)
|
||||
err error
|
||||
}
|
||||
|
||||
@@ -982,7 +983,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module
|
||||
used = nil
|
||||
}
|
||||
done := builds.BuilderBuild{State: builds.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used,
|
||||
Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts)} // the forms it is written in (ADR 0618 §1)
|
||||
Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts), Packages: pinsOf(lanes)} // the forms it is written in (ADR 0618 §1)
|
||||
for _, l := range lanes {
|
||||
if l.kept != "" {
|
||||
done.KeptOn = l.node // design 47 G §671: the build node that keeps the build
|
||||
@@ -1144,6 +1145,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
|
||||
for _, fb := range req.Builds {
|
||||
delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465)
|
||||
}
|
||||
l.pins = maps.Clone(ask.Packages) // kept with the done key, so its step's end reports them (H-A-207)
|
||||
}
|
||||
if id := asked.resume(language); id != "" {
|
||||
ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it
|
||||
@@ -1326,6 +1328,20 @@ func (b *Builder) takeOutOfStep(key string) []string {
|
||||
// importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593
|
||||
// §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that
|
||||
// kept one reports only what it built.
|
||||
// pinsOf is the mesh packages' versions the lanes were built with (H-A-207), nil when none was pinned.
|
||||
func pinsOf(lanes []*lane) map[string]string {
|
||||
var out map[string]string
|
||||
for _, l := range lanes {
|
||||
for p, v := range l.pins {
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
out[p] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func importsOf(lanes []*lane) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, l := range lanes {
|
||||
|
||||
@@ -41,7 +41,18 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui
|
||||
}
|
||||
var failed []string
|
||||
for _, e := range run.Entries {
|
||||
outcome, said := b.buildEntry(ctx, e)
|
||||
var pins map[string]string
|
||||
outcome, said := b.buildEntry(ctx, e, &pins)
|
||||
// The end reports the pins each entry was built with, so the runner asks its held test steps with them (H-A-207).
|
||||
for path, version := range pins {
|
||||
if version == "" {
|
||||
continue
|
||||
}
|
||||
if end.Packages == nil {
|
||||
end.Packages = map[string]string{}
|
||||
}
|
||||
end.Packages[path] = version
|
||||
}
|
||||
switch outcome {
|
||||
case meshbuilds.StepEndedOutcomeCouldNotRun:
|
||||
return finish(outcome, said)
|
||||
@@ -55,8 +66,9 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui
|
||||
return finish(meshbuilds.StepEndedOutcomePassed, "")
|
||||
}
|
||||
|
||||
// buildEntry builds one entry at its fingerprint and waits for its end.
|
||||
func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outcome, said string) {
|
||||
// buildEntry builds one entry at its fingerprint and waits for its end; pins is set, when it is done, to the pins its
|
||||
// done key keeps it was built with, the registry's included (H-A-207).
|
||||
func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) {
|
||||
entry, err := b.StepEntry(ctx, e.Fingerprint)
|
||||
if err != nil {
|
||||
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry %s at %s cannot be read: %v", e.Module, e.Fingerprint, err)
|
||||
@@ -78,6 +90,8 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
|
||||
defer b.dropEnd(key, ended)
|
||||
if done, failedWords, found := b.keyEnd(ctx, key); found {
|
||||
if done {
|
||||
// A key done before reports the pins it was built with, kept with it, never the ones a publish gives now.
|
||||
*pins = b.keptPins(ctx, key)
|
||||
return b.passedWithItsVersion(ctx, entry)
|
||||
}
|
||||
return meshbuilds.StepEndedOutcomeFailed, failedWords
|
||||
@@ -88,6 +102,7 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
|
||||
select {
|
||||
case log := <-ended:
|
||||
if log.Outcome == builds.BuildDone {
|
||||
*pins = b.keptPins(ctx, key)
|
||||
return b.passedWithItsVersion(ctx, entry)
|
||||
}
|
||||
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error)
|
||||
@@ -136,6 +151,20 @@ func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.Ca
|
||||
return meshbuilds.StepEndedOutcomePassed, ""
|
||||
}
|
||||
|
||||
// keptPins are the pins a done key keeps it was built with (H-A-207); none when it keeps none or cannot be read, so the
|
||||
// runner ends the steps that waited for them could not run, loud.
|
||||
func (b *Builder) keptPins(ctx context.Context, key string) map[string]string {
|
||||
raw, _, ok, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil || !ok {
|
||||
return nil
|
||||
}
|
||||
v, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil || v.State != builds.BuildDone {
|
||||
return nil
|
||||
}
|
||||
return v.Packages
|
||||
}
|
||||
|
||||
// keyEnd reads the build key: whether it is done, and its error when it failed for good; found only for a key that has ended.
|
||||
func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed string, found bool) {
|
||||
raw, _, ok, err := b.Claimer.Store.Get(ctx, key)
|
||||
|
||||
@@ -189,6 +189,32 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
|
||||
!reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
|
||||
t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks)
|
||||
}
|
||||
// The end reports the pins the build was built with, the one the publish gave included (H-A-207).
|
||||
if !reflect.DeepEqual(end.Packages, map[string]string{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
|
||||
t.Errorf("the end reports the pins %v; want the ones it was built with", end.Packages)
|
||||
}
|
||||
// A done key reports the pins it keeps it was built with, never the newer one a publish gives now (H-A-207).
|
||||
r.b.PackagePublished(unpinnedX, "v0.1.5")
|
||||
again := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
if again.Outcome != meshbuilds.StepEndedOutcomePassed || again.Packages[unpinnedX] != "v0.1.3" {
|
||||
t.Errorf("the done key's end = %+v; want passed reporting x at v0.1.3, as built", again)
|
||||
}
|
||||
// A done key that keeps no pins reports none, so the runner ends the waiting steps loud.
|
||||
v, err := builds.DecodeBuilderBuild(r.st.vals[r.key])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v.Packages = nil
|
||||
raw, err := builds.EncodeBuilderBuild(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.st.vals[r.key] = raw
|
||||
if bare := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); len(bare.Packages) != 0 {
|
||||
t.Errorf("a done key keeping no pins reported %v; want none", bare.Packages)
|
||||
}
|
||||
}
|
||||
|
||||
func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) {
|
||||
|
||||
@@ -17,8 +17,9 @@ import (
|
||||
// eventPrefix starts every event subject of the seat.
|
||||
const eventPrefix = "mesh.seat." + seat.Seat + ".event."
|
||||
|
||||
// perSource are the events said per source revision, on <event>.<branch>.<source> (design 47 H §746, H-A-72 (3)).
|
||||
var perSource = []string{seat.EventPipelineStarted, seat.EventPipelineEnded, seat.EventWaitHeld, seat.EventWaitEnded}
|
||||
// perRevision are the events said per pipeline, on <event>.<branch>.<revision>, its catalog revision's number as its key
|
||||
// names it (design 47 H §746, H-A-72 (3), H-A-221).
|
||||
var perRevision = []string{seat.EventPipelineStarted, seat.EventPipelineEnded, seat.EventWaitHeld, seat.EventWaitEnded}
|
||||
|
||||
// Subject is where the runner says a step's event or a run's start: mesh.seat.mesh-pipeline-runner.event.<event>.
|
||||
// <module>.<branch>.<commit>, each name escaped as a key's name is, so a branch holding a dot is one token (design 47 H
|
||||
@@ -27,41 +28,42 @@ func Subject(event, module, branch, commit string) string {
|
||||
return eventPrefix + event + "." + keys.EscapeName(module) + "." + keys.EscapeName(branch) + "." + keys.EscapeName(commit)
|
||||
}
|
||||
|
||||
// SourceSubject is where the runner says a pipeline's start and end and its waits: mesh.seat.mesh-pipeline-runner.
|
||||
// event.<event>.<branch>.<source>, the branch escaped as a key's name is (design 47 H §746, H-A-72 (3)).
|
||||
func SourceSubject(event, branch string, source int64) string {
|
||||
return eventPrefix + event + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10)
|
||||
// RevisionSubject is where the runner says a pipeline's start and end and its waits: mesh.seat.mesh-pipeline-runner.
|
||||
// event.<event>.<branch>.<revision>, the pipeline's catalog revision number, the branch escaped as a key's name is (design
|
||||
// 47 H §746, H-A-72 (3), H-A-221).
|
||||
func RevisionSubject(event, branch string, revision int64) string {
|
||||
return eventPrefix + event + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10)
|
||||
}
|
||||
|
||||
// Of is what an event's subject names: a step's event or a run's start names the module, the branch and the commit; an
|
||||
// event per source revision names the branch and the source.
|
||||
// event per pipeline names the branch and the pipeline's catalog revision (H-A-221).
|
||||
type Of struct {
|
||||
Event, Module, Branch, Commit string
|
||||
Source int64
|
||||
Revision int64
|
||||
}
|
||||
|
||||
// Parse reads a subject Subject or SourceSubject built: an event of the seat's, and its names unescaped.
|
||||
// Parse reads a subject Subject or RevisionSubject built: an event of the seat's, and its names unescaped.
|
||||
func Parse(subject string) (Of, error) {
|
||||
rest, ok := strings.CutPrefix(subject, eventPrefix)
|
||||
toks := strings.Split(rest, ".")
|
||||
bySource := len(toks) > 0 && slices.Contains(perSource, toks[0])
|
||||
byRevision := len(toks) > 0 && slices.Contains(perRevision, toks[0])
|
||||
want := 4
|
||||
if bySource {
|
||||
if byRevision {
|
||||
want = 3
|
||||
}
|
||||
if !ok || len(toks) != want || !slices.Contains(seat.Events(), toks[0]) {
|
||||
return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q is no event subject of the seat %s", subject, seat.Seat)
|
||||
}
|
||||
if bySource {
|
||||
if byRevision {
|
||||
branch, err := keys.UnescapeName(toks[1])
|
||||
if err != nil || branch == "" {
|
||||
return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: branch: %v", subject, err)
|
||||
}
|
||||
source, err := strconv.ParseInt(toks[2], 10, 64)
|
||||
if err != nil || source < 1 {
|
||||
return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: no source revision", subject)
|
||||
revision, err := strconv.ParseInt(toks[2], 10, 64)
|
||||
if err != nil || revision < 1 {
|
||||
return Of{}, fmt.Errorf("meshpipelinerunnerclient: %q: no revision", subject)
|
||||
}
|
||||
return Of{Event: toks[0], Branch: branch, Source: source}, nil
|
||||
return Of{Event: toks[0], Branch: branch, Revision: revision}, nil
|
||||
}
|
||||
names := make([]string, 0, 3)
|
||||
for i, t := range toks[1:] {
|
||||
|
||||
@@ -17,14 +17,14 @@ func TestSubjectsAndBodies(t *testing.T) {
|
||||
if of, err := Parse(s); err != nil || of != (Of{Event: seat.EventStepEnded, Module: "nats", Branch: "release.1", Commit: "c1"}) {
|
||||
t.Errorf("parse %s: %+v, %v", s, of, err)
|
||||
}
|
||||
ps := SourceSubject(seat.EventPipelineEnded, "release.1", 3589)
|
||||
ps := RevisionSubject(seat.EventPipelineEnded, "release.1", 3589)
|
||||
if ps != "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.release=2e1.3589" {
|
||||
t.Errorf("pipeline-ended subject %s", ps)
|
||||
}
|
||||
if of, err := Parse(ps); err != nil || of != (Of{Event: seat.EventPipelineEnded, Branch: "release.1", Source: 3589}) {
|
||||
if of, err := Parse(ps); err != nil || of != (Of{Event: seat.EventPipelineEnded, Branch: "release.1", Revision: 3589}) {
|
||||
t.Errorf("parse %s: %+v, %v", ps, of, err)
|
||||
}
|
||||
if of, err := Parse(SourceSubject(seat.EventWaitHeld, "main", 7)); err != nil || of.Source != 7 || of.Module != "" {
|
||||
if of, err := Parse(RevisionSubject(seat.EventWaitHeld, "main", 7)); err != nil || of.Revision != 7 || of.Module != "" {
|
||||
t.Errorf("wait-held: %+v, %v", of, err)
|
||||
}
|
||||
for _, bad := range []string{"not.an.event.subject",
|
||||
|
||||
@@ -95,6 +95,8 @@ type PackagePublished struct {
|
||||
Digest string `json:"digest"`
|
||||
// the module
|
||||
Module string `json:"module"`
|
||||
// the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field
|
||||
Revision int64 `json:"revision,omitempty"`
|
||||
// the source revision of that branch the build was made from
|
||||
Source int64 `json:"source"`
|
||||
V int64 `json:"v"`
|
||||
@@ -114,9 +116,11 @@ type StepEnded struct {
|
||||
// the node the builder ran it on, as in the subject
|
||||
Node string `json:"node"`
|
||||
// never passed for a build that built nothing
|
||||
Outcome string `json:"outcome"`
|
||||
Pipeline string `json:"pipeline"`
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
Outcome string `json:"outcome"`
|
||||
// the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207)
|
||||
Packages map[string]string `json:"packages,omitempty"`
|
||||
Pipeline string `json:"pipeline"`
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
// why it failed or could not run: the entry not built, the part not heard
|
||||
Said string `json:"said,omitempty"`
|
||||
Started *time.Time `json:"started,omitempty"`
|
||||
|
||||
@@ -905,8 +905,10 @@ type StepRun struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
// the step is asked again by the runner's rerun
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
// the source revision of the watched branch the pipeline runs for; absent for a run on request
|
||||
// the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request
|
||||
Revision int64 `json:"revision,omitempty"`
|
||||
// the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request
|
||||
Source int64 `json:"source,omitempty"`
|
||||
// the pipeline step
|
||||
Step string `json:"step"`
|
||||
Toolchain *ToolchainUsed `json:"toolchain,omitempty"`
|
||||
|
||||
@@ -903,8 +903,10 @@ type StepRun struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
// the step is asked again by the runner's rerun
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
// the source revision of the watched branch the pipeline runs for; absent for a run on request
|
||||
// the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request
|
||||
Revision int64 `json:"revision,omitempty"`
|
||||
// the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request
|
||||
Source int64 `json:"source,omitempty"`
|
||||
// the pipeline step
|
||||
Step string `json:"step"`
|
||||
Toolchain *ToolchainUsed `json:"toolchain,omitempty"`
|
||||
|
||||
@@ -903,8 +903,10 @@ type StepRun struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
// the step is asked again by the runner's rerun
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
// the source revision of the watched branch the pipeline runs for; absent for a run on request
|
||||
// the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request
|
||||
Revision int64 `json:"revision,omitempty"`
|
||||
// the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request
|
||||
Source int64 `json:"source,omitempty"`
|
||||
// the pipeline step
|
||||
Step string `json:"step"`
|
||||
Toolchain *ToolchainUsed `json:"toolchain,omitempty"`
|
||||
|
||||
@@ -1521,7 +1521,11 @@
|
||||
},
|
||||
"revision": {
|
||||
"type": "integer",
|
||||
"description": "the source revision of the watched branch the pipeline runs for; absent for a run on request"
|
||||
"description": "the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request"
|
||||
},
|
||||
"source": {
|
||||
"type": "integer",
|
||||
"description": "the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request"
|
||||
},
|
||||
"fingerprint": {
|
||||
"type": "string",
|
||||
|
||||
@@ -180,6 +180,10 @@
|
||||
"type": "integer",
|
||||
"description": "the source revision of that branch the build was made from"
|
||||
},
|
||||
"revision": {
|
||||
"type": "integer",
|
||||
"description": "the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field"
|
||||
},
|
||||
"dependencies": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
@@ -363,6 +367,13 @@
|
||||
"$ref": "#/defs/BuildEntry"
|
||||
},
|
||||
"description": "each entry the step built, by module and fingerprint"
|
||||
},
|
||||
"packages": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
@@ -69,6 +69,8 @@ export interface PackagePublished {
|
||||
"digest": string;
|
||||
/** the module */
|
||||
"module": string;
|
||||
/** the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field */
|
||||
"revision"?: number;
|
||||
/** the source revision of that branch the build was made from */
|
||||
"source": number;
|
||||
"v": number;
|
||||
@@ -89,6 +91,8 @@ export interface StepEnded {
|
||||
"node": string;
|
||||
/** never passed for a build that built nothing */
|
||||
"outcome": "passed" | "failed" | "nothing to run" | "could not run";
|
||||
/** the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207) */
|
||||
"packages"?: Record<string, string>;
|
||||
"pipeline": string;
|
||||
"rerun"?: boolean;
|
||||
/** why it failed or could not run: the entry not built, the part not heard */
|
||||
|
||||
@@ -510,8 +510,10 @@ export interface StepRun {
|
||||
"pipeline": string;
|
||||
/** the step is asked again by the runner's rerun */
|
||||
"rerun"?: boolean;
|
||||
/** the source revision of the watched branch the pipeline runs for; absent for a run on request */
|
||||
/** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */
|
||||
"revision"?: number;
|
||||
/** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */
|
||||
"source"?: number;
|
||||
/** the pipeline step */
|
||||
"step": "lint" | "unit" | "integration";
|
||||
"toolchain"?: ToolchainUsed;
|
||||
|
||||
+3
-1
@@ -510,8 +510,10 @@ export interface StepRun {
|
||||
"pipeline": string;
|
||||
/** the step is asked again by the runner's rerun */
|
||||
"rerun"?: boolean;
|
||||
/** the source revision of the watched branch the pipeline runs for; absent for a run on request */
|
||||
/** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */
|
||||
"revision"?: number;
|
||||
/** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */
|
||||
"source"?: number;
|
||||
/** the pipeline step */
|
||||
"step": "lint" | "unit" | "integration";
|
||||
"toolchain"?: ToolchainUsed;
|
||||
|
||||
+3
-1
@@ -510,8 +510,10 @@ export interface StepRun {
|
||||
"pipeline": string;
|
||||
/** the step is asked again by the runner's rerun */
|
||||
"rerun"?: boolean;
|
||||
/** the source revision of the watched branch the pipeline runs for; absent for a run on request */
|
||||
/** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */
|
||||
"revision"?: number;
|
||||
/** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */
|
||||
"source"?: number;
|
||||
/** the pipeline step */
|
||||
"step": "lint" | "unit" | "integration";
|
||||
"toolchain"?: ToolchainUsed;
|
||||
|
||||
@@ -71,3 +71,28 @@ func TestPlaceLostRunsReadsEachCandidateNewest(t *testing.T) {
|
||||
t.Errorf("asked %+v, want the candidate's group alone", asked[1])
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-213 (2), (4): a catalog-revision heard is handed whole to the start of a revision's pipeline; a start that fails is
|
||||
// an error, so the bus delivers the event again, and a body that is no event is ended.
|
||||
func TestARevisionHeardIsHandedToItsStart(t *testing.T) {
|
||||
var got []catalog.CatalogRevisionEvent
|
||||
fail := false
|
||||
hear := revisedHear(func(ev catalog.CatalogRevisionEvent) (int, error) {
|
||||
got = append(got, ev)
|
||||
if fail {
|
||||
return 0, errors.New("not read")
|
||||
}
|
||||
return 1, nil
|
||||
}, func(string, ...any) {})
|
||||
data := []byte(`{"v":1,"kind":"trunk","branch":"main","revision":10,"source":41,"call-kind":"release","call":"release of mesh-sdk 0.1.17"}`)
|
||||
if err := hear(t.Context(), data); err != nil || len(got) != 1 || got[0].Revision != 10 || got[0].CallKind != "release" || got[0].Source != 41 {
|
||||
t.Fatalf("heard %+v (%v), want the release revision 10", got, err)
|
||||
}
|
||||
fail = true
|
||||
if err := hear(t.Context(), data); err == nil {
|
||||
t.Error("a start that failed is not an error, so the event is never heard again")
|
||||
}
|
||||
if err := hear(t.Context(), []byte("no event")); err != nil {
|
||||
t.Errorf("a body that is no event = %v, want it ended", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,6 +98,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
var advancer atomic.Pointer[func(string)]
|
||||
var placer atomic.Pointer[func(catalog.CatalogRevisionEvent) error]
|
||||
var registrar atomic.Pointer[func(catalog.Registration)]
|
||||
var reviser atomic.Pointer[func(catalog.CatalogRevisionEvent) (int, error)]
|
||||
rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) {
|
||||
if f := resumer.Load(); f != nil {
|
||||
(*f)(branch)
|
||||
@@ -115,6 +116,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if f := registrar.Load(); f != nil {
|
||||
(*f)(reg)
|
||||
}
|
||||
}, func(ev catalog.CatalogRevisionEvent) (int, error) {
|
||||
if f := reviser.Load(); f != nil {
|
||||
return (*f)(ev)
|
||||
}
|
||||
return 0, fmt.Errorf("the revision %d of %s is heard before the runner can start its pipeline", ev.Revision, ev.Branch)
|
||||
}, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -142,7 +148,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
logger.Printf("widens: each read bound %s, %d catalog revisions kept", readBound, revisionsKept)
|
||||
hearer.Widens = widenerOf(calls.Call, readBound, revisionsKept, logger.Printf).Widens
|
||||
widener := widenerOf(calls.Call, readBound, revisionsKept, logger.Printf)
|
||||
hearer.Widens = widener.Widens
|
||||
hearer.LogLast = stepLog{ctx: ctx, js: js, module: cred.Module}.Last
|
||||
hearer.Where = routerRecordOf(cred.Module, calls.Call, calls.Ask)
|
||||
hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf}
|
||||
@@ -154,6 +161,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
|
||||
hearer.AskSteps = starter.AskSteps
|
||||
hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) }
|
||||
hearer.AskPinHeld = func(key, module string, pins map[string]string) {
|
||||
if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil {
|
||||
logger.Printf("the steps of %s's %s that waited for its pins are not asked: %v", key, module, err)
|
||||
}
|
||||
}
|
||||
hearer.Release = func(key string) {
|
||||
if _, err := starter.Release(hearer.Store, key, rv.Catalog.View); err != nil {
|
||||
logger.Printf("the movers of %s waiting for a build are not placed: %v", key, err)
|
||||
@@ -197,6 +209,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
}
|
||||
registrar.Store(®ister)
|
||||
// A release's, a take-in again's or a recompute's revision starts one pipeline, its revision read by its number
|
||||
// (H-A-213 (2), (4)).
|
||||
revise := func(ev catalog.CatalogRevisionEvent) (int, error) { return starter.Revised(ev, widener.Revision) }
|
||||
reviser.Store(&revise)
|
||||
starter.Store = store
|
||||
// The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in
|
||||
// the state queue, placed at the start from it and on every end.
|
||||
|
||||
@@ -101,7 +101,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
}
|
||||
}
|
||||
answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs})
|
||||
answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries})
|
||||
answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 100, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries})
|
||||
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
|
||||
// The router of the operator's asks holds the seat operator-channel; its record of each ask is in its state asks.
|
||||
const router = "operator-router"
|
||||
@@ -218,7 +218,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
}
|
||||
take := func(source int64, movers ...string) {
|
||||
t.Helper()
|
||||
ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: 1, Call: fmt.Sprintf("%016x", source)}
|
||||
ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: source, Call: fmt.Sprintf("%016x", source)}
|
||||
for _, m := range movers {
|
||||
ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m})
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -11,10 +12,10 @@ import (
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one
|
||||
// pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is
|
||||
// written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard
|
||||
// after a stop.
|
||||
// hearPublished hears each package-published (design 47 H §746, §747): in the running pipeline of the revision that
|
||||
// published it, the movers waiting for it are placed; it starts no pipeline (a release's revision does, H-A-213 (2)). The
|
||||
// message is acknowledged once that is written, on a goroutine of its own, so a slow write holds no other event; the
|
||||
// durable consumer delivers a publish heard after a stop.
|
||||
func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error {
|
||||
return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) {
|
||||
var ev builds.PackagePublished
|
||||
@@ -24,18 +25,38 @@ func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, l
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
n, err := starter.Published(ev, v.View)
|
||||
if err != nil {
|
||||
logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err)
|
||||
if err := starter.Published(ev, v.View); err != nil {
|
||||
logf("the publish of %s %s on %s: %v; heard again", ev.Artifact, ev.Version, ev.Branch, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
logf("the publish of %s %s on %s started %d pipelines", ev.Artifact, ev.Version, ev.Branch, n)
|
||||
logf("the publish of %s %s on %s is heard", ev.Artifact, ev.Version, ev.Branch)
|
||||
_ = m.Ack()
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
// revisedHear is the handler of catalog-revision that starts the pipeline of a revision whose call kind is not a
|
||||
// take-in's (design 47 H-A-212, H-A-213 (2), H-A-217): every other revision is left to the views that hear it. An error
|
||||
// has the event heard again; a second start of its key writes nothing.
|
||||
func revisedHear(revised func(ev catalog.CatalogRevisionEvent) (int, error), logf func(string, ...any)) func(context.Context, []byte) error {
|
||||
return func(_ context.Context, data []byte) error {
|
||||
var ev catalog.CatalogRevisionEvent
|
||||
if err := json.Unmarshal(data, &ev); err != nil {
|
||||
logf("a catalog-revision is not read: %v", err)
|
||||
return nil
|
||||
}
|
||||
n, err := revised(ev)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n > 0 {
|
||||
logf("the revision %d of %s (%s, %s) started %d pipeline", ev.Revision, ev.Branch, ev.CallKind, ev.Call, n)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged
|
||||
// once it is written, and heard again when a write failed, so the key already written starts nothing twice.
|
||||
func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error {
|
||||
|
||||
@@ -128,7 +128,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
if bs.Shared != nil {
|
||||
shared = bs.Shared(m.Data())
|
||||
}
|
||||
_, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared)
|
||||
var pins map[string]string
|
||||
if bs.Pins != nil {
|
||||
pins = bs.Pins(m.Data())
|
||||
}
|
||||
_, err = h.OnStepEndedPinned(bs, m.Subject(), ev, shared, pins)
|
||||
switch {
|
||||
case errors.Is(err, pipeline.ErrRefused):
|
||||
logf("%v", err)
|
||||
|
||||
+29
-33
@@ -37,7 +37,7 @@ import (
|
||||
// ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step
|
||||
// that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a
|
||||
// run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed;
|
||||
// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended,
|
||||
// a run that keeps saying its progress past its bound is not ended,
|
||||
// and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the
|
||||
// same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its
|
||||
// other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and
|
||||
@@ -88,7 +88,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil {
|
||||
askStream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -116,6 +117,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
|
||||
entries := []catalog.CatalogEntry{
|
||||
{Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app",
|
||||
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25"},
|
||||
Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
{Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}},
|
||||
{Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
|
||||
@@ -325,12 +327,23 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
|
||||
// One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its
|
||||
// ask is taken (A-469b), its holder present and nothing failing it (A-466y).
|
||||
const p3 = "main.3"
|
||||
// A pipeline is keyed by its catalog revision (H-A-213 (4)): source 3's take-in of revision 1 is main.1.
|
||||
const p3 = "main.1"
|
||||
takeIn("app", "c1", "src2:aa", 3, 1)
|
||||
placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747)
|
||||
passBuild(p3, "app", "c1", "src2:aa")
|
||||
unitRun, unit := runOfStep(p3, "app:program", "unit")
|
||||
integRun, integ := runOfStep(p3, "app:program", "integration")
|
||||
// A test ask fetches with the entry's pins, as the builder gives them the build (H-A-203).
|
||||
last, err := askStream.GetLastMsgForSubject(ctx, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var testAsk nodebuildgo.StepAsk
|
||||
if err := json.Unmarshal(last.Data, &testAsk); err != nil || testAsk.Step == nil ||
|
||||
testAsk.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25" || testAsk.Step.Revision != 1 || testAsk.Step.Source != 3 {
|
||||
t.Fatalf("the last test ask = %s (%v), want its fetch to carry the entry's pin, revision 1 and source 3 (H-A-219)", last.Data, err)
|
||||
}
|
||||
if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 {
|
||||
t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers)
|
||||
}
|
||||
@@ -471,39 +484,20 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
// A second revision with the same digests asks its own runs, so a failed test runs again (§724).
|
||||
revision.Store(2)
|
||||
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4})
|
||||
const p4, p5 = "main.4", "main.5"
|
||||
const p4 = "main.2"
|
||||
takeIn("app", "c1", "src2:aa", 4, 2)
|
||||
placed(p4)
|
||||
run4, r4 := runOfStep(p4, "app:program", "integration")
|
||||
if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask {
|
||||
t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask)
|
||||
}
|
||||
// A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads
|
||||
// the run, the first is cancelled and leaves it, and the run's ask is not dropped.
|
||||
takeIn("app", "c1", "src2:aa", 5, 2)
|
||||
placed(p5)
|
||||
waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 })
|
||||
if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" {
|
||||
t.Fatalf("cancel: %s", e)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
r, _ := runOf(run4)
|
||||
return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5
|
||||
})
|
||||
if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask {
|
||||
t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r)
|
||||
}
|
||||
mu.Lock()
|
||||
dropped := append([]string(nil), drops...)
|
||||
mu.Unlock()
|
||||
if len(dropped) != 0 {
|
||||
t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String())
|
||||
}
|
||||
// A replaced reader leaving the ask to the other readers of another pipeline (§726) is the unit tests': one revision is
|
||||
// one pipeline (H-A-213 (4)).
|
||||
// A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720).
|
||||
taken(r4.Ask)
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun &&
|
||||
step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun
|
||||
return step(read(p4), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun &&
|
||||
step(read(p4), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun
|
||||
})
|
||||
if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") {
|
||||
t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r)
|
||||
@@ -516,8 +510,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
|
||||
// A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not
|
||||
// asked (§535, §647; A-470g, A-470h).
|
||||
const web = "main.7"
|
||||
takeIn("web", "w1", "src2:ww", 7, 2)
|
||||
revision.Store(3)
|
||||
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 7})
|
||||
const web = "main.3"
|
||||
takeIn("web", "w1", "src2:ww", 7, 3)
|
||||
waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 })
|
||||
if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" {
|
||||
t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e)
|
||||
@@ -568,10 +564,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
revision.Store(3)
|
||||
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5})
|
||||
const p6 = "main.6"
|
||||
takeIn("app", "c1", "src2:aa", 6, 3)
|
||||
revision.Store(4)
|
||||
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 4, Kind: "trunk", Branch: "main", Source: 6})
|
||||
const p6 = "main.4"
|
||||
takeIn("app", "c1", "src2:aa", 6, 4)
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
s := step(read(p6), "app:program", "unit")
|
||||
return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib")
|
||||
|
||||
@@ -98,7 +98,7 @@ func (h *holderView) Holders(seatName string) ([]string, bool) {
|
||||
// after, then start hub once.
|
||||
func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State,
|
||||
call meshcatalogclient.Call, onReady, onAdvance func(branch string), candidate func(ctx context.Context, ev catalog.CatalogRevisionEvent) error,
|
||||
registered func(reg catalog.Registration), logf func(string, ...any)) (*runnerViews, error) {
|
||||
registered func(reg catalog.Registration), revised func(ev catalog.CatalogRevisionEvent) (int, error), logf func(string, ...any)) (*runnerViews, error) {
|
||||
findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -153,6 +153,11 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
|
||||
if err := meshcatalogclient.WireCandidatesProcess(reader, candidate); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A trunk revision of a release, a take-in again or a recompute starts one pipeline of its own, read by its number
|
||||
// (H-A-212, H-A-213 (2), H-A-217), heard on the same event's one subscription.
|
||||
if err := reader.AdoptEvent(catalog.Seat, catalog.EventCatalogRevision, revisedHear(revised, logf)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader.Start(ctx, js, cred, status)
|
||||
return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings, Holders: &holderView{reader: reader, catalog: cv, logf: logf}}, nil
|
||||
}
|
||||
|
||||
@@ -166,7 +166,8 @@ func catalogRevision(call func(string, any) (json.RawMessage, error), kept *revi
|
||||
if h, ok := kept.get(branch, a.Revision.Revision); ok {
|
||||
return h, nil
|
||||
}
|
||||
half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{}}
|
||||
half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{},
|
||||
Call: a.Revision.Call, Movers: a.Revision.Movers}
|
||||
in.Revision = a.Revision.Revision // the pages of one revision, though a newer one is said between them
|
||||
}
|
||||
for _, e := range a.Entries {
|
||||
|
||||
@@ -231,11 +231,11 @@ func (a *Ager) newest(p seat.Pipeline) (bool, error) {
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the pipelines of %s are not listed: %w", p.Branch, err)
|
||||
}
|
||||
// Newest for one of its movers: no later pipeline of the branch moves it.
|
||||
// Newest for one of its movers: no pipeline of a later revision of the branch moves it (H-A-213 (4)).
|
||||
for _, m := range p.Movers {
|
||||
newer := false
|
||||
for _, k := range pkeys {
|
||||
if q, _, err := a.Pipelines.Get(k); err == nil && q.Source > p.Source && Moves(q, m.Module) {
|
||||
if q, _, err := a.Pipelines.Get(k); err == nil && keyRevision(Begin{Revision: q.Revision, Source: q.Source}) > keyRevision(Begin{Revision: p.Revision, Source: p.Source}) && Moves(q, m.Module) {
|
||||
newer = true
|
||||
break
|
||||
}
|
||||
|
||||
@@ -142,3 +142,23 @@ func TestOneStepEndAgesAtMostTwentyEntries(t *testing.T) {
|
||||
t.Fatalf("%d outputs deleted, sequence %d; want 20 and 20", len(f.outputs), f.aged["7d"])
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-213 (4): a branch's newest pipeline for a mover is the one of the highest revision number, never of the highest source:
|
||||
// a release revision of the same source is newer than its take-in's.
|
||||
func TestTheNewestPipelineIsOfTheHighestRevision(t *testing.T) {
|
||||
f := newAgedFixture()
|
||||
for key, p := range map[string]seat.Pipeline{
|
||||
"main.41": {Pipeline: "main.41", Branch: "main", Source: 41, Revision: 41, Movers: []seat.Mover{{Module: "app"}}},
|
||||
"main.42": {Pipeline: "main.42", Branch: "main", Source: 41, Revision: 42, Movers: []seat.Mover{{Module: "app"}}},
|
||||
} {
|
||||
f.pipes.memStore[key] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{p, 1}
|
||||
}
|
||||
for key, want := range map[string]bool{"main.41": false, "main.42": true} {
|
||||
if got, err := f.ager.newest(f.pipes.memStore[key].p); got != want || err != nil {
|
||||
t.Errorf("%s newest = %v (%v), want %v", key, got, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -113,9 +113,19 @@ func TestAReleasedVersionIsTheMoversVersion(t *testing.T) {
|
||||
other := store[pkey]
|
||||
other.p.Pipeline, other.p.Branch, other.p.Movers = "beta.5", "beta", append([]seat.Mover(nil), other.p.Movers...)
|
||||
store["beta.5"] = other
|
||||
// A pipeline is keyed by its revision (H-A-213 (4)): main.8, a later revision of source 5, is written too; main.6, of
|
||||
// source 6, is not.
|
||||
for key, source := range map[string]int64{"main.8": 5, "main.6": 6} {
|
||||
again := store[pkey]
|
||||
again.p.Pipeline, again.p.Source, again.p.Movers = key, source, append([]seat.Mover(nil), again.p.Movers...)
|
||||
store[key] = again
|
||||
}
|
||||
ev := catalog.VersionReleasedEvent{Module: "app", Version: "0.4.1", Source: 5}
|
||||
if n, err := h.Released(ev, "main"); n != 1 || err != nil {
|
||||
t.Fatalf("released wrote %d (%v), want app's mover of main.5", n, err)
|
||||
if n, err := h.Released(ev, "main"); n != 2 || err != nil {
|
||||
t.Fatalf("released wrote %d (%v), want app's mover of main.5 and main.8", n, err)
|
||||
}
|
||||
if m, _ := MoverOf(store["main.6"].p, "app"); m.Version != "" {
|
||||
t.Errorf("main.6's mover = %+v, want no version: another source", m)
|
||||
}
|
||||
if m, _ := MoverOf(store[pkey].p, "app"); m.Version != "0.4.1" {
|
||||
t.Errorf("app's mover = %+v, want version 0.4.1", m)
|
||||
|
||||
@@ -70,14 +70,14 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) {
|
||||
if n, err := s.TakenIn(ev, view); n != 1 || err != nil {
|
||||
t.Fatalf("the take-in started %d (%v)", n, err)
|
||||
}
|
||||
p, _, _ := mem.Get("main.4")
|
||||
p, _, _ := mem.Get("main.7")
|
||||
if !placed(p, "mesh-sdk") || !placed(p, "builder") || placed(p, "app") || placed(p, "tool") {
|
||||
t.Fatalf("placed at the start: %+v, want mesh-sdk and builder only", p.Steps)
|
||||
}
|
||||
if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); n != 0 || err != nil {
|
||||
t.Fatalf("the publish started %d (%v), want none", n, err)
|
||||
if err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); err != nil {
|
||||
t.Fatalf("the publish is not heard: %v", err)
|
||||
}
|
||||
p, _, _ = mem.Get("main.4")
|
||||
p, _, _ = mem.Get("main.7")
|
||||
if sdk, _ := MoverOf(p, "mesh-sdk"); !placed(p, "app") || placed(p, "tool") || len(mem) != 1 || !slices.Contains(sdk.Published, "mesh-sdk:go") {
|
||||
t.Fatalf("after mesh-sdk's publish: %d pipelines, app placed %v, tool placed %v, mesh-sdk %+v", len(mem), placed(p, "app"), placed(p, "tool"), sdk)
|
||||
}
|
||||
@@ -87,22 +87,22 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) {
|
||||
b := stepsOf(p, "builder")[0]
|
||||
bs := stepSeatOf(t, builds.Seat)
|
||||
end := func(outcome string) StepEnded {
|
||||
e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.4","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`))
|
||||
e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.7","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return e
|
||||
}
|
||||
before := mem["main.4"]
|
||||
before := mem["main.7"]
|
||||
if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("passed")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning {
|
||||
if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning {
|
||||
t.Errorf("after builder's build passed, tool's steps = %+v, want placed and asked", stepsOf(p, "tool"))
|
||||
}
|
||||
// The same pipeline, builder's build failed: tool is placed with every step not asked, each saying its own step-ended
|
||||
// naming the dependency (H-A-101 (2)).
|
||||
mem["main.4"] = before
|
||||
mem["main.7"] = before
|
||||
var notAsked []seat.StepEvent
|
||||
s.Say = func(subject string, body any) error {
|
||||
if ev, ok := body.(seat.StepEvent); ok && strings.Contains(subject, ".step-ended.tool.") {
|
||||
@@ -113,7 +113,7 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) {
|
||||
if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("failed")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked {
|
||||
if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked {
|
||||
t.Errorf("after builder's build failed, tool's steps = %+v, want placed not asked", stepsOf(p, "tool"))
|
||||
}
|
||||
if len(notAsked) == 0 || len(notAsked) != len(slices.DeleteFunc(stepsOf(p, "tool"), func(st seat.StepResult) bool { return st.Outcome != seat.StepResultOutcomeNotAsked })) {
|
||||
|
||||
@@ -51,7 +51,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing.
|
||||
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
|
||||
t.Fatalf("the take-in started %d (%v)", n, err)
|
||||
}
|
||||
p, _, _ := store.Get("main.947")
|
||||
p, _, _ := store.Get("main.357")
|
||||
if p.Revision != 357 || len(p.Steps) != 0 || len(asks.asks) != 0 {
|
||||
t.Fatalf("a take-in of revision 357 with the view at 356 started %+v with %d asks, want revision 357 and no step", p, len(asks.asks))
|
||||
}
|
||||
@@ -60,7 +60,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing.
|
||||
}
|
||||
viewRevision = 357
|
||||
n, err := s.Waiting(store.memStore, "main", view)
|
||||
p, _, _ = store.Get("main.947")
|
||||
p, _, _ = store.Get("main.357")
|
||||
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Revision != 357 {
|
||||
t.Errorf("the view reaching 357 placed %d (%v): %d steps, %d asks at revision %d", n, err, len(p.Steps), len(asks.asks), p.Revision)
|
||||
}
|
||||
@@ -87,7 +87,7 @@ func TestAViewThatReachesTheRevisionDuringTheStartsPlacesTheHeldBackPipelines(t
|
||||
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
|
||||
t.Fatalf("started %d (%v)", n, err)
|
||||
}
|
||||
if p, _, _ := store.Get("main.947"); len(p.Steps) == 0 || len(asks.asks) == 0 {
|
||||
if p, _, _ := store.Get("main.357"); len(p.Steps) == 0 || len(asks.asks) == 0 {
|
||||
t.Errorf("the pipeline has %d steps and %d asks after the view reached 357 during the start, want them placed", len(p.Steps), len(asks.asks))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,10 @@ package pipeline
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -140,6 +143,13 @@ func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[
|
||||
steps = append(steps, s)
|
||||
continue
|
||||
}
|
||||
// A Go step is asked only once every pin of its entry has a version (H-A-206): until then it waits on the
|
||||
// publish, as the build does (H-A-165), and the publish re-pins the entry and moves the mover (H-A-204).
|
||||
if path := unpinnedPackage(entry); path != "" && language == sdkstep.AskGo {
|
||||
s.Seat, s.Outcome, s.Started, s.Why = bs, seat.StepResultOutcomeWaiting, &now, PinWait+path
|
||||
steps = append(steps, s)
|
||||
continue
|
||||
}
|
||||
s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now
|
||||
s.Ask = AskID(startOf(p, now), p.Pipeline, m.Module, a.Ref, step)
|
||||
body := stepAsk(p, m, entry, a, language, step, s.Ask, asker, now)
|
||||
@@ -227,6 +237,132 @@ type Asked struct {
|
||||
Commit string
|
||||
}
|
||||
|
||||
// PinWait opens the why of a Go step held because its entry names a package with no version yet (H-A-206), the build's
|
||||
// own wait form (H-A-165); the package's path follows.
|
||||
const PinWait = "waits on the publish of "
|
||||
|
||||
// unpinnedPackage is the first package, in path order, the entry names with no version, empty when every one has one.
|
||||
func unpinnedPackage(e catalog.CatalogEntry) string {
|
||||
for _, path := range slices.Sorted(maps.Keys(e.Packages)) {
|
||||
if e.Packages[path] == "" {
|
||||
return path
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// pinHeld says whether a step waits for its entry's pins (H-A-206).
|
||||
func pinHeld(s seat.StepResult) bool {
|
||||
return s.Outcome == seat.StepResultOutcomeWaiting && strings.HasPrefix(s.Why, PinWait)
|
||||
}
|
||||
|
||||
// endPinHeld ends each of build's mover's steps waiting for its entry's pins (H-A-206, H-A-207): as the build ended when
|
||||
// it did not pass, so no step waits on a publish the build no longer waits for; could not run, loud, when the build passed
|
||||
// and its end names no pins to run them with. It answers the steps it ended.
|
||||
func endPinHeld(p *seat.Pipeline, build seat.StepResult, pins map[string]string, now time.Time) []int {
|
||||
outcome, why := build.Outcome, "the build ended "+build.Outcome+" while this step waited: %s; "+build.Why
|
||||
if stepPassed(build) {
|
||||
if len(pins) > 0 {
|
||||
return nil
|
||||
}
|
||||
outcome, why = seat.StepResultOutcomeCouldNotRun, "the build passed, but its end names no pins to run this step with (H-A-207); it %s"
|
||||
}
|
||||
var ended []int
|
||||
for i := range p.Steps {
|
||||
st := &p.Steps[i]
|
||||
if st.Module != build.Module || !pinHeld(*st) {
|
||||
continue
|
||||
}
|
||||
st.Outcome, st.Why, st.Ended = outcome, fmt.Sprintf(why, st.Why), &now
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
|
||||
ended = append(ended, i)
|
||||
}
|
||||
return ended
|
||||
}
|
||||
|
||||
// readRevision is revision's read, an error when none is given.
|
||||
func readRevision(revision func(branch string, revision int64) (Half, error), branch string, n int64) (Half, error) {
|
||||
if revision == nil {
|
||||
return Half{}, errors.New("the runner reads no catalog revision")
|
||||
}
|
||||
return revision(branch, n)
|
||||
}
|
||||
|
||||
// heldOn says whether module has a step of p waiting for its entry's pins.
|
||||
func heldOn(p seat.Pipeline, module string) bool {
|
||||
return slices.ContainsFunc(p.Steps, func(st seat.StepResult) bool { return st.Module == module && pinHeld(st) })
|
||||
}
|
||||
|
||||
// AskPinHeld asks the steps of pipeline key's mover module that waited for its entry's pins (H-A-206) with the pins its
|
||||
// build passed with (H-A-207): each running under its own ask, its fetch the entry's as Place builds it with those pins.
|
||||
// The entry is read at the pipeline's own catalog revision through revision, which keeps what it read (H-A-205), never
|
||||
// the branch's newest: a revision that cannot be read, or whose entry is of another fingerprint than the mover's, or that
|
||||
// names no artifact of the step, ends the step could not run, saying so. It answers how many it asked.
|
||||
func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]string, revision func(branch string, revision int64) (Half, error)) (int, error) {
|
||||
for {
|
||||
p, rev, err := store.Get(key)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the pipeline %s is not read: %w", key, err)
|
||||
}
|
||||
m, moves := MoverOf(p, module)
|
||||
if !moves || !heldOn(p, module) {
|
||||
return 0, nil
|
||||
}
|
||||
var entry catalog.CatalogEntry
|
||||
unread := ""
|
||||
switch half, err := readRevision(revision, p.Branch, p.Revision); {
|
||||
case err != nil:
|
||||
unread = fmt.Sprintf("revision %d of %s, the pipeline's, cannot be read for its entry: %v", p.Revision, p.Branch, err)
|
||||
case half.Entries[module].Fp != m.Fingerprint:
|
||||
unread = fmt.Sprintf("revision %d of %s holds %s at %q, not the mover's %s", p.Revision, p.Branch, module, half.Entries[module].Fp, m.Fingerprint)
|
||||
default:
|
||||
entry = half.Entries[module]
|
||||
}
|
||||
entry.Packages = pins
|
||||
now := s.now()
|
||||
var asks []Asked
|
||||
var ended []int
|
||||
for i := range p.Steps {
|
||||
st := &p.Steps[i]
|
||||
if st.Module != module || !pinHeld(*st) {
|
||||
continue
|
||||
}
|
||||
j := slices.IndexFunc(entry.Artifacts, func(a catalog.Artifact) bool { return a.Ref == st.Artifact })
|
||||
language, err := sdkstep.AskLanguageOfSeat(st.Seat)
|
||||
if unread != "" || j < 0 || err != nil {
|
||||
why := unread
|
||||
if why == "" {
|
||||
why = fmt.Sprintf("revision %d of %s names no artifact %s of %s to ask it of", p.Revision, p.Branch, st.Artifact, module)
|
||||
}
|
||||
st.Outcome, st.Ended = seat.StepResultOutcomeCouldNotRun, &now
|
||||
st.Why = "its build passed with its pins, but " + why + " (H-A-207, H-A-205)"
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
|
||||
ended = append(ended, i)
|
||||
continue
|
||||
}
|
||||
a := entry.Artifacts[j]
|
||||
st.Outcome, st.Why, st.Started = seat.StepResultOutcomeRunning, "", &now
|
||||
st.Ask = AskID(startOf(p, now), p.Pipeline, module, a.Ref, st.Step)
|
||||
ask := Asked{Seat: st.Seat, ID: st.Ask, Ask: stepAsk(p, m, entry, a, language, st.Step, st.Ask, s.Name, now), Toolchain: a.Language, Commit: m.Commit}
|
||||
if a.Package != nil {
|
||||
ask.Package = a.Package.Name
|
||||
}
|
||||
asks = append(asks, ask)
|
||||
}
|
||||
if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return 0, fmt.Errorf("the pipeline %s is not written: %w", key, err)
|
||||
}
|
||||
s.logf("%s: %s's build passed with its pins: %d steps that waited for them are asked, %d could not run (H-A-207)", key, module, len(asks), len(ended))
|
||||
s.sayNotAsked(p, ended)
|
||||
if s.Steps == nil || len(asks) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return len(asks), s.askSteps(key, asks)
|
||||
}
|
||||
}
|
||||
|
||||
func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
|
||||
// A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the
|
||||
// artifact's named paths in it when the entry has them.
|
||||
@@ -234,9 +370,21 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar
|
||||
if paths := artifactPaths(e, a.Ref); len(paths) > 0 {
|
||||
fetch.Paths = paths
|
||||
}
|
||||
// The entry's pins, the same the builder gives the build (H-A-203, H-A-183): a test runs against what the build was
|
||||
// built against, and the Go build module refuses a requirement of another repository with no pin (H-A-192). A
|
||||
// package the entry names with no pin yet is left out, as the builder pins it only from its publish.
|
||||
for path, version := range e.Packages {
|
||||
if version == "" {
|
||||
continue
|
||||
}
|
||||
if fetch.Packages == nil {
|
||||
fetch.Packages = map[string]string{}
|
||||
}
|
||||
fetch.Packages[path] = version
|
||||
}
|
||||
return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now,
|
||||
Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: m.Commit, Fetch: fetch, Fingerprint: m.Fingerprint,
|
||||
Module: m.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}}
|
||||
Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}}
|
||||
}
|
||||
|
||||
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its
|
||||
|
||||
@@ -2,6 +2,8 @@ package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -44,7 +46,7 @@ func placeFixture() (catalog.Registration, map[string]catalog.CatalogEntry) {
|
||||
Artifacts: map[string]catalog.StepStatuses{"tools": {Lint: "bypass"}},
|
||||
}}
|
||||
return reg, map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"},
|
||||
"app": {Module: "app", Path: "modules/app", Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25"}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"},
|
||||
{Ref: "app:web", Language: "rust"}}, Inputs: &catalog.EntryInputs{Paths: map[string][]string{"app:program": {"cmd/app"}}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
|
||||
}
|
||||
@@ -100,10 +102,131 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) {
|
||||
a := asks.asks[1].(nodebuildgo.StepAsk)
|
||||
if asks.at[1] != "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner" || a.Kind != "step" || a.Asker != "the-runner" || a.ID != p.Steps[1].Ask || asks.ids[1] != a.ID ||
|
||||
a.Step.Commit != "c1" || a.Step.Fingerprint != "src2:aa" || a.Step.Pipeline != "main.4" || a.Step.Artifact != "app:program" ||
|
||||
a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Revision != 4 ||
|
||||
a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Source != 4 ||
|
||||
len(a.Step.Fetch.Paths) != 1 || a.Step.Fetch.Paths[0] != "cmd/app" {
|
||||
t.Errorf("the ask = %+v at %s", a, asks.at[1])
|
||||
}
|
||||
// Every step ask fetches with the entry's pins, as the builder gives them the build (H-A-203, H-A-192).
|
||||
for _, x := range asks.asks[1:] {
|
||||
if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && (len(b.Step.Fetch.Packages) != 1 || b.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25") {
|
||||
t.Errorf("the %s ask of %s fetches the pins %v, want the entry's", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages)
|
||||
}
|
||||
}
|
||||
// An entry naming a package with no version yet asks no Go step: each waits on its publish, as the build does, said in
|
||||
// the journal (H-A-206); the build is still asked.
|
||||
waiting := map[string]catalog.CatalogEntry{}
|
||||
for k, e := range entries {
|
||||
waiting[k] = e
|
||||
}
|
||||
held := waiting["app"]
|
||||
held.Packages = map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": ""}
|
||||
held.Fp = "src2:ab"
|
||||
waiting["app"] = held
|
||||
var journal []string
|
||||
heldAsks := &stepAsks{}
|
||||
hs := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: heldAsks, Name: "the-runner",
|
||||
Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) },
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }}
|
||||
if ok, err := hs.Start(Begin{Branch: "main", Source: 5, Revision: 3, Entries: waiting, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}}); !ok || err != nil {
|
||||
t.Fatalf("start = %v, %v", ok, err)
|
||||
}
|
||||
for _, x := range heldAsks.asks {
|
||||
if b, ok := x.(nodebuildgo.StepAsk); ok {
|
||||
t.Errorf("the %s of %s is asked with a package unpinned", b.Step.Step, b.Step.Artifact)
|
||||
}
|
||||
}
|
||||
var hp seat.Pipeline
|
||||
if err := json.Unmarshal(store["main.3"], &hp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := hp.Steps[1]; st.Step != "lint" || st.Outcome != seat.StepResultOutcomeWaiting || st.Ask != "" || st.Why != "waits on the publish of git.novox.be/novox/x/go" {
|
||||
t.Errorf("the held lint step = %+v; want it waiting on the publish", st)
|
||||
}
|
||||
// The build ending without passing past its bound ends each held step as it ended (H-A-206).
|
||||
failed := hp.Steps[0]
|
||||
failed.Outcome, failed.Why = seat.StepResultOutcomeFailed, "package git.novox.be/novox/x/go has no released version"
|
||||
failedP := hp
|
||||
failedP.Steps = append([]seat.StepResult(nil), hp.Steps...)
|
||||
if ended := endPinHeld(&failedP, failed, nil, time.Now()); len(ended) == 0 || failedP.Steps[1].Outcome != seat.StepResultOutcomeFailed || failedP.Steps[1].Ended == nil {
|
||||
t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", failedP.Steps[1], len(ended))
|
||||
}
|
||||
// A build passed whose end names no pins ends them could not run, loud (H-A-207).
|
||||
passed := hp.Steps[0]
|
||||
passed.Outcome = seat.StepResultOutcomePassed
|
||||
bare := hp
|
||||
bare.Steps = append([]seat.StepResult(nil), hp.Steps...)
|
||||
if ended := endPinHeld(&bare, passed, nil, time.Now()); len(ended) == 0 || bare.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun ||
|
||||
!strings.Contains(bare.Steps[1].Why, "names no pins") {
|
||||
t.Errorf("after a build passed naming no pins the held lint step = %+v; want could not run naming it", bare.Steps[1])
|
||||
}
|
||||
// A build passed naming its pins asks the held steps with them: its end heard, the runner asks each with the pins the
|
||||
// build was built with (H-A-207).
|
||||
pinsBuilt := map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": "v0.1.4"}
|
||||
ms := memStore{}
|
||||
ms["main.5"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{hp, 1}
|
||||
// The entry is read at the pipeline's own revision (3), never the newer revision 4 that moved the module (H-A-205).
|
||||
moved := held
|
||||
moved.Fp, moved.Path = "src2:ac", "modules/moved"
|
||||
revisions := func(branch string, n int64) (Half, error) {
|
||||
switch n {
|
||||
case 3:
|
||||
return Half{Revision: 3, Entries: waiting}, nil
|
||||
case 0, 4:
|
||||
return Half{Revision: 4, Entries: map[string]catalog.CatalogEntry{"app": moved}}, nil
|
||||
}
|
||||
return Half{}, fmt.Errorf("no revision %d", n)
|
||||
}
|
||||
var askedWith map[string]string
|
||||
h := &Hearer{Store: ms, Say: func(string, any) error { return nil }, Name: "the-runner",
|
||||
AskPinHeld: func(key, module string, pins map[string]string) {
|
||||
askedWith = pins
|
||||
if _, err := hs.AskPinHeld(ms, key, module, pins, revisions); err != nil {
|
||||
t.Errorf("ask held: %v", err)
|
||||
}
|
||||
}}
|
||||
buildEnd := StepEnded{ID: hp.Steps[0].Ask, Pipeline: "main.5", Step: "build", Node: "n1", Outcome: seat.StepResultOutcomePassed,
|
||||
Module: "app", Commit: "c1", Fingerprint: "src2:ab"}
|
||||
if _, err := h.OnStepEndedPinned(StepSeats[3], StepSeats[3].Prefix+"n1", buildEnd, nil, pinsBuilt); err != nil {
|
||||
t.Fatalf("the build's end: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(askedWith, pinsBuilt) {
|
||||
t.Errorf("the held steps were asked with %v; want the build's pins", askedWith)
|
||||
}
|
||||
var askedHeld int
|
||||
for _, x := range heldAsks.asks {
|
||||
if b, ok := x.(nodebuildgo.StepAsk); ok {
|
||||
askedHeld++
|
||||
if !reflect.DeepEqual(b.Step.Fetch.Packages, pinsBuilt) {
|
||||
t.Errorf("the %s of %s fetches %v; want the build's pins", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages)
|
||||
}
|
||||
if b.Step.Fetch.Dir != "modules/app" {
|
||||
t.Errorf("the %s of %s fetches %s; want the pipeline's revision's directory", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
after, _, _ := ms.Get("main.5")
|
||||
if askedHeld == 0 || after.Steps[1].Outcome == seat.StepResultOutcomeWaiting || after.Steps[1].Ask == "" {
|
||||
t.Errorf("after the build passed %d held steps were asked, lint = %+v; want each asked", askedHeld, after.Steps[1])
|
||||
}
|
||||
// A mover whose fingerprint the pipeline's revision does not hold ends its held steps could not run, naming it.
|
||||
other := hp
|
||||
other.Pipeline, other.Steps, other.Movers = "main.4", append([]seat.StepResult(nil), hp.Steps...), []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:zz", Repository: "o/app"}}
|
||||
ms["main.4"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{other, 1}
|
||||
if n, err := hs.AskPinHeld(ms, "main.4", "app", pinsBuilt, revisions); n != 0 || err != nil {
|
||||
t.Errorf("asked %d held steps of a mover its revision does not hold (%v); want none", n, err)
|
||||
}
|
||||
if o, _, _ := ms.Get("main.4"); o.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(o.Steps[1].Why, "not the mover's src2:zz") {
|
||||
t.Errorf("the held lint step of a moved fingerprint = %+v; want could not run naming it", o.Steps[1])
|
||||
}
|
||||
if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") {
|
||||
t.Errorf("the journal says %q; want the held step named", journal)
|
||||
}
|
||||
// An artifact that names no paths still fetches its module alone: the directory with no paths (principle 10, A-447).
|
||||
for _, x := range asks.asks[2:] {
|
||||
if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && b.Step.Artifact == "app:tools" && (b.Step.Fetch.Dir != "modules/app" || len(b.Step.Fetch.Paths) != 0) {
|
||||
@@ -409,7 +532,8 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) {
|
||||
Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }}
|
||||
start := func(source, revision int64) seat.Pipeline {
|
||||
t.Helper()
|
||||
if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil {
|
||||
// Keyed by its source, so pipelines of one revision are several (one kept under its source before H-A-213 (4)).
|
||||
if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil {
|
||||
t.Fatalf("start %d = %v, %v", source, ok, err)
|
||||
}
|
||||
p, _, _ := store.Get(Key("main", source))
|
||||
@@ -428,6 +552,12 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) {
|
||||
if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" {
|
||||
t.Fatalf("the test steps asked = %q, want the program's three alone", got)
|
||||
}
|
||||
// Every step ask names the catalog revision in revision and the gate's source in source (H-A-219).
|
||||
for _, a := range asks.asks {
|
||||
if b, ok := a.(nodebuildgo.StepAsk); ok && (b.Step.Revision != 9 || b.Step.Source != 4) {
|
||||
t.Errorf("the %s ask of %s names revision %d and source %d, want 9 and 4", b.Step.Step, b.Step.Artifact, b.Step.Revision, b.Step.Source)
|
||||
}
|
||||
}
|
||||
for _, level := range buildSteps {
|
||||
asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level)
|
||||
if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil {
|
||||
@@ -628,13 +758,13 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
p, _, _ := mem.Get("main.4")
|
||||
p, _, _ := mem.Get("main.7")
|
||||
if validates != 1 || compositions != 0 || stepNamed(p, "", seat.StepResultStepComposition).Outcome != seat.StepResultOutcomeWaiting {
|
||||
t.Fatalf("placed: %d validate and %d composition asks, steps %+v; want one validate, no composition yet", validates, compositions, p.Steps)
|
||||
}
|
||||
h := &Hearer{Store: mem, Name: "the-runner", Say: func(string, any) error { return nil }, AskSteps: s.AskSteps}
|
||||
bs := stepSeatOf(t, wants.Seat)
|
||||
raw := `{"id":"` + validateAsk + `","pipeline":"main.4","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}`
|
||||
raw := `{"id":"` + validateAsk + `","pipeline":"main.7","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}`
|
||||
ev, err := bs.Decode([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -643,7 +773,7 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) {
|
||||
if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, _, _ = mem.Get("main.4")
|
||||
p, _, _ = mem.Get("main.7")
|
||||
for _, st := range p.Steps {
|
||||
if st.Step == seat.StepResultStepValidate && st.Outcome != outcome {
|
||||
t.Errorf("%s: %s's validate is %s, want every mover's read from the one run", outcome, st.Module, st.Outcome)
|
||||
|
||||
@@ -164,7 +164,7 @@ func (q *Queue) leave(key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on <branch>.<source> with step pipeline and
|
||||
// queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on <branch>.<revision> with step pipeline and
|
||||
// on pipelines-at-once (design 47 H §748, H-A-52).
|
||||
func queueWait(p seat.Pipeline, at time.Time, how string) seat.Wait {
|
||||
return seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Revision: p.Revision, Run: p.Run,
|
||||
@@ -175,7 +175,7 @@ func (s *Starter) sayQueueWait(event string, p seat.Pipeline, at time.Time, how
|
||||
if s.Say == nil {
|
||||
return
|
||||
}
|
||||
if err := s.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), queueWait(p, at, how)); err != nil {
|
||||
if err := s.Say(runnerclient.RevisionSubject(event, p.Branch, subjectRevision(p)), queueWait(p, at, how)); err != nil {
|
||||
s.logf("%s of %s is not said: %v", event, p.Pipeline, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,7 +96,7 @@ func queueFixture() (*Starter, memStore, memQueue, *[]string, *[]seat.Wait, *ste
|
||||
func takeIn(t *testing.T, s *Starter, view func(string) (Half, bool)) func(source int64, modules ...string) {
|
||||
return func(source int64, modules ...string) {
|
||||
t.Helper()
|
||||
ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7}
|
||||
ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: source}
|
||||
for _, m := range modules {
|
||||
ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m})
|
||||
}
|
||||
|
||||
@@ -99,6 +99,10 @@ type Half struct {
|
||||
Revision int64
|
||||
Source int64
|
||||
Entries map[string]catalog.CatalogEntry
|
||||
// Call and Movers are the revision's own, as the revision verb answers them (H-A-213): what made it and its movers;
|
||||
// a view's half names none.
|
||||
Call string
|
||||
Movers []catalog.TakeInEntry
|
||||
}
|
||||
|
||||
// Views is the runner's views of the catalog half, one per watched branch the registrations name (design 47 G §21,
|
||||
|
||||
@@ -3,7 +3,6 @@ package pipeline
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
@@ -21,12 +20,13 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int,
|
||||
if branch != "" {
|
||||
b = keys.EscapeName(branch)
|
||||
}
|
||||
source := strconv.FormatInt(ev.Source, 10)
|
||||
// A pipeline is keyed by its revision (H-A-213 (4)), so the pipelines of a source are those naming it, of either key
|
||||
// form a source's pipelines were kept under.
|
||||
var pkeys []string
|
||||
for _, f := range []string{b + "." + source, b + "." + source + ".published.*.*"} {
|
||||
for _, f := range []string{b + ".*", b + ".*.published.*.*"} {
|
||||
k, err := h.Store.Keys(f)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the pipelines of source %s are not listed: %w", source, err)
|
||||
return 0, fmt.Errorf("the pipelines of source %d are not listed: %w", ev.Source, err)
|
||||
}
|
||||
pkeys = append(pkeys, k...)
|
||||
}
|
||||
@@ -37,6 +37,9 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int,
|
||||
if err != nil {
|
||||
return wrote, fmt.Errorf("the pipeline %s is not read: %w", key, err)
|
||||
}
|
||||
if p.Source != ev.Source || p.Run != "" {
|
||||
break
|
||||
}
|
||||
i := -1
|
||||
for j, m := range p.Movers {
|
||||
if m.Module == ev.Module {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -28,7 +27,7 @@ type Created interface {
|
||||
Create(key string, value []byte) error
|
||||
}
|
||||
|
||||
// Begin is what a start names: the pipeline's key (watched branch and source revision, design 47 H §746), the take-in's
|
||||
// Begin is what a start names: the pipeline's key (watched branch and catalog revision, design 47 H §746, H-A-213), the take-in's
|
||||
// call and catalog graph revision, its movers, and the catalog half of that branch.
|
||||
type Begin struct {
|
||||
Branch string
|
||||
@@ -39,28 +38,36 @@ type Begin struct {
|
||||
Movers []seat.Mover
|
||||
// Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them.
|
||||
Entries map[string]catalog.CatalogEntry
|
||||
// Published is the package's publish that started the pipeline, nil for any other start (§747).
|
||||
Published *seat.Published
|
||||
// Key is the pipeline's key when it is not <branch>.<source> (a publish's, §747); empty for a take-in's.
|
||||
// Key is the pipeline's key when it is not <branch>.<revision>; empty for every start the runner makes.
|
||||
Key string
|
||||
// unheld says the branch's catalog half is not held yet: the pipeline starts with no steps, placed once it is (A-336).
|
||||
unheld bool
|
||||
}
|
||||
|
||||
// Key is a take-in's pipeline key in the state pipelines, `<branch>.<source>`, the branch escaped as a key's name is
|
||||
// (design 47 H §746).
|
||||
func Key(branch string, source int64) string {
|
||||
return seat.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(source, 10))
|
||||
// Key is a pipeline's key in the state pipelines, `<branch>.<revision>`, the catalog revision's number the indexer counts,
|
||||
// the branch escaped as a key's name is (design 47 H §746, H-A-213 (4)); a take-in's number is unchanged where it equalled
|
||||
// its source. A pipeline kept under its source before (`<branch>.<source>`) has the same form and is read as it is.
|
||||
func Key(branch string, revision int64) string {
|
||||
return seat.PipelinesByBranchRevisionKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10))
|
||||
}
|
||||
|
||||
// PublishedKey is the key of a pipeline a package's publish starts, `<branch>.<source>.published.<package>.<version>`,
|
||||
// each part escaped as a key's name is (design 47 H §747).
|
||||
func PublishedKey(branch string, source int64, pkg, version string) string {
|
||||
return seat.PipelinesByBranchSourcePublishedPackageVersionKey(keys.EscapeName(branch), strconv.FormatInt(source, 10),
|
||||
keys.EscapeName(pkg), keys.EscapeName(version))
|
||||
// subjectRevision is the number p's pipeline-started, pipeline-ended and waits are said under, <branch>.<revision>: its
|
||||
// key's number (H-A-221).
|
||||
func subjectRevision(p seat.Pipeline) int64 {
|
||||
return keyRevision(Begin{Revision: p.Revision, Source: p.Source})
|
||||
}
|
||||
|
||||
// keyForms are the filters of every key form of the state pipelines: a take-in's, a run's and a publish's.
|
||||
// keyRevision is the number b's pipeline is keyed by (H-A-213 (4)): its revision, or its source where the start names no
|
||||
// revision (a take-in.taken of an indexer whose revision numbers were its sources, H-A-211).
|
||||
func keyRevision(b Begin) int64 {
|
||||
if b.Revision > 0 {
|
||||
return b.Revision
|
||||
}
|
||||
return b.Source
|
||||
}
|
||||
|
||||
// keyForms are the filters of every key form of the state pipelines: a revision's, a run's, and a publish's kept from before
|
||||
// a release revision started one pipeline (H-A-213 (2)).
|
||||
var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*"}
|
||||
|
||||
// AllKeys are every pipeline's key, of each key form (§746, §747; H-A-83).
|
||||
@@ -189,7 +196,7 @@ func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]A
|
||||
}
|
||||
var none []string
|
||||
for _, st := range p.Steps {
|
||||
if !open(st) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) {
|
||||
if (!open(st) && !pinHeld(st)) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) {
|
||||
continue
|
||||
}
|
||||
if holders, read := s.Holders(st.Seat); read && len(holders) == 0 {
|
||||
@@ -282,7 +289,7 @@ func (s *Starter) sayStarted(key string, asks []Asked) {
|
||||
func newPipeline(b Begin, key string, now time.Time) seat.Pipeline {
|
||||
movers := append([]seat.Mover(nil), b.Movers...)
|
||||
return seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Call: b.Call, Revision: b.Revision, Movers: movers,
|
||||
Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}, Published: b.Published}
|
||||
Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}}
|
||||
}
|
||||
|
||||
func (s *Starter) create(key string, p seat.Pipeline) error {
|
||||
@@ -299,12 +306,12 @@ func started(p seat.Pipeline, at time.Time) seat.PipelineStarted {
|
||||
Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run}
|
||||
}
|
||||
|
||||
// sayStartedPipeline says p's pipeline-started on <branch>.<source> (§746).
|
||||
// sayStartedPipeline says p's pipeline-started on <branch>.<revision> (§746).
|
||||
func (s *Starter) sayStartedPipeline(p seat.Pipeline, at time.Time) {
|
||||
if s.Say == nil {
|
||||
return
|
||||
}
|
||||
if err := s.Say(runnerclient.SourceSubject(seat.EventPipelineStarted, p.Branch, p.Source), started(p, at)); err != nil {
|
||||
if err := s.Say(runnerclient.RevisionSubject(seat.EventPipelineStarted, p.Branch, subjectRevision(p)), started(p, at)); err != nil {
|
||||
s.logf("pipeline-started of %s is not said: %v", p.Pipeline, err)
|
||||
}
|
||||
}
|
||||
@@ -337,6 +344,11 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
}
|
||||
steps, a := Place(*p, m, r, entries, now, s.Name)
|
||||
supersededWhy(*p, m.Module, entries, steps)
|
||||
for _, st := range steps {
|
||||
if pinHeld(st) {
|
||||
s.logf("%s: %s's %s of %s is not asked: it %s, as its build does (H-A-206)", p.Pipeline, m.Module, st.Step, st.Artifact, st.Why)
|
||||
}
|
||||
}
|
||||
if failed != "" {
|
||||
why := "its dependency " + failed + " did not build"
|
||||
for k := range steps {
|
||||
@@ -490,7 +502,7 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
now := s.now()
|
||||
key := b.Key
|
||||
if key == "" {
|
||||
key = Key(b.Branch, b.Source)
|
||||
key = Key(b.Branch, keyRevision(b))
|
||||
}
|
||||
if err := s.Fills.refuseStart(key); err != nil {
|
||||
return false, err
|
||||
@@ -887,14 +899,21 @@ func (s *Starter) Release(store Updater, key string, view func(branch string) (H
|
||||
}, s.Registration, view, "the publishes its movers waited for")
|
||||
}
|
||||
|
||||
// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the source
|
||||
// revision that published it, and places the movers that waited for it (design 47 H §746, H-A-84). It answers that
|
||||
// source's pipeline, read, and whether one is held.
|
||||
// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the catalog
|
||||
// revision that published it, found by the branch and revision the publish names (H-A-218), and places the movers that
|
||||
// waited for it (design 47 H §746, H-A-84). It answers that pipeline, read, and whether one is held.
|
||||
func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch string) (Half, bool)) (seat.Pipeline, bool, error) {
|
||||
if s.Store == nil {
|
||||
return seat.Pipeline{}, false, nil
|
||||
}
|
||||
key := Key(ev.Branch, ev.Source)
|
||||
key, ok := Key(ev.Branch, ev.Revision), true
|
||||
if ev.Revision <= 0 {
|
||||
// A publish of an older builder names no revision: its source's lowest-numbered revision, the take-in's (H-A-218).
|
||||
key, ok = s.sourcePipeline(ev.Branch, ev.Source)
|
||||
}
|
||||
if !ok {
|
||||
return seat.Pipeline{}, false, nil
|
||||
}
|
||||
for {
|
||||
p, rev, err := s.Store.Get(key)
|
||||
if err != nil {
|
||||
@@ -918,49 +937,81 @@ func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch st
|
||||
}
|
||||
}
|
||||
|
||||
// publishedKind is the edge a dependent builds on a published package by: B published before A builds, a package A
|
||||
// imports (§53).
|
||||
const publishedKind = 1
|
||||
|
||||
// Published hears a package-published (design 47 H §747, H-A-52): in the running pipeline of the source revision that
|
||||
// published it, a mover that builds on it waits for it there (§746) and nothing starts; every module the catalog half
|
||||
// holds a kind-1 edge from the package into that this pipeline does not move (a publish again of an older build, a
|
||||
// dependent it does not move) is a mover of one pipeline keyed `<branch>.<source>.published.<package>.<version>`, at the
|
||||
// publishing source's catalog revision. The dependents come from the edges the runner holds, never from a comparison.
|
||||
func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) (int, error) {
|
||||
if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 {
|
||||
return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev)
|
||||
}
|
||||
publishing, held, err := s.heardPublished(ev, view)
|
||||
// sourcePipeline is the key of branch's pipeline of the lowest-numbered revision of source, the take-in's (H-A-213 (4),
|
||||
// H-A-218): a pipeline is keyed by its revision, so one of a source is found by the source it names, never by its key.
|
||||
func (s *Starter) sourcePipeline(branch string, source int64) (string, bool) {
|
||||
pkeys, err := s.Store.Keys(keys.EscapeName(branch) + ".*")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
s.logf("the pipelines of %s are not listed, so source %d's is not found: %v", branch, source, err)
|
||||
return "", false
|
||||
}
|
||||
half, ok := view(ev.Branch)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("the catalog half of %s is not held yet: the publish of %s %s is heard again once it is", ev.Branch, ev.Artifact, ev.Version)
|
||||
}
|
||||
var movers []seat.Mover
|
||||
for _, module := range slices.Sorted(maps.Keys(half.Entries)) {
|
||||
e := half.Entries[module]
|
||||
if module == ev.Module || Moves(publishing, module) || !slices.ContainsFunc(e.Edges, func(x catalog.Edge) bool {
|
||||
return x.Kind == publishedKind && x.From == ev.Artifact
|
||||
}) {
|
||||
found, low := "", int64(0)
|
||||
for _, k := range pkeys {
|
||||
p, _, err := s.Store.Get(k)
|
||||
if err != nil || p.Branch != branch || p.Source != source || p.Run != "" {
|
||||
continue
|
||||
}
|
||||
movers = append(movers, seat.Mover{Module: module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
|
||||
if r := keyRevision(Begin{Revision: p.Revision, Source: p.Source}); found == "" || r < low {
|
||||
found, low = k, r
|
||||
}
|
||||
}
|
||||
if len(movers) == 0 {
|
||||
return found, found != ""
|
||||
}
|
||||
|
||||
// Published hears a package-published (design 47 H §746, H-A-52): in the running pipeline of the revision that published
|
||||
// it, a mover that builds on it waits for it there and is placed once it is heard. It starts no pipeline: a re-pin on a
|
||||
// module's release is a catalog revision of its own, whose pipeline Revised starts (H-A-212, H-A-213 (2)).
|
||||
func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) error {
|
||||
if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 {
|
||||
return fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev)
|
||||
}
|
||||
_, _, err := s.heardPublished(ev, view)
|
||||
return err
|
||||
}
|
||||
|
||||
// revisedKinds are the call kinds of a catalog revision whose pipeline Revised starts: every kind but a take-in's, whose
|
||||
// pipeline its take-in.taken starts (H-A-213 (2), H-A-217).
|
||||
var revisedKinds = []string{catalog.CatalogRevisionEventCallKindRelease, catalog.CatalogRevisionEventCallKindTakeInAgain,
|
||||
catalog.CatalogRevisionEventCallKindRecompute}
|
||||
|
||||
// Revised starts, on a catalog-revision of the trunk kind whose call kind is release, take-in again or recompute, ONE
|
||||
// pipeline keyed <branch>.<revision> by compare-and-set on its absence (design 47 H §746, H-A-212, H-A-213 (2) and (4),
|
||||
// H-A-217): the revision is read by its number through the catalog seat's revision verb, never the newest half, and its
|
||||
// movers are the revision's movers, its steps placed from the revision's entries. A revision of another kind, or one with
|
||||
// no mover, starts nothing; a second hearing writes nothing.
|
||||
func (s *Starter) Revised(ev catalog.CatalogRevisionEvent, read func(branch string, revision int64) (Half, error)) (int, error) {
|
||||
if ev.Kind != "trunk" || !slices.Contains(revisedKinds, ev.CallKind) {
|
||||
return 0, nil
|
||||
}
|
||||
revision := half.Revision
|
||||
if held && publishing.Revision > 0 {
|
||||
revision = publishing.Revision
|
||||
branch, err := keys.UnescapeName(ev.Branch)
|
||||
if err != nil || branch == "" || ev.Revision <= 0 {
|
||||
return 0, fmt.Errorf("a catalog-revision names its watched branch and revision: branch %q revision %d", ev.Branch, ev.Revision)
|
||||
}
|
||||
if read == nil {
|
||||
return 0, fmt.Errorf("the revision %d of %s cannot be read: the runner holds no revision reader", ev.Revision, branch)
|
||||
}
|
||||
half, err := read(branch, ev.Revision)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the revision %d of %s is not read, so its pipeline is not started: %w", ev.Revision, branch, err)
|
||||
}
|
||||
if half.Revision != ev.Revision {
|
||||
return 0, fmt.Errorf("the revision verb answered revision %d of %s for %d", half.Revision, branch, ev.Revision)
|
||||
}
|
||||
if len(half.Movers) == 0 {
|
||||
s.logf("the revision %d of %s (%s) names no mover: it starts no pipeline", ev.Revision, branch, ev.Call)
|
||||
return 0, nil
|
||||
}
|
||||
movers := make([]seat.Mover, 0, len(half.Movers))
|
||||
for _, e := range half.Movers {
|
||||
movers = append(movers, seat.Mover{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint, Repository: e.Repository})
|
||||
}
|
||||
movers = Tiers(movers, half.Entries, s.logf)
|
||||
started, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Call: publishing.Call, Revision: revision, Movers: movers,
|
||||
Entries: half.Entries, Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version},
|
||||
Key: PublishedKey(ev.Branch, ev.Source, ev.Artifact, ev.Version)})
|
||||
if started {
|
||||
call := half.Call
|
||||
if call == "" {
|
||||
call = ev.Call
|
||||
}
|
||||
ok, err := s.Start(Begin{Branch: branch, Source: half.Source, Call: call, Revision: half.Revision, Movers: movers, Entries: half.Entries})
|
||||
if ok {
|
||||
return 1, err
|
||||
}
|
||||
return 0, err
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
type created map[string][]byte
|
||||
@@ -55,12 +56,11 @@ func TestStarts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// R10/AC1 and AC2 (design 47 H §747, H-A-52): a package-published of a mover of its source's running pipeline whose
|
||||
// dependent that pipeline moves starts nothing: the dependent waits there. A publish again of an older build starts one
|
||||
// pipeline keyed <branch>.<source>.published.<package>.<version>, its movers the package's kind-1 dependents from the
|
||||
// edges the runner holds, at the publishing source's revision, and says pipeline-started with the publish; an ordering
|
||||
// edge starts nothing; heard twice, the second writes nothing.
|
||||
func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) {
|
||||
// H-A-212, H-A-213 (2) and (4), H-A-217: a package-published starts no pipeline, the movers waiting for it in its
|
||||
// pipeline placed there; a trunk catalog-revision of a release, a take-in again or a recompute starts ONE pipeline keyed
|
||||
// <branch>.<revision>, the revision read by its number, its movers the revision's, its source and call the revision's;
|
||||
// heard twice the second writes nothing; a take-in's revision, a candidate's and one with no mover start nothing.
|
||||
func TestARevisionOfAReleaseStartsOnePipeline(t *testing.T) {
|
||||
mem := memStore{}
|
||||
var said []seat.PipelineStarted
|
||||
s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(_ string, body any) error {
|
||||
@@ -70,37 +70,60 @@ func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) {
|
||||
return nil
|
||||
}}
|
||||
edges := []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 1}}
|
||||
view := func(string) (Half, bool) {
|
||||
return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{
|
||||
"mesh-sdk": {Module: "mesh-sdk"},
|
||||
"app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges},
|
||||
"tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges},
|
||||
"ordered": {Module: "ordered", Edges: []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 4}}}}}, true
|
||||
entries := map[string]catalog.CatalogEntry{
|
||||
"mesh-sdk": {Module: "mesh-sdk"},
|
||||
"app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges},
|
||||
"tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges}}
|
||||
view := func(string) (Half, bool) { return Half{Revision: 9, Entries: entries}, true }
|
||||
ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 30, Module: "mesh-sdk", Version: "0.1.9"}
|
||||
if err := s.Published(ev, view); err != nil || len(mem) != 0 {
|
||||
t.Fatalf("a publish of an older build wrote %d pipelines (%v), want none", len(mem), err)
|
||||
}
|
||||
mem[Key("main", 41)] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{seat.Pipeline{Pipeline: "main.41", Branch: "main", Source: 41, Revision: 8, Outcome: seat.PipelineOutcomeRunning,
|
||||
Movers: []seat.Mover{{Module: "mesh-sdk"}, {Module: "app"}, {Module: "tool"}}}, 1}
|
||||
ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}
|
||||
if n, err := s.Published(ev, view); n != 0 || err != nil || len(mem) != 1 {
|
||||
t.Fatalf("a publish in a pipeline moving its dependents started %d (%v), %d pipelines", n, err, len(mem))
|
||||
var asked []int64
|
||||
movers := []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}, {Module: "tool", Commit: "c2", Fingerprint: "src2:bc"}}
|
||||
read := func(branch string, revision int64) (Half, error) {
|
||||
asked = append(asked, revision)
|
||||
if branch != "release.1" {
|
||||
return Half{}, fmt.Errorf("branch %q", branch)
|
||||
}
|
||||
r := Half{Revision: revision, Source: 41, Entries: entries, Call: "release of mesh-sdk 0.1.17"}
|
||||
if revision != 12 {
|
||||
r.Movers = movers
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
// A publish again of an older build, at source 30: its pipeline is gone, so every kind-1 dependent is a mover.
|
||||
ev.Source, ev.Version = 30, "0.1.9"
|
||||
release := catalog.CatalogRevisionEvent{Kind: "trunk", Branch: keys.EscapeName("release.1"), Revision: 10, Source: 41, CallKind: "release", Call: "release of mesh-sdk 0.1.17"}
|
||||
for i := range 2 {
|
||||
if n, err := s.Published(ev, view); n != 1-i || err != nil {
|
||||
if n, err := s.Revised(release, read); n != 1-i || err != nil {
|
||||
t.Fatalf("hearing %d started %d (%v)", i+1, n, err)
|
||||
}
|
||||
}
|
||||
key := PublishedKey("main", 30, "mesh-sdk:go", "0.1.9")
|
||||
p, _, err := mem.Get(key)
|
||||
if err != nil || len(p.Movers) != 2 || !Moves(p, "app") || !Moves(p, "tool") || Moves(p, "ordered") || p.Revision != 9 ||
|
||||
p.Published == nil || p.Published.Version != "0.1.9" || len(said) != 1 || said[0].Published == nil {
|
||||
t.Fatalf("the pipeline %s = %+v (%v), said %+v", key, p, err, said)
|
||||
p, _, err := mem.Get(Key("release.1", 10))
|
||||
if err != nil || len(mem) != 1 || len(p.Movers) != 2 || p.Revision != 10 || p.Source != 41 || p.Call != "release of mesh-sdk 0.1.17" ||
|
||||
len(said) != 1 || len(asked) != 2 || asked[0] != 10 {
|
||||
t.Fatalf("the pipeline %s = %+v (%v), %d written, said %d, read %v", Key("release.1", 10), p, err, len(mem), len(said), asked)
|
||||
}
|
||||
if app, _ := MoverOf(p, "app"); app.Commit != "c1" || app.Fingerprint != "src2:aa" || app.Repository != "o/app" {
|
||||
t.Errorf("the mover app = %+v", app)
|
||||
if app, _ := MoverOf(p, "app"); app.Fingerprint != "src2:ab" || app.Repository != "o/app" || app.Tier != 1 {
|
||||
t.Errorf("the mover app = %+v, want the revision's", app)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
ev catalog.CatalogRevisionEvent
|
||||
want int
|
||||
}{
|
||||
{"take-in again", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 11, CallKind: "take-in-again"}, 1},
|
||||
{"recompute with no mover", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 12, CallKind: "recompute"}, 0},
|
||||
{"recompute", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 13, CallKind: "recompute"}, 1},
|
||||
{"take-in", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 14, CallKind: "take-in"}, 0},
|
||||
{"candidate", catalog.CatalogRevisionEvent{Kind: "candidate", Branch: release.Branch, Revision: 15, CallKind: "release"}, 0},
|
||||
} {
|
||||
if n, err := s.Revised(c.ev, read); n != c.want || err != nil {
|
||||
t.Errorf("%s: started %d (%v), want %d", c.name, n, err, c.want)
|
||||
}
|
||||
}
|
||||
if _, err := s.Revised(catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 16, CallKind: "release"},
|
||||
func(string, int64) (Half, error) { return Half{}, errors.New("no answer") }); err == nil {
|
||||
t.Error("a revision that cannot be read is not an error, so it is never heard again")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -128,7 +151,7 @@ func (failing) Create(string, []byte) error { return errors.New("bus away") }
|
||||
|
||||
// R4/AC1 (design 47 H §746, acceptance test 112): a take-in.taken naming mesh-sdk and one importer starts one pipeline
|
||||
// keyed <branch>.<source> by compare-and-set, its movers the event's entries, at the take-in's revision, and says one
|
||||
// pipeline-started on <branch>.<source> naming the call and the movers; R4/AC2: heard twice, the second writes nothing. A
|
||||
// pipeline-started on <branch>.<revision> naming the call and the movers; R4/AC2: heard twice, the second writes nothing. A
|
||||
// take-in with no entries and one naming no source revision start nothing.
|
||||
func TestATakeInIsOnePipeline(t *testing.T) {
|
||||
store := created{}
|
||||
@@ -153,13 +176,13 @@ func TestATakeInIsOnePipeline(t *testing.T) {
|
||||
t.Fatalf("written %d pipelines and said %d starts, want one each", len(store), len(said))
|
||||
}
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(store["main.7"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 {
|
||||
t.Fatalf("the pipeline main.7 = %+v (%v)", p, err)
|
||||
if err := json.Unmarshal(store["main.12"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 {
|
||||
t.Fatalf("the pipeline main.12 = %+v (%v)", p, err)
|
||||
}
|
||||
if m, _ := MoverOf(p, "mesh-sdk"); m.Commit != "c1" || m.Fingerprint != "src2:aa" || m.Repository != "o/sdk" || m.Tier != 1 {
|
||||
t.Errorf("the mover mesh-sdk = %+v", m)
|
||||
}
|
||||
if subjects[0] != "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.7" || said[0].Call != ev.Call || len(said[0].Movers) != 2 {
|
||||
if subjects[0] != "mesh.seat.mesh-pipeline-runner.event.pipeline-started.main.12" || said[0].Call != ev.Call || len(said[0].Movers) != 2 {
|
||||
t.Errorf("pipeline-started %+v on %s", said[0], subjects[0])
|
||||
}
|
||||
if _, err := s.TakenIn(catalog.TakeInTaken{Branch: "main", Entries: ev.Entries}, view); err == nil {
|
||||
@@ -243,25 +266,6 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
|
||||
if n != 1 || err != nil || len(p.Steps) == 0 {
|
||||
t.Errorf("the half of next placed %d (%v): %d steps, want the take-in's pipeline placed", n, err, len(p.Steps))
|
||||
}
|
||||
// A-336: a package published heard before its branch's half is held starts nothing, heard again once it is: its
|
||||
// dependents are the half's edges (§747).
|
||||
held = false
|
||||
app := entries["app"]
|
||||
app.Edges = []catalog.Edge{{From: "mesh-sdk:go", To: "app:program", Kind: 1}}
|
||||
entries["app"] = app
|
||||
published := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "next", Source: 7, Module: "mesh-sdk", Version: "0.1.17"}
|
||||
if n, err := s.Published(published, late); n != 0 || err == nil {
|
||||
t.Fatalf("the publish before the half started %d (%v), want none and an error, heard again", n, err)
|
||||
}
|
||||
held = true
|
||||
if n, err := s.Published(published, late); n != 1 || err != nil {
|
||||
t.Fatalf("the publish with the half held started %d (%v)", n, err)
|
||||
}
|
||||
p, _, _ = store.Get(PublishedKey("next", 7, "mesh-sdk:go", "0.1.17"))
|
||||
if len(p.Steps) == 0 {
|
||||
t.Errorf("the publish's pipeline placed %d steps, want its steps", len(p.Steps))
|
||||
}
|
||||
|
||||
// A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running,
|
||||
// no step placed) is placed and asked as a newly started one when the half is held; no hold is read any more.
|
||||
var old seat.Pipeline
|
||||
@@ -318,8 +322,8 @@ func TestTakenInStartsOnlyItsEntries(t *testing.T) {
|
||||
if err != nil || n != 1 {
|
||||
t.Fatalf("the take-in started %d (%v), want its one entry", n, err)
|
||||
}
|
||||
if p, _, err := mem.Get("main.9"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") {
|
||||
t.Errorf("main.9 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err)
|
||||
if p, _, err := mem.Get("main.12"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") {
|
||||
t.Errorf("main.12 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -335,3 +339,27 @@ func TestATakeInOfTheOldFormIsRefused(t *testing.T) {
|
||||
t.Errorf("the new form = %+v (%v)", ev, err)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-218: a package-published is taken into the pipeline of the branch and revision it names; one naming no revision (an
|
||||
// older builder) into its source's lowest-numbered revision, the take-in's.
|
||||
func TestAPublishIsTakenIntoThePipelineOfItsRevision(t *testing.T) {
|
||||
mem := memStore{}
|
||||
for _, r := range []int64{7, 8} {
|
||||
mem[Key("main", r)] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{seat.Pipeline{Pipeline: Key("main", r), Branch: "main", Source: 4, Revision: r, Outcome: seat.PipelineOutcomeRunning,
|
||||
Movers: []seat.Mover{{Module: "mesh-sdk"}}}, 1}
|
||||
}
|
||||
s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(string, any) error { return nil }}
|
||||
view := func(string) (Half, bool) { return Half{Revision: 8}, true }
|
||||
published := func(key string) []string { m, _ := MoverOf(mem[key].p, "mesh-sdk"); return m.Published }
|
||||
ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Revision: 8, Version: "0.1.18"}
|
||||
if err := s.Published(ev, view); err != nil || len(published("main.8")) != 1 || len(published("main.7")) != 0 {
|
||||
t.Fatalf("a publish of revision 8 (%v): main.8 %v, main.7 %v; want main.8's mover alone", err, published("main.8"), published("main.7"))
|
||||
}
|
||||
ev.Revision, ev.Artifact = 0, "mesh-sdk:ts"
|
||||
if err := s.Published(ev, view); err != nil || len(published("main.7")) != 1 || len(published("main.8")) != 1 {
|
||||
t.Errorf("a publish naming no revision (%v): main.7 %v, main.8 %v; want main.7's, the source's lowest", err, published("main.7"), published("main.8"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +31,8 @@ type StepSeat struct {
|
||||
// Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for
|
||||
// a seat whose ends name none.
|
||||
Shared func([]byte) []seat.Shared
|
||||
// Pins reads the pins a build's end names it was built with (H-A-207); nil for a seat whose ends name none.
|
||||
Pins func([]byte) map[string]string
|
||||
}
|
||||
|
||||
// StepSeats are the three build seats, from their contracts.
|
||||
@@ -39,7 +41,7 @@ var StepSeats = []StepSeat{
|
||||
{Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")},
|
||||
{Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")},
|
||||
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
|
||||
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
|
||||
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded, Pins: buildPins},
|
||||
// The seat mesh-wants answers the composition step (§22, §24).
|
||||
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared},
|
||||
}
|
||||
@@ -77,6 +79,15 @@ func decodeWantsEnded(raw []byte) (StepEnded, error) {
|
||||
Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil
|
||||
}
|
||||
|
||||
// buildPins is the pins a build step's end of the seat mesh-builds names its entry was built with (H-A-207).
|
||||
func buildPins(raw []byte) map[string]string {
|
||||
var b builds.StepEnded
|
||||
if json.Unmarshal(raw, &b) != nil {
|
||||
return nil
|
||||
}
|
||||
return b.Packages
|
||||
}
|
||||
|
||||
// decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one
|
||||
// entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is
|
||||
// refused (A-99).
|
||||
@@ -155,6 +166,9 @@ type Hearer struct {
|
||||
// Release places the movers of a pipeline whose awaited publishes are heard now, after a build step passed (a kind-3
|
||||
// edge, §746; Starter.Release); nil places none.
|
||||
Release func(key string)
|
||||
// AskPinHeld asks a mover's steps that waited for its entry's pins with the pins its build passed with (H-A-207;
|
||||
// Starter.AskPinHeld); nil leaves them waiting.
|
||||
AskPinHeld func(key, module string, pins map[string]string)
|
||||
// Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748;
|
||||
// Starter.Turn); nil takes none.
|
||||
Turn func(key string)
|
||||
@@ -197,6 +211,12 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726,
|
||||
// A-469m).
|
||||
func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) {
|
||||
return h.OnStepEndedPinned(bs, subject, ev, shared, nil)
|
||||
}
|
||||
|
||||
// OnStepEndedPinned is OnStepEndedShared of a build's end naming the pins it was built with (H-A-207): a passed build's
|
||||
// mover's steps that waited for its entry's pins are asked with them.
|
||||
func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared, pins map[string]string) (seat.Pipeline, error) {
|
||||
node := strings.TrimPrefix(subject, bs.Prefix)
|
||||
if node == subject || node == "" || strings.Contains(node, ".") {
|
||||
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
|
||||
@@ -272,6 +292,11 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
|
||||
// Every mover's validate reads the one run (§745, H-A-51); its end asks each mover's composition, or ends it not
|
||||
// asked when validate did not pass.
|
||||
var due []Asked
|
||||
askHeld := false
|
||||
if s.Step == seat.StepResultStepBuild {
|
||||
readers = append(readers, endPinHeld(&p, *s, pins, now)...)
|
||||
askHeld = stepPassed(*s) && heldOn(p, s.Module)
|
||||
}
|
||||
if s.Step == seat.StepResultStepValidate {
|
||||
readers = append(readers, sameValidate(&p, i, now)...)
|
||||
var notAsked []int
|
||||
@@ -324,6 +349,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
|
||||
if release && h.Release != nil {
|
||||
h.Release(ev.Pipeline)
|
||||
}
|
||||
if askHeld && h.AskPinHeld != nil {
|
||||
h.AskPinHeld(ev.Pipeline, s.Module, pins)
|
||||
}
|
||||
if h.AskSteps != nil && len(due) > 0 {
|
||||
if err := h.AskSteps(ev.Pipeline, due); err != nil {
|
||||
h.logf("the compositions of %s are written running and not asked, asked again at the next start: %v", ev.Pipeline, err)
|
||||
@@ -413,7 +441,7 @@ func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) erro
|
||||
}
|
||||
|
||||
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall;
|
||||
// an end is said once, on <branch>.<source> (design 47 H §746). The runner says no group-ended (§764).
|
||||
// an end is said once, on <branch>.<revision> (design 47 H §746). The runner says no group-ended (§764).
|
||||
func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
|
||||
for _, i := range asked {
|
||||
if err := h.say(seat.EventWaitHeld, p, &p.Steps[i]); err != nil {
|
||||
@@ -461,7 +489,7 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error
|
||||
subject, ev := stepEvent(event, p, *step, at)
|
||||
return h.Say(subject, ev)
|
||||
case seat.EventPipelineEnded:
|
||||
return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), pipelineEnded(p))
|
||||
return h.Say(runnerclient.RevisionSubject(event, p.Branch, subjectRevision(p)), pipelineEnded(p))
|
||||
default:
|
||||
w := seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Revision: p.Revision, Source: p.Source, Run: p.Run,
|
||||
Step: seat.WaitStepApproval, On: seat.WaitOnPerson, Since: at}
|
||||
@@ -469,7 +497,7 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error
|
||||
m, _ := MoverOf(p, step.Module)
|
||||
w.Module, w.Commit, w.Fingerprint, w.Step, w.Ask = m.Module, m.Commit, m.Fingerprint, step.Step, step.Ask
|
||||
}
|
||||
return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), w)
|
||||
return h.Say(runnerclient.RevisionSubject(event, p.Branch, subjectRevision(p)), w)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -113,7 +113,7 @@ func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline {
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
|
||||
}
|
||||
if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil {
|
||||
if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil {
|
||||
t.Fatalf("start = %v, %v", ok, err)
|
||||
}
|
||||
p, _, _ := f.store.Get(Key("main", source))
|
||||
|
||||
@@ -582,11 +582,11 @@ func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) {
|
||||
if !ok || half.Source <= 0 {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the catalog half of %s holds no entry of %s at a source revision: nothing was started", in.Branch, in.Module)
|
||||
}
|
||||
key := Key(in.Branch, half.Source)
|
||||
key := Key(in.Branch, keyRevision(Begin{Revision: half.Revision, Source: half.Source}))
|
||||
if err := v.record(seat.VerbStart, "started the pipeline "+key+" by hand", in.Why); err != nil {
|
||||
return seat.StartAnswer{}, err
|
||||
}
|
||||
// A hand start keys the newest revision's <branch>.<source> with the module as its one mover (H-A-52).
|
||||
// A hand start keys the newest revision's <branch>.<revision> with the module as its one mover (H-A-52, H-A-213 (4)).
|
||||
started, err := v.Starter.Start(Begin{Branch: in.Branch, Source: half.Source, Revision: half.Revision, Entries: half.Entries,
|
||||
Movers: []seat.Mover{{Module: in.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository, Tier: 1}}})
|
||||
if err != nil {
|
||||
|
||||
@@ -287,7 +287,7 @@ func TestStartAndAskAgain(t *testing.T) {
|
||||
_, entries := placeFixture()
|
||||
v.View = func(string) (Half, bool) { return Half{Source: 7, Revision: 3, Entries: entries}, true }
|
||||
a, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "no pipeline for the head"})
|
||||
if err != nil || a.Pipeline != "main.7" || made["main.7"] == nil {
|
||||
if err != nil || a.Pipeline != "main.3" || made["main.3"] == nil {
|
||||
t.Fatalf("start = %+v, %v; written %v", a, err, len(made))
|
||||
}
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "again"}); err == nil {
|
||||
|
||||
Reference in New Issue
Block a user