Merge the runner branch into the builder branch, so both merge into main as one

The builder's step end reports the pins it built with (H-A-207, from the runner
branch) and names the run's catalog revision on its origin (H-A-218, H-A-220); step.go
keeps both. The seat mesh-builds carries packages on step-ended and revision on
package-published once each; seatgen -check is clean.

Builds: design 47 H H-A-207, H-A-218, H-A-220
This commit is contained in:
2026-11-21 00:22:19 +01:00
38 changed files with 790 additions and 203 deletions
@@ -242,6 +242,11 @@ type BuilderBuild struct {
// Imports are, on a done key, the mesh packages the built programs import, as the build module answered them
// (BuildAnswer.Imports; novox/hq ADR 0593 §2): the builder carries them onto the build's end (BuildLogEntry.Imports).
Imports []string `json:"imports,omitempty"`
// Packages are, on a done key only, the mesh packages' versions its Go build was built with, by module path, the ones
// the registry gave a package the entry named with none included (H-A-207): the build step's end reports them, so the
// runner asks the steps that waited for the pins with what the build was built against. A reader that does not know
// the field reads the build as reporting none.
Packages map[string]string `json:"packages,omitempty"`
// Needs are, on a done key only, the reader capabilities (`form:<name>`) the build's reference forms need of a
// node-engine, beside the ones its manifest needs (novox/hq ADR 0618 §1). A reader that does not know the field
// reads the build as needing none.
@@ -843,6 +843,7 @@ type lane struct {
tcVersion string // and the version it ran
kept string // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty
outOfStep []string // the generated files its build module's compare found out of step, passed or failed
pins map[string]string // the mesh packages' versions its Go ask was built with (H-A-207)
err error
}
@@ -992,7 +993,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module
used = nil
}
done := builds.BuilderBuild{State: builds.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used,
Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts)} // the forms it is written in (ADR 0618 §1)
Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts), Packages: pinsOf(lanes)} // the forms it is written in (ADR 0618 §1)
for _, l := range lanes {
if l.kept != "" {
done.KeptOn = l.node // design 47 G §671: the build node that keeps the build
@@ -1154,6 +1155,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
for _, fb := range req.Builds {
delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465)
}
l.pins = maps.Clone(ask.Packages) // kept with the done key, so its step's end reports them (H-A-207)
}
if id := asked.resume(language); id != "" {
ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it
@@ -1336,6 +1338,20 @@ func (b *Builder) takeOutOfStep(key string) []string {
// importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593
// §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that
// kept one reports only what it built.
// pinsOf is the mesh packages' versions the lanes were built with (H-A-207), nil when none was pinned.
func pinsOf(lanes []*lane) map[string]string {
var out map[string]string
for _, l := range lanes {
for p, v := range l.pins {
if out == nil {
out = map[string]string{}
}
out[p] = v
}
}
return out
}
func importsOf(lanes []*lane) []string {
seen := map[string]bool{}
for _, l := range lanes {
@@ -42,7 +42,18 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui
b.askedRevision(run)
var failed []string
for _, e := range run.Entries {
outcome, said := b.buildEntry(ctx, e)
var pins map[string]string
outcome, said := b.buildEntry(ctx, e, &pins)
// The end reports the pins each entry was built with, so the runner asks its held test steps with them (H-A-207).
for path, version := range pins {
if version == "" {
continue
}
if end.Packages == nil {
end.Packages = map[string]string{}
}
end.Packages[path] = version
}
switch outcome {
case meshbuilds.StepEndedOutcomeCouldNotRun:
return finish(outcome, said)
@@ -77,8 +88,9 @@ func (b *Builder) askedRevision(run meshbuilds.StepRun) {
}
}
// buildEntry builds one entry at its fingerprint and waits for its end.
func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outcome, said string) {
// buildEntry builds one entry at its fingerprint and waits for its end; pins is set, when it is done, to the pins its
// done key keeps it was built with, the registry's included (H-A-207).
func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) {
entry, err := b.StepEntry(ctx, e.Fingerprint)
if err != nil {
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry %s at %s cannot be read: %v", e.Module, e.Fingerprint, err)
@@ -100,6 +112,8 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
defer b.dropEnd(key, ended)
if done, failedWords, found := b.keyEnd(ctx, key); found {
if done {
// A key done before reports the pins it was built with, kept with it, never the ones a publish gives now.
*pins = b.keptPins(ctx, key)
return b.passedWithItsVersion(ctx, entry)
}
return meshbuilds.StepEndedOutcomeFailed, failedWords
@@ -110,6 +124,7 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
select {
case log := <-ended:
if log.Outcome == builds.BuildDone {
*pins = b.keptPins(ctx, key)
return b.passedWithItsVersion(ctx, entry)
}
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error)
@@ -158,6 +173,20 @@ func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.Ca
return meshbuilds.StepEndedOutcomePassed, ""
}
// keptPins are the pins a done key keeps it was built with (H-A-207); none when it keeps none or cannot be read, so the
// runner ends the steps that waited for them could not run, loud.
func (b *Builder) keptPins(ctx context.Context, key string) map[string]string {
raw, _, ok, err := b.Claimer.Store.Get(ctx, key)
if err != nil || !ok {
return nil
}
v, err := builds.DecodeBuilderBuild(raw)
if err != nil || v.State != builds.BuildDone {
return nil
}
return v.Packages
}
// keyEnd reads the build key: whether it is done, and its error when it failed for good; found only for a key that has ended.
func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed string, found bool) {
raw, _, ok, err := b.Claimer.Store.Get(ctx, key)
@@ -190,6 +190,32 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
!reflect.DeepEqual(asks[0].Packages, buildstep.PackageVersions{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
t.Errorf("end = %+v, asks %+v; want passed, built with x at v0.1.3", end, asks)
}
// The end reports the pins the build was built with, the one the publish gave included (H-A-207).
if !reflect.DeepEqual(end.Packages, map[string]string{unpinnedX: "v0.1.3", "git.novox.be/novox/mesh-sdk/go": "v0.1.20"}) {
t.Errorf("the end reports the pins %v; want the ones it was built with", end.Packages)
}
// A done key reports the pins it keeps it was built with, never the newer one a publish gives now (H-A-207).
r.b.PackagePublished(unpinnedX, "v0.1.5")
again := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
if again.Outcome != meshbuilds.StepEndedOutcomePassed || again.Packages[unpinnedX] != "v0.1.3" {
t.Errorf("the done key's end = %+v; want passed reporting x at v0.1.3, as built", again)
}
// A done key that keeps no pins reports none, so the runner ends the waiting steps loud.
v, err := builds.DecodeBuilderBuild(r.st.vals[r.key])
if err != nil {
t.Fatal(err)
}
v.Packages = nil
raw, err := builds.EncodeBuilderBuild(v)
if err != nil {
t.Fatal(err)
}
r.st.vals[r.key] = raw
if bare := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); len(bare.Packages) != 0 {
t.Errorf("a done key keeping no pins reported %v; want none", bare.Packages)
}
}
func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) {
@@ -116,9 +116,11 @@ type StepEnded struct {
// the node the builder ran it on, as in the subject
Node string `json:"node"`
// never passed for a build that built nothing
Outcome string `json:"outcome"`
Pipeline string `json:"pipeline"`
Rerun bool `json:"rerun,omitempty"`
Outcome string `json:"outcome"`
// the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207)
Packages map[string]string `json:"packages,omitempty"`
Pipeline string `json:"pipeline"`
Rerun bool `json:"rerun,omitempty"`
// why it failed or could not run: the entry not built, the part not heard
Said string `json:"said,omitempty"`
Started *time.Time `json:"started,omitempty"`
@@ -905,8 +905,10 @@ type StepRun struct {
Pipeline string `json:"pipeline"`
// the step is asked again by the runner's rerun
Rerun bool `json:"rerun,omitempty"`
// the source revision of the watched branch the pipeline runs for; absent for a run on request
// the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request
Revision int64 `json:"revision,omitempty"`
// the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request
Source int64 `json:"source,omitempty"`
// the pipeline step
Step string `json:"step"`
Toolchain *ToolchainUsed `json:"toolchain,omitempty"`
@@ -903,8 +903,10 @@ type StepRun struct {
Pipeline string `json:"pipeline"`
// the step is asked again by the runner's rerun
Rerun bool `json:"rerun,omitempty"`
// the source revision of the watched branch the pipeline runs for; absent for a run on request
// the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request
Revision int64 `json:"revision,omitempty"`
// the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request
Source int64 `json:"source,omitempty"`
// the pipeline step
Step string `json:"step"`
Toolchain *ToolchainUsed `json:"toolchain,omitempty"`
@@ -903,8 +903,10 @@ type StepRun struct {
Pipeline string `json:"pipeline"`
// the step is asked again by the runner's rerun
Rerun bool `json:"rerun,omitempty"`
// the source revision of the watched branch the pipeline runs for; absent for a run on request
// the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request
Revision int64 `json:"revision,omitempty"`
// the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request
Source int64 `json:"source,omitempty"`
// the pipeline step
Step string `json:"step"`
Toolchain *ToolchainUsed `json:"toolchain,omitempty"`
@@ -1521,7 +1521,11 @@
},
"revision": {
"type": "integer",
"description": "the source revision of the watched branch the pipeline runs for; absent for a run on request"
"description": "the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request"
},
"source": {
"type": "integer",
"description": "the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request"
},
"fingerprint": {
"type": "string",
@@ -367,6 +367,13 @@
"$ref": "#/defs/BuildEntry"
},
"description": "each entry the step built, by module and fingerprint"
},
"packages": {
"type": "object",
"additionalProperties": {
"type": "string"
},
"description": "the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207)"
}
},
"required": [
@@ -91,6 +91,8 @@ export interface StepEnded {
"node": string;
/** never passed for a build that built nothing */
"outcome": "passed" | "failed" | "nothing to run" | "could not run";
/** the pins the step's entries were built with, each mesh package's version by its module path, the ones the builder took from the registry for a package the entry named with none included; the runner asks the entry's held test steps with them (H-A-207) */
"packages"?: Record<string, string>;
"pipeline": string;
"rerun"?: boolean;
/** why it failed or could not run: the entry not built, the part not heard */
@@ -510,8 +510,10 @@ export interface StepRun {
"pipeline": string;
/** the step is asked again by the runner's rerun */
"rerun"?: boolean;
/** the source revision of the watched branch the pipeline runs for; absent for a run on request */
/** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */
"revision"?: number;
/** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */
"source"?: number;
/** the pipeline step */
"step": "lint" | "unit" | "integration";
"toolchain"?: ToolchainUsed;
@@ -510,8 +510,10 @@ export interface StepRun {
"pipeline": string;
/** the step is asked again by the runner's rerun */
"rerun"?: boolean;
/** the source revision of the watched branch the pipeline runs for; absent for a run on request */
/** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */
"revision"?: number;
/** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */
"source"?: number;
/** the pipeline step */
"step": "lint" | "unit" | "integration";
"toolchain"?: ToolchainUsed;
@@ -510,8 +510,10 @@ export interface StepRun {
"pipeline": string;
/** the step is asked again by the runner's rerun */
"rerun"?: boolean;
/** the source revision of the watched branch the pipeline runs for; absent for a run on request */
/** the catalog revision number of the watched branch the pipeline runs on, the indexer's count, which a build module reads a revision by (design 47 H-A-219); absent for a run on request */
"revision"?: number;
/** the gate's source revision the pipeline's catalog revision was taken in from (design 47 H-A-219); absent for a run on request */
"source"?: number;
/** the pipeline step */
"step": "lint" | "unit" | "integration";
"toolchain"?: ToolchainUsed;
@@ -71,3 +71,28 @@ func TestPlaceLostRunsReadsEachCandidateNewest(t *testing.T) {
t.Errorf("asked %+v, want the candidate's group alone", asked[1])
}
}
// H-A-213 (2), (4): a catalog-revision heard is handed whole to the start of a revision's pipeline; a start that fails is
// an error, so the bus delivers the event again, and a body that is no event is ended.
func TestARevisionHeardIsHandedToItsStart(t *testing.T) {
var got []catalog.CatalogRevisionEvent
fail := false
hear := revisedHear(func(ev catalog.CatalogRevisionEvent) (int, error) {
got = append(got, ev)
if fail {
return 0, errors.New("not read")
}
return 1, nil
}, func(string, ...any) {})
data := []byte(`{"v":1,"kind":"trunk","branch":"main","revision":10,"source":41,"call-kind":"release","call":"release of mesh-sdk 0.1.17"}`)
if err := hear(t.Context(), data); err != nil || len(got) != 1 || got[0].Revision != 10 || got[0].CallKind != "release" || got[0].Source != 41 {
t.Fatalf("heard %+v (%v), want the release revision 10", got, err)
}
fail = true
if err := hear(t.Context(), data); err == nil {
t.Error("a start that failed is not an error, so the event is never heard again")
}
if err := hear(t.Context(), []byte("no event")); err != nil {
t.Errorf("a body that is no event = %v, want it ended", err)
}
}
@@ -98,6 +98,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
var advancer atomic.Pointer[func(string)]
var placer atomic.Pointer[func(catalog.CatalogRevisionEvent) error]
var registrar atomic.Pointer[func(catalog.Registration)]
var reviser atomic.Pointer[func(catalog.CatalogRevisionEvent) (int, error)]
rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) {
if f := resumer.Load(); f != nil {
(*f)(branch)
@@ -115,6 +116,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if f := registrar.Load(); f != nil {
(*f)(reg)
}
}, func(ev catalog.CatalogRevisionEvent) (int, error) {
if f := reviser.Load(); f != nil {
return (*f)(ev)
}
return 0, fmt.Errorf("the revision %d of %s is heard before the runner can start its pipeline", ev.Revision, ev.Branch)
}, logger.Printf)
if err != nil {
return err
@@ -142,7 +148,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
logger.Printf("widens: each read bound %s, %d catalog revisions kept", readBound, revisionsKept)
hearer.Widens = widenerOf(calls.Call, readBound, revisionsKept, logger.Printf).Widens
widener := widenerOf(calls.Call, readBound, revisionsKept, logger.Printf)
hearer.Widens = widener.Widens
hearer.LogLast = stepLog{ctx: ctx, js: js, module: cred.Module}.Last
hearer.Where = routerRecordOf(cred.Module, calls.Call, calls.Ask)
hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf}
@@ -154,6 +161,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
hearer.AskSteps = starter.AskSteps
hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) }
hearer.AskPinHeld = func(key, module string, pins map[string]string) {
if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil {
logger.Printf("the steps of %s's %s that waited for its pins are not asked: %v", key, module, err)
}
}
hearer.Release = func(key string) {
if _, err := starter.Release(hearer.Store, key, rv.Catalog.View); err != nil {
logger.Printf("the movers of %s waiting for a build are not placed: %v", key, err)
@@ -197,6 +209,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
}
registrar.Store(&register)
// A release's, a take-in again's or a recompute's revision starts one pipeline, its revision read by its number
// (H-A-213 (2), (4)).
revise := func(ev catalog.CatalogRevisionEvent) (int, error) { return starter.Revised(ev, widener.Revision) }
reviser.Store(&revise)
starter.Store = store
// The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in
// the state queue, placed at the start from it and on every end.
@@ -101,7 +101,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
}
}
answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs})
answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 1, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries})
answer("mesh.seat.mesh-catalog.call.revision.*", catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 100, Kind: "trunk", Branch: "main", Source: 2}, Entries: entries})
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
// The router of the operator's asks holds the seat operator-channel; its record of each ask is in its state asks.
const router = "operator-router"
@@ -218,7 +218,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
}
take := func(source int64, movers ...string) {
t.Helper()
ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: 1, Call: fmt.Sprintf("%016x", source)}
ev := catalog.TakeInTaken{Branch: "main", Source: source, Revision: source, Call: fmt.Sprintf("%016x", source)}
for _, m := range movers {
ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m})
}
@@ -1,6 +1,7 @@
package main
import (
"context"
"encoding/json"
"github.com/nats-io/nats.go/jetstream"
@@ -11,10 +12,10 @@ import (
"mesh-delivery-pipeline/internal/pipeline"
)
// hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one
// pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is
// written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard
// after a stop.
// hearPublished hears each package-published (design 47 H §746, §747): in the running pipeline of the revision that
// published it, the movers waiting for it are placed; it starts no pipeline (a release's revision does, H-A-213 (2)). The
// message is acknowledged once that is written, on a goroutine of its own, so a slow write holds no other event; the
// durable consumer delivers a publish heard after a stop.
func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error {
return hub.Handle(builds.SubjectPackagePublished, func(m jetstream.Msg) {
var ev builds.PackagePublished
@@ -24,18 +25,38 @@ func hearPublished(hub handlers, starter *pipeline.Starter, v *pipeline.Views, l
return
}
go func() {
n, err := starter.Published(ev, v.View)
if err != nil {
logf("the publish of %s %s on %s: %d pipelines started, then %v; heard again", ev.Artifact, ev.Version, ev.Branch, n, err)
if err := starter.Published(ev, v.View); err != nil {
logf("the publish of %s %s on %s: %v; heard again", ev.Artifact, ev.Version, ev.Branch, err)
_ = m.Nak()
return
}
logf("the publish of %s %s on %s started %d pipelines", ev.Artifact, ev.Version, ev.Branch, n)
logf("the publish of %s %s on %s is heard", ev.Artifact, ev.Version, ev.Branch)
_ = m.Ack()
}()
})
}
// revisedHear is the handler of catalog-revision that starts the pipeline of a revision whose call kind is not a
// take-in's (design 47 H-A-212, H-A-213 (2), H-A-217): every other revision is left to the views that hear it. An error
// has the event heard again; a second start of its key writes nothing.
func revisedHear(revised func(ev catalog.CatalogRevisionEvent) (int, error), logf func(string, ...any)) func(context.Context, []byte) error {
return func(_ context.Context, data []byte) error {
var ev catalog.CatalogRevisionEvent
if err := json.Unmarshal(data, &ev); err != nil {
logf("a catalog-revision is not read: %v", err)
return nil
}
n, err := revised(ev)
if err != nil {
return err
}
if n > 0 {
logf("the revision %d of %s (%s, %s) started %d pipeline", ev.Revision, ev.Branch, ev.CallKind, ev.Call, n)
}
return nil
}
}
// hearTakenIn starts the pipeline of each take-in.taken heard (design 47 H §746): one per take-in, the message acknowledged
// once it is written, and heard again when a write failed, so the key already written starts nothing twice.
func hearTakenIn(hub handlers, starter *pipeline.Starter, v *pipeline.Views, logf func(string, ...any)) error {
@@ -128,7 +128,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
if bs.Shared != nil {
shared = bs.Shared(m.Data())
}
_, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared)
var pins map[string]string
if bs.Pins != nil {
pins = bs.Pins(m.Data())
}
_, err = h.OnStepEndedPinned(bs, m.Subject(), ev, shared, pins)
switch {
case errors.Is(err, pipeline.ErrRefused):
logf("%v", err)
@@ -37,7 +37,7 @@ import (
// ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step
// that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a
// run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed;
// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended,
// a run that keeps saying its progress past its bound is not ended,
// and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the
// same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its
// other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and
@@ -88,7 +88,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil {
askStream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy})
if err != nil {
t.Fatal(err)
}
@@ -116,6 +117,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
entries := []catalog.CatalogEntry{
{Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app",
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25"},
Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
{Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}},
{Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
@@ -325,12 +327,23 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
// One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its
// ask is taken (A-469b), its holder present and nothing failing it (A-466y).
const p3 = "main.3"
// A pipeline is keyed by its catalog revision (H-A-213 (4)): source 3's take-in of revision 1 is main.1.
const p3 = "main.1"
takeIn("app", "c1", "src2:aa", 3, 1)
placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747)
passBuild(p3, "app", "c1", "src2:aa")
unitRun, unit := runOfStep(p3, "app:program", "unit")
integRun, integ := runOfStep(p3, "app:program", "integration")
// A test ask fetches with the entry's pins, as the builder gives them the build (H-A-203).
last, err := askStream.GetLastMsgForSubject(ctx, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner")
if err != nil {
t.Fatal(err)
}
var testAsk nodebuildgo.StepAsk
if err := json.Unmarshal(last.Data, &testAsk); err != nil || testAsk.Step == nil ||
testAsk.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25" || testAsk.Step.Revision != 1 || testAsk.Step.Source != 3 {
t.Fatalf("the last test ask = %s (%v), want its fetch to carry the entry's pin, revision 1 and source 3 (H-A-219)", last.Data, err)
}
if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 {
t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers)
}
@@ -471,39 +484,20 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
// A second revision with the same digests asks its own runs, so a failed test runs again (§724).
revision.Store(2)
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4})
const p4, p5 = "main.4", "main.5"
const p4 = "main.2"
takeIn("app", "c1", "src2:aa", 4, 2)
placed(p4)
run4, r4 := runOfStep(p4, "app:program", "integration")
if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask {
t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask)
}
// A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads
// the run, the first is cancelled and leaves it, and the run's ask is not dropped.
takeIn("app", "c1", "src2:aa", 5, 2)
placed(p5)
waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 })
if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" {
t.Fatalf("cancel: %s", e)
}
waitFor(t, ctx, logs, func() bool {
r, _ := runOf(run4)
return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5
})
if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask {
t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r)
}
mu.Lock()
dropped := append([]string(nil), drops...)
mu.Unlock()
if len(dropped) != 0 {
t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String())
}
// A replaced reader leaving the ask to the other readers of another pipeline (§726) is the unit tests': one revision is
// one pipeline (H-A-213 (4)).
// A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720).
taken(r4.Ask)
waitFor(t, ctx, logs, func() bool {
return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun &&
step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun
return step(read(p4), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun &&
step(read(p4), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun
})
if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") {
t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r)
@@ -516,8 +510,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
// A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not
// asked (§535, §647; A-470g, A-470h).
const web = "main.7"
takeIn("web", "w1", "src2:ww", 7, 2)
revision.Store(3)
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 7})
const web = "main.3"
takeIn("web", "w1", "src2:ww", 7, 3)
waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 })
if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" {
t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e)
@@ -568,10 +564,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
t.Fatal(err)
}
}
revision.Store(3)
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5})
const p6 = "main.6"
takeIn("app", "c1", "src2:aa", 6, 3)
revision.Store(4)
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 4, Kind: "trunk", Branch: "main", Source: 6})
const p6 = "main.4"
takeIn("app", "c1", "src2:aa", 6, 4)
waitFor(t, ctx, logs, func() bool {
s := step(read(p6), "app:program", "unit")
return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib")
@@ -98,7 +98,7 @@ func (h *holderView) Holders(seatName string) ([]string, bool) {
// after, then start hub once.
func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State,
call meshcatalogclient.Call, onReady, onAdvance func(branch string), candidate func(ctx context.Context, ev catalog.CatalogRevisionEvent) error,
registered func(reg catalog.Registration), logf func(string, ...any)) (*runnerViews, error) {
registered func(reg catalog.Registration), revised func(ev catalog.CatalogRevisionEvent) (int, error), logf func(string, ...any)) (*runnerViews, error) {
findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf)
if err != nil {
return nil, err
@@ -153,6 +153,11 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
if err := meshcatalogclient.WireCandidatesProcess(reader, candidate); err != nil {
return nil, err
}
// A trunk revision of a release, a take-in again or a recompute starts one pipeline of its own, read by its number
// (H-A-212, H-A-213 (2), H-A-217), heard on the same event's one subscription.
if err := reader.AdoptEvent(catalog.Seat, catalog.EventCatalogRevision, revisedHear(revised, logf)); err != nil {
return nil, err
}
reader.Start(ctx, js, cred, status)
return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings, Holders: &holderView{reader: reader, catalog: cv, logf: logf}}, nil
}
@@ -166,7 +166,8 @@ func catalogRevision(call func(string, any) (json.RawMessage, error), kept *revi
if h, ok := kept.get(branch, a.Revision.Revision); ok {
return h, nil
}
half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{}}
half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{},
Call: a.Revision.Call, Movers: a.Revision.Movers}
in.Revision = a.Revision.Revision // the pages of one revision, though a newer one is said between them
}
for _, e := range a.Entries {
@@ -231,11 +231,11 @@ func (a *Ager) newest(p seat.Pipeline) (bool, error) {
if err != nil {
return false, fmt.Errorf("the pipelines of %s are not listed: %w", p.Branch, err)
}
// Newest for one of its movers: no later pipeline of the branch moves it.
// Newest for one of its movers: no pipeline of a later revision of the branch moves it (H-A-213 (4)).
for _, m := range p.Movers {
newer := false
for _, k := range pkeys {
if q, _, err := a.Pipelines.Get(k); err == nil && q.Source > p.Source && Moves(q, m.Module) {
if q, _, err := a.Pipelines.Get(k); err == nil && keyRevision(Begin{Revision: q.Revision, Source: q.Source}) > keyRevision(Begin{Revision: p.Revision, Source: p.Source}) && Moves(q, m.Module) {
newer = true
break
}
@@ -142,3 +142,23 @@ func TestOneStepEndAgesAtMostTwentyEntries(t *testing.T) {
t.Fatalf("%d outputs deleted, sequence %d; want 20 and 20", len(f.outputs), f.aged["7d"])
}
}
// H-A-213 (4): a branch's newest pipeline for a mover is the one of the highest revision number, never of the highest source:
// a release revision of the same source is newer than its take-in's.
func TestTheNewestPipelineIsOfTheHighestRevision(t *testing.T) {
f := newAgedFixture()
for key, p := range map[string]seat.Pipeline{
"main.41": {Pipeline: "main.41", Branch: "main", Source: 41, Revision: 41, Movers: []seat.Mover{{Module: "app"}}},
"main.42": {Pipeline: "main.42", Branch: "main", Source: 41, Revision: 42, Movers: []seat.Mover{{Module: "app"}}},
} {
f.pipes.memStore[key] = struct {
p seat.Pipeline
rev uint64
}{p, 1}
}
for key, want := range map[string]bool{"main.41": false, "main.42": true} {
if got, err := f.ager.newest(f.pipes.memStore[key].p); got != want || err != nil {
t.Errorf("%s newest = %v (%v), want %v", key, got, err, want)
}
}
}
@@ -113,9 +113,19 @@ func TestAReleasedVersionIsTheMoversVersion(t *testing.T) {
other := store[pkey]
other.p.Pipeline, other.p.Branch, other.p.Movers = "beta.5", "beta", append([]seat.Mover(nil), other.p.Movers...)
store["beta.5"] = other
// A pipeline is keyed by its revision (H-A-213 (4)): main.8, a later revision of source 5, is written too; main.6, of
// source 6, is not.
for key, source := range map[string]int64{"main.8": 5, "main.6": 6} {
again := store[pkey]
again.p.Pipeline, again.p.Source, again.p.Movers = key, source, append([]seat.Mover(nil), again.p.Movers...)
store[key] = again
}
ev := catalog.VersionReleasedEvent{Module: "app", Version: "0.4.1", Source: 5}
if n, err := h.Released(ev, "main"); n != 1 || err != nil {
t.Fatalf("released wrote %d (%v), want app's mover of main.5", n, err)
if n, err := h.Released(ev, "main"); n != 2 || err != nil {
t.Fatalf("released wrote %d (%v), want app's mover of main.5 and main.8", n, err)
}
if m, _ := MoverOf(store["main.6"].p, "app"); m.Version != "" {
t.Errorf("main.6's mover = %+v, want no version: another source", m)
}
if m, _ := MoverOf(store[pkey].p, "app"); m.Version != "0.4.1" {
t.Errorf("app's mover = %+v, want version 0.4.1", m)
@@ -70,14 +70,14 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) {
if n, err := s.TakenIn(ev, view); n != 1 || err != nil {
t.Fatalf("the take-in started %d (%v)", n, err)
}
p, _, _ := mem.Get("main.4")
p, _, _ := mem.Get("main.7")
if !placed(p, "mesh-sdk") || !placed(p, "builder") || placed(p, "app") || placed(p, "tool") {
t.Fatalf("placed at the start: %+v, want mesh-sdk and builder only", p.Steps)
}
if n, err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); n != 0 || err != nil {
t.Fatalf("the publish started %d (%v), want none", n, err)
if err := s.Published(builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Version: "0.2.0"}, view); err != nil {
t.Fatalf("the publish is not heard: %v", err)
}
p, _, _ = mem.Get("main.4")
p, _, _ = mem.Get("main.7")
if sdk, _ := MoverOf(p, "mesh-sdk"); !placed(p, "app") || placed(p, "tool") || len(mem) != 1 || !slices.Contains(sdk.Published, "mesh-sdk:go") {
t.Fatalf("after mesh-sdk's publish: %d pipelines, app placed %v, tool placed %v, mesh-sdk %+v", len(mem), placed(p, "app"), placed(p, "tool"), sdk)
}
@@ -87,22 +87,22 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) {
b := stepsOf(p, "builder")[0]
bs := stepSeatOf(t, builds.Seat)
end := func(outcome string) StepEnded {
e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.4","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`))
e, err := bs.Decode([]byte(`{"id":"` + b.Ask + `","pipeline":"main.7","step":"build","node":"n1","outcome":"` + outcome + `","entries":[{"module":"builder","commit":"c2","fingerprint":"src2:bb"}]}`))
if err != nil {
t.Fatal(err)
}
return e
}
before := mem["main.4"]
before := mem["main.7"]
if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("passed")); err != nil {
t.Fatal(err)
}
if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning {
if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeRunning {
t.Errorf("after builder's build passed, tool's steps = %+v, want placed and asked", stepsOf(p, "tool"))
}
// The same pipeline, builder's build failed: tool is placed with every step not asked, each saying its own step-ended
// naming the dependency (H-A-101 (2)).
mem["main.4"] = before
mem["main.7"] = before
var notAsked []seat.StepEvent
s.Say = func(subject string, body any) error {
if ev, ok := body.(seat.StepEvent); ok && strings.Contains(subject, ".step-ended.tool.") {
@@ -113,7 +113,7 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) {
if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", end("failed")); err != nil {
t.Fatal(err)
}
if p, _, _ = mem.Get("main.4"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked {
if p, _, _ = mem.Get("main.7"); !placed(p, "tool") || stepsOf(p, "tool")[0].Outcome != seat.StepResultOutcomeNotAsked {
t.Errorf("after builder's build failed, tool's steps = %+v, want placed not asked", stepsOf(p, "tool"))
}
if len(notAsked) == 0 || len(notAsked) != len(slices.DeleteFunc(stepsOf(p, "tool"), func(st seat.StepResult) bool { return st.Outcome != seat.StepResultOutcomeNotAsked })) {
@@ -51,7 +51,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing.
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
t.Fatalf("the take-in started %d (%v)", n, err)
}
p, _, _ := store.Get("main.947")
p, _, _ := store.Get("main.357")
if p.Revision != 357 || len(p.Steps) != 0 || len(asks.asks) != 0 {
t.Fatalf("a take-in of revision 357 with the view at 356 started %+v with %d asks, want revision 357 and no step", p, len(asks.asks))
}
@@ -60,7 +60,7 @@ func TestATakeInStartsItsPipelinesAtItsOwnRevisionAndWaitsForTheView(t *testing.
}
viewRevision = 357
n, err := s.Waiting(store.memStore, "main", view)
p, _, _ = store.Get("main.947")
p, _, _ = store.Get("main.357")
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Revision != 357 {
t.Errorf("the view reaching 357 placed %d (%v): %d steps, %d asks at revision %d", n, err, len(p.Steps), len(asks.asks), p.Revision)
}
@@ -87,7 +87,7 @@ func TestAViewThatReachesTheRevisionDuringTheStartsPlacesTheHeldBackPipelines(t
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
t.Fatalf("started %d (%v)", n, err)
}
if p, _, _ := store.Get("main.947"); len(p.Steps) == 0 || len(asks.asks) == 0 {
if p, _, _ := store.Get("main.357"); len(p.Steps) == 0 || len(asks.asks) == 0 {
t.Errorf("the pipeline has %d steps and %d asks after the view reached 357 during the start, want them placed", len(p.Steps), len(asks.asks))
}
}
@@ -3,7 +3,10 @@ package pipeline
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"maps"
"slices"
"sort"
"strconv"
"strings"
@@ -140,6 +143,13 @@ func place(p seat.Pipeline, m seat.Mover, reg catalog.Registration, entries map[
steps = append(steps, s)
continue
}
// A Go step is asked only once every pin of its entry has a version (H-A-206): until then it waits on the
// publish, as the build does (H-A-165), and the publish re-pins the entry and moves the mover (H-A-204).
if path := unpinnedPackage(entry); path != "" && language == sdkstep.AskGo {
s.Seat, s.Outcome, s.Started, s.Why = bs, seat.StepResultOutcomeWaiting, &now, PinWait+path
steps = append(steps, s)
continue
}
s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now
s.Ask = AskID(startOf(p, now), p.Pipeline, m.Module, a.Ref, step)
body := stepAsk(p, m, entry, a, language, step, s.Ask, asker, now)
@@ -227,6 +237,132 @@ type Asked struct {
Commit string
}
// PinWait opens the why of a Go step held because its entry names a package with no version yet (H-A-206), the build's
// own wait form (H-A-165); the package's path follows.
const PinWait = "waits on the publish of "
// unpinnedPackage is the first package, in path order, the entry names with no version, empty when every one has one.
func unpinnedPackage(e catalog.CatalogEntry) string {
for _, path := range slices.Sorted(maps.Keys(e.Packages)) {
if e.Packages[path] == "" {
return path
}
}
return ""
}
// pinHeld says whether a step waits for its entry's pins (H-A-206).
func pinHeld(s seat.StepResult) bool {
return s.Outcome == seat.StepResultOutcomeWaiting && strings.HasPrefix(s.Why, PinWait)
}
// endPinHeld ends each of build's mover's steps waiting for its entry's pins (H-A-206, H-A-207): as the build ended when
// it did not pass, so no step waits on a publish the build no longer waits for; could not run, loud, when the build passed
// and its end names no pins to run them with. It answers the steps it ended.
func endPinHeld(p *seat.Pipeline, build seat.StepResult, pins map[string]string, now time.Time) []int {
outcome, why := build.Outcome, "the build ended "+build.Outcome+" while this step waited: %s; "+build.Why
if stepPassed(build) {
if len(pins) > 0 {
return nil
}
outcome, why = seat.StepResultOutcomeCouldNotRun, "the build passed, but its end names no pins to run this step with (H-A-207); it %s"
}
var ended []int
for i := range p.Steps {
st := &p.Steps[i]
if st.Module != build.Module || !pinHeld(*st) {
continue
}
st.Outcome, st.Why, st.Ended = outcome, fmt.Sprintf(why, st.Why), &now
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
ended = append(ended, i)
}
return ended
}
// readRevision is revision's read, an error when none is given.
func readRevision(revision func(branch string, revision int64) (Half, error), branch string, n int64) (Half, error) {
if revision == nil {
return Half{}, errors.New("the runner reads no catalog revision")
}
return revision(branch, n)
}
// heldOn says whether module has a step of p waiting for its entry's pins.
func heldOn(p seat.Pipeline, module string) bool {
return slices.ContainsFunc(p.Steps, func(st seat.StepResult) bool { return st.Module == module && pinHeld(st) })
}
// AskPinHeld asks the steps of pipeline key's mover module that waited for its entry's pins (H-A-206) with the pins its
// build passed with (H-A-207): each running under its own ask, its fetch the entry's as Place builds it with those pins.
// The entry is read at the pipeline's own catalog revision through revision, which keeps what it read (H-A-205), never
// the branch's newest: a revision that cannot be read, or whose entry is of another fingerprint than the mover's, or that
// names no artifact of the step, ends the step could not run, saying so. It answers how many it asked.
func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]string, revision func(branch string, revision int64) (Half, error)) (int, error) {
for {
p, rev, err := store.Get(key)
if err != nil {
return 0, fmt.Errorf("the pipeline %s is not read: %w", key, err)
}
m, moves := MoverOf(p, module)
if !moves || !heldOn(p, module) {
return 0, nil
}
var entry catalog.CatalogEntry
unread := ""
switch half, err := readRevision(revision, p.Branch, p.Revision); {
case err != nil:
unread = fmt.Sprintf("revision %d of %s, the pipeline's, cannot be read for its entry: %v", p.Revision, p.Branch, err)
case half.Entries[module].Fp != m.Fingerprint:
unread = fmt.Sprintf("revision %d of %s holds %s at %q, not the mover's %s", p.Revision, p.Branch, module, half.Entries[module].Fp, m.Fingerprint)
default:
entry = half.Entries[module]
}
entry.Packages = pins
now := s.now()
var asks []Asked
var ended []int
for i := range p.Steps {
st := &p.Steps[i]
if st.Module != module || !pinHeld(*st) {
continue
}
j := slices.IndexFunc(entry.Artifacts, func(a catalog.Artifact) bool { return a.Ref == st.Artifact })
language, err := sdkstep.AskLanguageOfSeat(st.Seat)
if unread != "" || j < 0 || err != nil {
why := unread
if why == "" {
why = fmt.Sprintf("revision %d of %s names no artifact %s of %s to ask it of", p.Revision, p.Branch, st.Artifact, module)
}
st.Outcome, st.Ended = seat.StepResultOutcomeCouldNotRun, &now
st.Why = "its build passed with its pins, but " + why + " (H-A-207, H-A-205)"
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
ended = append(ended, i)
continue
}
a := entry.Artifacts[j]
st.Outcome, st.Why, st.Started = seat.StepResultOutcomeRunning, "", &now
st.Ask = AskID(startOf(p, now), p.Pipeline, module, a.Ref, st.Step)
ask := Asked{Seat: st.Seat, ID: st.Ask, Ask: stepAsk(p, m, entry, a, language, st.Step, st.Ask, s.Name, now), Toolchain: a.Language, Commit: m.Commit}
if a.Package != nil {
ask.Package = a.Package.Name
}
asks = append(asks, ask)
}
if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return 0, fmt.Errorf("the pipeline %s is not written: %w", key, err)
}
s.logf("%s: %s's build passed with its pins: %d steps that waited for them are asked, %d could not run (H-A-207)", key, module, len(asks), len(ended))
s.sayNotAsked(p, ended)
if s.Steps == nil || len(asks) == 0 {
return 0, nil
}
return len(asks), s.askSteps(key, asks)
}
}
func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
// A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the
// artifact's named paths in it when the entry has them.
@@ -234,9 +370,21 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar
if paths := artifactPaths(e, a.Ref); len(paths) > 0 {
fetch.Paths = paths
}
// The entry's pins, the same the builder gives the build (H-A-203, H-A-183): a test runs against what the build was
// built against, and the Go build module refuses a requirement of another repository with no pin (H-A-192). A
// package the entry names with no pin yet is left out, as the builder pins it only from its publish.
for path, version := range e.Packages {
if version == "" {
continue
}
if fetch.Packages == nil {
fetch.Packages = map[string]string{}
}
fetch.Packages[path] = version
}
return nodebuildgo.StepAsk{V: 1, ID: id, Asker: asker, Kind: "step", Language: language, At: &now,
Step: &nodebuildgo.StepRun{Artifact: a.Ref, Branch: p.Branch, Commit: m.Commit, Fetch: fetch, Fingerprint: m.Fingerprint,
Module: m.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}}
Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}}
}
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its
@@ -2,6 +2,8 @@ package pipeline
import (
"encoding/json"
"fmt"
"reflect"
"strconv"
"strings"
"testing"
@@ -44,7 +46,7 @@ func placeFixture() (catalog.Registration, map[string]catalog.CatalogEntry) {
Artifacts: map[string]catalog.StepStatuses{"tools": {Lint: "bypass"}},
}}
return reg, map[string]catalog.CatalogEntry{
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"},
"app": {Module: "app", Path: "modules/app", Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25"}, Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"},
{Ref: "app:web", Language: "rust"}}, Inputs: &catalog.EntryInputs{Paths: map[string][]string{"app:program": {"cmd/app"}}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
}
@@ -100,10 +102,131 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) {
a := asks.asks[1].(nodebuildgo.StepAsk)
if asks.at[1] != "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner" || a.Kind != "step" || a.Asker != "the-runner" || a.ID != p.Steps[1].Ask || asks.ids[1] != a.ID ||
a.Step.Commit != "c1" || a.Step.Fingerprint != "src2:aa" || a.Step.Pipeline != "main.4" || a.Step.Artifact != "app:program" ||
a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Revision != 4 ||
a.Step.Step != "lint" || a.Step.Fetch.Repository != "o/app" || a.Step.Fetch.Dir != "modules/app" || a.Step.Source != 4 ||
len(a.Step.Fetch.Paths) != 1 || a.Step.Fetch.Paths[0] != "cmd/app" {
t.Errorf("the ask = %+v at %s", a, asks.at[1])
}
// Every step ask fetches with the entry's pins, as the builder gives them the build (H-A-203, H-A-192).
for _, x := range asks.asks[1:] {
if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && (len(b.Step.Fetch.Packages) != 1 || b.Step.Fetch.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25") {
t.Errorf("the %s ask of %s fetches the pins %v, want the entry's", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages)
}
}
// An entry naming a package with no version yet asks no Go step: each waits on its publish, as the build does, said in
// the journal (H-A-206); the build is still asked.
waiting := map[string]catalog.CatalogEntry{}
for k, e := range entries {
waiting[k] = e
}
held := waiting["app"]
held.Packages = map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": ""}
held.Fp = "src2:ab"
waiting["app"] = held
var journal []string
heldAsks := &stepAsks{}
hs := &Starter{Pipelines: store, Say: func(string, any) error { return nil }, Steps: heldAsks, Name: "the-runner",
Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) },
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" }}
if ok, err := hs.Start(Begin{Branch: "main", Source: 5, Revision: 3, Entries: waiting, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}}}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
for _, x := range heldAsks.asks {
if b, ok := x.(nodebuildgo.StepAsk); ok {
t.Errorf("the %s of %s is asked with a package unpinned", b.Step.Step, b.Step.Artifact)
}
}
var hp seat.Pipeline
if err := json.Unmarshal(store["main.3"], &hp); err != nil {
t.Fatal(err)
}
if st := hp.Steps[1]; st.Step != "lint" || st.Outcome != seat.StepResultOutcomeWaiting || st.Ask != "" || st.Why != "waits on the publish of git.novox.be/novox/x/go" {
t.Errorf("the held lint step = %+v; want it waiting on the publish", st)
}
// The build ending without passing past its bound ends each held step as it ended (H-A-206).
failed := hp.Steps[0]
failed.Outcome, failed.Why = seat.StepResultOutcomeFailed, "package git.novox.be/novox/x/go has no released version"
failedP := hp
failedP.Steps = append([]seat.StepResult(nil), hp.Steps...)
if ended := endPinHeld(&failedP, failed, nil, time.Now()); len(ended) == 0 || failedP.Steps[1].Outcome != seat.StepResultOutcomeFailed || failedP.Steps[1].Ended == nil {
t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", failedP.Steps[1], len(ended))
}
// A build passed whose end names no pins ends them could not run, loud (H-A-207).
passed := hp.Steps[0]
passed.Outcome = seat.StepResultOutcomePassed
bare := hp
bare.Steps = append([]seat.StepResult(nil), hp.Steps...)
if ended := endPinHeld(&bare, passed, nil, time.Now()); len(ended) == 0 || bare.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun ||
!strings.Contains(bare.Steps[1].Why, "names no pins") {
t.Errorf("after a build passed naming no pins the held lint step = %+v; want could not run naming it", bare.Steps[1])
}
// A build passed naming its pins asks the held steps with them: its end heard, the runner asks each with the pins the
// build was built with (H-A-207).
pinsBuilt := map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": "v0.1.4"}
ms := memStore{}
ms["main.5"] = struct {
p seat.Pipeline
rev uint64
}{hp, 1}
// The entry is read at the pipeline's own revision (3), never the newer revision 4 that moved the module (H-A-205).
moved := held
moved.Fp, moved.Path = "src2:ac", "modules/moved"
revisions := func(branch string, n int64) (Half, error) {
switch n {
case 3:
return Half{Revision: 3, Entries: waiting}, nil
case 0, 4:
return Half{Revision: 4, Entries: map[string]catalog.CatalogEntry{"app": moved}}, nil
}
return Half{}, fmt.Errorf("no revision %d", n)
}
var askedWith map[string]string
h := &Hearer{Store: ms, Say: func(string, any) error { return nil }, Name: "the-runner",
AskPinHeld: func(key, module string, pins map[string]string) {
askedWith = pins
if _, err := hs.AskPinHeld(ms, key, module, pins, revisions); err != nil {
t.Errorf("ask held: %v", err)
}
}}
buildEnd := StepEnded{ID: hp.Steps[0].Ask, Pipeline: "main.5", Step: "build", Node: "n1", Outcome: seat.StepResultOutcomePassed,
Module: "app", Commit: "c1", Fingerprint: "src2:ab"}
if _, err := h.OnStepEndedPinned(StepSeats[3], StepSeats[3].Prefix+"n1", buildEnd, nil, pinsBuilt); err != nil {
t.Fatalf("the build's end: %v", err)
}
if !reflect.DeepEqual(askedWith, pinsBuilt) {
t.Errorf("the held steps were asked with %v; want the build's pins", askedWith)
}
var askedHeld int
for _, x := range heldAsks.asks {
if b, ok := x.(nodebuildgo.StepAsk); ok {
askedHeld++
if !reflect.DeepEqual(b.Step.Fetch.Packages, pinsBuilt) {
t.Errorf("the %s of %s fetches %v; want the build's pins", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages)
}
if b.Step.Fetch.Dir != "modules/app" {
t.Errorf("the %s of %s fetches %s; want the pipeline's revision's directory", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Dir)
}
}
}
after, _, _ := ms.Get("main.5")
if askedHeld == 0 || after.Steps[1].Outcome == seat.StepResultOutcomeWaiting || after.Steps[1].Ask == "" {
t.Errorf("after the build passed %d held steps were asked, lint = %+v; want each asked", askedHeld, after.Steps[1])
}
// A mover whose fingerprint the pipeline's revision does not hold ends its held steps could not run, naming it.
other := hp
other.Pipeline, other.Steps, other.Movers = "main.4", append([]seat.StepResult(nil), hp.Steps...), []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:zz", Repository: "o/app"}}
ms["main.4"] = struct {
p seat.Pipeline
rev uint64
}{other, 1}
if n, err := hs.AskPinHeld(ms, "main.4", "app", pinsBuilt, revisions); n != 0 || err != nil {
t.Errorf("asked %d held steps of a mover its revision does not hold (%v); want none", n, err)
}
if o, _, _ := ms.Get("main.4"); o.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(o.Steps[1].Why, "not the mover's src2:zz") {
t.Errorf("the held lint step of a moved fingerprint = %+v; want could not run naming it", o.Steps[1])
}
if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") {
t.Errorf("the journal says %q; want the held step named", journal)
}
// An artifact that names no paths still fetches its module alone: the directory with no paths (principle 10, A-447).
for _, x := range asks.asks[2:] {
if b, ok := x.(nodebuildgo.StepAsk); ok && b.Step != nil && b.Step.Artifact == "app:tools" && (b.Step.Fetch.Dir != "modules/app" || len(b.Step.Fetch.Paths) != 0) {
@@ -409,7 +532,8 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) {
Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }}
start := func(source, revision int64) seat.Pipeline {
t.Helper()
if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil {
// Keyed by its source, so pipelines of one revision are several (one kept under its source before H-A-213 (4)).
if ok, err := s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil {
t.Fatalf("start %d = %v, %v", source, ok, err)
}
p, _, _ := store.Get(Key("main", source))
@@ -428,6 +552,12 @@ func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) {
if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" {
t.Fatalf("the test steps asked = %q, want the program's three alone", got)
}
// Every step ask names the catalog revision in revision and the gate's source in source (H-A-219).
for _, a := range asks.asks {
if b, ok := a.(nodebuildgo.StepAsk); ok && (b.Step.Revision != 9 || b.Step.Source != 4) {
t.Errorf("the %s ask of %s names revision %d and source %d, want 9 and 4", b.Step.Step, b.Step.Artifact, b.Step.Revision, b.Step.Source)
}
}
for _, level := range buildSteps {
asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level)
if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil {
@@ -628,13 +758,13 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) {
}
}
}
p, _, _ := mem.Get("main.4")
p, _, _ := mem.Get("main.7")
if validates != 1 || compositions != 0 || stepNamed(p, "", seat.StepResultStepComposition).Outcome != seat.StepResultOutcomeWaiting {
t.Fatalf("placed: %d validate and %d composition asks, steps %+v; want one validate, no composition yet", validates, compositions, p.Steps)
}
h := &Hearer{Store: mem, Name: "the-runner", Say: func(string, any) error { return nil }, AskSteps: s.AskSteps}
bs := stepSeatOf(t, wants.Seat)
raw := `{"id":"` + validateAsk + `","pipeline":"main.4","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}`
raw := `{"id":"` + validateAsk + `","pipeline":"main.7","step":"validate","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"` + outcome + `","said":"manifest-refused: app names an unknown field"}`
ev, err := bs.Decode([]byte(raw))
if err != nil {
t.Fatal(err)
@@ -643,7 +773,7 @@ func TestValidateComesBeforeCompositionAndAFailureAsksNone(t *testing.T) {
if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil {
t.Fatal(err)
}
p, _, _ = mem.Get("main.4")
p, _, _ = mem.Get("main.7")
for _, st := range p.Steps {
if st.Step == seat.StepResultStepValidate && st.Outcome != outcome {
t.Errorf("%s: %s's validate is %s, want every mover's read from the one run", outcome, st.Module, st.Outcome)
@@ -96,7 +96,7 @@ func queueFixture() (*Starter, memStore, memQueue, *[]string, *[]seat.Wait, *ste
func takeIn(t *testing.T, s *Starter, view func(string) (Half, bool)) func(source int64, modules ...string) {
return func(source int64, modules ...string) {
t.Helper()
ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7}
ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: source}
for _, m := range modules {
ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m})
}
@@ -99,6 +99,10 @@ type Half struct {
Revision int64
Source int64
Entries map[string]catalog.CatalogEntry
// Call and Movers are the revision's own, as the revision verb answers them (H-A-213): what made it and its movers;
// a view's half names none.
Call string
Movers []catalog.TakeInEntry
}
// Views is the runner's views of the catalog half, one per watched branch the registrations name (design 47 G §21,
@@ -3,7 +3,6 @@ package pipeline
import (
"errors"
"fmt"
"strconv"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/keys"
@@ -21,12 +20,13 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int,
if branch != "" {
b = keys.EscapeName(branch)
}
source := strconv.FormatInt(ev.Source, 10)
// A pipeline is keyed by its revision (H-A-213 (4)), so the pipelines of a source are those naming it, of either key
// form a source's pipelines were kept under.
var pkeys []string
for _, f := range []string{b + "." + source, b + "." + source + ".published.*.*"} {
for _, f := range []string{b + ".*", b + ".*.published.*.*"} {
k, err := h.Store.Keys(f)
if err != nil {
return 0, fmt.Errorf("the pipelines of source %s are not listed: %w", source, err)
return 0, fmt.Errorf("the pipelines of source %d are not listed: %w", ev.Source, err)
}
pkeys = append(pkeys, k...)
}
@@ -37,6 +37,9 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int,
if err != nil {
return wrote, fmt.Errorf("the pipeline %s is not read: %w", key, err)
}
if p.Source != ev.Source || p.Run != "" {
break
}
i := -1
for j, m := range p.Movers {
if m.Module == ev.Module {
@@ -4,7 +4,6 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"sort"
"strconv"
@@ -28,7 +27,7 @@ type Created interface {
Create(key string, value []byte) error
}
// Begin is what a start names: the pipeline's key (watched branch and source revision, design 47 H §746), the take-in's
// Begin is what a start names: the pipeline's key (watched branch and catalog revision, design 47 H §746, H-A-213), the take-in's
// call and catalog graph revision, its movers, and the catalog half of that branch.
type Begin struct {
Branch string
@@ -39,28 +38,30 @@ type Begin struct {
Movers []seat.Mover
// Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them.
Entries map[string]catalog.CatalogEntry
// Published is the package's publish that started the pipeline, nil for any other start (§747).
Published *seat.Published
// Key is the pipeline's key when it is not <branch>.<source> (a publish's, §747); empty for a take-in's.
// Key is the pipeline's key when it is not <branch>.<revision>; empty for every start the runner makes.
Key string
// unheld says the branch's catalog half is not held yet: the pipeline starts with no steps, placed once it is (A-336).
unheld bool
}
// Key is a take-in's pipeline key in the state pipelines, `<branch>.<source>`, the branch escaped as a key's name is
// (design 47 H §746).
func Key(branch string, source int64) string {
return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10))
// Key is a pipeline's key in the state pipelines, `<branch>.<revision>`, the catalog revision's number the indexer counts,
// the branch escaped as a key's name is (design 47 H §746, H-A-213 (4)); a take-in's number is unchanged where it equalled
// its source. A pipeline kept under its source before (`<branch>.<source>`) has the same form and is read as it is.
func Key(branch string, revision int64) string {
return seat.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10))
}
// PublishedKey is the key of a pipeline a package's publish starts, `<branch>.<source>.published.<package>.<version>`,
// each part escaped as a key's name is (design 47 H §747).
func PublishedKey(branch string, source int64, pkg, version string) string {
return seat.PipelinesByBranchSourcePublishedPackageVersionKey(keys.EscapeName(branch), strconv.FormatInt(source, 10),
keys.EscapeName(pkg), keys.EscapeName(version))
// keyRevision is the number b's pipeline is keyed by (H-A-213 (4)): its revision, or its source where the start names no
// revision (a take-in.taken of an indexer whose revision numbers were its sources, H-A-211).
func keyRevision(b Begin) int64 {
if b.Revision > 0 {
return b.Revision
}
return b.Source
}
// keyForms are the filters of every key form of the state pipelines: a take-in's, a run's and a publish's.
// keyForms are the filters of every key form of the state pipelines: a revision's, a run's, and a publish's kept from before
// a release revision started one pipeline (H-A-213 (2)).
var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*"}
// AllKeys are every pipeline's key, of each key form (§746, §747; H-A-83).
@@ -189,7 +190,7 @@ func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]A
}
var none []string
for _, st := range p.Steps {
if !open(st) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) {
if (!open(st) && !pinHeld(st)) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) {
continue
}
if holders, read := s.Holders(st.Seat); read && len(holders) == 0 {
@@ -282,7 +283,7 @@ func (s *Starter) sayStarted(key string, asks []Asked) {
func newPipeline(b Begin, key string, now time.Time) seat.Pipeline {
movers := append([]seat.Mover(nil), b.Movers...)
return seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Call: b.Call, Revision: b.Revision, Movers: movers,
Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}, Published: b.Published}
Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}}
}
func (s *Starter) create(key string, p seat.Pipeline) error {
@@ -337,6 +338,11 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
}
steps, a := Place(*p, m, r, entries, now, s.Name)
supersededWhy(*p, m.Module, entries, steps)
for _, st := range steps {
if pinHeld(st) {
s.logf("%s: %s's %s of %s is not asked: it %s, as its build does (H-A-206)", p.Pipeline, m.Module, st.Step, st.Artifact, st.Why)
}
}
if failed != "" {
why := "its dependency " + failed + " did not build"
for k := range steps {
@@ -490,7 +496,7 @@ func (s *Starter) Start(b Begin) (bool, error) {
now := s.now()
key := b.Key
if key == "" {
key = Key(b.Branch, b.Source)
key = Key(b.Branch, keyRevision(b))
}
if err := s.Fills.refuseStart(key); err != nil {
return false, err
@@ -887,14 +893,21 @@ func (s *Starter) Release(store Updater, key string, view func(branch string) (H
}, s.Registration, view, "the publishes its movers waited for")
}
// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the source
// revision that published it, and places the movers that waited for it (design 47 H §746, H-A-84). It answers that
// source's pipeline, read, and whether one is held.
// heardPublished writes a package-published into the publishing mover's published, in the running pipeline of the catalog
// revision that published it, found by the branch and revision the publish names (H-A-218), and places the movers that
// waited for it (design 47 H §746, H-A-84). It answers that pipeline, read, and whether one is held.
func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch string) (Half, bool)) (seat.Pipeline, bool, error) {
if s.Store == nil {
return seat.Pipeline{}, false, nil
}
key := Key(ev.Branch, ev.Source)
key, ok := Key(ev.Branch, ev.Revision), true
if ev.Revision <= 0 {
// A publish of an older builder names no revision: its source's lowest-numbered revision, the take-in's (H-A-218).
key, ok = s.sourcePipeline(ev.Branch, ev.Source)
}
if !ok {
return seat.Pipeline{}, false, nil
}
for {
p, rev, err := s.Store.Get(key)
if err != nil {
@@ -918,49 +931,81 @@ func (s *Starter) heardPublished(ev builds.PackagePublished, view func(branch st
}
}
// publishedKind is the edge a dependent builds on a published package by: B published before A builds, a package A
// imports (§53).
const publishedKind = 1
// Published hears a package-published (design 47 H §747, H-A-52): in the running pipeline of the source revision that
// published it, a mover that builds on it waits for it there (§746) and nothing starts; every module the catalog half
// holds a kind-1 edge from the package into that this pipeline does not move (a publish again of an older build, a
// dependent it does not move) is a mover of one pipeline keyed `<branch>.<source>.published.<package>.<version>`, at the
// publishing source's catalog revision. The dependents come from the edges the runner holds, never from a comparison.
func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) (int, error) {
if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 {
return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev)
}
publishing, held, err := s.heardPublished(ev, view)
// sourcePipeline is the key of branch's pipeline of the lowest-numbered revision of source, the take-in's (H-A-213 (4),
// H-A-218): a pipeline is keyed by its revision, so one of a source is found by the source it names, never by its key.
func (s *Starter) sourcePipeline(branch string, source int64) (string, bool) {
pkeys, err := s.Store.Keys(keys.EscapeName(branch) + ".*")
if err != nil {
return 0, err
s.logf("the pipelines of %s are not listed, so source %d's is not found: %v", branch, source, err)
return "", false
}
half, ok := view(ev.Branch)
if !ok {
return 0, fmt.Errorf("the catalog half of %s is not held yet: the publish of %s %s is heard again once it is", ev.Branch, ev.Artifact, ev.Version)
}
var movers []seat.Mover
for _, module := range slices.Sorted(maps.Keys(half.Entries)) {
e := half.Entries[module]
if module == ev.Module || Moves(publishing, module) || !slices.ContainsFunc(e.Edges, func(x catalog.Edge) bool {
return x.Kind == publishedKind && x.From == ev.Artifact
}) {
found, low := "", int64(0)
for _, k := range pkeys {
p, _, err := s.Store.Get(k)
if err != nil || p.Branch != branch || p.Source != source || p.Run != "" {
continue
}
movers = append(movers, seat.Mover{Module: module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
if r := keyRevision(Begin{Revision: p.Revision, Source: p.Source}); found == "" || r < low {
found, low = k, r
}
}
if len(movers) == 0 {
return found, found != ""
}
// Published hears a package-published (design 47 H §746, H-A-52): in the running pipeline of the revision that published
// it, a mover that builds on it waits for it there and is placed once it is heard. It starts no pipeline: a re-pin on a
// module's release is a catalog revision of its own, whose pipeline Revised starts (H-A-212, H-A-213 (2)).
func (s *Starter) Published(ev builds.PackagePublished, view func(branch string) (Half, bool)) error {
if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 {
return fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev)
}
_, _, err := s.heardPublished(ev, view)
return err
}
// revisedKinds are the call kinds of a catalog revision whose pipeline Revised starts: every kind but a take-in's, whose
// pipeline its take-in.taken starts (H-A-213 (2), H-A-217).
var revisedKinds = []string{catalog.CatalogRevisionEventCallKindRelease, catalog.CatalogRevisionEventCallKindTakeInAgain,
catalog.CatalogRevisionEventCallKindRecompute}
// Revised starts, on a catalog-revision of the trunk kind whose call kind is release, take-in again or recompute, ONE
// pipeline keyed <branch>.<revision> by compare-and-set on its absence (design 47 H §746, H-A-212, H-A-213 (2) and (4),
// H-A-217): the revision is read by its number through the catalog seat's revision verb, never the newest half, and its
// movers are the revision's movers, its steps placed from the revision's entries. A revision of another kind, or one with
// no mover, starts nothing; a second hearing writes nothing.
func (s *Starter) Revised(ev catalog.CatalogRevisionEvent, read func(branch string, revision int64) (Half, error)) (int, error) {
if ev.Kind != "trunk" || !slices.Contains(revisedKinds, ev.CallKind) {
return 0, nil
}
revision := half.Revision
if held && publishing.Revision > 0 {
revision = publishing.Revision
branch, err := keys.UnescapeName(ev.Branch)
if err != nil || branch == "" || ev.Revision <= 0 {
return 0, fmt.Errorf("a catalog-revision names its watched branch and revision: branch %q revision %d", ev.Branch, ev.Revision)
}
if read == nil {
return 0, fmt.Errorf("the revision %d of %s cannot be read: the runner holds no revision reader", ev.Revision, branch)
}
half, err := read(branch, ev.Revision)
if err != nil {
return 0, fmt.Errorf("the revision %d of %s is not read, so its pipeline is not started: %w", ev.Revision, branch, err)
}
if half.Revision != ev.Revision {
return 0, fmt.Errorf("the revision verb answered revision %d of %s for %d", half.Revision, branch, ev.Revision)
}
if len(half.Movers) == 0 {
s.logf("the revision %d of %s (%s) names no mover: it starts no pipeline", ev.Revision, branch, ev.Call)
return 0, nil
}
movers := make([]seat.Mover, 0, len(half.Movers))
for _, e := range half.Movers {
movers = append(movers, seat.Mover{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint, Repository: e.Repository})
}
movers = Tiers(movers, half.Entries, s.logf)
started, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Call: publishing.Call, Revision: revision, Movers: movers,
Entries: half.Entries, Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version},
Key: PublishedKey(ev.Branch, ev.Source, ev.Artifact, ev.Version)})
if started {
call := half.Call
if call == "" {
call = ev.Call
}
ok, err := s.Start(Begin{Branch: branch, Source: half.Source, Call: call, Revision: half.Revision, Movers: movers, Entries: half.Entries})
if ok {
return 1, err
}
return 0, err
@@ -11,6 +11,7 @@ import (
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/keys"
)
type created map[string][]byte
@@ -55,12 +56,11 @@ func TestStarts(t *testing.T) {
}
}
// R10/AC1 and AC2 (design 47 H §747, H-A-52): a package-published of a mover of its source's running pipeline whose
// dependent that pipeline moves starts nothing: the dependent waits there. A publish again of an older build starts one
// pipeline keyed <branch>.<source>.published.<package>.<version>, its movers the package's kind-1 dependents from the
// edges the runner holds, at the publishing source's revision, and says pipeline-started with the publish; an ordering
// edge starts nothing; heard twice, the second writes nothing.
func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) {
// H-A-212, H-A-213 (2) and (4), H-A-217: a package-published starts no pipeline, the movers waiting for it in its
// pipeline placed there; a trunk catalog-revision of a release, a take-in again or a recompute starts ONE pipeline keyed
// <branch>.<revision>, the revision read by its number, its movers the revision's, its source and call the revision's;
// heard twice the second writes nothing; a take-in's revision, a candidate's and one with no mover start nothing.
func TestARevisionOfAReleaseStartsOnePipeline(t *testing.T) {
mem := memStore{}
var said []seat.PipelineStarted
s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(_ string, body any) error {
@@ -70,37 +70,60 @@ func TestAPublishStartsOnlyWhatItsPipelineDoesNotMove(t *testing.T) {
return nil
}}
edges := []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 1}}
view := func(string) (Half, bool) {
return Half{Revision: 9, Entries: map[string]catalog.CatalogEntry{
"mesh-sdk": {Module: "mesh-sdk"},
"app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges},
"tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges},
"ordered": {Module: "ordered", Edges: []catalog.Edge{{From: "mesh-sdk:go", To: "x", Kind: 4}}}}}, true
entries := map[string]catalog.CatalogEntry{
"mesh-sdk": {Module: "mesh-sdk"},
"app": {Module: "app", Commit: "c1", Fp: "src2:aa", Repository: "o/app", Edges: edges},
"tool": {Module: "tool", Commit: "c2", Fp: "src2:bb", Edges: edges}}
view := func(string) (Half, bool) { return Half{Revision: 9, Entries: entries}, true }
ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 30, Module: "mesh-sdk", Version: "0.1.9"}
if err := s.Published(ev, view); err != nil || len(mem) != 0 {
t.Fatalf("a publish of an older build wrote %d pipelines (%v), want none", len(mem), err)
}
mem[Key("main", 41)] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{Pipeline: "main.41", Branch: "main", Source: 41, Revision: 8, Outcome: seat.PipelineOutcomeRunning,
Movers: []seat.Mover{{Module: "mesh-sdk"}, {Module: "app"}, {Module: "tool"}}}, 1}
ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "main", Source: 41, Module: "mesh-sdk", Version: "0.1.16"}
if n, err := s.Published(ev, view); n != 0 || err != nil || len(mem) != 1 {
t.Fatalf("a publish in a pipeline moving its dependents started %d (%v), %d pipelines", n, err, len(mem))
var asked []int64
movers := []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:ab", Repository: "o/app"}, {Module: "tool", Commit: "c2", Fingerprint: "src2:bc"}}
read := func(branch string, revision int64) (Half, error) {
asked = append(asked, revision)
if branch != "release.1" {
return Half{}, fmt.Errorf("branch %q", branch)
}
r := Half{Revision: revision, Source: 41, Entries: entries, Call: "release of mesh-sdk 0.1.17"}
if revision != 12 {
r.Movers = movers
}
return r, nil
}
// A publish again of an older build, at source 30: its pipeline is gone, so every kind-1 dependent is a mover.
ev.Source, ev.Version = 30, "0.1.9"
release := catalog.CatalogRevisionEvent{Kind: "trunk", Branch: keys.EscapeName("release.1"), Revision: 10, Source: 41, CallKind: "release", Call: "release of mesh-sdk 0.1.17"}
for i := range 2 {
if n, err := s.Published(ev, view); n != 1-i || err != nil {
if n, err := s.Revised(release, read); n != 1-i || err != nil {
t.Fatalf("hearing %d started %d (%v)", i+1, n, err)
}
}
key := PublishedKey("main", 30, "mesh-sdk:go", "0.1.9")
p, _, err := mem.Get(key)
if err != nil || len(p.Movers) != 2 || !Moves(p, "app") || !Moves(p, "tool") || Moves(p, "ordered") || p.Revision != 9 ||
p.Published == nil || p.Published.Version != "0.1.9" || len(said) != 1 || said[0].Published == nil {
t.Fatalf("the pipeline %s = %+v (%v), said %+v", key, p, err, said)
p, _, err := mem.Get(Key("release.1", 10))
if err != nil || len(mem) != 1 || len(p.Movers) != 2 || p.Revision != 10 || p.Source != 41 || p.Call != "release of mesh-sdk 0.1.17" ||
len(said) != 1 || len(asked) != 2 || asked[0] != 10 {
t.Fatalf("the pipeline %s = %+v (%v), %d written, said %d, read %v", Key("release.1", 10), p, err, len(mem), len(said), asked)
}
if app, _ := MoverOf(p, "app"); app.Commit != "c1" || app.Fingerprint != "src2:aa" || app.Repository != "o/app" {
t.Errorf("the mover app = %+v", app)
if app, _ := MoverOf(p, "app"); app.Fingerprint != "src2:ab" || app.Repository != "o/app" || app.Tier != 1 {
t.Errorf("the mover app = %+v, want the revision's", app)
}
for _, c := range []struct {
name string
ev catalog.CatalogRevisionEvent
want int
}{
{"take-in again", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 11, CallKind: "take-in-again"}, 1},
{"recompute with no mover", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 12, CallKind: "recompute"}, 0},
{"recompute", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 13, CallKind: "recompute"}, 1},
{"take-in", catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 14, CallKind: "take-in"}, 0},
{"candidate", catalog.CatalogRevisionEvent{Kind: "candidate", Branch: release.Branch, Revision: 15, CallKind: "release"}, 0},
} {
if n, err := s.Revised(c.ev, read); n != c.want || err != nil {
t.Errorf("%s: started %d (%v), want %d", c.name, n, err, c.want)
}
}
if _, err := s.Revised(catalog.CatalogRevisionEvent{Kind: "trunk", Branch: release.Branch, Revision: 16, CallKind: "release"},
func(string, int64) (Half, error) { return Half{}, errors.New("no answer") }); err == nil {
t.Error("a revision that cannot be read is not an error, so it is never heard again")
}
}
@@ -153,8 +176,8 @@ func TestATakeInIsOnePipeline(t *testing.T) {
t.Fatalf("written %d pipelines and said %d starts, want one each", len(store), len(said))
}
var p seat.Pipeline
if err := json.Unmarshal(store["main.7"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 {
t.Fatalf("the pipeline main.7 = %+v (%v)", p, err)
if err := json.Unmarshal(store["main.12"], &p); err != nil || p.Call != ev.Call || p.Revision != 12 || len(p.Movers) != 2 {
t.Fatalf("the pipeline main.12 = %+v (%v)", p, err)
}
if m, _ := MoverOf(p, "mesh-sdk"); m.Commit != "c1" || m.Fingerprint != "src2:aa" || m.Repository != "o/sdk" || m.Tier != 1 {
t.Errorf("the mover mesh-sdk = %+v", m)
@@ -243,25 +266,6 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
if n != 1 || err != nil || len(p.Steps) == 0 {
t.Errorf("the half of next placed %d (%v): %d steps, want the take-in's pipeline placed", n, err, len(p.Steps))
}
// A-336: a package published heard before its branch's half is held starts nothing, heard again once it is: its
// dependents are the half's edges (§747).
held = false
app := entries["app"]
app.Edges = []catalog.Edge{{From: "mesh-sdk:go", To: "app:program", Kind: 1}}
entries["app"] = app
published := builds.PackagePublished{Artifact: "mesh-sdk:go", Branch: "next", Source: 7, Module: "mesh-sdk", Version: "0.1.17"}
if n, err := s.Published(published, late); n != 0 || err == nil {
t.Fatalf("the publish before the half started %d (%v), want none and an error, heard again", n, err)
}
held = true
if n, err := s.Published(published, late); n != 1 || err != nil {
t.Fatalf("the publish with the half held started %d (%v)", n, err)
}
p, _, _ = store.Get(PublishedKey("next", 7, "mesh-sdk:go", "0.1.17"))
if len(p.Steps) == 0 {
t.Errorf("the publish's pipeline placed %d steps, want its steps", len(p.Steps))
}
// A-446: a pipeline an earlier runner left set aside for its new version (the record keeps a `held` field, outcome running,
// no step placed) is placed and asked as a newly started one when the half is held; no hold is read any more.
var old seat.Pipeline
@@ -318,8 +322,8 @@ func TestTakenInStartsOnlyItsEntries(t *testing.T) {
if err != nil || n != 1 {
t.Fatalf("the take-in started %d (%v), want its one entry", n, err)
}
if p, _, err := mem.Get("main.9"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") {
t.Errorf("main.9 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err)
if p, _, err := mem.Get("main.12"); err != nil || len(p.Movers) != 1 || !Moves(p, "moved") {
t.Errorf("main.12 = %+v (%v), want the one mover moved: batched is not named by the take-in", p, err)
}
}
@@ -335,3 +339,27 @@ func TestATakeInOfTheOldFormIsRefused(t *testing.T) {
t.Errorf("the new form = %+v (%v)", ev, err)
}
}
// H-A-218: a package-published is taken into the pipeline of the branch and revision it names; one naming no revision (an
// older builder) into its source's lowest-numbered revision, the take-in's.
func TestAPublishIsTakenIntoThePipelineOfItsRevision(t *testing.T) {
mem := memStore{}
for _, r := range []int64{7, 8} {
mem[Key("main", r)] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{Pipeline: Key("main", r), Branch: "main", Source: 4, Revision: r, Outcome: seat.PipelineOutcomeRunning,
Movers: []seat.Mover{{Module: "mesh-sdk"}}}, 1}
}
s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: func(string, any) error { return nil }}
view := func(string) (Half, bool) { return Half{Revision: 8}, true }
published := func(key string) []string { m, _ := MoverOf(mem[key].p, "mesh-sdk"); return m.Published }
ev := builds.PackagePublished{Artifact: "mesh-sdk:go", Module: "mesh-sdk", Branch: "main", Source: 4, Revision: 8, Version: "0.1.18"}
if err := s.Published(ev, view); err != nil || len(published("main.8")) != 1 || len(published("main.7")) != 0 {
t.Fatalf("a publish of revision 8 (%v): main.8 %v, main.7 %v; want main.8's mover alone", err, published("main.8"), published("main.7"))
}
ev.Revision, ev.Artifact = 0, "mesh-sdk:ts"
if err := s.Published(ev, view); err != nil || len(published("main.7")) != 1 || len(published("main.8")) != 1 {
t.Errorf("a publish naming no revision (%v): main.7 %v, main.8 %v; want main.7's, the source's lowest", err, published("main.7"), published("main.8"))
}
}
@@ -31,6 +31,8 @@ type StepSeat struct {
// Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for
// a seat whose ends name none.
Shared func([]byte) []seat.Shared
// Pins reads the pins a build's end names it was built with (H-A-207); nil for a seat whose ends name none.
Pins func([]byte) map[string]string
}
// StepSeats are the three build seats, from their contracts.
@@ -39,7 +41,7 @@ var StepSeats = []StepSeat{
{Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")},
{Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")},
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded, Pins: buildPins},
// The seat mesh-wants answers the composition step (§22, §24).
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared},
}
@@ -77,6 +79,15 @@ func decodeWantsEnded(raw []byte) (StepEnded, error) {
Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil
}
// buildPins is the pins a build step's end of the seat mesh-builds names its entry was built with (H-A-207).
func buildPins(raw []byte) map[string]string {
var b builds.StepEnded
if json.Unmarshal(raw, &b) != nil {
return nil
}
return b.Packages
}
// decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one
// entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is
// refused (A-99).
@@ -155,6 +166,9 @@ type Hearer struct {
// Release places the movers of a pipeline whose awaited publishes are heard now, after a build step passed (a kind-3
// edge, §746; Starter.Release); nil places none.
Release func(key string)
// AskPinHeld asks a mover's steps that waited for its entry's pins with the pins its build passed with (H-A-207;
// Starter.AskPinHeld); nil leaves them waiting.
AskPinHeld func(key, module string, pins map[string]string)
// Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748;
// Starter.Turn); nil takes none.
Turn func(key string)
@@ -197,6 +211,12 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726,
// A-469m).
func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) {
return h.OnStepEndedPinned(bs, subject, ev, shared, nil)
}
// OnStepEndedPinned is OnStepEndedShared of a build's end naming the pins it was built with (H-A-207): a passed build's
// mover's steps that waited for its entry's pins are asked with them.
func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared, pins map[string]string) (seat.Pipeline, error) {
node := strings.TrimPrefix(subject, bs.Prefix)
if node == subject || node == "" || strings.Contains(node, ".") {
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
@@ -272,6 +292,11 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
// Every mover's validate reads the one run (§745, H-A-51); its end asks each mover's composition, or ends it not
// asked when validate did not pass.
var due []Asked
askHeld := false
if s.Step == seat.StepResultStepBuild {
readers = append(readers, endPinHeld(&p, *s, pins, now)...)
askHeld = stepPassed(*s) && heldOn(p, s.Module)
}
if s.Step == seat.StepResultStepValidate {
readers = append(readers, sameValidate(&p, i, now)...)
var notAsked []int
@@ -324,6 +349,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
if release && h.Release != nil {
h.Release(ev.Pipeline)
}
if askHeld && h.AskPinHeld != nil {
h.AskPinHeld(ev.Pipeline, s.Module, pins)
}
if h.AskSteps != nil && len(due) > 0 {
if err := h.AskSteps(ev.Pipeline, due); err != nil {
h.logf("the compositions of %s are written running and not asked, asked again at the next start: %v", ev.Pipeline, err)
@@ -113,7 +113,7 @@ func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline {
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
}
if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil {
if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Revision: revision, Key: Key("main", source), Entries: entries, Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}); !ok || err != nil {
t.Fatalf("start = %v, %v", ok, err)
}
p, _, _ := f.store.Get(Key("main", source))
@@ -582,11 +582,11 @@ func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) {
if !ok || half.Source <= 0 {
return seat.StartAnswer{}, fmt.Errorf("the catalog half of %s holds no entry of %s at a source revision: nothing was started", in.Branch, in.Module)
}
key := Key(in.Branch, half.Source)
key := Key(in.Branch, keyRevision(Begin{Revision: half.Revision, Source: half.Source}))
if err := v.record(seat.VerbStart, "started the pipeline "+key+" by hand", in.Why); err != nil {
return seat.StartAnswer{}, err
}
// A hand start keys the newest revision's <branch>.<source> with the module as its one mover (H-A-52).
// A hand start keys the newest revision's <branch>.<revision> with the module as its one mover (H-A-52, H-A-213 (4)).
started, err := v.Starter.Start(Begin{Branch: in.Branch, Source: half.Source, Revision: half.Revision, Entries: half.Entries,
Movers: []seat.Mover{{Module: in.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository, Tier: 1}}})
if err != nil {
@@ -287,7 +287,7 @@ func TestStartAndAskAgain(t *testing.T) {
_, entries := placeFixture()
v.View = func(string) (Half, bool) { return Half{Source: 7, Revision: 3, Entries: entries}, true }
a, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "no pipeline for the head"})
if err != nil || a.Pipeline != "main.7" || made["main.7"] == nil {
if err != nil || a.Pipeline != "main.3" || made["main.3"] == nil {
t.Fatalf("start = %+v, %v; written %v", a, err, len(made))
}
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Why: "again"}); err == nil {