Design 47 H1: a held test step is asked with the pins its build passed with (H-A-207)

When the build of an entry with an empty pin passed, its held lint, unit
and integration steps had no pin to run with and nothing ended them, so a
pipeline stood running and, one pipeline at a time, the queue stopped. The
runner reads the pins the build's end names and, when the build passed,
asks each held step with them, its fetch the entry's as placed; a passed
build whose end names no pins ends the held steps could not run, naming
why, as does an artifact the branch's half no longer names.

Builds: design 47 H H-A-207, H-A-206, H-A-203
This commit is contained in:
2026-11-20 23:07:33 +01:00
parent cb26645614
commit 64cba95a10
5 changed files with 164 additions and 11 deletions
@@ -154,6 +154,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
hearer.AskSteps = starter.AskSteps
hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) }
hearer.AskPinHeld = func(key, module string, pins map[string]string) {
if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, rv.Catalog.View); err != nil {
logger.Printf("the steps of %s's %s that waited for its pins are not asked: %v", key, module, err)
}
}
hearer.Release = func(key string) {
if _, err := starter.Release(hearer.Store, key, rv.Catalog.View); err != nil {
logger.Printf("the movers of %s waiting for a build are not placed: %v", key, err)
@@ -128,7 +128,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
if bs.Shared != nil {
shared = bs.Shared(m.Data())
}
_, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared)
var pins map[string]string
if bs.Pins != nil {
pins = bs.Pins(m.Data())
}
_, err = h.OnStepEndedPinned(bs, m.Subject(), ev, shared, pins)
switch {
case errors.Is(err, pipeline.ErrRefused):
logf("%v", err)
@@ -3,6 +3,7 @@ package pipeline
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"maps"
"slices"
@@ -255,12 +256,16 @@ func pinHeld(s seat.StepResult) bool {
return s.Outcome == seat.StepResultOutcomeWaiting && strings.HasPrefix(s.Why, PinWait)
}
// endPinHeld ends, as the mover's build ended without passing, each of its steps waiting for its entry's pins (H-A-206):
// with the build's outcome and why, so no step waits on a publish the build no longer waits for. It answers the steps it
// ended; a build that passed ends none, the publish re-pinning the entry and moving the mover.
func endPinHeld(p *seat.Pipeline, build seat.StepResult, now time.Time) []int {
// endPinHeld ends each of build's mover's steps waiting for its entry's pins (H-A-206, H-A-207): as the build ended when
// it did not pass, so no step waits on a publish the build no longer waits for; could not run, loud, when the build passed
// and its end names no pins to run them with. It answers the steps it ended.
func endPinHeld(p *seat.Pipeline, build seat.StepResult, pins map[string]string, now time.Time) []int {
outcome, why := build.Outcome, "the build ended "+build.Outcome+" while this step waited: %s; "+build.Why
if stepPassed(build) {
return nil
if len(pins) > 0 {
return nil
}
outcome, why = seat.StepResultOutcomeCouldNotRun, "the build passed, but its end names no pins to run this step with (H-A-207); it %s"
}
var ended []int
for i := range p.Steps {
@@ -268,13 +273,78 @@ func endPinHeld(p *seat.Pipeline, build seat.StepResult, now time.Time) []int {
if st.Module != build.Module || !pinHeld(*st) {
continue
}
st.Outcome, st.Why, st.Ended = build.Outcome, "the build ended "+build.Outcome+" while this step waited: "+st.Why+"; "+build.Why, &now
st.Outcome, st.Why, st.Ended = outcome, fmt.Sprintf(why, st.Why), &now
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
ended = append(ended, i)
}
return ended
}
// heldOn says whether module has a step of p waiting for its entry's pins.
func heldOn(p seat.Pipeline, module string) bool {
return slices.ContainsFunc(p.Steps, func(st seat.StepResult) bool { return st.Module == module && pinHeld(st) })
}
// AskPinHeld asks the steps of pipeline key's mover module that waited for its entry's pins (H-A-206) with the pins its
// build passed with (H-A-207): each running under its own ask, its fetch the entry's as Place builds it with those pins.
// A step whose artifact the branch's half no longer names ends could not run, saying so. It answers how many it asked.
func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]string, view func(branch string) (Half, bool)) (int, error) {
for {
p, rev, err := store.Get(key)
if err != nil {
return 0, fmt.Errorf("the pipeline %s is not read: %w", key, err)
}
m, moves := MoverOf(p, module)
if !moves || !heldOn(p, module) {
return 0, nil
}
var entry catalog.CatalogEntry
if view != nil {
if half, ok := view(p.Branch); ok {
entry = half.Entries[module]
}
}
entry.Packages = pins
now := s.now()
var asks []Asked
var ended []int
for i := range p.Steps {
st := &p.Steps[i]
if st.Module != module || !pinHeld(*st) {
continue
}
j := slices.IndexFunc(entry.Artifacts, func(a catalog.Artifact) bool { return a.Ref == st.Artifact })
language, err := sdkstep.AskLanguageOfSeat(st.Seat)
if j < 0 || err != nil {
st.Outcome, st.Ended = seat.StepResultOutcomeCouldNotRun, &now
st.Why = fmt.Sprintf("its build passed with its pins, but the catalog half of %s names no artifact %s of %s to ask it of (H-A-207)", p.Branch, st.Artifact, module)
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
ended = append(ended, i)
continue
}
a := entry.Artifacts[j]
st.Outcome, st.Why, st.Started = seat.StepResultOutcomeRunning, "", &now
st.Ask = AskID(startOf(p, now), p.Pipeline, module, a.Ref, st.Step)
ask := Asked{Seat: st.Seat, ID: st.Ask, Ask: stepAsk(p, m, entry, a, language, st.Step, st.Ask, s.Name, now), Toolchain: a.Language, Commit: m.Commit}
if a.Package != nil {
ask.Package = a.Package.Name
}
asks = append(asks, ask)
}
if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return 0, fmt.Errorf("the pipeline %s is not written: %w", key, err)
}
s.logf("%s: %s's build passed with its pins: %d steps that waited for them are asked, %d could not run (H-A-207)", key, module, len(asks), len(ended))
s.sayNotAsked(p, ended)
if s.Steps == nil || len(asks) == 0 {
return 0, nil
}
return len(asks), s.askSteps(key, asks)
}
}
func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
// A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the
// artifact's named paths in it when the entry has them.
@@ -3,6 +3,7 @@ package pipeline
import (
"encoding/json"
"fmt"
"reflect"
"strconv"
"strings"
"testing"
@@ -143,8 +144,56 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) {
// The build ending without passing past its bound ends each held step as it ended (H-A-206).
failed := hp.Steps[0]
failed.Outcome, failed.Why = seat.StepResultOutcomeFailed, "package git.novox.be/novox/x/go has no released version"
if ended := endPinHeld(&hp, failed, time.Now()); len(ended) == 0 || hp.Steps[1].Outcome != seat.StepResultOutcomeFailed || hp.Steps[1].Ended == nil {
t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", hp.Steps[1], len(ended))
failedP := hp
failedP.Steps = append([]seat.StepResult(nil), hp.Steps...)
if ended := endPinHeld(&failedP, failed, nil, time.Now()); len(ended) == 0 || failedP.Steps[1].Outcome != seat.StepResultOutcomeFailed || failedP.Steps[1].Ended == nil {
t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", failedP.Steps[1], len(ended))
}
// A build passed whose end names no pins ends them could not run, loud (H-A-207).
passed := hp.Steps[0]
passed.Outcome = seat.StepResultOutcomePassed
bare := hp
bare.Steps = append([]seat.StepResult(nil), hp.Steps...)
if ended := endPinHeld(&bare, passed, nil, time.Now()); len(ended) == 0 || bare.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun ||
!strings.Contains(bare.Steps[1].Why, "names no pins") {
t.Errorf("after a build passed naming no pins the held lint step = %+v; want could not run naming it", bare.Steps[1])
}
// A build passed naming its pins asks the held steps with them: its end heard, the runner asks each with the pins the
// build was built with (H-A-207).
pinsBuilt := map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": "v0.1.4"}
ms := memStore{}
ms["main.5"] = struct {
p seat.Pipeline
rev uint64
}{hp, 1}
var askedWith map[string]string
h := &Hearer{Store: ms, Say: func(string, any) error { return nil }, Name: "the-runner",
AskPinHeld: func(key, module string, pins map[string]string) {
askedWith = pins
if _, err := hs.AskPinHeld(ms, key, module, pins, func(string) (Half, bool) { return Half{Entries: waiting}, true }); err != nil {
t.Errorf("ask held: %v", err)
}
}}
buildEnd := StepEnded{ID: hp.Steps[0].Ask, Pipeline: "main.5", Step: "build", Node: "n1", Outcome: seat.StepResultOutcomePassed,
Module: "app", Commit: "c1", Fingerprint: "src2:ab"}
if _, err := h.OnStepEndedPinned(StepSeats[3], StepSeats[3].Prefix+"n1", buildEnd, nil, pinsBuilt); err != nil {
t.Fatalf("the build's end: %v", err)
}
if !reflect.DeepEqual(askedWith, pinsBuilt) {
t.Errorf("the held steps were asked with %v; want the build's pins", askedWith)
}
var askedHeld int
for _, x := range heldAsks.asks {
if b, ok := x.(nodebuildgo.StepAsk); ok {
askedHeld++
if !reflect.DeepEqual(b.Step.Fetch.Packages, pinsBuilt) {
t.Errorf("the %s of %s fetches %v; want the build's pins", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages)
}
}
}
after, _, _ := ms.Get("main.5")
if askedHeld == 0 || after.Steps[1].Outcome == seat.StepResultOutcomeWaiting || after.Steps[1].Ask == "" {
t.Errorf("after the build passed %d held steps were asked, lint = %+v; want each asked", askedHeld, after.Steps[1])
}
if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") {
t.Errorf("the journal says %q; want the held step named", journal)
@@ -31,6 +31,8 @@ type StepSeat struct {
// Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for
// a seat whose ends name none.
Shared func([]byte) []seat.Shared
// Pins reads the pins a build's end names it was built with (H-A-207); nil for a seat whose ends name none.
Pins func([]byte) map[string]string
}
// StepSeats are the three build seats, from their contracts.
@@ -39,7 +41,7 @@ var StepSeats = []StepSeat{
{Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")},
{Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")},
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded, Pins: buildPins},
// The seat mesh-wants answers the composition step (§22, §24).
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared},
}
@@ -77,6 +79,15 @@ func decodeWantsEnded(raw []byte) (StepEnded, error) {
Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil
}
// buildPins is the pins a build step's end of the seat mesh-builds names its entry was built with (H-A-207).
func buildPins(raw []byte) map[string]string {
var b builds.StepEnded
if json.Unmarshal(raw, &b) != nil {
return nil
}
return b.Packages
}
// decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one
// entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is
// refused (A-99).
@@ -155,6 +166,9 @@ type Hearer struct {
// Release places the movers of a pipeline whose awaited publishes are heard now, after a build step passed (a kind-3
// edge, §746; Starter.Release); nil places none.
Release func(key string)
// AskPinHeld asks a mover's steps that waited for its entry's pins with the pins its build passed with (H-A-207;
// Starter.AskPinHeld); nil leaves them waiting.
AskPinHeld func(key, module string, pins map[string]string)
// Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748;
// Starter.Turn); nil takes none.
Turn func(key string)
@@ -197,6 +211,12 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726,
// A-469m).
func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) {
return h.OnStepEndedPinned(bs, subject, ev, shared, nil)
}
// OnStepEndedPinned is OnStepEndedShared of a build's end naming the pins it was built with (H-A-207): a passed build's
// mover's steps that waited for its entry's pins are asked with them.
func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared, pins map[string]string) (seat.Pipeline, error) {
node := strings.TrimPrefix(subject, bs.Prefix)
if node == subject || node == "" || strings.Contains(node, ".") {
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
@@ -272,8 +292,10 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
// Every mover's validate reads the one run (§745, H-A-51); its end asks each mover's composition, or ends it not
// asked when validate did not pass.
var due []Asked
askHeld := false
if s.Step == seat.StepResultStepBuild {
readers = append(readers, endPinHeld(&p, *s, now)...)
readers = append(readers, endPinHeld(&p, *s, pins, now)...)
askHeld = stepPassed(*s) && heldOn(p, s.Module)
}
if s.Step == seat.StepResultStepValidate {
readers = append(readers, sameValidate(&p, i, now)...)
@@ -327,6 +349,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
if release && h.Release != nil {
h.Release(ev.Pipeline)
}
if askHeld && h.AskPinHeld != nil {
h.AskPinHeld(ev.Pipeline, s.Module, pins)
}
if h.AskSteps != nil && len(due) > 0 {
if err := h.AskSteps(ev.Pipeline, due); err != nil {
h.logf("the compositions of %s are written running and not asked, asked again at the next start: %v", ev.Pipeline, err)