Design 47 H1: a held test step is asked with the pins its build passed with (H-A-207)
When the build of an entry with an empty pin passed, its held lint, unit and integration steps had no pin to run with and nothing ended them, so a pipeline stood running and, one pipeline at a time, the queue stopped. The runner reads the pins the build's end names and, when the build passed, asks each held step with them, its fetch the entry's as placed; a passed build whose end names no pins ends the held steps could not run, naming why, as does an artifact the branch's half no longer names. Builds: design 47 H H-A-207, H-A-206, H-A-203
This commit is contained in:
@@ -154,6 +154,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
|
||||
hearer.AskSteps = starter.AskSteps
|
||||
hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) }
|
||||
hearer.AskPinHeld = func(key, module string, pins map[string]string) {
|
||||
if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, rv.Catalog.View); err != nil {
|
||||
logger.Printf("the steps of %s's %s that waited for its pins are not asked: %v", key, module, err)
|
||||
}
|
||||
}
|
||||
hearer.Release = func(key string) {
|
||||
if _, err := starter.Release(hearer.Store, key, rv.Catalog.View); err != nil {
|
||||
logger.Printf("the movers of %s waiting for a build are not placed: %v", key, err)
|
||||
|
||||
@@ -128,7 +128,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
if bs.Shared != nil {
|
||||
shared = bs.Shared(m.Data())
|
||||
}
|
||||
_, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared)
|
||||
var pins map[string]string
|
||||
if bs.Pins != nil {
|
||||
pins = bs.Pins(m.Data())
|
||||
}
|
||||
_, err = h.OnStepEndedPinned(bs, m.Subject(), ev, shared, pins)
|
||||
switch {
|
||||
case errors.Is(err, pipeline.ErrRefused):
|
||||
logf("%v", err)
|
||||
|
||||
@@ -3,6 +3,7 @@ package pipeline
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
@@ -255,12 +256,16 @@ func pinHeld(s seat.StepResult) bool {
|
||||
return s.Outcome == seat.StepResultOutcomeWaiting && strings.HasPrefix(s.Why, PinWait)
|
||||
}
|
||||
|
||||
// endPinHeld ends, as the mover's build ended without passing, each of its steps waiting for its entry's pins (H-A-206):
|
||||
// with the build's outcome and why, so no step waits on a publish the build no longer waits for. It answers the steps it
|
||||
// ended; a build that passed ends none, the publish re-pinning the entry and moving the mover.
|
||||
func endPinHeld(p *seat.Pipeline, build seat.StepResult, now time.Time) []int {
|
||||
// endPinHeld ends each of build's mover's steps waiting for its entry's pins (H-A-206, H-A-207): as the build ended when
|
||||
// it did not pass, so no step waits on a publish the build no longer waits for; could not run, loud, when the build passed
|
||||
// and its end names no pins to run them with. It answers the steps it ended.
|
||||
func endPinHeld(p *seat.Pipeline, build seat.StepResult, pins map[string]string, now time.Time) []int {
|
||||
outcome, why := build.Outcome, "the build ended "+build.Outcome+" while this step waited: %s; "+build.Why
|
||||
if stepPassed(build) {
|
||||
return nil
|
||||
if len(pins) > 0 {
|
||||
return nil
|
||||
}
|
||||
outcome, why = seat.StepResultOutcomeCouldNotRun, "the build passed, but its end names no pins to run this step with (H-A-207); it %s"
|
||||
}
|
||||
var ended []int
|
||||
for i := range p.Steps {
|
||||
@@ -268,13 +273,78 @@ func endPinHeld(p *seat.Pipeline, build seat.StepResult, now time.Time) []int {
|
||||
if st.Module != build.Module || !pinHeld(*st) {
|
||||
continue
|
||||
}
|
||||
st.Outcome, st.Why, st.Ended = build.Outcome, "the build ended "+build.Outcome+" while this step waited: "+st.Why+"; "+build.Why, &now
|
||||
st.Outcome, st.Why, st.Ended = outcome, fmt.Sprintf(why, st.Why), &now
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
|
||||
ended = append(ended, i)
|
||||
}
|
||||
return ended
|
||||
}
|
||||
|
||||
// heldOn says whether module has a step of p waiting for its entry's pins.
|
||||
func heldOn(p seat.Pipeline, module string) bool {
|
||||
return slices.ContainsFunc(p.Steps, func(st seat.StepResult) bool { return st.Module == module && pinHeld(st) })
|
||||
}
|
||||
|
||||
// AskPinHeld asks the steps of pipeline key's mover module that waited for its entry's pins (H-A-206) with the pins its
|
||||
// build passed with (H-A-207): each running under its own ask, its fetch the entry's as Place builds it with those pins.
|
||||
// A step whose artifact the branch's half no longer names ends could not run, saying so. It answers how many it asked.
|
||||
func (s *Starter) AskPinHeld(store Updater, key, module string, pins map[string]string, view func(branch string) (Half, bool)) (int, error) {
|
||||
for {
|
||||
p, rev, err := store.Get(key)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the pipeline %s is not read: %w", key, err)
|
||||
}
|
||||
m, moves := MoverOf(p, module)
|
||||
if !moves || !heldOn(p, module) {
|
||||
return 0, nil
|
||||
}
|
||||
var entry catalog.CatalogEntry
|
||||
if view != nil {
|
||||
if half, ok := view(p.Branch); ok {
|
||||
entry = half.Entries[module]
|
||||
}
|
||||
}
|
||||
entry.Packages = pins
|
||||
now := s.now()
|
||||
var asks []Asked
|
||||
var ended []int
|
||||
for i := range p.Steps {
|
||||
st := &p.Steps[i]
|
||||
if st.Module != module || !pinHeld(*st) {
|
||||
continue
|
||||
}
|
||||
j := slices.IndexFunc(entry.Artifacts, func(a catalog.Artifact) bool { return a.Ref == st.Artifact })
|
||||
language, err := sdkstep.AskLanguageOfSeat(st.Seat)
|
||||
if j < 0 || err != nil {
|
||||
st.Outcome, st.Ended = seat.StepResultOutcomeCouldNotRun, &now
|
||||
st.Why = fmt.Sprintf("its build passed with its pins, but the catalog half of %s names no artifact %s of %s to ask it of (H-A-207)", p.Branch, st.Artifact, module)
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
|
||||
ended = append(ended, i)
|
||||
continue
|
||||
}
|
||||
a := entry.Artifacts[j]
|
||||
st.Outcome, st.Why, st.Started = seat.StepResultOutcomeRunning, "", &now
|
||||
st.Ask = AskID(startOf(p, now), p.Pipeline, module, a.Ref, st.Step)
|
||||
ask := Asked{Seat: st.Seat, ID: st.Ask, Ask: stepAsk(p, m, entry, a, language, st.Step, st.Ask, s.Name, now), Toolchain: a.Language, Commit: m.Commit}
|
||||
if a.Package != nil {
|
||||
ask.Package = a.Package.Name
|
||||
}
|
||||
asks = append(asks, ask)
|
||||
}
|
||||
if _, err := store.Update(key, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return 0, fmt.Errorf("the pipeline %s is not written: %w", key, err)
|
||||
}
|
||||
s.logf("%s: %s's build passed with its pins: %d steps that waited for them are asked, %d could not run (H-A-207)", key, module, len(asks), len(ended))
|
||||
s.sayNotAsked(p, ended)
|
||||
if s.Steps == nil || len(asks) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return len(asks), s.askSteps(key, asks)
|
||||
}
|
||||
}
|
||||
|
||||
func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Artifact, language, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
|
||||
// A step is for its module alone (principle 10, A-447): the fetch always names the module's directory, and the
|
||||
// artifact's named paths in it when the entry has them.
|
||||
|
||||
@@ -3,6 +3,7 @@ package pipeline
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -143,8 +144,56 @@ func TestStartPlacesTheStepsThePolicyNamesAndAsksEachBuildSeat(t *testing.T) {
|
||||
// The build ending without passing past its bound ends each held step as it ended (H-A-206).
|
||||
failed := hp.Steps[0]
|
||||
failed.Outcome, failed.Why = seat.StepResultOutcomeFailed, "package git.novox.be/novox/x/go has no released version"
|
||||
if ended := endPinHeld(&hp, failed, time.Now()); len(ended) == 0 || hp.Steps[1].Outcome != seat.StepResultOutcomeFailed || hp.Steps[1].Ended == nil {
|
||||
t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", hp.Steps[1], len(ended))
|
||||
failedP := hp
|
||||
failedP.Steps = append([]seat.StepResult(nil), hp.Steps...)
|
||||
if ended := endPinHeld(&failedP, failed, nil, time.Now()); len(ended) == 0 || failedP.Steps[1].Outcome != seat.StepResultOutcomeFailed || failedP.Steps[1].Ended == nil {
|
||||
t.Errorf("after the build failed the held lint step = %+v (%d ended); want it failed", failedP.Steps[1], len(ended))
|
||||
}
|
||||
// A build passed whose end names no pins ends them could not run, loud (H-A-207).
|
||||
passed := hp.Steps[0]
|
||||
passed.Outcome = seat.StepResultOutcomePassed
|
||||
bare := hp
|
||||
bare.Steps = append([]seat.StepResult(nil), hp.Steps...)
|
||||
if ended := endPinHeld(&bare, passed, nil, time.Now()); len(ended) == 0 || bare.Steps[1].Outcome != seat.StepResultOutcomeCouldNotRun ||
|
||||
!strings.Contains(bare.Steps[1].Why, "names no pins") {
|
||||
t.Errorf("after a build passed naming no pins the held lint step = %+v; want could not run naming it", bare.Steps[1])
|
||||
}
|
||||
// A build passed naming its pins asks the held steps with them: its end heard, the runner asks each with the pins the
|
||||
// build was built with (H-A-207).
|
||||
pinsBuilt := map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25", "git.novox.be/novox/x/go": "v0.1.4"}
|
||||
ms := memStore{}
|
||||
ms["main.5"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{hp, 1}
|
||||
var askedWith map[string]string
|
||||
h := &Hearer{Store: ms, Say: func(string, any) error { return nil }, Name: "the-runner",
|
||||
AskPinHeld: func(key, module string, pins map[string]string) {
|
||||
askedWith = pins
|
||||
if _, err := hs.AskPinHeld(ms, key, module, pins, func(string) (Half, bool) { return Half{Entries: waiting}, true }); err != nil {
|
||||
t.Errorf("ask held: %v", err)
|
||||
}
|
||||
}}
|
||||
buildEnd := StepEnded{ID: hp.Steps[0].Ask, Pipeline: "main.5", Step: "build", Node: "n1", Outcome: seat.StepResultOutcomePassed,
|
||||
Module: "app", Commit: "c1", Fingerprint: "src2:ab"}
|
||||
if _, err := h.OnStepEndedPinned(StepSeats[3], StepSeats[3].Prefix+"n1", buildEnd, nil, pinsBuilt); err != nil {
|
||||
t.Fatalf("the build's end: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(askedWith, pinsBuilt) {
|
||||
t.Errorf("the held steps were asked with %v; want the build's pins", askedWith)
|
||||
}
|
||||
var askedHeld int
|
||||
for _, x := range heldAsks.asks {
|
||||
if b, ok := x.(nodebuildgo.StepAsk); ok {
|
||||
askedHeld++
|
||||
if !reflect.DeepEqual(b.Step.Fetch.Packages, pinsBuilt) {
|
||||
t.Errorf("the %s of %s fetches %v; want the build's pins", b.Step.Step, b.Step.Artifact, b.Step.Fetch.Packages)
|
||||
}
|
||||
}
|
||||
}
|
||||
after, _, _ := ms.Get("main.5")
|
||||
if askedHeld == 0 || after.Steps[1].Outcome == seat.StepResultOutcomeWaiting || after.Steps[1].Ask == "" {
|
||||
t.Errorf("after the build passed %d held steps were asked, lint = %+v; want each asked", askedHeld, after.Steps[1])
|
||||
}
|
||||
if !strings.Contains(strings.Join(journal, "\n"), "app's lint of app:program is not asked: it waits on the publish of git.novox.be/novox/x/go") {
|
||||
t.Errorf("the journal says %q; want the held step named", journal)
|
||||
|
||||
@@ -31,6 +31,8 @@ type StepSeat struct {
|
||||
// Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for
|
||||
// a seat whose ends name none.
|
||||
Shared func([]byte) []seat.Shared
|
||||
// Pins reads the pins a build's end names it was built with (H-A-207); nil for a seat whose ends name none.
|
||||
Pins func([]byte) map[string]string
|
||||
}
|
||||
|
||||
// StepSeats are the three build seats, from their contracts.
|
||||
@@ -39,7 +41,7 @@ var StepSeats = []StepSeat{
|
||||
{Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")},
|
||||
{Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")},
|
||||
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
|
||||
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
|
||||
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded, Pins: buildPins},
|
||||
// The seat mesh-wants answers the composition step (§22, §24).
|
||||
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared},
|
||||
}
|
||||
@@ -77,6 +79,15 @@ func decodeWantsEnded(raw []byte) (StepEnded, error) {
|
||||
Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil
|
||||
}
|
||||
|
||||
// buildPins is the pins a build step's end of the seat mesh-builds names its entry was built with (H-A-207).
|
||||
func buildPins(raw []byte) map[string]string {
|
||||
var b builds.StepEnded
|
||||
if json.Unmarshal(raw, &b) != nil {
|
||||
return nil
|
||||
}
|
||||
return b.Packages
|
||||
}
|
||||
|
||||
// decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one
|
||||
// entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is
|
||||
// refused (A-99).
|
||||
@@ -155,6 +166,9 @@ type Hearer struct {
|
||||
// Release places the movers of a pipeline whose awaited publishes are heard now, after a build step passed (a kind-3
|
||||
// edge, §746; Starter.Release); nil places none.
|
||||
Release func(key string)
|
||||
// AskPinHeld asks a mover's steps that waited for its entry's pins with the pins its build passed with (H-A-207;
|
||||
// Starter.AskPinHeld); nil leaves them waiting.
|
||||
AskPinHeld func(key, module string, pins map[string]string)
|
||||
// Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748;
|
||||
// Starter.Turn); nil takes none.
|
||||
Turn func(key string)
|
||||
@@ -197,6 +211,12 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726,
|
||||
// A-469m).
|
||||
func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) {
|
||||
return h.OnStepEndedPinned(bs, subject, ev, shared, nil)
|
||||
}
|
||||
|
||||
// OnStepEndedPinned is OnStepEndedShared of a build's end naming the pins it was built with (H-A-207): a passed build's
|
||||
// mover's steps that waited for its entry's pins are asked with them.
|
||||
func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared, pins map[string]string) (seat.Pipeline, error) {
|
||||
node := strings.TrimPrefix(subject, bs.Prefix)
|
||||
if node == subject || node == "" || strings.Contains(node, ".") {
|
||||
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
|
||||
@@ -272,8 +292,10 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
|
||||
// Every mover's validate reads the one run (§745, H-A-51); its end asks each mover's composition, or ends it not
|
||||
// asked when validate did not pass.
|
||||
var due []Asked
|
||||
askHeld := false
|
||||
if s.Step == seat.StepResultStepBuild {
|
||||
readers = append(readers, endPinHeld(&p, *s, now)...)
|
||||
readers = append(readers, endPinHeld(&p, *s, pins, now)...)
|
||||
askHeld = stepPassed(*s) && heldOn(p, s.Module)
|
||||
}
|
||||
if s.Step == seat.StepResultStepValidate {
|
||||
readers = append(readers, sameValidate(&p, i, now)...)
|
||||
@@ -327,6 +349,9 @@ func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, sh
|
||||
if release && h.Release != nil {
|
||||
h.Release(ev.Pipeline)
|
||||
}
|
||||
if askHeld && h.AskPinHeld != nil {
|
||||
h.AskPinHeld(ev.Pipeline, s.Module, pins)
|
||||
}
|
||||
if h.AskSteps != nil && len(due) > 0 {
|
||||
if err := h.AskSteps(ev.Pipeline, due); err != nil {
|
||||
h.logf("the compositions of %s are written running and not asked, asked again at the next start: %v", ev.Pipeline, err)
|
||||
|
||||
Reference in New Issue
Block a user