H-A-320 (1): a journal line per step end, step ask and build answer

A working 36-mover pipeline was read as stalled twice: the runner wrote no
line when a step ended or was asked, and the builder none when it answered a
step ask. The runner now writes "<pipeline> <mover> <step> ended <outcome>
(<node>, <duration>)" at each step end it writes (the asking step, every
reader of its test run, and each reader EndRun ends), and "<pipeline> <mover>
<step> asked on <seat>" for each ask put (the re-ask paths keep their own
line). The builder writes "<module>@<fingerprint> build step answered
<outcome> for <pipeline> (existing build / built now)".

Builds: design 47 H H-A-320 (1)
This commit is contained in:
2026-11-21 21:50:48 +01:00
parent e196064e51
commit dc83ef3e27
10 changed files with 130 additions and 16 deletions
@@ -43,7 +43,7 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui
var failed []string
for _, e := range run.Entries {
var pins map[string]string
outcome, said := b.buildEntry(ctx, e, &pins)
outcome, said := b.buildEntry(ctx, run.Pipeline, e, &pins)
// The end reports the pins each entry was built with, so the runner asks its held test steps with them (H-A-207).
for path, version := range pins {
if version == "" {
@@ -88,9 +88,10 @@ func (b *Builder) askedRevision(run meshbuilds.StepRun) {
}
}
// buildEntry builds one entry at its fingerprint and waits for its end; pins is set, when it is done, to the pins its
// done key keeps it was built with, the registry's included (H-A-207).
func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) {
// buildEntry builds one entry at its fingerprint for pipeline and waits for its end; pins is set, when it is done, to the pins
// its done key keeps it was built with, the registry's included (H-A-207). The answer from a build is one journal line,
// naming the entry, its outcome, the pipeline and whether the build existed or was built now (H-A-320 (1)).
func (b *Builder) buildEntry(ctx context.Context, pipeline string, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) {
entry, err := b.StepEntry(ctx, e.Fingerprint)
if err != nil {
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry %s at %s cannot be read: %v", e.Module, e.Fingerprint, err)
@@ -110,13 +111,18 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins
}
ended := b.awaitEnd(key)
defer b.dropEnd(key, ended)
answered := func(outcome, said, how string) (string, string) {
b.logf("%s@%s build step answered %s for %s (%s)", entry.Module, entry.Fp, outcome, pipeline, how)
return outcome, said
}
if done, failedWords, found := b.keyEnd(ctx, key); found {
if done {
// A key done before reports the pins it was built with, kept with it, never the ones a publish gives now.
*pins = b.keptPins(ctx, key)
return b.passedWithItsVersion(ctx, entry)
outcome, said := b.passedWithItsVersion(ctx, entry)
return answered(outcome, said, "existing build")
}
return meshbuilds.StepEndedOutcomeFailed, failedWords
return answered(meshbuilds.StepEndedOutcomeFailed, failedWords, "existing build")
}
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalogclient.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
@@ -125,9 +131,10 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins
case log := <-ended:
if log.Outcome == builds.BuildDone {
*pins = b.keptPins(ctx, key)
return b.passedWithItsVersion(ctx, entry)
outcome, said := b.passedWithItsVersion(ctx, entry)
return answered(outcome, said, "built now")
}
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error)
return answered(meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error), "built now")
case <-ctx.Done():
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the builder stopped before its build ended", entry.Module, entry.Fp)
}
@@ -7,6 +7,7 @@ import (
"reflect"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -171,6 +172,18 @@ func awaitWaiting[T any](t *testing.T, waiting <-chan struct{}, ended <-chan T)
// another series and the module's release number pin nothing.
func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
r, _, waiting := unpinnedRig(t)
// Each answer from a build is one journal line, built now or existing (H-A-320 (1)).
var mu sync.Mutex
var answers []string
logf := r.b.Logf
r.b.Logf = func(format string, args ...any) {
logf(format, args...)
if l := fmt.Sprintf(format, args...); strings.Contains(l, "build step answered") {
mu.Lock()
answers = append(answers, l)
mu.Unlock()
}
}
r.b.PackagePublished(unpinnedX, "v0.2.9") // another series than the 0.1 the module accepts: no pin
r.b.PackagePublished(unpinnedX, "0.1.4-feat-x.2") // a group's tagged version: no pin
ended := make(chan meshbuilds.StepEnded, 1)
@@ -206,6 +219,12 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
if again.Outcome != meshbuilds.StepEndedOutcomePassed || again.Packages[unpinnedX] != "v0.1.3" {
t.Errorf("the done key's end = %+v; want passed reporting x at v0.1.3, as built", again)
}
mu.Lock()
want := []string{"m@" + fp1 + " build step answered passed for main.9.m (built now)", "m@" + fp1 + " build step answered passed for main.9.m (existing build)"}
if !reflect.DeepEqual(answers, want) {
t.Errorf("the answers journaled %q, want %q", answers, want)
}
mu.Unlock()
// A done key that keeps no pins reports none, so the runner ends the waiting steps loud.
v, err := builds.DecodeBuilderBuild(r.st.vals[r.key])
if err != nil {
@@ -176,7 +176,7 @@ func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error {
return nil // its second bound ends the step
}
v.Starter.carry(key, []Asked{*a})
return askAll(v.Starter.Steps, key, []Asked{*a}, func(commit string) bool { return v.Starter.recovered(key, commit) }, nil)
return askAll(v.Starter.Steps, key, []Asked{*a}, func(commit string) bool { return v.Starter.recovered(key, commit) }, nil, nil)
}
// HolderStarted puts again every open ask of each silent seat module holds (H-A-243 (1)): its holder takes its asks on a plain
@@ -234,7 +234,7 @@ func (v Verbs) HolderStarted(module string) (int, error) {
continue
}
st.carry(key, due)
if err := askAll(st.Steps, key, due, func(commit string) bool { return st.recovered(key, commit) }, nil); err != nil && first == nil {
if err := askAll(st.Steps, key, due, func(commit string) bool { return st.recovered(key, commit) }, nil, nil); err != nil && first == nil {
first = err
}
asked += len(due)
@@ -446,8 +446,10 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar
}
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. arm, when not nil, arms
// each ask's bound before it is put (H-A-235, H-A-238 (5)); an ask whose bound is not armed is not put.
func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool, arm func(Asked) error) error {
// each ask's bound before it is put (H-A-235, H-A-238 (5)); an ask whose bound is not armed is not put. logf, when not nil,
// writes one journal line per ask put, naming the pipeline, the mover, the step and the seat (H-A-320 (1)); a caller that
// says the ask in a line of its own (an ask put again) gives none.
func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool, arm func(Asked) error, logf func(string, ...any)) error {
var first error
for _, a := range asks {
accept, ok := acceptStep[a.Seat]
@@ -464,6 +466,10 @@ func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit stri
if err == nil {
err = asker.AskStep(accept, a.ID, a.Ask)
}
if err == nil && logf != nil {
mover, step := askedStep(a)
logf("%s %s %s asked on %s", key, mover, step, a.Seat)
}
if err != nil && first == nil {
first = fmt.Errorf("the step %s of %s is not asked of %s: %w", a.ID, key, a.Seat, err)
}
@@ -471,6 +477,31 @@ func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit stri
return first
}
// askedStep is the mover and the step an ask names, read from its body; "-" where it names none (the validate ask names no
// mover, H-A-51).
func askedStep(a Asked) (mover, step string) {
switch b := a.Ask.(type) {
case nodebuildgo.StepAsk:
if b.Step != nil {
mover, step = b.Step.Module, b.Step.Step
}
case builds.StepRun:
step = b.Step
if len(b.Entries) == 1 {
mover = b.Entries[0].Module
}
case wants.StepRun:
mover, step = b.Module, b.Step
}
if mover == "" {
mover = "-"
}
if step == "" {
step = "-"
}
return mover, step
}
// artifactPaths are the named paths the entry keeps for the artifact, by its reference or its name.
func artifactPaths(e catalog.CatalogEntry, ref string) []string {
if e.Inputs == nil {
@@ -372,6 +372,7 @@ func (h *Hearer) endReader(reader RunReader, run string, r Run) error {
} else if err != nil {
return fmt.Errorf("pipeline %s cannot be written: %w", reader.Pipeline, err)
}
h.logEnded(reader.Pipeline, *s)
h.Failures.StepEnded(h.Store, p, *s)
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return fmt.Errorf("pipeline %s's %s is written but not said: %w", reader.Pipeline, seat.EventStepEnded, err)
@@ -265,7 +265,7 @@ func (s *Starter) askSteps(key string, asks []Asked) error {
var err error
if len(asks) > 0 {
s.carry(key, asks)
err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) }, s.Asks.arm)
err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) }, s.Asks.arm, s.logf)
}
s.sayStarted(key, placed)
return err
@@ -345,6 +345,10 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh
} else if err != nil {
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
}
h.logEnded(ev.Pipeline, *s)
for _, j := range readers {
h.logEnded(ev.Pipeline, p.Steps[j])
}
// The run's end is written as its step-ended is heard, before any finding the step says, so no wait elsewhere (a
// finding's event the bus is slow to take) delays it past its bound (A-470o).
runErr := h.endRunOf(p, *s, ev)
@@ -552,6 +556,22 @@ func endEntry(key string, s seat.StepResult, at time.Time, kept ...string) seat.
Output: s.Output, Kept: kept, At: at}
}
// logEnded writes the journal line of a step's end written (H-A-320 (1)): the pipeline, the mover, the step, its outcome, the
// node that ended it and how long it ran from its start.
func (h *Hearer) logEnded(key string, s seat.StepResult) {
mover, node, took := s.Module, s.Node, "no start"
if mover == "" {
mover = "-"
}
if node == "" {
node = "no node"
}
if s.Started != nil && s.Ended != nil {
took = s.Ended.Sub(*s.Started).Round(time.Millisecond).String()
}
h.logf("%s %s %s ended %s (%s, %s)", key, mover, s.Step, s.Outcome, node, took)
}
func (h *Hearer) logf(format string, args ...any) {
if h.Logf != nil {
h.Logf(format, args...)
@@ -10,6 +10,7 @@ import (
"git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
@@ -363,10 +364,21 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
}{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} {
var to []string
asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil })
if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }, nil); err != nil || strings.Join(to, ",") != c.want {
if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }, nil, nil); err != nil || strings.Join(to, ",") != c.want {
t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want)
}
}
// Each ask put is one journal line naming the pipeline, the mover, the step and the seat: the build ask on mesh-builds and a
// check ask alike (H-A-320 (1)); an ask refused writes none.
var lines []string
put := []Asked{{Seat: builds.Seat, ID: "b1", Ask: builds.StepRun{Step: builds.StepRunStepBuild, Entries: []builds.BuildEntry{{Module: "app"}}}},
{Seat: nodebuildgo.Seat, ID: "u1", Ask: nodebuildgo.StepAsk{Step: &nodebuildgo.StepRun{Module: "app", Step: "unit"}}},
{Seat: "no-seat", ID: "x1", Ask: 3}}
_ = askAll(askerFunc(func(string, string, any) error { return nil }), "main.4", put, nil, nil,
func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) })
if want := "main.4 app build asked on mesh-builds\nmain.4 app unit asked on node-build-go"; strings.Join(lines, "\n") != want {
t.Errorf("journal %q, want %q", lines, want)
}
// Starter.recovered reads the commit; a failed read asks on check and says so.
var said []string
s := &Starter{Logf: func(f string, a ...any) { said = append(said, f) }, Recovered: func(c string) (bool, error) {
@@ -278,6 +278,12 @@ func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) {
a := f.start(t, 4, 9)
b := f.start(t, 6, 9) // the same candidate: it reads a's runs
unit := stepNamed(a, "app:program", "unit")
// One journal line per ask put, naming the pipeline, the mover, the step and the seat (H-A-320 (1)).
if n := countLines(f.logged, a.Pipeline+" app unit asked on node-build-go"); n != 1 || len(f.asks.ids) == 0 ||
countLines(f.logged, " asked on ") != len(f.asks.ids) {
t.Errorf("journal %q for %d asks put; want one asked-on line per ask, a's unit ask once", f.logged, len(f.asks.ids))
}
f.logged = nil
ended := time.Now()
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
Pipeline: a.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed",
@@ -308,6 +314,13 @@ func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) {
if saidReaders != 4 {
t.Errorf("said %d unit step-ended, want one per reading step", saidReaders)
}
// One journal line per step's end, the asking step's and each reader's, naming its pipeline, mover, step, outcome and node
// (H-A-320 (1)).
for _, key := range []string{a.Pipeline, b.Pipeline} {
if n := countLines(f.logged, key+" app unit ended failed (ace, "); n != 2 {
t.Errorf("%s's unit ends journaled %d times in %q, want once per artifact", key, n, f.logged)
}
}
// a, cancelled, leaves the lint run to b's readers; b, cancelled after, leaves none: the ask is dropped and the run ends.
lint := stepNamed(a, "app:program", "lint")
v := Verbs{Hearer: f.h, Acts: &actLog{}}
@@ -671,3 +684,14 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
}
})
}
// countLines is how many of the journal's lines hold part.
func countLines(lines []string, part string) int {
n := 0
for _, l := range lines {
if strings.Contains(l, part) {
n++
}
}
return n
}
@@ -326,7 +326,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
}
if ask != nil {
v.Starter.carry(in.Pipeline, []Asked{*ask})
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil {
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm, v.Starter.logf); err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
}
}
@@ -473,7 +473,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
break
}
v.Starter.carry(in.Pipeline, []Asked{*ask})
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil {
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm, v.Starter.logf); err != nil {
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
}
}