H-A-320 (1): a journal line per step end, step ask and build answer
A working 36-mover pipeline was read as stalled twice: the runner wrote no line when a step ended or was asked, and the builder none when it answered a step ask. The runner now writes "<pipeline> <mover> <step> ended <outcome> (<node>, <duration>)" at each step end it writes (the asking step, every reader of its test run, and each reader EndRun ends), and "<pipeline> <mover> <step> asked on <seat>" for each ask put (the re-ask paths keep their own line). The builder writes "<module>@<fingerprint> build step answered <outcome> for <pipeline> (existing build / built now)". Builds: design 47 H H-A-320 (1)
This commit is contained in:
@@ -43,7 +43,7 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui
|
||||
var failed []string
|
||||
for _, e := range run.Entries {
|
||||
var pins map[string]string
|
||||
outcome, said := b.buildEntry(ctx, e, &pins)
|
||||
outcome, said := b.buildEntry(ctx, run.Pipeline, e, &pins)
|
||||
// The end reports the pins each entry was built with, so the runner asks its held test steps with them (H-A-207).
|
||||
for path, version := range pins {
|
||||
if version == "" {
|
||||
@@ -88,9 +88,10 @@ func (b *Builder) askedRevision(run meshbuilds.StepRun) {
|
||||
}
|
||||
}
|
||||
|
||||
// buildEntry builds one entry at its fingerprint and waits for its end; pins is set, when it is done, to the pins its
|
||||
// done key keeps it was built with, the registry's included (H-A-207).
|
||||
func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) {
|
||||
// buildEntry builds one entry at its fingerprint for pipeline and waits for its end; pins is set, when it is done, to the pins
|
||||
// its done key keeps it was built with, the registry's included (H-A-207). The answer from a build is one journal line,
|
||||
// naming the entry, its outcome, the pipeline and whether the build existed or was built now (H-A-320 (1)).
|
||||
func (b *Builder) buildEntry(ctx context.Context, pipeline string, e meshbuilds.BuildEntry, pins *map[string]string) (outcome, said string) {
|
||||
entry, err := b.StepEntry(ctx, e.Fingerprint)
|
||||
if err != nil {
|
||||
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry %s at %s cannot be read: %v", e.Module, e.Fingerprint, err)
|
||||
@@ -110,13 +111,18 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins
|
||||
}
|
||||
ended := b.awaitEnd(key)
|
||||
defer b.dropEnd(key, ended)
|
||||
answered := func(outcome, said, how string) (string, string) {
|
||||
b.logf("%s@%s build step answered %s for %s (%s)", entry.Module, entry.Fp, outcome, pipeline, how)
|
||||
return outcome, said
|
||||
}
|
||||
if done, failedWords, found := b.keyEnd(ctx, key); found {
|
||||
if done {
|
||||
// A key done before reports the pins it was built with, kept with it, never the ones a publish gives now.
|
||||
*pins = b.keptPins(ctx, key)
|
||||
return b.passedWithItsVersion(ctx, entry)
|
||||
outcome, said := b.passedWithItsVersion(ctx, entry)
|
||||
return answered(outcome, said, "existing build")
|
||||
}
|
||||
return meshbuilds.StepEndedOutcomeFailed, failedWords
|
||||
return answered(meshbuilds.StepEndedOutcomeFailed, failedWords, "existing build")
|
||||
}
|
||||
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalogclient.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
|
||||
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
|
||||
@@ -125,9 +131,10 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins
|
||||
case log := <-ended:
|
||||
if log.Outcome == builds.BuildDone {
|
||||
*pins = b.keptPins(ctx, key)
|
||||
return b.passedWithItsVersion(ctx, entry)
|
||||
outcome, said := b.passedWithItsVersion(ctx, entry)
|
||||
return answered(outcome, said, "built now")
|
||||
}
|
||||
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error)
|
||||
return answered(meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error), "built now")
|
||||
case <-ctx.Done():
|
||||
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the builder stopped before its build ended", entry.Module, entry.Fp)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -171,6 +172,18 @@ func awaitWaiting[T any](t *testing.T, waiting <-chan struct{}, ended <-chan T)
|
||||
// another series and the module's release number pin nothing.
|
||||
func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
|
||||
r, _, waiting := unpinnedRig(t)
|
||||
// Each answer from a build is one journal line, built now or existing (H-A-320 (1)).
|
||||
var mu sync.Mutex
|
||||
var answers []string
|
||||
logf := r.b.Logf
|
||||
r.b.Logf = func(format string, args ...any) {
|
||||
logf(format, args...)
|
||||
if l := fmt.Sprintf(format, args...); strings.Contains(l, "build step answered") {
|
||||
mu.Lock()
|
||||
answers = append(answers, l)
|
||||
mu.Unlock()
|
||||
}
|
||||
}
|
||||
r.b.PackagePublished(unpinnedX, "v0.2.9") // another series than the 0.1 the module accepts: no pin
|
||||
r.b.PackagePublished(unpinnedX, "0.1.4-feat-x.2") // a group's tagged version: no pin
|
||||
ended := make(chan meshbuilds.StepEnded, 1)
|
||||
@@ -206,6 +219,12 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
|
||||
if again.Outcome != meshbuilds.StepEndedOutcomePassed || again.Packages[unpinnedX] != "v0.1.3" {
|
||||
t.Errorf("the done key's end = %+v; want passed reporting x at v0.1.3, as built", again)
|
||||
}
|
||||
mu.Lock()
|
||||
want := []string{"m@" + fp1 + " build step answered passed for main.9.m (built now)", "m@" + fp1 + " build step answered passed for main.9.m (existing build)"}
|
||||
if !reflect.DeepEqual(answers, want) {
|
||||
t.Errorf("the answers journaled %q, want %q", answers, want)
|
||||
}
|
||||
mu.Unlock()
|
||||
// A done key that keeps no pins reports none, so the runner ends the waiting steps loud.
|
||||
v, err := builds.DecodeBuilderBuild(r.st.vals[r.key])
|
||||
if err != nil {
|
||||
|
||||
@@ -176,7 +176,7 @@ func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error {
|
||||
return nil // its second bound ends the step
|
||||
}
|
||||
v.Starter.carry(key, []Asked{*a})
|
||||
return askAll(v.Starter.Steps, key, []Asked{*a}, func(commit string) bool { return v.Starter.recovered(key, commit) }, nil)
|
||||
return askAll(v.Starter.Steps, key, []Asked{*a}, func(commit string) bool { return v.Starter.recovered(key, commit) }, nil, nil)
|
||||
}
|
||||
|
||||
// HolderStarted puts again every open ask of each silent seat module holds (H-A-243 (1)): its holder takes its asks on a plain
|
||||
@@ -234,7 +234,7 @@ func (v Verbs) HolderStarted(module string) (int, error) {
|
||||
continue
|
||||
}
|
||||
st.carry(key, due)
|
||||
if err := askAll(st.Steps, key, due, func(commit string) bool { return st.recovered(key, commit) }, nil); err != nil && first == nil {
|
||||
if err := askAll(st.Steps, key, due, func(commit string) bool { return st.recovered(key, commit) }, nil, nil); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
asked += len(due)
|
||||
|
||||
@@ -446,8 +446,10 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar
|
||||
}
|
||||
|
||||
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. arm, when not nil, arms
|
||||
// each ask's bound before it is put (H-A-235, H-A-238 (5)); an ask whose bound is not armed is not put.
|
||||
func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool, arm func(Asked) error) error {
|
||||
// each ask's bound before it is put (H-A-235, H-A-238 (5)); an ask whose bound is not armed is not put. logf, when not nil,
|
||||
// writes one journal line per ask put, naming the pipeline, the mover, the step and the seat (H-A-320 (1)); a caller that
|
||||
// says the ask in a line of its own (an ask put again) gives none.
|
||||
func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool, arm func(Asked) error, logf func(string, ...any)) error {
|
||||
var first error
|
||||
for _, a := range asks {
|
||||
accept, ok := acceptStep[a.Seat]
|
||||
@@ -464,6 +466,10 @@ func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit stri
|
||||
if err == nil {
|
||||
err = asker.AskStep(accept, a.ID, a.Ask)
|
||||
}
|
||||
if err == nil && logf != nil {
|
||||
mover, step := askedStep(a)
|
||||
logf("%s %s %s asked on %s", key, mover, step, a.Seat)
|
||||
}
|
||||
if err != nil && first == nil {
|
||||
first = fmt.Errorf("the step %s of %s is not asked of %s: %w", a.ID, key, a.Seat, err)
|
||||
}
|
||||
@@ -471,6 +477,31 @@ func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit stri
|
||||
return first
|
||||
}
|
||||
|
||||
// askedStep is the mover and the step an ask names, read from its body; "-" where it names none (the validate ask names no
|
||||
// mover, H-A-51).
|
||||
func askedStep(a Asked) (mover, step string) {
|
||||
switch b := a.Ask.(type) {
|
||||
case nodebuildgo.StepAsk:
|
||||
if b.Step != nil {
|
||||
mover, step = b.Step.Module, b.Step.Step
|
||||
}
|
||||
case builds.StepRun:
|
||||
step = b.Step
|
||||
if len(b.Entries) == 1 {
|
||||
mover = b.Entries[0].Module
|
||||
}
|
||||
case wants.StepRun:
|
||||
mover, step = b.Module, b.Step
|
||||
}
|
||||
if mover == "" {
|
||||
mover = "-"
|
||||
}
|
||||
if step == "" {
|
||||
step = "-"
|
||||
}
|
||||
return mover, step
|
||||
}
|
||||
|
||||
// artifactPaths are the named paths the entry keeps for the artifact, by its reference or its name.
|
||||
func artifactPaths(e catalog.CatalogEntry, ref string) []string {
|
||||
if e.Inputs == nil {
|
||||
|
||||
@@ -372,6 +372,7 @@ func (h *Hearer) endReader(reader RunReader, run string, r Run) error {
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("pipeline %s cannot be written: %w", reader.Pipeline, err)
|
||||
}
|
||||
h.logEnded(reader.Pipeline, *s)
|
||||
h.Failures.StepEnded(h.Store, p, *s)
|
||||
if err := h.say(seat.EventStepEnded, p, s); err != nil {
|
||||
return fmt.Errorf("pipeline %s's %s is written but not said: %w", reader.Pipeline, seat.EventStepEnded, err)
|
||||
|
||||
@@ -265,7 +265,7 @@ func (s *Starter) askSteps(key string, asks []Asked) error {
|
||||
var err error
|
||||
if len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) }, s.Asks.arm)
|
||||
err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) }, s.Asks.arm, s.logf)
|
||||
}
|
||||
s.sayStarted(key, placed)
|
||||
return err
|
||||
|
||||
@@ -345,6 +345,10 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh
|
||||
} else if err != nil {
|
||||
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
|
||||
}
|
||||
h.logEnded(ev.Pipeline, *s)
|
||||
for _, j := range readers {
|
||||
h.logEnded(ev.Pipeline, p.Steps[j])
|
||||
}
|
||||
// The run's end is written as its step-ended is heard, before any finding the step says, so no wait elsewhere (a
|
||||
// finding's event the bus is slow to take) delays it past its bound (A-470o).
|
||||
runErr := h.endRunOf(p, *s, ev)
|
||||
@@ -552,6 +556,22 @@ func endEntry(key string, s seat.StepResult, at time.Time, kept ...string) seat.
|
||||
Output: s.Output, Kept: kept, At: at}
|
||||
}
|
||||
|
||||
// logEnded writes the journal line of a step's end written (H-A-320 (1)): the pipeline, the mover, the step, its outcome, the
|
||||
// node that ended it and how long it ran from its start.
|
||||
func (h *Hearer) logEnded(key string, s seat.StepResult) {
|
||||
mover, node, took := s.Module, s.Node, "no start"
|
||||
if mover == "" {
|
||||
mover = "-"
|
||||
}
|
||||
if node == "" {
|
||||
node = "no node"
|
||||
}
|
||||
if s.Started != nil && s.Ended != nil {
|
||||
took = s.Ended.Sub(*s.Started).Round(time.Millisecond).String()
|
||||
}
|
||||
h.logf("%s %s %s ended %s (%s, %s)", key, mover, s.Step, s.Outcome, node, took)
|
||||
}
|
||||
|
||||
func (h *Hearer) logf(format string, args ...any) {
|
||||
if h.Logf != nil {
|
||||
h.Logf(format, args...)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
@@ -363,10 +364,21 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
|
||||
}{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} {
|
||||
var to []string
|
||||
asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil })
|
||||
if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }, nil); err != nil || strings.Join(to, ",") != c.want {
|
||||
if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }, nil, nil); err != nil || strings.Join(to, ",") != c.want {
|
||||
t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want)
|
||||
}
|
||||
}
|
||||
// Each ask put is one journal line naming the pipeline, the mover, the step and the seat: the build ask on mesh-builds and a
|
||||
// check ask alike (H-A-320 (1)); an ask refused writes none.
|
||||
var lines []string
|
||||
put := []Asked{{Seat: builds.Seat, ID: "b1", Ask: builds.StepRun{Step: builds.StepRunStepBuild, Entries: []builds.BuildEntry{{Module: "app"}}}},
|
||||
{Seat: nodebuildgo.Seat, ID: "u1", Ask: nodebuildgo.StepAsk{Step: &nodebuildgo.StepRun{Module: "app", Step: "unit"}}},
|
||||
{Seat: "no-seat", ID: "x1", Ask: 3}}
|
||||
_ = askAll(askerFunc(func(string, string, any) error { return nil }), "main.4", put, nil, nil,
|
||||
func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) })
|
||||
if want := "main.4 app build asked on mesh-builds\nmain.4 app unit asked on node-build-go"; strings.Join(lines, "\n") != want {
|
||||
t.Errorf("journal %q, want %q", lines, want)
|
||||
}
|
||||
// Starter.recovered reads the commit; a failed read asks on check and says so.
|
||||
var said []string
|
||||
s := &Starter{Logf: func(f string, a ...any) { said = append(said, f) }, Recovered: func(c string) (bool, error) {
|
||||
|
||||
@@ -278,6 +278,12 @@ func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) {
|
||||
a := f.start(t, 4, 9)
|
||||
b := f.start(t, 6, 9) // the same candidate: it reads a's runs
|
||||
unit := stepNamed(a, "app:program", "unit")
|
||||
// One journal line per ask put, naming the pipeline, the mover, the step and the seat (H-A-320 (1)).
|
||||
if n := countLines(f.logged, a.Pipeline+" app unit asked on node-build-go"); n != 1 || len(f.asks.ids) == 0 ||
|
||||
countLines(f.logged, " asked on ") != len(f.asks.ids) {
|
||||
t.Errorf("journal %q for %d asks put; want one asked-on line per ask, a's unit ask once", f.logged, len(f.asks.ids))
|
||||
}
|
||||
f.logged = nil
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: a.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed",
|
||||
@@ -308,6 +314,13 @@ func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) {
|
||||
if saidReaders != 4 {
|
||||
t.Errorf("said %d unit step-ended, want one per reading step", saidReaders)
|
||||
}
|
||||
// One journal line per step's end, the asking step's and each reader's, naming its pipeline, mover, step, outcome and node
|
||||
// (H-A-320 (1)).
|
||||
for _, key := range []string{a.Pipeline, b.Pipeline} {
|
||||
if n := countLines(f.logged, key+" app unit ended failed (ace, "); n != 2 {
|
||||
t.Errorf("%s's unit ends journaled %d times in %q, want once per artifact", key, n, f.logged)
|
||||
}
|
||||
}
|
||||
// a, cancelled, leaves the lint run to b's readers; b, cancelled after, leaves none: the ask is dropped and the run ends.
|
||||
lint := stepNamed(a, "app:program", "lint")
|
||||
v := Verbs{Hearer: f.h, Acts: &actLog{}}
|
||||
@@ -671,3 +684,14 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// countLines is how many of the journal's lines hold part.
|
||||
func countLines(lines []string, part string) int {
|
||||
n := 0
|
||||
for _, l := range lines {
|
||||
if strings.Contains(l, part) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
@@ -326,7 +326,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
}
|
||||
if ask != nil {
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil {
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm, v.Starter.logf); err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -473,7 +473,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
|
||||
break
|
||||
}
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil {
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm, v.Starter.logf); err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user