package steprun import ( "context" "encoding/json" "errors" "fmt" "slices" "strings" "time" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" ) // **A pipeline step, run and said** (novox/hq ADR 0666 §7, §8, §17, §21; design 47 *Pipeline steps and the policy*, // *Asked on a work queue, ended by an event*, *Carried, not run*). The runner asks one lint or test step of an artifact // as the accepted kind step; the holder fetches the change's heads, runs the step and says its end as the seat's event // step-ended on its own node's subject before the ask is acknowledged, so a holder restarted mid-step gets the ask again // from its work queue and finishes it (ADR 0664 §1), its end said under the ask's id. A test step runs only the packages // with no kept result for their digest at its level, says the others carried, and keeps what it passed; a step with // nothing to do ends `nothing to run`, never passed; one that cannot run ends `could not run`, saying why. // StepAnswer is what a step ask answers on its step key: the end it said. type StepAnswer struct { Ended sdkstep.StepEnded `json:"ended"` } // LintAnswer is what a module's lint body answers: the failures of its language's lint, none when it passed, and // Nothing when the tree holds nothing its lint reads. type LintAnswer struct { Failed []sdkstep.Failure `json:"failed,omitempty"` Nothing bool `json:"nothing,omitempty"` } // runStep runs ask's pipeline step and says its end. A transport fault, a decline or the process stopping is handed back // unsaid, for the bus to give the ask again, here or on another node. func (s AskSteps) runStep(ctx context.Context, ask sdkstep.StepAsk) (any, error) { if s.SayStep == nil { return nil, Refuse("this build module says no step's end") } end, err := sdkstep.EndOf(ask, s.Node, sdkstep.OutcomePassed) if err != nil { return nil, Refuse("%v", err) } if err := s.stepOutcome(ctx, ask, &end); err != nil { if !sdkstep.IsPassing(err) || runOf(ctx) < sdkstep.AskRuns { return nil, err } // Its last run met a transport fault (design 47 H2 §799, §800): the step ends here and its end is said, a starved // setup could-not-run, never failed, a fetch or a publish met away failed with the why it opens with. end.Outcome, end.Said = sdkstep.OutcomeFailed, err.Error() if errors.Is(err, errSetupStarved) { end.Outcome = sdkstep.OutcomeCouldNotRun } end.Tests, end.Carried, end.Kept, end.NotRun = nil, nil, nil, nil } if err := s.SayStep(end); err != nil { return nil, &sdkstep.Passing{Err: fmt.Errorf("the end of the step %s could not be said: %w", ask.ID, err)} } return StepAnswer{Ended: end}, nil } // stepOutcome runs the step into end; the error it answers is one that keeps the end unsaid. func (s AskSteps) stepOutcome(ctx context.Context, ask sdkstep.StepAsk, end *sdkstep.StepEnded) error { var err error if ask.Install != nil { err = s.installStep(ctx, ask, end) } else if ask.Step.Step == sdkstep.StepLint { err = s.lintStep(ctx, *ask.Step, end) } else { err = s.testStep(ctx, ask, end) } var dec *Declining switch { case err == nil: return nil case sdkstep.IsPassing(err), errors.As(err, &dec), ctx.Err() != nil: return err default: end.Outcome, end.Said = sdkstep.OutcomeCouldNotRun, err.Error() end.Tests, end.Carried, end.Kept = nil, nil, nil return nil } } // installStep runs the module's install body and names the archive it published in the end (design 47 H2 §800, H-A-399 // (2), H-A-403); an install that answers no archive could not run. func (s AskSteps) installStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkstep.StepEnded) error { if s.Install == nil { return fmt.Errorf("this build module installs nothing") } i := ask.Install Phase(ctx, "install") out, err := s.Install(ctx, map[string]any{"module": i.Module, "commit": i.Commit, "fingerprint": i.Fingerprint, "pipeline": i.Pipeline, "lock": i.Lock, "artifact": sdkstep.InstallArtifact(i.Module), "version": ask.Version}) if err != nil { return err } r, err := raw(out) if err != nil { return err } var a struct { Archive string `json:"archive"` } if err := json.Unmarshal(r, &a); err != nil || a.Archive == "" { return fmt.Errorf("the install answered no archive: %s", r) } end.Outcome, end.Archive = sdkstep.OutcomePassed, a.Archive return nil } // lintStep fetches the heads and runs the module's lint body over the artifact's tree. func (s AskSteps) lintStep(ctx context.Context, run sdkstep.StepRun, end *sdkstep.StepEnded) error { if s.Lint == nil { end.Outcome = sdkstep.OutcomeNothingToRun return nil } if _, err := s.onToolchain(ctx, run.Toolchain); err != nil { return err } fetch := run.Fetch _, f, err := s.fetch(ctx, &fetch) if err != nil { return err } defer s.release(f.Workspace) if len(f.Conflicts) > 0 { return fmt.Errorf("the heads do not merge: %s", strings.Join(f.Conflicts, ", ")) } Phase(ctx, "lint") out, err := s.Lint(ctx, map[string]any{"workspace": f.Workspace, "dir": fetch.Dir}) if err != nil { return err } r, err := raw(out) if err != nil { return err } var la LintAnswer if err := json.Unmarshal(r, &la); err != nil { return fmt.Errorf("the lint answered what is not a lint answer: %s: %w", r, err) } switch { case len(la.Failed) > 0: end.Outcome, end.Said = sdkstep.OutcomeFailed, failedSaid(la.Failed) case la.Nothing: end.Outcome = sdkstep.OutcomeNothingToRun default: end.Outcome = sdkstep.OutcomePassed } return nil } // testStep runs one test level as a check of that level alone, taking the step's kept results in the checker's place. func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkstep.StepEnded) error { run := *ask.Step level := run.Step s.Kept = func(_ context.Context, lvl, digest string) (int, bool, error) { for _, k := range run.Kept { if k.Level == lvl && k.Digest == digest { return 0, true, nil } } return 0, false, nil } job := sdkstep.CheckJob{Fetch: *withPackages(&run.Fetch, ask.Packages), Levels: []string{level}, Kept: true, Toolchain: run.Toolchain, Archive: run.Archive} // The level's text, taken out of its test answers, is the run's output (design 47 H2 H-A-412). var text strings.Builder out, err := s.checkTexts(ctx, job, &text) if err != nil { return err } answer, _ := out.(sdkstep.CheckAnswer) if len(answer.Levels) == 0 { var conflicts struct { Conflicts []string `json:"conflicts"` } _ = json.Unmarshal(mustRaw(answer.Fetch), &conflicts) return fmt.Errorf("the heads do not merge: %s", strings.Join(conflicts.Conflicts, ", ")) } la := answer.Levels[0] if len(la.Unmet) > 0 { // The node lacks what the packages' tests need: the step is declined with the needs and what this node lacks of them, // for the runner to put it again on a node that has them (design 47 H2 H-A-375, H-A-445). var what, needs []string for _, u := range la.Unmet { what = append(what, u.Package+" ("+strings.Join(u.Needs, ", ")+")") for _, n := range u.Needs { if !slices.Contains(needs, n) { needs = append(needs, n) } } } return &Declining{Text: fmt.Sprintf("the %s tests of %s need what this node lacks", level, strings.Join(what, ", ")), Needs: needs, Lacks: sdkstep.Lacks(needs, s.Has)} } tests, passed, ran := 0, true, false var why []string var failed []sdkstep.Failure bound := 0.0 for _, t := range la.Tests { var tr struct { Passed bool `json:"passed"` Tests int `json:"tests"` NoTests bool `json:"no_tests"` Why string `json:"why"` // NotRun are the tests that did not run, with their reasons (design 47 H2 §800): the part's not-run, read by // the build module's reader (ReadGoTest's NotRun). NotRun []sdkstep.NotRun `json:"not-run"` Failed []sdkstep.Failure `json:"failed"` // Bound is the level's bound in seconds, the setup bound being half of it (H-A-368). Bound float64 `json:"bound"` } if err := json.Unmarshal(t, &tr); err != nil { return fmt.Errorf("the %s level answered what is no test result: %w", level, err) } tests += tr.Tests ran = ran || (!tr.NoTests && tr.Tests > 0) || !tr.Passed end.NotRun = append(end.NotRun, tr.NotRun...) bound = max(bound, tr.Bound) if !tr.Passed { passed = false failed = append(failed, tr.Failed...) if tr.Why != "" { why = append(why, tr.Why) } } } s.keepRunOutput(ask.ID, text.String(), failed) if !passed && starvedOnly(failed) { // Design 47 H2 §800, H-A-368: a setup whose wait on the bus passed its bound runs again by §799's delays, never // failed; the third such run ends could-not-run (runStep). end.NotRun = nil return &sdkstep.Passing{Err: fmt.Errorf("%w %s under load (%s)", errSetupStarved, setupBound(bound), firstOf(failed[0].Cause))} } for _, k := range la.Kept { end.Carried = append(end.Carried, keptOf(run, k.Package, k.Digest)) } switch { case !passed: end.Outcome, end.Said = sdkstep.OutcomeFailed, strings.Join(why, "; ") if end.Said == "" { end.Said = "the " + level + " tests failed" } case !ran && len(end.Carried) == 0: end.Outcome = sdkstep.OutcomeNothingToRun default: end.Outcome = sdkstep.OutcomePassed if la.Digest != nil { for _, p := range la.Digest.Packages { if !carried(end.Carried, p.Package) { end.Kept = append(end.Kept, keptOf(run, p.Package, p.Digest)) } } } } if end.Outcome != sdkstep.OutcomeNothingToRun { end.Tests = &sdkstep.TestCounts{} if level == sdkstep.StepUnit { end.Tests.Unit = tests } else { end.Tests.Integration = tests } } return nil } // keepRunOutput keeps a test step's level text, or a failed build's error text (H-A-482), as the run's output `.` in the outputs objects, with the header of the failing tests and their words from the level's results (design 47 // H2 §791, H-A-412 (1)). A holder that // keeps no outputs, or a write that fails, is said in the journal: the step's end stands on the results. func (s AskSteps) keepRunOutput(ask, text string, failed []sdkstep.Failure) { var failing []sdkstep.FailingTest for _, f := range failed { failing = append(failing, sdkstep.FailingTest{Test: strings.TrimSpace(f.Package + " " + f.Test), Words: f.Cause}) } if err := keepOutput(s.Outputs, s.Node, ask, text, failing); err != nil && s.Logf != nil { s.Logf("the output of the ask %s could not be kept: %v", ask, err) } } // errSetupStarved opens the why of a level whose every failure is a starved setup (design 47 H2 §800): its words are the // could-not-run end's at the third run. var errSetupStarved = errors.New("the bus did not answer in setup within") // starvedOnly says a level failed only by starved setups: every failure's words open stdio.SetupStarved (H-A-368). An // assertion that fails is never `again` (§800), so one such failure among them makes the level failed. func starvedOnly(failed []sdkstep.Failure) bool { for _, f := range failed { if !strings.HasPrefix(strings.TrimSpace(f.Cause), stdio.SetupStarved) { return false } } return len(failed) > 0 } // setupBound is the setup bound of a level bound in seconds: half of it, 30 s when the level names none (H-A-368). func setupBound(levelSeconds float64) time.Duration { if levelSeconds <= 0 { return 30 * time.Second } return time.Duration(levelSeconds * float64(time.Second) / 2).Round(time.Second) } func keptOf(run sdkstep.StepRun, pkg, digest string) sdkstep.KeptResult { return sdkstep.KeptResult{Package: pkg, Digest: digest, Level: run.Step, Toolchain: run.Toolchain, Pipeline: run.Pipeline} } func carried(kept []sdkstep.KeptResult, pkg string) bool { for _, k := range kept { if k.Package == pkg { return true } } return false } func mustRaw(v any) json.RawMessage { r, _ := raw(v) return r } // failedSaid names each failure in one line, the first ten. func failedSaid(failed []sdkstep.Failure) string { var names []string for i, f := range failed { if i == 10 { names = append(names, fmt.Sprintf("and %d more", len(failed)-10)) break } names = append(names, strings.TrimSpace(f.Package+" "+f.Test+": "+firstOf(f.Cause))) } return fmt.Sprintf("%d failed: %s", len(failed), strings.Join(names, "; ")) } func firstOf(s string) string { l, _, _ := strings.Cut(strings.TrimSpace(s), "\n") return l } // SayStepOn is a holder's SayStep: the end, encoded and checked by the domain SDK (sdkstep.EncodeStepEnded), published // on the language's build seat's step-ended subject of node with publish (stdio.SeatPublish), de-duplicated by the // ask's id, so a step run again after a restart is said once. func SayStepOn(language, node string, publish func(subject string, body any, id string) error) func(sdkstep.StepEnded) error { return func(end sdkstep.StepEnded) error { subject, err := sdkstep.StepEndedSubject(language, node) if err != nil { return err } body, err := sdkstep.EncodeStepEnded(end) if err != nil { return err } return publish(subject, json.RawMessage(body), "step-ended."+end.ID) } } // SayOn is a holder's Puller.Say: an event of the language's build seat published on that seat's subject of node with // publish (stdio.SeatPublish), `mesh.seat.node-build-.event..`. func SayOn(language, node string, publish func(subject string, body any, id string) error) func(event string, body any, id string) error { return func(event string, body any, id string) error { seat, err := sdkstep.BuildSeat(language) if err != nil { return err } return publish("mesh.seat."+seat+".event."+event+"."+node, body, id) } }