package steprun import ( "context" "encoding/json" "errors" "fmt" "strings" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" ) // **A pipeline step, run and said** (novox/hq ADR 0666 §7, §8, §17, §21; design 47 *Pipeline steps and the policy*, // *Asked on a work queue, ended by an event*, *Carried, not run*). The runner asks one lint or test step of an artifact // as the accepted kind step; the holder fetches the change's heads, runs the step and says its end as the seat's event // step-ended on its own node's subject before the ask is acknowledged, so a holder restarted mid-step gets the ask again // from its work queue and finishes it (ADR 0664 §1), its end said under the ask's id. A test step runs only the packages // with no kept result for their digest at its level, says the others carried, and keeps what it passed; a step with // nothing to do ends `nothing to run`, never passed; one that cannot run ends `could not run`, saying why. // StepAnswer is what a step ask answers on its step key: the end it said. type StepAnswer struct { Ended sdkstep.StepEnded `json:"ended"` } // LintAnswer is what a module's lint body answers: the failures of its language's lint, none when it passed, and // Nothing when the tree holds nothing its lint reads. type LintAnswer struct { Failed []sdkstep.Failure `json:"failed,omitempty"` Nothing bool `json:"nothing,omitempty"` } // runStep runs ask's pipeline step and says its end. A transport fault, a decline or the process stopping is handed back // unsaid, for the bus to give the ask again, here or on another node. func (s AskSteps) runStep(ctx context.Context, ask sdkstep.StepAsk) (any, error) { if s.SayStep == nil { return nil, Refuse("this build module says no step's end") } end, err := sdkstep.EndOf(ask, s.Node, sdkstep.OutcomePassed) if err != nil { return nil, Refuse("%v", err) } if err := s.stepOutcome(ctx, ask, &end); err != nil { return nil, err } if err := s.SayStep(end); err != nil { return nil, &sdkstep.Passing{Err: fmt.Errorf("the end of the step %s could not be said: %w", ask.ID, err)} } return StepAnswer{Ended: end}, nil } // stepOutcome runs the step into end; the error it answers is one that keeps the end unsaid. func (s AskSteps) stepOutcome(ctx context.Context, ask sdkstep.StepAsk, end *sdkstep.StepEnded) error { var err error if ask.Step.Step == sdkstep.StepLint { err = s.lintStep(ctx, *ask.Step, end) } else { err = s.testStep(ctx, ask, end) } var dec *Declining switch { case err == nil: return nil case sdkstep.IsPassing(err), errors.As(err, &dec), ctx.Err() != nil: return err default: end.Outcome, end.Said = sdkstep.OutcomeCouldNotRun, err.Error() end.Tests, end.Carried, end.Kept = nil, nil, nil return nil } } // lintStep fetches the heads and runs the module's lint body over the artifact's tree. func (s AskSteps) lintStep(ctx context.Context, run sdkstep.StepRun, end *sdkstep.StepEnded) error { if s.Lint == nil { end.Outcome = sdkstep.OutcomeNothingToRun return nil } if _, err := s.onToolchain(ctx, run.Toolchain); err != nil { return err } fetch := run.Fetch _, f, err := s.fetch(ctx, &fetch) if err != nil { return err } defer s.release(f.Workspace) if len(f.Conflicts) > 0 { return fmt.Errorf("the heads do not merge: %s", strings.Join(f.Conflicts, ", ")) } Phase(ctx, "lint") out, err := s.Lint(ctx, map[string]any{"workspace": f.Workspace, "dir": fetch.Dir}) if err != nil { return err } r, err := raw(out) if err != nil { return err } var la LintAnswer if err := json.Unmarshal(r, &la); err != nil { return fmt.Errorf("the lint answered what is not a lint answer: %s: %w", r, err) } switch { case len(la.Failed) > 0: end.Outcome, end.Said = sdkstep.OutcomeFailed, failedSaid(la.Failed) case la.Nothing: end.Outcome = sdkstep.OutcomeNothingToRun default: end.Outcome = sdkstep.OutcomePassed } return nil } // testStep runs one test level as a check of that level alone, taking the step's kept results in the checker's place. func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkstep.StepEnded) error { run := *ask.Step level := run.Step s.Kept = func(_ context.Context, lvl, digest string) (int, bool, error) { for _, k := range run.Kept { if k.Level == lvl && k.Digest == digest { return 0, true, nil } } return 0, false, nil } job := sdkstep.CheckJob{Fetch: *withPackages(&run.Fetch, ask.Packages), Levels: []string{level}, Kept: true, Toolchain: run.Toolchain} out, err := s.check(ctx, job) if err != nil { return err } answer, _ := out.(sdkstep.CheckAnswer) if len(answer.Levels) == 0 { var conflicts struct { Conflicts []string `json:"conflicts"` } _ = json.Unmarshal(mustRaw(answer.Fetch), &conflicts) return fmt.Errorf("the heads do not merge: %s", strings.Join(conflicts.Conflicts, ", ")) } la := answer.Levels[0] if len(la.Unmet) > 0 { var what []string for _, u := range la.Unmet { what = append(what, u.Package+" ("+strings.Join(u.Needs, ", ")+")") } return fmt.Errorf("the %s tests of %s need what this node lacks", level, strings.Join(what, ", ")) } tests, passed, ran := 0, true, false var why []string for _, t := range la.Tests { var tr struct { Passed bool `json:"passed"` Tests int `json:"tests"` NoTests bool `json:"no_tests"` Why string `json:"why"` } if err := json.Unmarshal(t, &tr); err != nil { return fmt.Errorf("the %s level answered what is no test result: %w", level, err) } tests += tr.Tests ran = ran || (!tr.NoTests && tr.Tests > 0) || !tr.Passed if !tr.Passed { passed = false if tr.Why != "" { why = append(why, tr.Why) } } } for _, k := range la.Kept { end.Carried = append(end.Carried, keptOf(run, k.Package, k.Digest)) } switch { case !passed: end.Outcome, end.Said = sdkstep.OutcomeFailed, strings.Join(why, "; ") if end.Said == "" { end.Said = "the " + level + " tests failed" } case !ran && len(end.Carried) == 0: end.Outcome = sdkstep.OutcomeNothingToRun default: end.Outcome = sdkstep.OutcomePassed if la.Digest != nil { for _, p := range la.Digest.Packages { if !carried(end.Carried, p.Package) { end.Kept = append(end.Kept, keptOf(run, p.Package, p.Digest)) } } } } if end.Outcome != sdkstep.OutcomeNothingToRun { end.Tests = &sdkstep.TestCounts{} if level == sdkstep.StepUnit { end.Tests.Unit = tests } else { end.Tests.Integration = tests } } return nil } func keptOf(run sdkstep.StepRun, pkg, digest string) sdkstep.KeptResult { return sdkstep.KeptResult{Package: pkg, Digest: digest, Level: run.Step, Toolchain: run.Toolchain, Pipeline: run.Pipeline} } func carried(kept []sdkstep.KeptResult, pkg string) bool { for _, k := range kept { if k.Package == pkg { return true } } return false } func mustRaw(v any) json.RawMessage { r, _ := raw(v) return r } // failedSaid names each failure in one line, the first ten. func failedSaid(failed []sdkstep.Failure) string { var names []string for i, f := range failed { if i == 10 { names = append(names, fmt.Sprintf("and %d more", len(failed)-10)) break } names = append(names, strings.TrimSpace(f.Package+" "+f.Test+": "+firstOf(f.Cause))) } return fmt.Sprintf("%d failed: %s", len(failed), strings.Join(names, "; ")) } func firstOf(s string) string { l, _, _ := strings.Cut(strings.TrimSpace(s), "\n") return l } // SayStepOn is a holder's SayStep: the end, encoded and checked by the domain SDK (sdkstep.EncodeStepEnded), published // on the language's build seat's step-ended subject of node with publish (stdio.SeatPublish), de-duplicated by the // ask's id, so a step run again after a restart is said once. func SayStepOn(language, node string, publish func(subject string, body any, id string) error) func(sdkstep.StepEnded) error { return func(end sdkstep.StepEnded) error { subject, err := sdkstep.StepEndedSubject(language, node) if err != nil { return err } body, err := sdkstep.EncodeStepEnded(end) if err != nil { return err } return publish(subject, json.RawMessage(body), "step-ended."+end.ID) } }