Name what failed in a repository check from its whole output, not its tail (issue 460)

A long run's logs pushed the --- FAIL lines out of the last 200 lines the
summary was named from, so a failed check could not say which test failed.
Pick the lines that name a failure as the output streams, keep them whole at
the end of the verdict's report, and say the whole output in the build's log.

internal/builder/check.go

@@ -146,6 +146,9 @@ type Layer struct {
 	Verdict string
 	Summary string
 	Modules []string
+	// Failed is what the repository's own check printed that names what failed, picked from the whole of
+	// its output and said at the end of the verdict's report (novox/hq issue 460). Empty when it passed.
+	Failed string
 }
 
 // CheckScript is what a repository declares its own merge check as: run from its root, with the
@@ -477,7 +480,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 	}
 
 	v.Verdict, v.Summary = v.Gate.Verdict, v.Gate.Summary
-	v.Report, v.Took = out.String(), time.Since(began)
+	v.Report, v.Took = withWhatFailed(out.String(), v.Repo), time.Since(began)
 	say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary,
 		strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second))
 	return v, nil
@@ -515,17 +518,29 @@ func ownCheck(ctx context.Context, spec CheckSpec, parts []ScriptPart, tree stri
 		say("check", "running its %s in the mesh's %s toolchain", part.Script, part.Toolchain)
 		fmt.Fprintf(out, "--- its %s (%s toolchain)\n", part.Script, part.Toolchain)
 		var own tail
+		var picked failures
+		logged := &toTheLog{say: say}
 		cmd := command(image, part.Script)
 		inItsOwnGroup(cmd)
-		w := io.MultiWriter(out, &own)
+		w := io.MultiWriter(out, &own, &picked, logged)
 		cmd.Stdout, cmd.Stderr = w, w
-		switch err := cmd.Run(); {
+		err := cmd.Run()
+		logged.close(picked.String())
+		switch {
 		case timedOut():
-			return &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", part.Script, CheckTimeout)}
+			return &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", part.Script, CheckTimeout),
+				Failed: picked.String()}
 		case ctx.Err() != nil:
 			return nil
 		case err != nil:
-			return &Layer{Verdict: "fail", Summary: passedSoFar(ran) + "its " + part.Script + " failed: " + whatFailed(own.String())}
+			// Named from the whole run, not the tail: what failed may be thousands of lines from the end
+			// (novox/hq issue 460).
+			said := picked.said()
+			if said == "" {
+				said = whatFailed(own.String())
+			}
+			return &Layer{Verdict: "fail", Summary: passedSoFar(ran) + "its " + part.Script + " failed: " + said,
+				Failed: picked.String()}
 		}
 		ran = append(ran, fmt.Sprintf("%s (%s)", part.Script, part.Toolchain))
 	}
@@ -535,6 +550,84 @@ func ownCheck(ctx context.Context, spec CheckSpec, parts []ScriptPart, tree stri
 	return &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed, each part in its toolchain: " + strings.Join(ran, ", ")}
 }
 
+// withWhatFailed is a check's report with what its repository's own check names as failed said last, whole:
+// the report's tail is the last reportLines lines, and the controller keeps the end of a report it cuts, so
+// what failed travels whatever came before or after it (novox/hq issue 460).
+func withWhatFailed(report string, repo *Layer) string {
+	if repo == nil || repo.Failed == "" {
+		return report
+	}
+	return report + "\n--- what failed, picked from the whole of its output (the whole is in the build's log)\n" +
+		repo.Failed
+}
+
+// logLines bounds the lines of a repository's own check that go into the build's log. The bus keeps 10 000
+// messages per subject (EVENTS' MaxMsgsPerSubject), and a build's log is one subject: past that, its first
+// lines would be lost. The build's own lines are a few hundred at most, so the check's output takes 8 000.
+const logLines = 8000
+
+// logLineBytes bounds one line of it, so a line that pastes a document does not fill a message.
+const logLineBytes = 4 << 10
+
+// toTheLog says each line a repository's own check prints in the build's log, as it prints it — the whole
+// output kept for the build and read with `builds` and its id (novox/hq issue 460), where before only the
+// verdict's tail of it was kept. Past logLines it says how many lines it left out, and the lines that name
+// what failed after them.
+type toTheLog struct {
+	say     func(step, format string, args ...any)
+	partial []byte
+	said    int
+	dropped int
+}
+
+func (l *toTheLog) Write(p []byte) (int, error) {
+	l.partial = append(l.partial, p...)
+	for {
+		i := bytes.IndexByte(l.partial, '\n')
+		if i < 0 {
+			break
+		}
+		l.line(string(bytes.TrimRight(l.partial[:i], "\r")))
+		l.partial = l.partial[i+1:]
+	}
+	if len(l.partial) > logLineBytes {
+		l.line(string(l.partial))
+		l.partial = nil
+	}
+	return len(p), nil
+}
+
+func (l *toTheLog) line(line string) {
+	if l.said >= logLines {
+		l.dropped++
+		return
+	}
+	if len(line) > logLineBytes {
+		line = line[:logLineBytes] + fmt.Sprintf(" … (%d bytes more)", len(line)-logLineBytes)
+	}
+	l.said++
+	l.say("output", "%s", line)
+}
+
+// close says the last line, and, when lines were left out, how many and what failed.
+func (l *toTheLog) close(failed string) {
+	if len(l.partial) > 0 {
+		l.line(string(l.partial))
+		l.partial = nil
+	}
+	if l.dropped == 0 {
+		return
+	}
+	l.say("output", "… %d more line(s) of its output are not in this log, which keeps its first %d", l.dropped, logLines)
+	if failed == "" {
+		return
+	}
+	l.say("output", "--- what failed, picked from the whole of its output")
+	for _, line := range strings.Split(failed, "\n") {
+		l.say("output", "%s", line)
+	}
+}
+
 // passedSoFar is what of a check's parts passed before the one that did not, said first.
 func passedSoFar(ran []string) string {
 	if len(ran) == 0 {
@@ -1018,9 +1111,17 @@ func whatFailed(s string) string {
 			return "not gofmt'd by the toolchain's gofmt: " + strings.Join(files, ", ")
 		}
 	}
-	for _, prefix := range []string{"--- FAIL:", "FAIL\t", "panic:"} {
+	// A panic, a data race or a build error outside any test says more than the package's bare FAIL line
+	// (novox/hq issue 460).
+	for _, said := range []func(string) bool{
+		func(l string) bool { return strings.HasPrefix(l, "--- FAIL:") },
+		func(l string) bool { return strings.HasPrefix(l, "panic:") || strings.HasPrefix(l, "fatal error:") },
+		func(l string) bool { return l == "WARNING: DATA RACE" },
+		goError.MatchString,
+		func(l string) bool { return strings.HasPrefix(l, "FAIL\t") },
+	} {
 		for _, line := range lines {
-			if line = strings.TrimSpace(line); strings.HasPrefix(line, prefix) {
+			if line = strings.TrimSpace(line); said(line) {
 				return line
 			}
 		}