Merge pull request 'Run each part of a repository's own check in the toolchain it declares (hq issue 302)' (#125) from fix/a-check-runs-each-toolchain-it-declares into main

internal/builder/check.go

@@ -39,6 +39,11 @@ import (
 //     quality, run when present in the toolchain it declares (`# mesh-check-toolchain: go|typescript`
 //     among its first lines; go when it declares none). A repository that reaches the build seat with
 //     none is said as a warning: it is the mesh's, and nothing of its own is tested before it merges.
+//     **A repository in two languages declares the other part too** (`# mesh-check-also: <toolchain>
+//     <script>`, one line each, among the same first lines): each named script is run from the root in
+//     its own toolchain's container, after merge-check.sh, and the layer passes only when every part
+//     does (novox/hq issue 302 — the lab's Go replays went unchecked because its script runs in the
+//     TypeScript toolchain, which holds no Go compiler).
 //
 // One check:
 //
@@ -178,6 +183,32 @@ const noScript = "the repository declares no " + CheckScript + ": none of its ow
 // toolchainLine is how a merge-check.sh declares the toolchain it runs in.
 var toolchainLine = regexp.MustCompile(`^#\s*mesh-check-toolchain:\s*([a-z0-9-]+)\s*$`)
 
+// alsoLine is how a merge-check.sh declares a further part of the repository's check, run in a toolchain
+// of its own: `# mesh-check-also: <toolchain> <script>`.
+var alsoLine = regexp.MustCompile(`^#\s*mesh-check-also:\s*([a-z0-9-]+)\s+(\S+)\s*$`)
+
+// ScriptPart is one part of a repository's own check: a script, run from the repository's root, and the
+// toolchain it runs in.
+type ScriptPart struct {
+	Toolchain string
+	Script    string
+}
+
+// ScriptParts is every part of a repository's own check: its merge-check.sh in the toolchain it declares,
+// then each further part it declares among its first twenty lines (`# mesh-check-also: <toolchain>
+// <script>`), in the order declared. A declared script that is not a path inside the repository is
+// answered as a part all the same, for the check to refuse by name rather than to drop.
+func ScriptParts(script []byte) []ScriptPart {
+	parts := []ScriptPart{{Toolchain: ScriptToolchain(script), Script: CheckScript}}
+	lines := bufio.NewScanner(bytes.NewReader(script))
+	for i := 0; i < 20 && lines.Scan(); i++ {
+		if m := alsoLine.FindStringSubmatch(strings.TrimSpace(lines.Text())); m != nil {
+			parts = append(parts, ScriptPart{Toolchain: m[1], Script: m[2]})
+		}
+	}
+	return parts
+}
+
 // ScriptToolchain is the language a merge-check.sh declares it runs in, among its first twenty lines;
 // go when it declares none.
 func ScriptToolchain(script []byte) string {
@@ -416,32 +447,11 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 	case timedOut():
 		v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("the check ran past %s before its %s ran", CheckTimeout, CheckScript)}
 	default:
-		language := ScriptToolchain(script)
+		v.Repo = ownCheck(ctx, spec, ScriptParts(script), tree, &out, timedOut, func(image string, script string) *exec.Cmd {
-		image := spec.Toolchains[language]
+			return exec.CommandContext(ctx, "docker", LabelledArgs("docker", in(image, tree, env, "sh", script), spec.ID)...)
-		if language == "go" && image == "" {
+		}, say)
-			image = spec.Toolchain
+		if v.Repo == nil {
-		}
-		if image == "" {
-			v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s runs in the %s toolchain, which the mesh does "+
-				"not hold", CheckScript, language)}
-			break
-		}
-		say("check", "running its %s in the mesh's %s toolchain", CheckScript, language)
-		fmt.Fprintf(&out, "--- its %s (%s toolchain)\n", CheckScript, language)
-		var own tail
-		cmd := exec.CommandContext(ctx, "docker", LabelledArgs("docker", in(image, tree, env, "sh", CheckScript), spec.ID)...)
-		inItsOwnGroup(cmd)
-		w := io.MultiWriter(&out, &own)
-		cmd.Stdout, cmd.Stderr = w, w
-		switch err := cmd.Run(); {
-		case timedOut():
-			v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", CheckScript, CheckTimeout)}
-		case ctx.Err() != nil:
 			return v, ctx.Err()
-		case err != nil:
-			v.Repo = &Layer{Verdict: "fail", Summary: "its " + CheckScript + " failed: " + whatFailed(own.String())}
-		default:
-			v.Repo = &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed"}
 		}
 	}
 
@@ -452,6 +462,66 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 	return v, nil
 }
 
+// ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the
+// layer's verdict. Nil when the check was ended from outside (not past its bound), which the caller
+// answers as the context's error.
+func ownCheck(ctx context.Context, spec CheckSpec, parts []ScriptPart, tree string, out *tail, timedOut func() bool,
+	command func(image, script string) *exec.Cmd, say func(step, format string, args ...any)) *Layer {
+	var ran []string
+	for i, part := range parts {
+		named := part.Script
+		if i > 0 {
+			named = CheckScript + "'s part " + part.Script
+		}
+		image := spec.Toolchains[part.Toolchain]
+		if part.Toolchain == "go" && image == "" {
+			image = spec.Toolchain
+		}
+		if image == "" {
+			return &Layer{Verdict: "error", Summary: fmt.Sprintf("%sits %s runs in the %s toolchain, which the mesh does "+
+				"not hold", passedSoFar(ran), named, part.Toolchain)}
+		}
+		if i > 0 && !filepath.IsLocal(part.Script) {
+			return &Layer{Verdict: "fail", Summary: fmt.Sprintf("%s declares a part %q that is not a path inside the "+
+				"repository", CheckScript, part.Script)}
+		}
+		if i > 0 {
+			if info, err := os.Stat(filepath.Join(tree, part.Script)); err != nil || info.IsDir() {
+				return &Layer{Verdict: "fail", Summary: fmt.Sprintf("%s declares a part %s, which the repository does "+
+					"not hold", CheckScript, part.Script)}
+			}
+		}
+		say("check", "running its %s in the mesh's %s toolchain", part.Script, part.Toolchain)
+		fmt.Fprintf(out, "--- its %s (%s toolchain)\n", part.Script, part.Toolchain)
+		var own tail
+		cmd := command(image, part.Script)
+		inItsOwnGroup(cmd)
+		w := io.MultiWriter(out, &own)
+		cmd.Stdout, cmd.Stderr = w, w
+		switch err := cmd.Run(); {
+		case timedOut():
+			return &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", part.Script, CheckTimeout)}
+		case ctx.Err() != nil:
+			return nil
+		case err != nil:
+			return &Layer{Verdict: "fail", Summary: passedSoFar(ran) + "its " + part.Script + " failed: " + whatFailed(own.String())}
+		}
+		ran = append(ran, fmt.Sprintf("%s (%s)", part.Script, part.Toolchain))
+	}
+	if len(ran) == 1 {
+		return &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed"}
+	}
+	return &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed, each part in its toolchain: " + strings.Join(ran, ", ")}
+}
+
+// passedSoFar is what of a check's parts passed before the one that did not, said first.
+func passedSoFar(ran []string) string {
+	if len(ran) == 0 {
+		return ""
+	}
+	return strings.Join(ran, ", ") + " passed; "
+}
+
 // gateLayer runs the gate: the touched manifests through `module check`, every machine composed with the
 // change, and the replays of what the mesh runs. It answers the gate's verdict and summary.
 func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFile string, env []string,

internal/builder/check_test.go

@@ -480,3 +480,76 @@ func TestARedeliveredCheckRemovesWhatItsEarlierDeliveryLeft(t *testing.T) {
 		t.Errorf("the check left %d container(s) behind", len(left))
 	}
 }
+
+// **A repository in two languages checks both, each in its own toolchain** (novox/hq issue 302): the lab's
+// merge-check.sh runs in the TypeScript toolchain, and its Go replays were said "NOT CHECKED HERE" on every
+// pull request, so a register that did not compile would have merged green. Its script declares the Go
+// part; the check runs it in the Go toolchain's container after the script, and the layer passes only
+// when both do.
+func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) {
+	script := "#!/bin/sh\n# mesh-check-toolchain: typescript\n# mesh-check-also: go replays/merge-check.sh\nnpm test\n"
+	parts := ScriptParts([]byte(script))
+	if len(parts) != 2 || parts[0] != (ScriptPart{"typescript", CheckScript}) ||
+		parts[1] != (ScriptPart{"go", "replays/merge-check.sh"}) {
+		t.Fatalf("the parts read as %+v", parts)
+	}
+	if got := ScriptParts([]byte("#!/bin/sh\nset -eu\n")); len(got) != 1 || got[0] != (ScriptPart{"go", CheckScript}) {
+		t.Fatalf("a script declaring nothing reads as %+v", got)
+	}
+
+	// Each part run where its toolchain is: here, the image the part was given is what the command says.
+	held := CheckSpec{Toolchain: "go-image", Toolchains: map[string]string{"typescript": "ts-image", "go": "go-image"}}
+	run := func(t *testing.T, spec CheckSpec, files map[string]string, parts []ScriptPart) (*Layer, []string) {
+		t.Helper()
+		tree := t.TempDir()
+		for name, body := range files {
+			if err := os.MkdirAll(filepath.Dir(filepath.Join(tree, name)), 0o755); err != nil {
+				t.Fatal(err)
+			}
+			if err := os.WriteFile(filepath.Join(tree, name), []byte(body), 0o755); err != nil {
+				t.Fatal(err)
+			}
+		}
+		var images []string
+		var out tail
+		layer := ownCheck(t.Context(), spec, parts, tree, &out, func() bool { return false },
+			func(image, script string) *exec.Cmd {
+				images = append(images, image+" "+script)
+				cmd := exec.CommandContext(t.Context(), "sh", script)
+				cmd.Dir = tree
+				return cmd
+			}, func(string, string, ...any) {})
+		return layer, images
+	}
+	twoParts := []ScriptPart{{"typescript", CheckScript}, {"go", "replays/merge-check.sh"}}
+
+	layer, images := run(t, held, map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts)
+	if layer.Verdict != "pass" || !strings.Contains(layer.Summary, "replays/merge-check.sh (go)") ||
+		strings.Join(images, ", ") != "ts-image merge-check.sh, go-image replays/merge-check.sh" {
+		t.Errorf("both parts passing answered %+v, ran %v", layer, images)
+	}
+	layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n",
+		"replays/merge-check.sh": "echo 'not gofmt'\"'\"'d:'; echo register.go; exit 1\n"}, twoParts)
+	if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "replays/merge-check.sh failed") ||
+		!strings.Contains(layer.Summary, "register.go") || !strings.Contains(layer.Summary, "merge-check.sh (typescript) passed") {
+		t.Errorf("a failing Go part answered %+v", layer)
+	}
+	layer, images = run(t, held, map[string]string{CheckScript: "exit 2\n", "replays/merge-check.sh": "exit 0\n"}, twoParts)
+	if layer.Verdict != "fail" || len(images) != 1 {
+		t.Errorf("a failing first part answered %+v and ran %v", layer, images)
+	}
+	layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, twoParts)
+	if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "does not hold") {
+		t.Errorf("a declared part the repository lacks answered %+v", layer)
+	}
+	layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, []ScriptPart{{"typescript", CheckScript},
+		{"go", "../elsewhere.sh"}})
+	if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "not a path inside") {
+		t.Errorf("a part outside the repository answered %+v", layer)
+	}
+	layer, _ = run(t, CheckSpec{Toolchains: map[string]string{"typescript": "ts-image"}},
+		map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts)
+	if layer.Verdict != "error" || !strings.Contains(layer.Summary, "go toolchain") {
+		t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer)
+	}
+}