mesh-controller/cmd/mesh-controller/replay867_test.go

package main

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/nats-io/nats.go"

	"github.com/novox/mesh-controller/internal/builder"
	"github.com/novox/mesh-controller/internal/catalogue"
	meshfacts "github.com/novox/mesh-controller/internal/facts"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// TestReplay867 is the replay of novox/hq issue 867, written with only what the controller and the build seat had
// before the fix, so mesh-lab's prover can lay it over the commit before. A mesh-host change (#114) and a
// mesh-controller change (#346) of one delivery group need each other: the node-engine change's own check reads the
// controller's lock beside it, which only the controller change brings; the controller change builds only with the
// validator word only the node-engine change brings. Each was read beside the other's repository at the commit the mesh
// runs, which holds neither change, so neither could pass its own merge check — and the pair could not merge.
//
// It runs the whole path: the forge announces each pull request (following.PullUpdated), the controller asks
// mesh-delivery which pull requests the group holds, the ask is queued for the build seat, and the build seat's check
// (builder.Check, with a container runtime that runs each container's command on this machine) judges it.
func TestReplay867(t *testing.T) { replay867(t, false) }

// TestReplay867Recheck is the same pair re-checked through mesh-delivery (`delivery check` with one member): the forge's
// announcement is not repeated, so the pull request reaches the controller without its head branch name.
func TestReplay867Recheck(t *testing.T) { replay867(t, true) }

func replay867(t *testing.T, recheck bool) {
	if _, err := exec.LookPath("go"); err != nil {
		t.Skip("no Go toolchain on this machine to build a judge with")
	}
	r := theGroupOfIssue867(t)
	for _, c := range []struct {
		what, repo string
		number     int
		head       string
		module     string
	}{{"the node-engine change", "mesh-host", 114, r.hostHead, "mesh-host"},
		{"the controller change", "mesh-controller", 346, r.controllerHead, "mesh-controller"}} {
		v := r.checked(t, c.repo, c.number, c.head, recheck)
		if v.Gate == nil || v.Gate.Verdict != "pass" || v.Repo == nil || v.Repo.Verdict != "pass" {
			t.Errorf("%s (%s#%d) did not pass its own merge check beside its group: gate %+v, repository %+v\n%s",
				c.what, c.repo, c.number, v.Gate, v.Repo, v.Report)
		}
	}
}

// theIssue867 is the two repositories as the trunk holds them and each pull request's head.
type theIssue867 struct {
	open                     *stores
	hostHead, controllerHead string
	js                       interface{ Conn() *nats.Conn }
	judgeCache               string
	registry                 string
	hostRepo, controllerRepo string
	groups                   string // what mesh-delivery's groups answers
}

func r867Git(t *testing.T, dir string, args ...string) string {
	t.Helper()
	cmd := exec.Command("git", args...)
	cmd.Dir = dir
	cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org",
		"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org")
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("git %v: %v\n%s", args, err, out)
	}
	return strings.TrimSpace(string(out))
}

func r867Commit(t *testing.T, dir string, files map[string]string) string {
	t.Helper()
	for name, body := range files {
		if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil {
			t.Fatal(err)
		}
	}
	r867Git(t, dir, "add", "-A")
	r867Git(t, dir, "commit", "--quiet", "-m", "x")
	return r867Git(t, dir, "rev-parse", "HEAD")
}

// r867Controller is a controller whose judge says who it is and the validator word it was built with.
func r867Controller(who, extra string) map[string]string {
	return map[string]string{
		"go.mod":             "module example.org/judge\n\ngo 1.22\n\nrequire github.com/novox/mesh-host v0.0.0\n",
		"vendor/modules.txt": "# github.com/novox/mesh-host v0.0.0\n## explicit; go 1.22\ngithub.com/novox/mesh-host/validate\n",
		"vendor/github.com/novox/mesh-host/validate/v.go":             "package validate\n\nconst Word = \"the vendored validator\"\n",
		"vendor/github.com/novox/mesh-host/internal/declaration/d.go": "package declaration\n",
		"cmd/mesh-controller/main.go": `package main

import (
	"fmt"
	"os"

	"github.com/novox/mesh-host/validate"
)

func main() {
	if len(os.Args) == 2 && os.Args[1] == "merge-gate" {
		fmt.Println("usage: merge-gate --facts <file> --store <postgres>")
		os.Exit(2)
	}
	fmt.Printf("{\"verdict\":\"pass\",\"summary\":\"%s with %s\"}\n", "` + who + `", validate.Word` + extra + `)
}
`,
	}
}

func theGroupOfIssue867(t *testing.T) *theIssue867 { return theGroupOfIssue867Into(t, "main") }

// theGroupOfIssue867Into is the group with both pull requests into base: main, or a release branch both repositories
// hold, cut from their trunks.
func theGroupOfIssue867Into(t *testing.T, base string) *theIssue867 {
	t.Helper()
	open := aMesh(t)
	r := &theIssue867{open: open, judgeCache: t.TempDir()}
	root := filepath.Join(t.TempDir(), "novox")
	controller, host := filepath.Join(root, "mesh-controller"), filepath.Join(root, "mesh-host")
	r.hostRepo, r.controllerRepo = host, controller
	for _, d := range []string{controller, host} {
		if err := os.MkdirAll(d, 0o755); err != nil {
			t.Fatal(err)
		}
		r867Git(t, d, "init", "--quiet", "-b", "main")
	}
	// The trunks.
	files := r867Controller("the controller", "")
	files["lock.txt"] = "foundation: old\n"
	files["module.json"] = `{"module":"mesh-controller"}`
	files[builder.CheckScript] = "true\n"
	r867Commit(t, controller, files)
	r867Commit(t, host, map[string]string{"validate/v.go": "package validate\n\nconst Word = \"the trunk's validator\"\n",
		"internal/declaration/d.go": "package declaration\n", "module.json": `{"module":"mesh-host"}`,
		builder.CheckScript: "set -e\ngrep -q 'foundation: new' \"$MESH_CHECK_BESIDE/mesh-controller/lock.txt\"\n"})
	if base != "main" {
		for _, d := range []string{controller, host} {
			r867Git(t, d, "branch", base, "main")
		}
	}
	// The two pull requests of one group, feat/867.
	r867Git(t, controller, "checkout", "--quiet", "-b", "feat/867", base)
	change := r867Controller("the controller", "+validate.More")
	change["lock.txt"] = "foundation: new\n"
	r.controllerHead = r867Commit(t, controller, change)
	r867Git(t, controller, "checkout", "--quiet", "main")
	r867Git(t, host, "checkout", "--quiet", "-b", "feat/867", base)
	r.hostHead = r867Commit(t, host, map[string]string{"validate/more.go": "package validate\n\nconst More = \" and more\"\n"})
	r867Git(t, host, "checkout", "--quiet", "main")

	followed := func(string) string { return "main" }
	for _, m := range []struct{ module, dir string }{{"mesh-controller", controller}, {"mesh-host", host}} {
		// The mesh runs each at its trunk's head, the commit a check reads it at beside another repository's change.
		running := r867Git(t, m.dir, "rev-parse", "main")
		if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: m.module, Version: "1"},
			inventory.Source{Repository: m.dir, Ref: followed(m.module), BuiltFrom: running, Head: running}); err != nil {
			t.Fatal(err)
		}
	}
	if base != "main" {
		// A module built from the controller's repository follows the release branch, as the mesh runs it there: a pull
		// request into the release branch moves it, so the composed check's gate runs.
		running := r867Git(t, controller, "rev-parse", "main")
		if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "zz-agent", Version: "1"},
			inventory.Source{Repository: controller, Ref: base, BuiltFrom: running, Head: running}); err != nil {
			t.Fatal(err)
		}
	}
	holdTheDeliverySeat(t, open)

	// mesh-delivery says the group: two open pull requests into main, each with its head.
	was := askDeliveryOwner
	t.Cleanup(func() { askDeliveryOwner = was })
	askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, _ map[string]any) (json.RawMessage, error) {
		if verb != "groups" {
			t.Fatalf("asked the owner %s", verb)
		}
		return json.RawMessage(r.groups), nil
	}
	r.groups = (`[{"id":"feat/867","members":["` + root + `/mesh-host@` + r.hostHead[:12] + ` (checked)","` +
		root + `/mesh-controller@` + r.controllerHead[:12] + ` (checked)"],"member_details":[` +
		`{"id":"novox/mesh-host@` + r.hostHead[:12] + `","repository":"novox/mesh-host","state":"checked","number":114,"head":"` +
		r.hostHead + `","base":"` + base + `","open":true},` +
		`{"id":"novox/mesh-controller@` + r.controllerHead[:12] + `","repository":"novox/mesh-controller","state":"checked","number":346,"head":"` +
		r.controllerHead + `","base":"` + base + `","open":true}]}]`)
	r.registry = r867Registry(t)
	r867Runtime(t)
	return r
}

// checked is a pull request announced by the forge, asked of the build seat by the controller, and checked there.
func (r *theIssue867) checked(t *testing.T, repo string, number int, head string, recheck bool) builderVerdict {
	t.Helper()
	js, ask := aBuildQueue(t)
	_ = ask
	servingBus.Store(js)
	t.Cleanup(func() { servingBus.Store(nil) })
	p := link.PullUpdated{Owner: "novox", Repo: repo, Number: number, Base: "main", Head: "feat/867", Commit: head,
		Paths: []string{"lock.txt", "validate/more.go", "cmd/mesh-controller/main.go"}, ModuleDirsSaid: true,
		MergeCheck: true, MergeCheckSaid: true}
	if recheck {
		// As `delivery check --members <one head>` builds it from mesh-delivery's member: no head branch name.
		p = orderMember{ID: "novox/" + repo + "@" + head[:12], Repository: "novox/" + repo, Base: "main", Head: head,
			Number: number, Paths: p.Paths, ModuleDirsSaid: true}.pull()
		p.MergeCheck, p.MergeCheckSaid = true, true
	}
	if err := (following{open: r.open, oneShot: true}).PullUpdated(t.Context(), p); err != nil {
		t.Fatal(err)
	}
	q, err := link.ReadQueue(t.Context(), js, link.TheBuildMachine)
	if err != nil {
		t.Fatal(err)
	}
	if len(q.Asks) == 0 {
		t.Fatalf("%s#%d: nothing was asked of the build seat", repo, number)
	}
	request := q.Asks[len(q.Asks)-1].Request
	// What the build seat makes of the ask: the fields the two share, by name.
	var spec builder.CheckSpec
	body, _ := json.Marshal(request.Check)
	if err := json.Unmarshal(body, &spec); err != nil {
		t.Fatal(err)
	}
	spec.ID, spec.Repository, spec.Ref = request.ID, request.Repository, request.Ref
	spec.Toolchain, spec.JudgeCache = "golang", r.judgeCache
	v, err := builder.Check(t.Context(), builder.Command, spec, t.TempDir(), r.registry, builder.GitCredential{}, nil)
	if err != nil {
		t.Fatal(err)
	}
	return v
}

type builderVerdict = builder.CheckVerdict

func r867Registry(t *testing.T) string {
	t.Helper()
	body, err := json.Marshal(meshfacts.Facts{Format: meshfacts.Format, Taken: time.Now().UTC(),
		Versions: meshfacts.Versions{Bus: "2.11.17", Store: "17.11"}, Machines: []meshfacts.Machine{{Name: "abcdef", Length: 6}}})
	if err != nil {
		t.Fatal(err)
	}
	sum := sha256.Sum256(body)
	digest := "sha256:" + hex.EncodeToString(sum[:])
	manifest, _ := json.Marshal(map[string]any{"schemaVersion": 2, "layers": []map[string]any{
		{"mediaType": meshfacts.MediaType, "digest": digest, "size": len(body)}}})
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		switch {
		case strings.Contains(req.URL.Path, "/manifests/"):
			w.Write(manifest)
		case strings.HasSuffix(req.URL.Path, "/blobs/"+digest):
			w.Write(body)
		default:
			http.NotFound(w, req)
		}
	}))
	t.Cleanup(srv.Close)
	return strings.TrimPrefix(srv.URL, "http://")
}

// r867Runtime puts a container runtime on PATH that runs every container's command on this machine, in the directory
// and with the environment the check gives it, and a store and a bus that answer.
func r867Runtime(t *testing.T) {
	t.Helper()
	bus, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { bus.Close() })
	go func() {
		for {
			c, err := bus.Accept()
			if err != nil {
				return
			}
			c.Close()
		}
	}()
	dir := t.TempDir()
	script := `#!/bin/sh
cmd="$1"; shift
case "$cmd" in
ps|rm|exec) exit 0 ;;
port) echo "` + bus.Addr().String() + `"; exit 0 ;;
run)
  workdir=""; detached=""
  while [ $# -gt 0 ]; do
    case "$1" in
      -d) detached=1; shift ;;
      --rm) shift ;;
      --workdir) workdir="$2"; shift 2 ;;
      --env) export "$2"; shift 2 ;;
      --network|--volume|--user|--label|--name|-p|-e) shift 2 ;;
      *) break ;;
    esac
  done
  if [ -n "$detached" ]; then echo a-container; exit 0; fi
  shift
  cd "$workdir" && exec "$@" ;;
esac
exit 1
`
	if err := os.WriteFile(filepath.Join(dir, "docker"), []byte(script), 0o755); err != nil {
		t.Fatal(err)
	}
	t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
	t.Setenv("GOTOOLCHAIN", "local")
}

// A recheck runs as a command of its own (`delivery check` with one member), where the controller is not the serving
// one and holds no connection: it dials the bus for the moment and asks mesh-delivery's `groups` as the daemon does.
// This drives the verb on a test bus with a fake mesh-delivery answering `groups`, with no stub of the seam between.
func TestTheRecheckVerbAsksMeshDeliveryOnABusOfItsOwn(t *testing.T) {
	realAsk := askDeliveryOwner
	r := theGroupOfIssue867(t)
	askDeliveryOwner = realAsk
	js, _ := aBuildQueue(t)
	if servingBus.Load() != nil {
		t.Fatal("this test is of a process that is not the serving controller")
	}
	asked := 0
	wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": r.groups}}})
	sub, err := js.Conn().Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "groups"), func(m *nats.Msg) {
		asked++
		body, _ := json.Marshal(link.Answer{Result: wrapped})
		_ = m.Respond(body)
	})
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = sub.Unsubscribe() })
	member := `[{"id":"novox/mesh-host@` + r.hostHead[:12] + `","repository":"novox/mesh-host","base":"main","head":"` +
		r.hostHead + `","number":114,"paths":["validate/more.go"],"module_dirs_said":true}]`
	if err := deliveryCommand(t.Context(), []string{"check", "--members", member}); err != nil {
		t.Fatal(err)
	}
	q, err := link.ReadQueue(t.Context(), js, link.TheBuildMachine)
	if err != nil || len(q.Asks) == 0 {
		t.Fatalf("nothing was asked of the build seat: %v", err)
	}
	c := q.Asks[len(q.Asks)-1].Request.Check
	if c.SiblingsUnknown != "" || len(c.Siblings) != 1 || c.Siblings[0].Repository != "novox/mesh-controller" ||
		c.Siblings[0].Number != 346 || c.Siblings[0].Head != r.controllerHead {
		t.Fatalf("the recheck learned siblings %+v, unknown %q", c.Siblings, c.SiblingsUnknown)
	}
	if asked != 1 {
		t.Fatalf("mesh-delivery was asked %d times", asked)
	}
}

// **A delivery group's composed check is judged by the group's own controller** (novox/hq ADR 0239 §14), into a release
// branch too: the controller member and the mesh-host member of the pair are consistent only together (the controller
// calls what only the node-engine change brings), and both merge into a release branch. The release judge (the
// controller of the release branch with the validator of its own) cannot build it; the group's own controller, as
// merged, with the node-engine member's validator, can. It drives askGroupCheck and the build seat's Check for real.
func TestAGroupsCompositionIntoAReleaseBranchIsJudgedByItsOwnController(t *testing.T) {
	groupIntoARelease(t, false)
}

// The shape the live check had: the node-engine member is the group's primary, so the composed request is asked with
// Judge "validator" (a union of modules that includes mesh-host's) and the release branch set; the release judge then
// put this change's validator into the old controller and "judge-with-this-validator" did not build.
func TestAGroupsCompositionJudgedWithTheValidatorIntoAReleaseBranchIsJudgedByItsOwnController(t *testing.T) {
	groupIntoARelease(t, true)
}

func groupIntoARelease(t *testing.T, hostPrimary bool) {
	if _, err := exec.LookPath("go"); err != nil {
		t.Skip("no Go toolchain on this machine to build a judge with")
	}
	const release = "feat/r"
	r := theGroupOfIssue867Into(t, release)
	aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {release}}, nil)
	// The pair renames what the validator says: the node-engine change drops Word for Word2, the controller change
	// calls Word2. The controller of the release branch calls Word, so it does not build with the new validator; the
	// controller change does not build with the validator it vendors. Only together are they consistent.
	r867Git(t, r.hostRepo, "checkout", "--quiet", "-b", "swap", release)
	r.hostHead = r867Commit(t, r.hostRepo, map[string]string{"validate/v.go": "package validate\n\nconst Word2 = \"the new validator\"\n"})
	r867Git(t, r.hostRepo, "checkout", "--quiet", "main")
	r867Git(t, r.controllerRepo, "checkout", "--quiet", "-b", "swap", release)
	swapped := strings.Replace(r867Controller("the controller", "")["cmd/mesh-controller/main.go"], "validate.Word)", "validate.Word2)", 1)
	r.controllerHead = r867Commit(t, r.controllerRepo, map[string]string{"cmd/mesh-controller/main.go": swapped})
	r867Git(t, r.controllerRepo, "checkout", "--quiet", "main")
	js, _ := aBuildQueue(t)
	servingBus.Store(js)
	t.Cleanup(func() { servingBus.Store(nil) })
	members := []orderMember{
		{ID: "novox/mesh-controller@" + r.controllerHead[:12], Repository: "novox/mesh-controller", Base: release,
			Head: r.controllerHead, Number: 346, Paths: []string{"lock.txt", "cmd/mesh-controller/main.go"}, ModuleDirsSaid: true},
		{ID: "novox/mesh-host@" + r.hostHead[:12], Repository: "novox/mesh-host", Base: release, Head: r.hostHead, Number: 114,
			Paths: []string{"validate/v.go"}, ModuleDirsSaid: true}}
	if hostPrimary {
		members[0], members[1] = members[1], members[0]
	}
	if _, err := askGroupCheck(t.Context(), r.open, "feat/867", members); err != nil {
		t.Fatal(err)
	}
	q, err := link.ReadQueue(t.Context(), js, link.TheBuildMachine)
	if err != nil || len(q.Asks) == 0 {
		t.Fatalf("nothing was asked of the build seat: %v", err)
	}
	request := q.Asks[len(q.Asks)-1].Request
	if request.Check.Release != release || request.Check.Group != "feat/867" || len(request.Check.Members) != 1 || len(request.Check.Modules) == 0 {
		t.Fatalf("the composed check is asked as release %q, group %q, %d other head(s)", request.Check.Release,
			request.Check.Group, len(request.Check.Members))
	}
	if hostPrimary {
		if request.Check.Repo != "mesh-host" {
			t.Fatalf("the primary is %s, not the node-engine", request.Check.Repo)
		}
		// As the live ask had it (the mesh-host module among the union's): the node-engine's validator judges.
		request.Check.Judge = link.JudgeValidator
	}
	// What the build seat makes of the ask.
	var wire map[string]json.RawMessage
	body, _ := json.Marshal(request.Check)
	_ = json.Unmarshal(body, &wire)
	delete(wire, "group")
	delete(wire, "members")
	body, _ = json.Marshal(wire)
	var spec builder.CheckSpec
	if err := json.Unmarshal(body, &spec); err != nil {
		t.Fatal(err)
	}
	for _, m := range request.Check.Members {
		spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository, Ref: m.Ref,
			Paths: m.Paths, Base: m.Base})
	}
	spec.ID, spec.Repository, spec.Ref = request.ID, request.Repository, request.Ref
	spec.Toolchain, spec.JudgeCache = "golang", r.judgeCache
	v, err := builder.Check(t.Context(), builder.Command, spec, t.TempDir(), r.registry, builder.GitCredential{}, nil)
	if err != nil {
		t.Fatal(err)
	}
	if v.Gate == nil || v.Gate.Verdict != "pass" {
		t.Fatalf("the group's composition into %s was not judged by its own controller: %+v\n%s", release, v.Gate, v.Report)
	}
	if mark := "[release " + release + ", judged by the group's controller " + v.JudgedBy + "] "; !strings.HasPrefix(v.Gate.Summary, mark) ||
		!strings.Contains(v.Report, "by the group's controller at "+v.JudgedBy) {
		t.Fatalf("the statuses and the report do not say the group's controller judged:\n%s\n%s", v.Gate.Summary, v.Report)
	}
	if !strings.Contains(v.Gate.Summary, "the new validator") {
		t.Fatalf("the gate was judged by %q, not by the group's controller with the node-engine member's validator", v.Gate.Summary)
	}
	if !strings.HasPrefix(v.JudgeFrom, "the group's own controller") || !strings.Contains(v.Validator, "the group's mesh-host merge") {
		t.Fatalf("the verdict does not say where the judge and the validator came from: %q / %q", v.JudgeFrom, v.Validator)
	}
	if v.Release != release || v.JudgedBy == "" {
		t.Fatalf("the verdict names release %q judged by %q", v.Release, v.JudgedBy)
	}
}