package main import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "net" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" "strings" "testing" "time" "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/builder" "github.com/novox/mesh-controller/internal/catalogue" meshfacts "github.com/novox/mesh-controller/internal/facts" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // TestReplay867 is the replay of novox/hq issue 867, written with only what the controller and the build seat had // before the fix, so mesh-lab's prover can lay it over the commit before. A mesh-host change (#114) and a // mesh-controller change (#346) of one delivery group need each other: the node-engine change's own check reads the // controller's lock beside it, which only the controller change brings; the controller change builds only with the // validator word only the node-engine change brings. Each was read beside the other's repository at the commit the mesh // runs, which holds neither change, so neither could pass its own merge check — and the pair could not merge. // // It runs the whole path: the forge announces each pull request (following.PullUpdated), the controller asks // mesh-delivery which pull requests the group holds, the ask is queued for the build seat, and the build seat's check // (builder.Check, with a container runtime that runs each container's command on this machine) judges it. func TestReplay867(t *testing.T) { replay867(t, false) } // TestReplay867Recheck is the same pair re-checked through mesh-delivery (`delivery check` with one member): the forge's // announcement is not repeated, so the pull request reaches the controller without its head branch name. func TestReplay867Recheck(t *testing.T) { replay867(t, true) } func replay867(t *testing.T, recheck bool) { if _, err := exec.LookPath("go"); err != nil { t.Skip("no Go toolchain on this machine to build a judge with") } r := theGroupOfIssue867(t) for _, c := range []struct { what, repo string number int head string module string }{{"the node-engine change", "mesh-host", 114, r.hostHead, "mesh-host"}, {"the controller change", "mesh-controller", 346, r.controllerHead, "mesh-controller"}} { v := r.checked(t, c.repo, c.number, c.head, recheck) if v.Gate == nil || v.Gate.Verdict != "pass" || v.Repo == nil || v.Repo.Verdict != "pass" { t.Errorf("%s (%s#%d) did not pass its own merge check beside its group: gate %+v, repository %+v\n%s", c.what, c.repo, c.number, v.Gate, v.Repo, v.Report) } } } // theIssue867 is the two repositories as the trunk holds them and each pull request's head. type theIssue867 struct { open *stores hostHead, controllerHead string js interface{ Conn() *nats.Conn } judgeCache string registry string hostRepo, controllerRepo string groups string // what mesh-delivery's groups answers } func r867Git(t *testing.T, dir string, args ...string) string { t.Helper() cmd := exec.Command("git", args...) cmd.Dir = dir cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("git %v: %v\n%s", args, err, out) } return strings.TrimSpace(string(out)) } func r867Commit(t *testing.T, dir string, files map[string]string) string { t.Helper() for name, body := range files { if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil { t.Fatal(err) } } r867Git(t, dir, "add", "-A") r867Git(t, dir, "commit", "--quiet", "-m", "x") return r867Git(t, dir, "rev-parse", "HEAD") } // r867Controller is a controller whose judge says who it is and the validator word it was built with. func r867Controller(who, extra string) map[string]string { return map[string]string{ "go.mod": "module example.org/judge\n\ngo 1.22\n\nrequire github.com/novox/mesh-host v0.0.0\n", "vendor/modules.txt": "# github.com/novox/mesh-host v0.0.0\n## explicit; go 1.22\ngithub.com/novox/mesh-host/validate\n", "vendor/github.com/novox/mesh-host/validate/v.go": "package validate\n\nconst Word = \"the vendored validator\"\n", "vendor/github.com/novox/mesh-host/internal/declaration/d.go": "package declaration\n", "cmd/mesh-controller/main.go": `package main import ( "fmt" "os" "github.com/novox/mesh-host/validate" ) func main() { if len(os.Args) == 2 && os.Args[1] == "merge-gate" { fmt.Println("usage: merge-gate --facts --store ") os.Exit(2) } fmt.Printf("{\"verdict\":\"pass\",\"summary\":\"%s with %s\"}\n", "` + who + `", validate.Word` + extra + `) } `, } } func theGroupOfIssue867(t *testing.T) *theIssue867 { return theGroupOfIssue867Into(t, "main") } // theGroupOfIssue867Into is the group with both pull requests into base: main, or a release branch both repositories // hold, cut from their trunks. func theGroupOfIssue867Into(t *testing.T, base string) *theIssue867 { t.Helper() open := aMesh(t) r := &theIssue867{open: open, judgeCache: t.TempDir()} root := filepath.Join(t.TempDir(), "novox") controller, host := filepath.Join(root, "mesh-controller"), filepath.Join(root, "mesh-host") r.hostRepo, r.controllerRepo = host, controller for _, d := range []string{controller, host} { if err := os.MkdirAll(d, 0o755); err != nil { t.Fatal(err) } r867Git(t, d, "init", "--quiet", "-b", "main") } // The trunks. files := r867Controller("the controller", "") files["lock.txt"] = "foundation: old\n" files["module.json"] = `{"module":"mesh-controller"}` files[builder.CheckScript] = "true\n" r867Commit(t, controller, files) r867Commit(t, host, map[string]string{"validate/v.go": "package validate\n\nconst Word = \"the trunk's validator\"\n", "internal/declaration/d.go": "package declaration\n", "module.json": `{"module":"mesh-host"}`, builder.CheckScript: "set -e\ngrep -q 'foundation: new' \"$MESH_CHECK_BESIDE/mesh-controller/lock.txt\"\n"}) if base != "main" { for _, d := range []string{controller, host} { r867Git(t, d, "branch", base, "main") } } // The two pull requests of one group, feat/867. r867Git(t, controller, "checkout", "--quiet", "-b", "feat/867", base) change := r867Controller("the controller", "+validate.More") change["lock.txt"] = "foundation: new\n" r.controllerHead = r867Commit(t, controller, change) r867Git(t, controller, "checkout", "--quiet", "main") r867Git(t, host, "checkout", "--quiet", "-b", "feat/867", base) r.hostHead = r867Commit(t, host, map[string]string{"validate/more.go": "package validate\n\nconst More = \" and more\"\n"}) r867Git(t, host, "checkout", "--quiet", "main") followed := func(string) string { return "main" } for _, m := range []struct{ module, dir string }{{"mesh-controller", controller}, {"mesh-host", host}} { // The mesh runs each at its trunk's head, the commit a check reads it at beside another repository's change. running := r867Git(t, m.dir, "rev-parse", "main") if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: m.module, Version: "1"}, inventory.Source{Repository: m.dir, Ref: followed(m.module), BuiltFrom: running, Head: running}); err != nil { t.Fatal(err) } } if base != "main" { // A module built from the controller's repository follows the release branch, as the mesh runs it there: a pull // request into the release branch moves it, so the composed check's gate runs. running := r867Git(t, controller, "rev-parse", "main") if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "zz-agent", Version: "1"}, inventory.Source{Repository: controller, Ref: base, BuiltFrom: running, Head: running}); err != nil { t.Fatal(err) } } holdTheDeliverySeat(t, open) // mesh-delivery says the group: two open pull requests into main, each with its head. was := askDeliveryOwner t.Cleanup(func() { askDeliveryOwner = was }) askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, _ map[string]any) (json.RawMessage, error) { if verb != "groups" { t.Fatalf("asked the owner %s", verb) } return json.RawMessage(r.groups), nil } r.groups = (`[{"id":"feat/867","members":["` + root + `/mesh-host@` + r.hostHead[:12] + ` (checked)","` + root + `/mesh-controller@` + r.controllerHead[:12] + ` (checked)"],"member_details":[` + `{"id":"novox/mesh-host@` + r.hostHead[:12] + `","repository":"novox/mesh-host","state":"checked","number":114,"head":"` + r.hostHead + `","base":"` + base + `","open":true},` + `{"id":"novox/mesh-controller@` + r.controllerHead[:12] + `","repository":"novox/mesh-controller","state":"checked","number":346,"head":"` + r.controllerHead + `","base":"` + base + `","open":true}]}]`) r.registry = r867Registry(t) r867Runtime(t) return r } // checked is a pull request announced by the forge, asked of the build seat by the controller, and checked there. func (r *theIssue867) checked(t *testing.T, repo string, number int, head string, recheck bool) builderVerdict { t.Helper() js, ask := aBuildQueue(t) _ = ask servingBus.Store(js) t.Cleanup(func() { servingBus.Store(nil) }) p := link.PullUpdated{Owner: "novox", Repo: repo, Number: number, Base: "main", Head: "feat/867", Commit: head, Paths: []string{"lock.txt", "validate/more.go", "cmd/mesh-controller/main.go"}, ModuleDirsSaid: true, MergeCheck: true, MergeCheckSaid: true} if recheck { // As `delivery check --members ` builds it from mesh-delivery's member: no head branch name. p = orderMember{ID: "novox/" + repo + "@" + head[:12], Repository: "novox/" + repo, Base: "main", Head: head, Number: number, Paths: p.Paths, ModuleDirsSaid: true}.pull() p.MergeCheck, p.MergeCheckSaid = true, true } if err := (following{open: r.open, oneShot: true}).PullUpdated(t.Context(), p); err != nil { t.Fatal(err) } q, err := link.ReadQueue(t.Context(), js, link.TheBuildMachine) if err != nil { t.Fatal(err) } if len(q.Asks) == 0 { t.Fatalf("%s#%d: nothing was asked of the build seat", repo, number) } request := q.Asks[len(q.Asks)-1].Request // What the build seat makes of the ask: the fields the two share, by name. var spec builder.CheckSpec body, _ := json.Marshal(request.Check) if err := json.Unmarshal(body, &spec); err != nil { t.Fatal(err) } spec.ID, spec.Repository, spec.Ref = request.ID, request.Repository, request.Ref spec.Toolchain, spec.JudgeCache = "golang", r.judgeCache v, err := builder.Check(t.Context(), builder.Command, spec, t.TempDir(), r.registry, builder.GitCredential{}, nil) if err != nil { t.Fatal(err) } return v } type builderVerdict = builder.CheckVerdict func r867Registry(t *testing.T) string { t.Helper() body, err := json.Marshal(meshfacts.Facts{Format: meshfacts.Format, Taken: time.Now().UTC(), Versions: meshfacts.Versions{Bus: "2.11.17", Store: "17.11"}, Machines: []meshfacts.Machine{{Name: "abcdef", Length: 6}}}) if err != nil { t.Fatal(err) } sum := sha256.Sum256(body) digest := "sha256:" + hex.EncodeToString(sum[:]) manifest, _ := json.Marshal(map[string]any{"schemaVersion": 2, "layers": []map[string]any{ {"mediaType": meshfacts.MediaType, "digest": digest, "size": len(body)}}}) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { switch { case strings.Contains(req.URL.Path, "/manifests/"): w.Write(manifest) case strings.HasSuffix(req.URL.Path, "/blobs/"+digest): w.Write(body) default: http.NotFound(w, req) } })) t.Cleanup(srv.Close) return strings.TrimPrefix(srv.URL, "http://") } // r867Runtime puts a container runtime on PATH that runs every container's command on this machine, in the directory // and with the environment the check gives it, and a store and a bus that answer. func r867Runtime(t *testing.T) { t.Helper() bus, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } t.Cleanup(func() { bus.Close() }) go func() { for { c, err := bus.Accept() if err != nil { return } c.Close() } }() dir := t.TempDir() script := `#!/bin/sh cmd="$1"; shift case "$cmd" in ps|rm|exec) exit 0 ;; port) echo "` + bus.Addr().String() + `"; exit 0 ;; run) workdir=""; detached="" while [ $# -gt 0 ]; do case "$1" in -d) detached=1; shift ;; --rm) shift ;; --workdir) workdir="$2"; shift 2 ;; --env) export "$2"; shift 2 ;; --network|--volume|--user|--label|--name|-p|-e) shift 2 ;; *) break ;; esac done if [ -n "$detached" ]; then echo a-container; exit 0; fi shift cd "$workdir" && exec "$@" ;; esac exit 1 ` if err := os.WriteFile(filepath.Join(dir, "docker"), []byte(script), 0o755); err != nil { t.Fatal(err) } t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) t.Setenv("GOTOOLCHAIN", "local") } // A recheck runs as a command of its own (`delivery check` with one member), where the controller is not the serving // one and holds no connection: it dials the bus for the moment and asks mesh-delivery's `groups` as the daemon does. // This drives the verb on a test bus with a fake mesh-delivery answering `groups`, with no stub of the seam between. func TestTheRecheckVerbAsksMeshDeliveryOnABusOfItsOwn(t *testing.T) { realAsk := askDeliveryOwner r := theGroupOfIssue867(t) askDeliveryOwner = realAsk js, _ := aBuildQueue(t) if servingBus.Load() != nil { t.Fatal("this test is of a process that is not the serving controller") } asked := 0 wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": r.groups}}}) sub, err := js.Conn().Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "groups"), func(m *nats.Msg) { asked++ body, _ := json.Marshal(link.Answer{Result: wrapped}) _ = m.Respond(body) }) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) member := `[{"id":"novox/mesh-host@` + r.hostHead[:12] + `","repository":"novox/mesh-host","base":"main","head":"` + r.hostHead + `","number":114,"paths":["validate/more.go"],"module_dirs_said":true}]` if err := deliveryCommand(t.Context(), []string{"check", "--members", member}); err != nil { t.Fatal(err) } q, err := link.ReadQueue(t.Context(), js, link.TheBuildMachine) if err != nil || len(q.Asks) == 0 { t.Fatalf("nothing was asked of the build seat: %v", err) } c := q.Asks[len(q.Asks)-1].Request.Check if c.SiblingsUnknown != "" || len(c.Siblings) != 1 || c.Siblings[0].Repository != "novox/mesh-controller" || c.Siblings[0].Number != 346 || c.Siblings[0].Head != r.controllerHead { t.Fatalf("the recheck learned siblings %+v, unknown %q", c.Siblings, c.SiblingsUnknown) } if asked != 1 { t.Fatalf("mesh-delivery was asked %d times", asked) } } // **A delivery group's composed check is judged by the group's own controller** (novox/hq ADR 0239 §14), into a release // branch too: the controller member and the mesh-host member of the pair are consistent only together (the controller // calls what only the node-engine change brings), and both merge into a release branch. The release judge (the // controller of the release branch with the validator of its own) cannot build it; the group's own controller, as // merged, with the node-engine member's validator, can. It drives askGroupCheck and the build seat's Check for real. func TestAGroupsCompositionIntoAReleaseBranchIsJudgedByItsOwnController(t *testing.T) { groupIntoARelease(t, false) } // The shape the live check had: the node-engine member is the group's primary, so the composed request is asked with // Judge "validator" (a union of modules that includes mesh-host's) and the release branch set; the release judge then // put this change's validator into the old controller and "judge-with-this-validator" did not build. func TestAGroupsCompositionJudgedWithTheValidatorIntoAReleaseBranchIsJudgedByItsOwnController(t *testing.T) { groupIntoARelease(t, true) } func groupIntoARelease(t *testing.T, hostPrimary bool) { if _, err := exec.LookPath("go"); err != nil { t.Skip("no Go toolchain on this machine to build a judge with") } const release = "feat/r" r := theGroupOfIssue867Into(t, release) aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {release}}, nil) // The pair renames what the validator says: the node-engine change drops Word for Word2, the controller change // calls Word2. The controller of the release branch calls Word, so it does not build with the new validator; the // controller change does not build with the validator it vendors. Only together are they consistent. r867Git(t, r.hostRepo, "checkout", "--quiet", "-b", "swap", release) r.hostHead = r867Commit(t, r.hostRepo, map[string]string{"validate/v.go": "package validate\n\nconst Word2 = \"the new validator\"\n"}) r867Git(t, r.hostRepo, "checkout", "--quiet", "main") r867Git(t, r.controllerRepo, "checkout", "--quiet", "-b", "swap", release) swapped := strings.Replace(r867Controller("the controller", "")["cmd/mesh-controller/main.go"], "validate.Word)", "validate.Word2)", 1) r.controllerHead = r867Commit(t, r.controllerRepo, map[string]string{"cmd/mesh-controller/main.go": swapped}) r867Git(t, r.controllerRepo, "checkout", "--quiet", "main") js, _ := aBuildQueue(t) servingBus.Store(js) t.Cleanup(func() { servingBus.Store(nil) }) members := []orderMember{ {ID: "novox/mesh-controller@" + r.controllerHead[:12], Repository: "novox/mesh-controller", Base: release, Head: r.controllerHead, Number: 346, Paths: []string{"lock.txt", "cmd/mesh-controller/main.go"}, ModuleDirsSaid: true}, {ID: "novox/mesh-host@" + r.hostHead[:12], Repository: "novox/mesh-host", Base: release, Head: r.hostHead, Number: 114, Paths: []string{"validate/v.go"}, ModuleDirsSaid: true}} if hostPrimary { members[0], members[1] = members[1], members[0] } if _, err := askGroupCheck(t.Context(), r.open, "feat/867", members); err != nil { t.Fatal(err) } q, err := link.ReadQueue(t.Context(), js, link.TheBuildMachine) if err != nil || len(q.Asks) == 0 { t.Fatalf("nothing was asked of the build seat: %v", err) } request := q.Asks[len(q.Asks)-1].Request if request.Check.Release != release || request.Check.Group != "feat/867" || len(request.Check.Members) != 1 || len(request.Check.Modules) == 0 { t.Fatalf("the composed check is asked as release %q, group %q, %d other head(s)", request.Check.Release, request.Check.Group, len(request.Check.Members)) } if hostPrimary { if request.Check.Repo != "mesh-host" { t.Fatalf("the primary is %s, not the node-engine", request.Check.Repo) } // As the live ask had it (the mesh-host module among the union's): the node-engine's validator judges. request.Check.Judge = link.JudgeValidator } // What the build seat makes of the ask. var wire map[string]json.RawMessage body, _ := json.Marshal(request.Check) _ = json.Unmarshal(body, &wire) delete(wire, "group") delete(wire, "members") body, _ = json.Marshal(wire) var spec builder.CheckSpec if err := json.Unmarshal(body, &spec); err != nil { t.Fatal(err) } for _, m := range request.Check.Members { spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository, Ref: m.Ref, Paths: m.Paths, Base: m.Base}) } spec.ID, spec.Repository, spec.Ref = request.ID, request.Repository, request.Ref spec.Toolchain, spec.JudgeCache = "golang", r.judgeCache v, err := builder.Check(t.Context(), builder.Command, spec, t.TempDir(), r.registry, builder.GitCredential{}, nil) if err != nil { t.Fatal(err) } if v.Gate == nil || v.Gate.Verdict != "pass" { t.Fatalf("the group's composition into %s was not judged by its own controller: %+v\n%s", release, v.Gate, v.Report) } if mark := "[release " + release + ", judged by the group's controller " + v.JudgedBy + "] "; !strings.HasPrefix(v.Gate.Summary, mark) || !strings.Contains(v.Report, "by the group's controller at "+v.JudgedBy) { t.Fatalf("the statuses and the report do not say the group's controller judged:\n%s\n%s", v.Gate.Summary, v.Report) } if !strings.Contains(v.Gate.Summary, "the new validator") { t.Fatalf("the gate was judged by %q, not by the group's controller with the node-engine member's validator", v.Gate.Summary) } if !strings.HasPrefix(v.JudgeFrom, "the group's own controller") || !strings.Contains(v.Validator, "the group's mesh-host merge") { t.Fatalf("the verdict does not say where the judge and the validator came from: %q / %q", v.JudgeFrom, v.Validator) } if v.Release != release || v.JudgedBy == "" { t.Fatalf("the verdict names release %q judged by %q", v.Release, v.JudgedBy) } }