diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index f007913..e22a0ee 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -249,7 +249,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say) if err == nil { result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report, - Took: v.Took.Round(time.Second).String()} + Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo)} } } else if err == nil { // The package-registry credential is a build input, so it is resolved before the clone: a @@ -293,6 +293,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri } result.Against = built.Against result.SourceFingerprint = built.Source + result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches for _, r := range built.Read { result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref}) } @@ -334,13 +335,22 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { c := request.Check spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref, Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{}, - Toolchain: builder.ToolchainOf(request.Held)} + Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base, + Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)} for dir, b := range c.Beside { spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref} } return spec } +// layerOf is one layer of a check as the outcome carries it. +func layerOf(l *builder.Layer) *link.CheckLayer { + if l == nil { + return nil + } + return &link.CheckLayer{Verdict: l.Verdict, Summary: l.Summary, Modules: l.Modules} +} + // packagesFrom is where a build resolves the mesh's own published packages — the SDK above all // (novox/hq ADR 0076, issue 053). // diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 86c460e..54d75cb 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/novox/mesh-controller/internal/conditions" "os" + "slices" "strings" "time" @@ -572,6 +573,19 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w", result.On, result.Repository, short(result.Commit), err) } + // **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's + // default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay + // --register` of one — is for checking, and is never a module's version; nothing could then send it. + // The branch the module already follows is its trunk — never the branch a build was asked at, or a + // build of a feature branch by name would make that branch its trunk. + follows := "" + if was, err := inv.SourceOf(ctx, manifest.Module); err == nil { + follows = followedBranch(was.Ref) + } + if err := publishable(result, follows); err != nil { + return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On, + manifest.Module, short(result.Commit), err) + } // **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard // twice, or replayed, would otherwise make the build a rollback put back what the module is again, // and the next push would send it. @@ -613,6 +627,34 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu return manifest, kept, nil } +// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published. +var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " + + "never published (ADR 0238) — merge it, and the merge builds it") + +// publishable says whether a build's outcome may become a module's version: its commit on the module's +// trunk, as the build seat read the forge at the build — the branch the module follows when its source +// names one, else its repository's default branch. A build seat that could not say — one older than the +// rule, building its own successor — is let through and said, so the rule can reach the mesh. +func publishable(result link.BuildResult, follows string) error { + if result.Check != nil || result.Checked != nil || result.DryRun { + return errors.New("a check or a dry run is never published") + } + if result.Trunk == "" { + fmt.Printf("%s: the build seat did not say whether %s is on its repository's trunk (it predates ADR 0238); "+ + "registered as before\n", result.ID, short(result.Commit)) + return nil + } + trunk := result.Trunk + if follows != "" { + trunk = follows + } + on := slices.Contains(result.Branches, trunk) || (trunk == result.Trunk && result.OnTrunk) + if !on { + return fmt.Errorf("%w (%s is not on %s)", errOffTheTrunk, short(result.Commit), trunk) + } + return nil +} + // buildAndShow builds and prints the manifest without recording anything. func buildAndShow(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error { repository, err := cloneFrom(ctx, source) diff --git a/cmd/mesh-controller/changeplan.go b/cmd/mesh-controller/changeplan.go new file mode 100644 index 0000000..9a86688 --- /dev/null +++ b/cmd/mesh-controller/changeplan.go @@ -0,0 +1,155 @@ +package main + +import ( + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The change plan (novox/hq ADR 0238): **one commit, one plan** — what a change does to the mesh, computed +// from its diffset (a repository, the branch it merges into, the commit at hand) by the planner, never by +// a mapping of its own. reachOfMerge answers what moves and what follows it; this adds where each module +// goes — the deploy plan, machine by machine in the build plan's order — and what is not an ordinary +// send. A pull request's check posts it with its verdict; the release a merge makes follows the same +// planner, so the two can be compared. + +// policyOf is a module's upgrade policy, as the controller holds it; false when it cannot be read. +type policyOf func(module string) (inventory.Upgrade, bool) + +// changePlanOf is the change plan of a reach: pure, so it is tested without a store. +func changePlanOf(repository, base, head string, r mergeReach, entries []inventory.Entry, policy policyOf) link.ChangePlan { + p := link.ChangePlan{Repository: repository, Base: base, Head: head, Moved: r.Moved(), Dependents: r.Dependents(), + New: r.Added, Unread: r.Unread, Tiers: r.Plan.Tiers} + byName := map[string]inventory.Entry{} + for _, e := range entries { + byName[e.Manifest.Module] = e + } + machines := map[string]*link.MachinePlan{} + machine := func(name string) *link.MachinePlan { + if machines[name] == nil { + machines[name] = &link.MachinePlan{Machine: name} + } + return machines[name] + } + for _, tier := range r.Plan.Tiers { + for _, name := range tier { + e, held := byName[name] + if !held { + continue + } + u, known := policy(name) + waits := known && !u.RollOut + for _, on := range e.On { + if waits { + machine(on).Waits = append(machine(on).Waits, name) + } else { + machine(on).Receives = append(machine(on).Receives, name) + } + } + switch { + case name == "nats": + p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send") + case waits && len(e.On) > 0: + p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name, + orNone(u.From))) + } + if len(e.Manifest.Provides) > 0 && len(e.On) > 0 { + var offers []string + for _, o := range e.Manifest.Provides { + offers = append(offers, o.Name) + } + p.Steps = append(p.Steps, fmt.Sprintf("%s provides %s: its consumers are sent again after it", + name, strings.Join(offers, ", "))) + } + if e.Manifest.Data != nil && len(e.On) > 0 { + p.Steps = append(p.Steps, fmt.Sprintf("%s keeps data (ADR 0233): what it holds is backed up before it moves", name)) + } + } + } + var names []string + for name := range machines { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + p.Machines = append(p.Machines, *machines[name]) + } + p.Summary = summaryOf(p) + return p +} + +// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves. +func summaryOf(p link.ChangePlan) string { + if len(p.Moved) == 0 && len(p.New) == 0 { + return "builds nothing: the change touches no module of the mesh's graph" + } + var parts []string + what := strings.Join(p.Moved, ", ") + if len(p.Dependents) > 0 { + what += fmt.Sprintf(" (+%d dependent(s))", len(p.Dependents)) + } + if len(p.New) > 0 { + if what != "" { + what += ", " + } + what += "new: " + strings.Join(p.New, ", ") + } + var to []string + for _, m := range p.Machines { + if len(m.Receives) > 0 { + to = append(to, m.Machine) + } + } + if len(to) > 0 { + parts = append(parts, "builds "+what+" → "+strings.Join(to, ", ")) + } else { + parts = append(parts, "builds "+what+", sent nowhere") + } + bus := "no bus step" + for _, s := range p.Steps { + if strings.HasPrefix(s, "a planned bus step") { + bus = "a bus step" + } + } + parts = append(parts, bus) + waiting := 0 + for _, m := range p.Machines { + waiting += len(m.Waits) + } + if waiting > 0 { + parts = append(parts, fmt.Sprintf("%d wait(s) for a person", waiting)) + } + return strings.Join(parts, "; ") +} + +// planText is a change plan as a person reads it, for the pull request's comment. +func planText(p link.ChangePlan) string { + var b strings.Builder + fmt.Fprintf(&b, "change plan of %s at %.8s into %s: %s\n", p.Repository, p.Head, p.Base, p.Summary) + for i, tier := range p.Tiers { + fmt.Fprintf(&b, " tier %d: %s\n", i, strings.Join(tier, ", ")) + } + for _, m := range p.Machines { + line := " " + m.Machine + ": " + if len(m.Receives) > 0 { + line += "receives " + strings.Join(m.Receives, ", ") + } + if len(m.Waits) > 0 { + if len(m.Receives) > 0 { + line += "; " + } + line += "waits for a person: " + strings.Join(m.Waits, ", ") + } + b.WriteString(line + "\n") + } + for _, s := range p.Steps { + b.WriteString(" - " + s + "\n") + } + if len(p.Unread) > 0 { + fmt.Fprintf(&b, " read by no module's build: %s\n", strings.Join(p.Unread, ", ")) + } + return b.String() +} diff --git a/cmd/mesh-controller/changeplan_test.go b/cmd/mesh-controller/changeplan_test.go new file mode 100644 index 0000000..a2632b7 --- /dev/null +++ b/cmd/mesh-controller/changeplan_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// **One commit, one change plan** (novox/hq ADR 0238): the planner's reach, laid out machine by machine in +// the build plan's order, with what is not an ordinary send said — the bus step, a module that waits for a +// person, a provider whose consumers follow it. +func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) { + const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git" + entry := func(name, path string, on ...string) inventory.Entry { + e := fromRepo(name, catalogue_, path) + e.Source.BuiltFrom = "old" + e.On = on + return e + } + nats := entry("nats", "modules/nats", "anchor") + nats.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} + gitea := entry("gitea", "modules/gitea", "anchor") + held := entry("photos", "modules/photos", "anchor", "laptop") + tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools") + tools.On = []string{"anchor", "laptop"} + entries := []inventory.Entry{nats, gitea, held, tools} + edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}} + policy := func(module string) (inventory.Upgrade, bool) { + if module == "photos" { + return inventory.Upgrade{RollOut: false, From: "a person"}, true + } + return inventory.Upgrade{RollOut: true}, true + } + plan := func(paths ...string) link.ChangePlan { + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths} + return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy) + } + + p := plan("modules/gitea/index.ts") + if p.Summary != "builds gitea → anchor; no bus step" { + t.Errorf("one module's change reads %q", p.Summary) + } + + p = plan("modules/nats/Dockerfile", "modules/photos/x.js") + if !strings.Contains(p.Summary, "a bus step") || !strings.Contains(p.Summary, "2 wait(s) for a person") || + !strings.Contains(p.Summary, "(+1 dependent(s))") { + t.Errorf("the bus and a held module read %q", p.Summary) + } + got := map[string]string{} + for _, m := range p.Machines { + got[m.Machine] = strings.Join(m.Receives, ",") + "|" + strings.Join(m.Waits, ",") + } + // The bus first, what stands on it after: the build plan's order, per machine. + if got["anchor"] != "nats,node-tools|photos" || got["laptop"] != "node-tools|photos" { + t.Errorf("the deploy plan reads %v", got) + } + text := strings.Join(p.Steps, "\n") + for _, want := range []string{"a planned bus step", "photos waits for a person", "nats provides mesh-bus"} { + if !strings.Contains(text, want) { + t.Errorf("the steps do not say %q:\n%s", want, text) + } + } + + p = plan("merge-check.sh") + if !strings.HasPrefix(p.Summary, "builds nothing") || len(p.Machines) != 0 || strings.Join(p.Unread, ",") != "merge-check.sh" { + t.Errorf("a root file's plan reads %+v", p) + } + if !strings.Contains(planText(p), "read by no module's build: merge-check.sh") { + t.Errorf("the plan's text does not say why nothing is built:\n%s", planText(p)) + } +} diff --git a/cmd/mesh-controller/check_test.go b/cmd/mesh-controller/check_test.go index 8947901..6754bd1 100644 --- a/cmd/mesh-controller/check_test.go +++ b/cmd/mesh-controller/check_test.go @@ -92,3 +92,14 @@ func TestRegistrationRefusesADefinitionNamingAnInstallation(t *testing.T) { t.Fatalf("a name declared on purpose passes; got %v", err) } } + +// **The controller's own manifest names every verb of its seat** (novox/hq ADR 0132): its tools lagged +// the seat's verbs for weeks, and `module check` — the gate's first step for a change touching it — +// refused it. Held here, so a verb added to the table without the manifest fails this repository's +// own suite rather than its next pull request's gate. +func TestTheControllersManifestServesEveryVerbOfItsSeat(t *testing.T) { + var out strings.Builder + if err := moduleCheck([]string{"../../module.json"}, &out); err != nil { + t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String()) + } +} diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index 26b6b6a..ee457a6 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -4,6 +4,9 @@ import ( "context" "encoding/json" "fmt" + "path" + "slices" + "sort" "strings" "time" @@ -12,46 +15,170 @@ import ( "github.com/novox/mesh-controller/internal/link" ) -// A pull request's merge check (novox/hq to-be 45 §9): the forge announces a pull request's new head, -// the controller asks the build seat to check it, and says the verdict as `checked`, which the forge's -// holder sets as the pull request's status. **Before merge, never after**: every check the mesh had ran -// after a merge, on a machine. +// A pull request's merge check (novox/hq to-be 45 §9, ADR 0237 as amended 2026-10-06): the forge +// announces every pull request's new head, the controller decides what is checked, asks the build seat to +// check it, and says the verdict as `checked`, which the forge's holder sets as the pull request's +// statuses. **Before merge, never after**: every check the mesh had ran after a merge, on a machine. // -// What is checked is decided here and run there. Here: whether the mesh builds anything from the -// repository into that branch — a repository it builds nothing from is not its to judge — and what the -// check reads beside it: the controller the mesh runs (its judge, for a catalogue change: a manifest -// that controller cannot read fails, which is version skew caught), the catalogue the mesh holds, the -// host it runs. There: the repository's own merge-check.sh, or the merge gate alone for a repository -// that declares none (internal/builder/check.go). +// **The mesh's module graph decides, not the repository.** The controller holds the graph — every module, +// the repository and directory it is built from — and maps the pull request's changed paths onto it by +// the planner's own answer (reachOfMerge, touchedBy — the one place a changed file is mapped onto modules, +// for the merge handler, the release planner, the merge gate and this check; issue 280): **a changed file +// touches exactly the modules whose build reads it** — a module's own directory (the whole repository for +// one built from its root), or a repository its recipe packages. A file no build reads — a script at the +// root, a README — touches no module. A directory the change adds a module.json in, which the graph does +// not hold yet (said by the head, issue 278), is a new module and is checked too. +// +// - touches a module: the build seat runs **the gate** — `mesh/merge-gate`, the touched manifests, every +// machine composed with the change, the replays — and the repository's own merge-check.sh beside it; +// - touches none, in a repository that is the mesh's (it sources a module on some branch, or shares the +// core's owner): the gate is a pass that says so — a fact, not a missing check — and the repository's +// own merge-check.sh runs as `mesh/repo-check`, a warning when it has none; +// - touches none, anywhere else: the gate is a pass that says so, and nothing more is said. +// +// What is checked is decided here and run there (internal/builder/check.go). // checkTimeout is how long one check may run on the build seat. Said here so the ask's watchdog (S6) // and the builder agree on what late means. const checkTimeout = 45 * time.Minute -// PullUpdated asks for a pull request's merge check. +// noModuleTouched is the gate's word for a change that touches nothing of the graph. +const noModuleTouched = "the change touches no module of the mesh's graph" + +// noMergeCheck is the repository layer's word for a repository of the mesh with no merge-check.sh. +const noMergeCheck = "the repository declares no merge-check.sh: none of its own tests run before it merges" + +// coreModules are the modules whose repositories are the mesh's core, by the directory a check finds +// each beside it — and whose owner is the mesh's own. +var coreModules = map[string]string{"mesh-controller": "mesh-controller", "mesh-host": "mesh-host", + "node-tools": "mesh-tools", "nats": "mesh-catalog"} + +// checkScope is what a pull request reaches of the mesh's graph, as the planner reckons it. +type checkScope struct { + // Modules are the modules a merge of the change would move itself — built from the repository into + // the pull request's base and reading a changed file, or packaging the repository's source — and + // Dependents those the plan would build after them; New the directories it adds a module in. + Modules []string + Dependents []string + New []string + // Manifests are the moved modules' and the new ones' manifests in the change's tree. + Manifests []string + // Width is how many modules a merge would build, in how many tiers; Unread the changed files no + // module's build reads. + Width, Tiers int + Unread []string + // Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the + // core's, or the change adds a module to it. + Mesh bool + // Judge is who judges the gate (link.JudgeSelf, link.JudgeValidator, or the running controller). + Judge string + // From is a module built from the repository, for how the mesh clones it; nil when none is. + From *inventory.Entry + // Reach is the planner's whole answer, which the change plan is made from. + Reach mergeReach +} + +func (s checkScope) gated() bool { return len(s.Modules)+len(s.New) > 0 } + +// pullScope is what a pull request reaches: **the planner's own answer** (reachOfMerge), asked as if the +// head were merged into the base — never a mapping of its own, so a change to what a merge touches +// changes what is checked with it (novox/hq ADR 0238). +func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][]inventory.ReadRepository, + edges []inventory.Edge) checkScope { + m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL, Commit: p.Commit, + Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, + ModuleDirsSaid: p.ModuleDirsSaid} + r := reachOfMerge(m, entries, read, edges) + s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread, + Width: len(r.Plan.Modules), Tiers: len(r.Plan.Tiers), Reach: r} + for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) { + s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile)) + switch e.Manifest.Module { + case "mesh-controller": + s.Judge = link.JudgeSelf + case "mesh-host": + if s.Judge == "" { + s.Judge = link.JudgeValidator + } + } + } + for _, d := range r.Added { + s.Manifests = append(s.Manifests, path.Join(d, moduleManifestFile)) + } + sort.Strings(s.Manifests) + s.Manifests = slices.Compact(s.Manifests) + + // Whose repository it is, for how it is cloned and whether its own check is the mesh's to run. + owners := map[string]bool{} + for i, e := range entries { + if e.Provided { + continue + } + if _, core := coreModules[e.Manifest.Module]; core { + if owner := sourceOwner(e.Source.Repository); owner != "" { + owners[owner] = true + } + } + if sameRepository(e.Source.Repository, m) && s.From == nil { + s.From = &entries[i] + } + } + s.Mesh = s.From != nil || owners[strings.ToLower(p.Owner)] || s.gated() + return s +} + +// sourceOwner is the owner of a recorded repository, a path on the git seat or a URL: novox/mesh-host → novox. +func sourceOwner(repository string) string { + parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/") + if len(parts) < 2 { + return "" + } + return strings.ToLower(parts[len(parts)-2]) +} + +// PullUpdated decides a pull request's merge check, and asks for it when there is something to run. func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error { inv := f.open.inventory entries, err := inv.Catalogued(ctx) if err != nil { return err } - moved := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL} - var from *inventory.Entry - for i, e := range entries { - if !e.Provided && sourceIs(e.Source, moved) { - from = &entries[i] - break - } - } - if from == nil { - fmt.Printf("%s/%s#%d (%.8s): the mesh builds nothing from it into %s, so it is not the mesh's to check\n", - p.Owner, p.Repo, p.Number, p.Commit, p.Base) - return nil - } - request, err := checkRequestFor(ctx, f.open, p, *from, entries) + read, err := inv.ReadRepositories(ctx) if err != nil { return err } + edges, err := inv.Dependencies(ctx) + if err != nil { + return err + } + scope := pullScope(p, entries, read, edges) + // The change plan of the commit at hand (ADR 0238): what a merge of it would build and send, posted + // with the verdict whatever the verdict is. + plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) { + u, err := inv.UpgradeOf(ctx, module) + return u, err == nil + }) + fmt.Print(planText(plan)) + direct := link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit, + ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched, + Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan} + switch { + case !scope.gated() && !scope.Mesh: + fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n", + p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched) + sayChecked(ctx, direct) + return nil + case !scope.gated() && p.MergeCheckSaid && !p.MergeCheck: + direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck} + fmt.Printf("%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck) + sayChecked(ctx, direct) + return nil + } + request, err := checkRequestFor(ctx, f.open, p, scope, entries) + if err != nil { + return err + } + request.Check.Plan = &plan seat := buildSeatHeld(ctx) ask, err := askOverOn(seat) if err != nil { @@ -61,14 +188,20 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error { if err := ask.Ask(ctx, request); err != nil { return err } - fmt.Printf("%s/%s#%d (%.8s): asked %s to check it before it merges, as %s\n", p.Owner, p.Repo, p.Number, - p.Commit, seat, request.ID) + what := "its own merge-check.sh alone: " + noModuleTouched + if scope.gated() { + what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))", + strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "), + scope.Width, scope.Tiers) + } + fmt.Printf("%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number, + p.Commit, seat, what, request.ID) return nil } // checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head, // and what is read beside it. -func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, from inventory.Entry, +func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope, entries []inventory.Entry) (link.BuildRequest, error) { shelf := map[string]catalogue.Manifest{} for _, e := range entries { @@ -84,7 +217,12 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, from } return clonedFromSeat(world, s.Seat, s.Repository) } - repository, err := clone(from.Source) + // As the mesh clones a module built from it; a repository no module is built from, from the forge. + source := inventory.Source{Seat: gitSeat, Repository: p.Owner + "/" + p.Repo} + if scope.From != nil { + source = scope.From.Source + } + repository, err := clone(source) if err != nil { return link.BuildRequest{}, err } @@ -94,10 +232,8 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, from } // Beside it, at what the mesh runs: each core repository by the module the mesh builds from it. beside := map[string]link.CheckedOut{} - byModule := map[string]string{"mesh-controller": "mesh-controller", "mesh-host": "mesh-host", - "node-tools": "mesh-tools", "nats": "mesh-catalog"} for _, e := range entries { - dir, core := byModule[e.Manifest.Module] + dir, core := coreModules[e.Manifest.Module] if !core || e.Provided || e.Source.Repository == "" { continue } @@ -130,9 +266,10 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, from Ref: p.Commit, Held: heldBy(ctx), Seats: seatBases(ctx), - Source: sourceOnSeat(from.Source), + Source: sourceOnSeat(source), Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, - Paths: p.Paths, Beside: beside}, + Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New, + Manifests: scope.Manifests, Judge: scope.Judge}, }, nil } @@ -162,6 +299,12 @@ const maxCheckReport = 60 << 10 // checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or // registered: a check builds nothing (issue 240's rule for a dry run, kept for a check). func checked(ctx context.Context, result link.BuildResult) { + sayChecked(ctx, checkedOf(result)) +} + +// checkedOf is what a check's outcome says: each layer, and an error — never a pass — for a check that +// could not run. +func checkedOf(result link.BuildResult) link.Checked { c := link.Checked{ID: result.ID, On: result.On, Commit: result.Ref} if result.Checked != nil { c.Owner, c.Repo, c.Number = result.Checked.Owner, result.Checked.Repo, result.Checked.Number @@ -169,8 +312,13 @@ func checked(ctx context.Context, result link.BuildResult) { switch { case result.Check != nil: c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report + c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck + if c.Gate == nil { + // A build seat from before the layers: its verdict is the gate's. + c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary} + } case result.Failed != "": - // The check could not run: an error, never read as a pass. + // The check could not run: an error, never read as a pass — on both layers it was asked for. c.Verdict, c.Summary = "error", "the check could not run: "+firstLine(result.Failed) default: c.Verdict, c.Summary = "error", "the build seat answered the check with no verdict" @@ -178,11 +326,46 @@ func checked(ctx context.Context, result link.BuildResult) { if c.Verdict == "" { c.Verdict = "error" } + if result.Check == nil { + c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary} + c.RepoCheck = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary} + } + if result.Checked != nil && c.Gate != nil && len(c.Gate.Modules) == 0 { + c.Gate.Modules = append(append([]string{}, result.Checked.Modules...), prefixedAll("new:", result.Checked.New)...) + } + if result.Checked != nil && c.Gate != nil && len(c.Gate.Dependents) == 0 { + c.Gate.Dependents = result.Checked.Dependents + } + if result.Checked != nil { + c.Plan = result.Checked.Plan + } + for _, l := range []*link.CheckLayer{c.Gate, c.RepoCheck} { + if l != nil && l.Verdict == "" { + l.Verdict = "error" + } + } if len(c.Report) > maxCheckReport { c.Report = "…" + c.Report[len(c.Report)-maxCheckReport:] } - fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: %s — %s\n", result.ID, c.Owner, c.Repo, c.Number, c.Commit, - orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary) + return c +} + +func prefixedAll(prefix string, items []string) []string { + out := make([]string, 0, len(items)) + for _, i := range items { + out = append(out, prefix+i) + } + return out +} + +// sayChecked says a merge check's verdict on the bus, where the forge's holder hears it. +func sayChecked(ctx context.Context, c link.Checked) { + repo := "none" + if c.RepoCheck != nil { + repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary + } + fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number, + c.Commit, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo) if checkEvents == nil { return } @@ -193,6 +376,6 @@ func checked(ctx context.Context, result link.BuildResult) { stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer stop() if err := checkEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyChecked, body); err != nil { - fmt.Printf("%s: the verdict could not be said, so the pull request is not told it: %v\n", result.ID, err) + fmt.Printf("%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err) } } diff --git a/cmd/mesh-controller/checks_test.go b/cmd/mesh-controller/checks_test.go new file mode 100644 index 0000000..7e0bc94 --- /dev/null +++ b/cmd/mesh-controller/checks_test.go @@ -0,0 +1,212 @@ +package main + +import ( + "errors" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// **The mesh's module graph decides what a pull request's check runs**, not the repository (novox/hq +// ADR 0237 as amended): a change is mapped onto the graph by the rule a merge is (issue 278), the gate +// runs when it touches a module — a new one included — and a repository that is the mesh's and touches +// none still has its own merge-check.sh run. +func TestThePullRequestIsMappedOntoTheModuleGraph(t *testing.T) { + const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git" + const controller = "http://forge.internal:20000/novox/mesh-controller.git" + const host = "http://forge.internal:20000/novox/mesh-host.git" + photos := fromRepo("photos", "http://forge.internal:20000/novox/photos.git", "") + photos.Source.Ref = "nox-mesh" + snake := fromRepo("snake", "jschoubben/snake", "") + snake.Source.Seat = "git" + entries := []inventory.Entry{ + fromRepo("gitea", catalogue, "modules/gitea"), + fromRepo("keycloak", catalogue, "modules/keycloak"), + fromRepo("nats", catalogue, "modules/nats"), + fromRepo("mesh-controller", controller, ""), + fromRepo("mesh-host", host, ""), + photos, snake, + } + pull := func(owner, repo, base string, paths []string, dirs ...string) link.PullUpdated { + return link.PullUpdated{Owner: owner, Repo: repo, Base: base, Commit: "abc", Paths: paths, + ModuleDirs: dirs, ModuleDirsSaid: true} + } + for _, c := range []struct { + what string + p link.PullUpdated + modules, new, manifest string + mesh bool + judge string + }{ + {"one module's own files", pull("novox", "mesh-catalog", "main", []string{"modules/gitea/index.ts"}, "modules/gitea"), + "gitea", "", "modules/gitea/module.json", true, ""}, + {"a file no module's build reads touches no module (issue 280)", + pull("novox", "mesh-catalog", "main", []string{"merge-check.sh", "README.md"}), "", "", "", true, ""}, + {"files the announcer could not list: everything built from it", + link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", PathsTruncated: true, + Paths: []string{"README.md"}}, "gitea,keycloak,nats", "", + "modules/gitea/module.json,modules/keycloak/module.json,modules/nats/module.json", true, ""}, + {"a new module, said by the head", pull("novox", "mesh-catalog", "main", + []string{"modules/newmod/index.ts", "modules/newmod/module.json"}, "modules/newmod"), + "", "modules/newmod", "modules/newmod/module.json", true, ""}, + {"the controller judges itself", pull("novox", "mesh-controller", "main", []string{"cmd/x.go"}), + "mesh-controller", "", "module.json", true, "self"}, + {"the node-engine is judged with its validator", pull("novox", "mesh-host", "main", []string{"validate/v.go"}), + "mesh-host", "", "module.json", true, "validator"}, + {"the core's owner, no module: the mesh's, its own check alone", pull("novox", "hq", "main", []string{"README.md"}), + "", "", "", true, ""}, + {"a branch nothing is built from: the mesh's repository, no module", pull("novox", "photos", "master", + []string{"server/x.js"}), "", "", "", true, ""}, + {"the branch a module is built from", pull("novox", "photos", "nox-mesh", []string{"server/x.js"}), + "photos", "", "module.json", true, ""}, + {"a repository on the forge's seat", pull("jschoubben", "snake", "main", []string{"index.html"}), + "snake", "", "module.json", true, ""}, + {"a repository of nobody's, touching nothing", pull("someone", "dotfiles", "main", []string{"x"}), + "", "", "", false, ""}, + {"a repository adding a module at its root", pull("someone", "newapp", "main", []string{"module.json", "x.js"}), + "", ".", "module.json", true, ""}, + } { + s := pullScope(c.p, entries, nil, nil) + got := []string{strings.Join(s.Modules, ","), strings.Join(s.New, ","), strings.Join(s.Manifests, ",")} + want := []string{c.modules, c.new, c.manifest} + for i, what := range []string{"modules", "new", "manifests"} { + if got[i] != want[i] { + t.Errorf("%s: %s %q, wanted %q", c.what, what, got[i], want[i]) + } + } + if s.Mesh != c.mesh || s.Judge != c.judge { + t.Errorf("%s: the mesh's %v judged by %q, wanted %v by %q", c.what, s.Mesh, s.Judge, c.mesh, c.judge) + } + if s.gated() != (c.modules != "" || c.new != "") { + t.Errorf("%s: gated %v", c.what, s.gated()) + } + } +} + +// A module whose build packages another repository's source is touched by a change to it. +func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) { + entries := []inventory.Entry{fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")} + read := map[string][]inventory.ReadRepository{"node-tools": {{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}} + s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "abc", Paths: []string{"go/x.go"}}, entries, read, nil) + if strings.Join(s.Modules, ",") != "node-tools" || len(s.Manifests) != 0 { + t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests) + } +} + +// Each layer is said; a check that could not run is an error on both, never a pass; a build seat from +// before the layers is read as the gate. +func TestAChecksLayersAreEachSaidAndAnErrorIsNeverAPass(t *testing.T) { + asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3, Modules: []string{"gitea"}} + c := checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"}) + if c.Verdict != "error" || c.Gate == nil || c.Gate.Verdict != "error" || c.RepoCheck == nil || c.RepoCheck.Verdict != "error" { + t.Fatalf("a check that could not run said %+v", c) + } + if strings.Join(c.Gate.Modules, ",") != "gitea" { + t.Errorf("the gate names %v", c.Gate.Modules) + } + c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "warning", Summary: "wide"}}) + if c.Gate == nil || c.Gate.Verdict != "warning" || c.RepoCheck != nil { + t.Fatalf("an outcome without layers said %+v", c) + } + c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "pass", + Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "its merge-check.sh failed"}}}) + if c.RepoCheck.Verdict != "fail" || c.Gate.Verdict != "pass" { + t.Fatalf("the layers said %+v / %+v", c.Gate, c.RepoCheck) + } +} + +// **The check asks the planner, never a mapping of its own** (novox/hq ADR 0238): what a pull request +// reaches is reachOfMerge's answer — the same that plans a merge — so a file at the root touches no +// module in both, and what stands on a touched module is named as its dependents in both. +func TestAPullRequestReachesWhatAMergeOfItWouldPlan(t *testing.T) { + const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git" + nats := fromRepo("nats", catalogue, "modules/nats") + nats.Source.BuiltFrom = "old" + gitea := fromRepo("gitea", catalogue, "modules/gitea") + gitea.Source.BuiltFrom = "old" + tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools") + entries := []inventory.Entry{nats, gitea, tools} + // node-tools stands on the bus's image; a code edge, so a plan takes it along. + edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}} + read := map[string][]inventory.ReadRepository{} + + for _, c := range []struct { + what string + paths []string + moved, dependents string + width int + unread string + }{ + {"a file at the root, read by no build", []string{"merge-check.sh"}, "", "", 0, "merge-check.sh"}, + {"the bus's own directory", []string{"modules/nats/Dockerfile"}, "nats", "node-tools", 2, ""}, + {"both, and a README", []string{"modules/gitea/index.ts", "modules/nats/x", "README.md"}, "gitea,nats", "node-tools", 3, "README.md"}, + } { + p := link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths} + s := pullScope(p, entries, read, edges) + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths} + r := reachOfMerge(m, entries, read, edges) + if got := strings.Join(s.Modules, ","); got != c.moved || got != strings.Join(r.Moved(), ",") { + t.Errorf("%s: the check reaches %q, the planner %v, wanted %q", c.what, got, r.Moved(), c.moved) + } + if got := strings.Join(s.Dependents, ","); got != c.dependents { + t.Errorf("%s: dependents %q, wanted %q", c.what, got, c.dependents) + } + if s.Width != c.width || s.Width != len(r.Plan.Modules) { + t.Errorf("%s: a merge would build %d, the planner says %d, wanted %d", c.what, s.Width, len(r.Plan.Modules), c.width) + } + if got := strings.Join(s.Unread, ","); got != c.unread { + t.Errorf("%s: unread %q, wanted %q", c.what, got, c.unread) + } + } + + // A repository whose build another module's recipe packages: that module is reached through the + // planner's `read`, and what stands on it after it. + read["gitea"] = []inventory.ReadRepository{{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}} + s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "head", + Paths: []string{"src/index.ts"}}, entries, read, edges) + if strings.Join(s.Modules, ",") != "gitea" || len(s.Manifests) != 0 { + t.Fatalf("a change to the source gitea packages reaches %v (manifests %v)", s.Modules, s.Manifests) + } +} + +// **Only a commit on the trunk is published** (novox/hq ADR 0238): a build of a commit off its repository's +// default branch — a pull request's head, a branch built by hand, a rebuild or replay of one — is recorded +// and never registered, so nothing can send it; a check or a dry run never is either. +func TestABuildOffTheTrunkIsNeverPublished(t *testing.T) { + on := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main", OnTrunk: true} + if err := publishable(on, ""); err != nil { + t.Errorf("a build on the trunk was refused: %v", err) + } + off := link.BuildResult{ID: "b", Commit: "abc", Trunk: "main"} + if err := publishable(off, ""); !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "main") { + t.Errorf("a build off the trunk was let through: %v", err) + } + for _, r := range []link.BuildResult{ + {ID: "c", Trunk: "main", OnTrunk: true, Check: &link.CheckOutcome{Verdict: "pass"}}, + {ID: "d", Trunk: "main", OnTrunk: true, Checked: &link.CheckRequest{}}, + {ID: "e", Trunk: "main", OnTrunk: true, DryRun: true}, + } { + if publishable(r, "") == nil { + t.Errorf("%s, a check or a dry run, was publishable", r.ID) + } + } + // A module that follows a branch other than the default: that branch is its trunk, and the default + // is not. + follows := link.BuildResult{ID: "g", Commit: "abc", Trunk: "master", Branches: []string{"nox-mesh"}} + if err := publishable(follows, "nox-mesh"); err != nil { + t.Errorf("a commit on the branch a module follows was refused: %v", err) + } + if err := publishable(link.BuildResult{ID: "h", Commit: "abc", Trunk: "master", OnTrunk: true, + Branches: []string{"master"}}, "nox-mesh"); err == nil { + t.Error("a commit on the default but not on the branch the module follows was published") + } + if err := publishable(link.BuildResult{ID: "i", Commit: "abc", Trunk: "main", Branches: []string{"feat/x"}}, ""); err == nil { + t.Error("a feature branch's commit was published") + } + // A build seat older than the rule says nothing: let through and said, so the rule can reach the mesh. + if err := publishable(link.BuildResult{ID: "f", Commit: "abc"}, ""); err != nil { + t.Errorf("a build seat that said nothing was refused: %v", err) + } +} diff --git a/cmd/mesh-controller/merge_gate.go b/cmd/mesh-controller/merge_gate.go index 2bd68c5..4e6b0fb 100644 --- a/cmd/mesh-controller/merge_gate.go +++ b/cmd/mesh-controller/merge_gate.go @@ -108,9 +108,9 @@ type mergeComposed struct { type mergeWidthOf struct { Modules []string `json:"modules"` Tiers int `json:"tiers"` - // Shared is the changed paths read as shared code: in no directory of a module the mesh holds, so - // everything built from the repository is rebuilt for them. - Shared []string `json:"shared,omitempty"` + // Unread is the changed paths no module's build reads — in no module's directory — which rebuild + // nothing (issue 280): said, so a reader sees why a change to them moves no module. + Unread []string `json:"unread,omitempty"` } // wideRebuild is how many modules a rebuild may take before the gate says so as a warning. @@ -206,9 +206,25 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error) } change := base sources := sourcesOfFacts(in.facts) + // **What the change moves is the planner's answer** (reachOfMerge, novox/hq ADR 0238), asked of the + // snapshot as if the change were merged: the definitions composed with the change are those of the + // modules a merge of it would rebuild, and of the modules it adds — not every definition in the tree, + // which may differ from what the mesh holds for reasons that are not this change's (a module pinned + // at an older commit, a main the mesh has not built yet). + var reach *mergeReach + if in.repository != "" && len(in.changed) > 0 { + r, err := reachOfChange(in.facts, in.repository, in.changed, in.tree) + if err != nil { + v.Notes = append(v.Notes, "what a merge of the change would rebuild could not be worked out, so every "+ + "definition in its tree is judged: "+err.Error()) + } else { + reach = &r + } + } if in.tree != "" { var failures []string - change, failures, err = shelfWithChange(base, sources, in.facts, in.repository, in.tree, v.Modules) + change, failures, err = shelfWithChange(base, sources, in.facts, in.repository, in.tree, v.Modules, + scopeOfReach(reach, in.repository)) if err != nil { return v, err } @@ -306,20 +322,16 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error) } } - if in.repository != "" && len(in.changed) > 0 { - w, err := rebuildWidth(in.facts, in.repository, in.changed, in.tree) - if err != nil { - v.Notes = append(v.Notes, "the width of the rebuild could not be worked out: "+err.Error()) - } else { - v.Width = &w - if len(w.Shared) > 0 && len(w.Modules) > wideRebuild { - v.Warnings = append(v.Warnings, fmt.Sprintf("a merge rebuilds %d module(s), because %s read as shared "+ - "code: in no directory of a module the mesh holds (issue 278)", len(w.Modules), - readableList(w.Shared))) - } else if len(w.Modules) > wideRebuild { - v.Warnings = append(v.Warnings, fmt.Sprintf("a merge rebuilds %d module(s) in %d tier(s)", - len(w.Modules), w.Tiers)) - } + if reach != nil { + w := widthOf(*reach) + v.Width = &w + if len(w.Unread) > 0 { + v.Notes = append(v.Notes, fmt.Sprintf("%s read by no module's build, and rebuild nothing (issue 280)", + readableList(w.Unread))) + } + if len(w.Modules) > wideRebuild { + v.Warnings = append(v.Warnings, fmt.Sprintf("a merge rebuilds %d module(s) in %d tier(s)", + len(w.Modules), w.Tiers)) } } @@ -443,8 +455,12 @@ var ignoredInTree = map[string]bool{".git": true, "vendor": true, "node_modules" // read by this controller's strict parser — the one the mesh runs, for a change to a catalogue — resolved // with stand-in builds, and read again as registration reads a built one. A module the repository held // and the tree no longer has is removed. Answers the shelf and what the tree itself fails. +// +// `scope` is the directories of the modules a merge of the change would rebuild or add, as the planner says +// (scopeOfReach); a definition elsewhere in the tree is left as the mesh holds it. Nil judges every one. func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inventory.Source, f snapshot.Facts, - repository, tree string, moved map[string]string) (map[string]catalogue.Manifest, []string, error) { + repository, tree string, moved map[string]string, scope map[string]bool) (map[string]catalogue.Manifest, []string, error) { + inScope := func(dir string) bool { return scope == nil || scope[dir] } out := make(map[string]catalogue.Manifest, len(base)) for k, m := range base { out[k] = m @@ -477,16 +493,23 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve } m, err := catalogue.ParseManifest(raw) if err != nil { - failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s", - orRoot(dir), version, oneLine(err.Error()))) + if inScope(dir) { + failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s", + orRoot(dir), version, oneLine(err.Error()))) + } return nil } if was, twice := found[m.Module]; twice { - failures = append(failures, fmt.Sprintf("%s and %s both define %s: two definitions of one module", - orRoot(was), orRoot(dir), m.Module)) + if inScope(dir) || inScope(was) { + failures = append(failures, fmt.Sprintf("%s and %s both define %s: two definitions of one module", + orRoot(was), orRoot(dir), m.Module)) + } return nil } found[m.Module] = dir + if !inScope(dir) { + return nil + } if s, held := sources[m.Module]; held && s.Repository != "" && !sameRepository(s.Repository, link.SourceMoved{Owner: ownerOf(repository), Repo: repoOf(repository)}) { failures = append(failures, fmt.Sprintf("%s defines %s, which the mesh builds from %s: two definitions "+ @@ -538,6 +561,9 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve if _, still := found[name]; still { continue } + if !inScope(strings.Trim(s.Path, "/")) { + continue + } delete(out, name) moved[name] = "removed" if on := running[name]; len(on) > 0 { @@ -961,25 +987,32 @@ func standInKey() (string, error) { return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes()), nil } -// mergeWidth is what a merge of the change would rebuild, computed the way the merge handler computes -// it, from the modules, sources and edges the snapshot carries. -func rebuildWidth(f snapshot.Facts, repository string, paths []string, tree string) (mergeWidthOf, error) { +// reachOfChange is what a merge of the change would move and build — **the planner's own answer** +// (reachOfMerge), asked of the modules, sources, reads and edges the snapshot carries, never a mapping of +// the gate's own: what a merge touches and what follows it are decided in one place, so the gate and the +// plan a merge makes cannot disagree (novox/hq ADR 0238). +func reachOfChange(f snapshot.Facts, repository string, paths []string, tree string) (mergeReach, error) { owner, repo, found := strings.Cut(repository, "/") if !found { - return mergeWidthOf{}, fmt.Errorf("%q is not owner/repository", repository) + return mergeReach{}, fmt.Errorf("%q is not owner/repository", repository) } m := link.SourceMoved{Owner: owner, Repo: repo, Base: "main", Commit: "gate", Paths: paths} // Which changed directories hold a module, read from the change's tree as the forge's announcer reads - // them at the merge commit (issue 278): a file inside one is that module's business, held or not. + // them at the head (issue 278): what the planner is told of a module the mesh does not hold yet. if tree != "" { m.ModuleDirs, m.ModuleDirsSaid = moduleDirsIn(tree, paths), true + for _, p := range paths { + if _, err := os.Stat(filepath.Join(tree, p)); os.IsNotExist(err) { + m.Removed = append(m.Removed, p) + } + } } var entries []inventory.Entry read := map[string][]inventory.ReadRepository{} for _, mod := range f.Modules { var manifest catalogue.Manifest if err := json.Unmarshal(mod.Manifest, &manifest); err != nil { - return mergeWidthOf{}, err + return mergeReach{}, err } entries = append(entries, inventory.Entry{Manifest: manifest, Provided: mod.Provided, Source: inventory.Source{Repository: mod.Repository, Path: mod.Path, BuiltFrom: mod.Commit}}) @@ -991,51 +1024,39 @@ func rebuildWidth(f snapshot.Facts, repository string, paths []string, tree stri for _, e := range f.Edges { edges = append(edges, inventory.Edge{From: e.From, To: e.To, Kind: e.Kind}) } - var from []inventory.Entry - for _, e := range entries { - if !e.Provided && sourceIs(e.Source, m) { - from = append(from, e) + return reachOfMerge(m, entries, read, edges), nil +} + +// widthOf is how wide the rebuild of a reach is. +func widthOf(r mergeReach) mergeWidthOf { + w := mergeWidthOf{Tiers: len(r.Plan.Tiers), Unread: r.Unread} + for name := range r.Plan.Modules { + w.Modules = append(w.Modules, name) + } + sort.Strings(w.Modules) + return w +} + +// scopeOfReach is the directories, in the change's repository, of the modules a reach rebuilds or adds; +// nil — every definition judged — when there is no reach. +func scopeOfReach(r *mergeReach, repository string) map[string]bool { + if r == nil { + return nil + } + scope := map[string]bool{} + same := link.SourceMoved{Owner: ownerOf(repository), Repo: repoOf(repository)} + for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) { + if sameRepository(e.Source.Repository, same) { + scope[strings.Trim(e.Source.Path, "/")] = true } } - touched := whatTheMergeTouched(from, entries, m) - var names []string - for _, e := range touched { - names = append(names, e.Manifest.Module) - } - for _, e := range entries { - if readsFrom(read[e.Manifest.Module], m) && !slices.Contains(names, e.Manifest.Module) { - names = append(names, e.Manifest.Module) + for _, d := range r.Added { + if d == "." { + d = "" } + scope[d] = true } - w := mergeWidthOf{} - if len(names) > 0 { - p := planOfMerge(m, names, edges) - w.Tiers = len(p.Tiers) - for name := range p.Modules { - w.Modules = append(w.Modules, name) - } - sort.Strings(w.Modules) - } - // The paths read as shared: in no directory of a module the mesh holds from this repository. - var dirs []string - for _, e := range from { - if d := strings.Trim(e.Source.Path, "/"); d != "" { - dirs = append(dirs, d+"/") - } - } - for _, d := range m.ModuleDirs { - dirs = append(dirs, strings.Trim(d, "/")+"/") - } - for _, p := range paths { - inModule := false - for _, d := range dirs { - inModule = inModule || strings.HasPrefix(p, d) - } - if !inModule && len(dirs) > 0 { - w.Shared = append(w.Shared, p) - } - } - return w, nil + return scope } // moduleDirsIn are the directories above the changed paths, never the root, that hold a module.json in diff --git a/cmd/mesh-controller/merge_gate_test.go b/cmd/mesh-controller/merge_gate_test.go index c5280ee..95fc9b4 100644 --- a/cmd/mesh-controller/merge_gate_test.go +++ b/cmd/mesh-controller/merge_gate_test.go @@ -180,26 +180,32 @@ func TestAModuleAMachineRunsRemovedFromItsSourceFails(t *testing.T) { } } -// **Issue 278**: a file of a module nobody holds read as shared code, and the merge rebuilt the -// catalogue. The gate says how wide a merge's rebuild is, and warns when shared code makes it wide. +// **Issues 278 and 280**: a file in no held module's directory read as shared code, and a merge rebuilt the +// catalogue. A file is a module's only by being inside it — no build reads one outside — so it rebuilds +// nothing, and the gate says why; a merge that does rebuild widely is warned of. func TestIssue278AWideRebuildIsSaidBeforeTheMerge(t *testing.T) { f, manifests := catalogueMesh(t) was := wideRebuild wideRebuild = 2 t.Cleanup(func() { wideRebuild = was }) - v := gateJudged(t, f, aTree(t, manifests), "modules/showcase/index.ts") - if v.Width == nil || len(v.Width.Modules) < 5 || len(v.Width.Shared) != 1 { - t.Fatalf("the width reads %+v", v.Width) + for _, file := range []string{"modules/showcase/index.ts", "merge-check.sh", "README.md"} { + v := gateJudged(t, f, aTree(t, manifests), file) + if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 1 || v.Verdict != "pass" { + t.Fatalf("%s, read by no build, reads %+v, %s", file, v.Width, v.Verdict) + } + if !strings.Contains(v.Report(), "read by no module's build") { + t.Errorf("why %s rebuilds nothing is not said:\n%s", file, v.Report()) + } } - if v.Verdict != "warning" || !strings.Contains(v.Report(), "shared") { - t.Fatalf("a rebuild of everything for one shared file is not said:\n%s", v.Report()) + v := gateJudged(t, f, aTree(t, manifests), "modules/album/x.ts", "modules/objects/x.go", "modules/resolver/x.go") + if v.Width == nil || len(v.Width.Modules) < 3 || v.Verdict != "warning" { + t.Fatalf("a rebuild wider than the bound is not warned of: %+v, %s", v.Width, v.Verdict) } - // The same file, with the reference module's definition in the tree: its directory is a module, held - // or not, and the file is its business alone (the fix of 278, read from the tree as the announcer does). + // With the reference module's definition in the tree, its directory is a module, held or not. withShowcase := withEdit(manifests, "showcase", `{"module":"showcase","version":"1"}`) v = gateJudged(t, f, aTree(t, withShowcase), "modules/showcase/index.ts") - if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Shared) != 0 { - t.Fatalf("a file of a module nobody holds still reads as shared: %+v", v.Width) + if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 0 { + t.Fatalf("a file of a module nobody holds reads %+v", v.Width) } v = gateJudged(t, f, aTree(t, manifests), "modules/album/module.json") if v.Width == nil || strings.Join(v.Width.Modules, ",") != "album" || v.Verdict != "pass" { diff --git a/cmd/mesh-controller/order_test.go b/cmd/mesh-controller/order_test.go index 5fe7b12..7c6a901 100644 --- a/cmd/mesh-controller/order_test.go +++ b/cmd/mesh-controller/order_test.go @@ -144,7 +144,7 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) { }{ {"one module's own files", merge([]string{"modules/gitea/index.ts", "modules/gitea/client.ts"}, false), "gitea"}, {"two modules' files", merge([]string{"modules/gitea/index.ts", "modules/keycloak/module.json"}, false), "gitea,keycloak"}, - {"a file they share", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"}, + {"a file at the root no build reads (issue 280)", merge([]string{"tsconfig.json"}, false), ""}, {"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""}, {"nothing said about the files", merge(nil, false), "gitea,keycloak"}, {"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"}, @@ -153,9 +153,9 @@ func TestAMergeRebuildsTheModulesItChanged(t *testing.T) { {"a new module beside a held one", merge([]string{"modules/gitea/x", "modules/newmod/module.json"}, false), "gitea"}, {"a new module's other files", merge([]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, false), ""}, {"a module removed", merge([]string{"modules/gone/module.json"}, false), ""}, - {"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), "gitea,keycloak"}, - {"a file directly among the modules", merge([]string{"modules/README.md"}, false), "gitea,keycloak"}, - {"the root's files still", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"}, + {"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), ""}, + {"a file directly among the modules", merge([]string{"modules/README.md"}, false), ""}, + {"a root file beside a module's", merge([]string{"merge-check.sh", "modules/keycloak/x.ts"}, false), "keycloak"}, } { if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want { t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want) @@ -245,12 +245,11 @@ func TestAModuleBuiltAtACommitStillFollowsItsBranch(t *testing.T) { } } -// novox/hq issue 278: whether a directory is a module is a fact of the repository at the merge commit, -// and the announcer says it. A merge touching the code of a module the mesh does not hold — the +// novox/hq issues 278 and 280: a merge touching the code of a module the mesh does not hold — the // catalogue's reference module, whose manifest it left alone — read as shared and rebuilt every module -// built from the repository (103 of them on 2026-10-06, 88 byte-identical). Said by the announcer, it -// rebuilds nothing; a directory holding no manifest is shared as before; and an announcer that does not -// say keeps the old rule. +// built from the repository (103 of them on 2026-10-06, 88 byte-identical); so did a merge-check.sh added at +// the root. No build reads a file outside its module's directory, so neither rebuilds anything, said by the +// announcer or not. func TestAChangeInsideAModuleIsThatModulesHeldOrNot(t *testing.T) { const repo = "http://forge.internal:20000/novox/mesh-catalog.git" gitea := fromRepo("gitea", repo, "modules/gitea") @@ -278,12 +277,12 @@ func TestAChangeInsideAModuleIsThatModulesHeldOrNot(t *testing.T) { {"beside a held one's change", merge(append([]string{"modules/gitea/index.ts"}, showcase...), []string{"modules/gitea", "modules/showcase"}, true), "gitea"}, {"deeper inside it", merge([]string{"modules/showcase/daemon/index.ts"}, []string{"modules/showcase"}, true), ""}, - {"a directory holding no manifest is still shared", merge([]string{"modules/lib/x.go"}, nil, true), "gitea,keycloak"}, + {"a directory holding no manifest, read by no build", merge([]string{"modules/lib/x.go"}, nil, true), ""}, {"one of two files in no module", merge([]string{"modules/showcase/index.ts", "modules/lib/x.go"}, - []string{"modules/showcase"}, true), "gitea,keycloak"}, - {"the root is never a module directory", merge([]string{"tsconfig.json"}, []string{"", "/", "."}, true), "gitea,keycloak"}, - {"not said: the old rule", merge(showcase, []string{"modules/showcase"}, false), "gitea,keycloak"}, - {"an old announcer saying nothing", merge(showcase, nil, false), "gitea,keycloak"}, + []string{"modules/showcase"}, true), ""}, + {"the root is never a module directory", merge([]string{"tsconfig.json"}, []string{"", "/", "."}, true), ""}, + {"not said: still no build reads it", merge(showcase, []string{"modules/showcase"}, false), ""}, + {"an old announcer saying nothing", merge(showcase, nil, false), ""}, {"a manifest the merge removed, said or not", merge([]string{"modules/gone/module.json", "modules/gone/x.ts"}, nil, true), ""}, } { if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want { diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 2e3e611..0ad9255 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -1357,18 +1357,20 @@ func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string for _, name := range modules { named[name] = true } - for _, e := range entries { - switch { - case named[e.Manifest.Module]: - from = append(from, e) - case len(named) == 0 && sourceIs(e.Source, m): - from = append(from, e) - case readsFrom(read[e.Manifest.Module], m): - packaging = append(packaging, e) - } - } if len(named) == 0 { - from = whatTheMergeTouched(from, entries, m) + // The changed files: the planner's own answer, as the merge handler, the merge gate and a pull + // request's check ask it (novox/hq ADR 0238). + r := reachOfMerge(m, entries, read, nil) + from, packaging = append(append([]inventory.Entry{}, r.Touched...), r.Deleted...), r.Packaging + } else { + for _, e := range entries { + switch { + case named[e.Manifest.Module]: + from = append(from, e) + case readsFrom(read[e.Manifest.Module], m): + packaging = append(packaging, e) + } + } } moved := append(append([]inventory.Entry{}, from...), packaging...) if len(moved) == 0 { diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index b95a59e..5919a00 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -5,8 +5,8 @@ import ( "errors" "flag" "fmt" - "path" "regexp" + "slices" "sort" "strings" "sync" @@ -616,71 +616,157 @@ func lastLookAt(entries []inventory.Entry, m link.SourceMoved) time.Time { return newest } -// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed. -// -// **A change inside no module's own directory is a change to what they share.** The forge lists the -// files a merge changed; a module is affected when one of them is inside its own directory, when it -// is built from the repository's root — everything there is its source — or when some changed file -// belongs to no module's directory at all, which is how a shared file, a build recipe or a -// dependency at the root rebuilds everything built from that repository. -// -// A change inside *another* module's directory is that module's business and not this one's, even -// when the mesh does not hold that module: `known` is every module this repository is known to hold, -// whatever branch it was registered from. -// -// **And a module the mesh has never seen is still a module** (novox/hq issue 252). A merge adding a -// new module to the catalogue repository — `modules/newmod/module.json` and its files — read as a -// change to shared code, because `modules/newmod` was nobody's known directory, and every module -// built from the repository was rebuilt and rolled out for a module none of them is. So the -// directories that hold modules are known too: the parents of the known modules' directories -// (`modules`, never the root). A changed path `//…` belongs to the module at -// `/` — held or not — and rebuilds nothing else, **provided it is shown to be a -// module**: its `module.json` is among the changed files (added, changed, or removed with it). A -// directory under the same parent whose manifest the merge did not touch may as well be a shared -// library (`modules/lib`), and that is still read as shared. Rebuilding too much remains the safe -// direction: the fault this whole path exists for is a mesh that believes it is current and is not -// (novox/hq 04-ISSUES/131). A file at the root, or directly in a parent, is shared as it always was. -// -// **Whether a directory is a module is a fact of the repository at the merge commit** (novox/hq issue -// 278), and the announcer now says it: a changed file inside a directory holding a `module.json` there -// is that module's, whatever the merge did to the manifest; only a file in no such directory is shared. -// From an announcer that does not say, the rule above stands. +// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: **a +// changed file touches exactly the modules whose build reads it** (novox/hq issue 280, ADR 0238). It is +// touchedBy's first answer; touchedBy is the one place the mesh maps a changed file onto its modules. func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved) []inventory.Entry { - // Nothing said about the files, or not all of them said: everything built from it is affected. - if len(m.Paths) == 0 || m.PathsTruncated { - return candidates - } - var dirs []string - parents := map[string]bool{} + touched, _, _ := touchedBy(candidates, known, m) + return touched +} + +// touchedBy maps a merge's changed files onto the mesh's modules — **the one place it is done**, for the +// merge handler, the release planner's what-if, the merge gate and a pull request's check alike (novox/hq +// ADR 0238), so planning and gating cannot disagree about what a change touches. +// +// **A changed file touches exactly the modules whose build reads it.** What a build reads is the module's +// own directory — the builder clones the repository and builds within that directory alone: the manifest, +// the recipes, the bundles' sources, the Docker context — or the whole repository for a module built from +// its root. A second repository a recipe packages (an artifact's `context`) is read too; that is the build +// record's `read`, answered by readsFrom in mergeCandidates. So a changed file inside a module's directory +// is that module's; a file in no module's directory — a script at the root, a README, CI configuration — +// is read by no build and touches nothing: it is answered as `unread`. +// +// **It used to be read as shared code**, rebuilding everything built from the repository, on the theory +// that a root file might be a build input (04-ISSUES/131). No build reads one: the theory cost a rebuild +// of 103 modules for a merge-check.sh added at the catalogue's root (issue 280), as it had for a module +// held by no machine (278) and a new module's directory (252), each patched as an exception to a rule that +// was wrong. Rebuilding too much is not the safe direction when every rebuild is a rollout. +// +// `added` is the directories the change holds a module in that no module of this repository is known +// from — said by the announcer at the head (issue 278), or a module.json among the changed files — `.` +// for the root: a new module, which a check judges before it merges and a merge does not build. +// +// Nothing said about the files, or not all of them said, is still everything: what is not known cannot +// be narrowed. +func touchedBy(candidates, known []inventory.Entry, m link.SourceMoved) (touched []inventory.Entry, added, unread []string) { + knownDirs := map[string]bool{} for _, e := range known { - if e.Source.Path != "" && sameRepository(e.Source.Repository, m) { - dir := strings.Trim(e.Source.Path, "/") - dirs = append(dirs, dir) - if parent := path.Dir(dir); parent != "." && parent != "/" { - parents[parent] = true - } + if !e.Provided && sameRepository(e.Source.Repository, m) { + knownDirs[strings.Trim(e.Source.Path, "/")] = true } } - changed := map[string]bool{} - for _, p := range m.Paths { - changed[strings.TrimPrefix(p, "/")] = true + newDir := map[string]bool{} + for _, d := range saidModuleDirs(m) { + if !knownDirs[d] { + newDir[d] = true + } } - modules := saidModuleDirs(m) for _, p := range m.Paths { - if insideAny(p, dirs) || inAModuleOfItsOwn(p, parents, changed) || insideAny(p, modules) { + p = strings.Trim(p, "/") + if p != moduleManifestFile && !strings.HasSuffix(p, "/"+moduleManifestFile) { continue } - return candidates - } - var out []inventory.Entry - for _, e := range candidates { - if e.Source.Path == "" || anyInside(m.Paths, e.Source.Path) { - out = append(out, e) + d := strings.TrimSuffix(strings.TrimSuffix(p, moduleManifestFile), "/") + if !knownDirs[d] { + if d == "" { + d = "." + } + newDir[d] = true } } + for d := range newDir { + added = append(added, d) + } + sort.Strings(added) + + if len(m.Paths) == 0 || m.PathsTruncated { + return candidates, added, nil + } + for _, e := range candidates { + if strings.Trim(e.Source.Path, "/") == "" || anyInside(m.Paths, e.Source.Path) { + touched = append(touched, e) + } + } + for _, p := range m.Paths { + read := newDir["."] + for _, e := range candidates { + read = read || strings.Trim(e.Source.Path, "/") == "" || inside(p, e.Source.Path) + } + for d := range newDir { + read = read || inside(p, d) + } + if !read { + unread = append(unread, p) + } + } + return touched, added, unread +} + +// mergeReach is what a merge of a repository's branch reaches, as the planner reckons it: the planner's +// one answer, given to the release planner's what-if, the merge gate and a pull request's check (novox/hq +// ADR 0238). The merge handler acts on the same pieces — mergeCandidates, touchedBy, splitDeleted, +// planOfMerge — as it goes. +type mergeReach struct { + // Touched are the modules built from the repository and branch whose build reads a changed file, and + // Deleted those of them whose manifest the merge removes. + Touched, Deleted []inventory.Entry + // Packaging are the modules whose build packages source from the repository. + Packaging []inventory.Entry + // Already counts the modules built from the repository that are built from this very commit. + Already int + // Added are the directories the change holds a new module in; Unread the changed files no build reads. + Added, Unread []string + // Plan is what moves and everything that follows it along the catalogue's dependencies, tiered — + // what a merge would build. Empty when nothing moves. + Plan inventory.Plan +} + +// Moved are the modules the merge moves itself, by name: touched, deleted and packaging. +func (r mergeReach) Moved() []string { + var out []string + for _, group := range [][]inventory.Entry{r.Touched, r.Deleted, r.Packaging} { + for _, e := range group { + out = append(out, e.Manifest.Module) + } + } + sort.Strings(out) + return slices.Compact(out) +} + +// Dependents are the modules the plan builds after the moved ones because they stand on them. +func (r mergeReach) Dependents() []string { + moved := map[string]bool{} + for _, name := range r.Moved() { + moved[name] = true + } + var out []string + for name := range r.Plan.Modules { + if !moved[name] { + out = append(out, name) + } + } + sort.Strings(out) return out } +// reachOfMerge is what a merge of these changed files into this branch would move and build, without +// acting: the merge handler's own judgement and the release plan's dependency walk. +func reachOfMerge(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository, + edges []inventory.Edge) mergeReach { + from, packaging, already := mergeCandidates(m, entries, read) + touched, added, unread := touchedBy(from, entries, m) + kept, deleted := splitDeleted(touched, m) + r := mergeReach{Touched: kept, Deleted: deleted, Packaging: packaging, Already: already, Added: added, Unread: unread} + var building []string + for _, e := range append(append([]inventory.Entry{}, kept...), packaging...) { + building = append(building, e.Manifest.Module) + } + if len(building) > 0 { + r.Plan = planOfMerge(m, building, edges) + } + return r +} + // saidModuleDirs is the directories the announcer says hold a module at the merge commit (novox/hq // issue 278): a changed file in one of them is that module's business and nobody else's — the // catalogue's reference module, held by no machine, whose code a merge touched beside its manifest, @@ -700,28 +786,6 @@ func saidModuleDirs(m link.SourceMoved) []string { return out } -// inAModuleOfItsOwn is whether a changed file is inside a module directory the mesh does not know — -// `//…` under a directory known to hold modules, whose `module.json` the same merge -// changed (novox/hq issue 252). Such a file is that module's business and nobody else's. -func inAModuleOfItsOwn(p string, parents, changed map[string]bool) bool { - p = strings.TrimPrefix(p, "/") - for parent := range parents { - rest, under := strings.CutPrefix(p, parent+"/") - if !under { - continue - } - name, _, inADirectory := strings.Cut(rest, "/") - if !inADirectory || name == "" { - // A file directly in the parent — `modules/README.md` — is about all of them. - continue - } - if changed[parent+"/"+name+"/module.json"] { - return true - } - } - return false -} - // inside is whether a changed file is in a directory: that directory itself, or under it. func inside(path, dir string) bool { dir = strings.Trim(dir, "/") @@ -729,16 +793,6 @@ func inside(path, dir string) bool { return path == dir || strings.HasPrefix(path, dir+"/") } -// insideAny is whether a changed file is in any of these directories. -func insideAny(path string, dirs []string) bool { - for _, dir := range dirs { - if inside(path, dir) { - return true - } - } - return false -} - // anyInside is whether any of these changed files is in a directory. func anyInside(paths []string, dir string) bool { for _, p := range paths { diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 9dabaa3..0484a40 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -68,6 +68,15 @@ type Result struct { // change to this module (novox/hq 04-ISSUES/131). Read []catalogue.ArtifactContext + // Trunk is the repository's default branch as the forge holds it at the build, and OnTrunk whether + // the commit built is reachable from it (novox/hq ADR 0238): a commit off the trunk is checked, + // never published. Trunk empty is "could not be said". + Trunk string + OnTrunk bool + // Branches are every branch of the forge the commit is on: a module may follow a branch other than + // the default, and that branch is its trunk. + Branches []string + // Source is the build's source fingerprint (source.go): what it was made from — the module's tree, // the contexts' trees, the bases and toolchains by digest — hashed. Empty where the source does not // pin the build. Two builds with one fingerprint are one build, whatever digests they made @@ -144,6 +153,16 @@ func Build(ctx context.Context, run Runner, publish Publisher, } commit = strings.TrimSpace(commit) say("commit", "%s", short(commit)) + // **Whether the commit is on the trunk** (novox/hq ADR 0238): only a commit on the repository's own + // default branch is published. Read from the clone just made — the forge's own word on which branch + // is its default and what it holds now — and said with the outcome, so the controller refuses to + // register a build of a commit off it. + trunk, onTrunk := trunkOf(ctx, run, tree, commit) + branches := branchesHolding(ctx, run, tree, commit) + if trunk != "" { + say("trunk", "%s is %son %s; on %s", short(commit), map[bool]string{true: "", false: "NOT "}[onTrunk], trunk, + orNoBranch(branches)) + } // A module is a repository and a path within it (novox/hq ADR 0069). The ordinary case is an // empty path, meaning the repository's root; a repository holding several modules names each @@ -250,7 +269,50 @@ func Build(ctx context.Context, run Runner, publish Publisher, say("source", "no source fingerprint: %s", orNoTree(src.unpinned)) } return Result{Manifest: resolved, Commit: commit, Built: built, - Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint}, nil + Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint, + Trunk: trunk, OnTrunk: onTrunk, Branches: branches}, nil +} + +// branchesHolding is every branch of a fresh clone's origin the commit is on, without `origin/`. +func branchesHolding(ctx context.Context, run Runner, clone, commit string) []string { + out, err := run(ctx, clone, "git", "branch", "--remotes", "--format=%(refname:short)", "--contains", commit) + if err != nil { + // empty-on-error: no branch said is the commit on none, which refuses its registration — never a pass + return nil + } + var branches []string + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if b, ok := strings.CutPrefix(line, "origin/"); ok && b != "HEAD" && b != "" { + branches = append(branches, b) + } + } + sort.Strings(branches) + return branches +} + +func orNoBranch(branches []string) string { + if len(branches) == 0 { + return "no branch" + } + return strings.Join(branches, ", ") +} + +// trunkOf is a fresh clone's trunk — the branch the forge names its default, as `origin/HEAD` says — and +// whether a commit is reachable from it. Empty when the clone does not say, which is "not known", never +// "on it". +func trunkOf(ctx context.Context, run Runner, clone, commit string) (string, bool) { + head, err := run(ctx, clone, "git", "symbolic-ref", "--quiet", "--short", "refs/remotes/origin/HEAD") + if err != nil { + return "", false + } + remote := strings.TrimSpace(head) + trunk := strings.TrimPrefix(remote, "origin/") + if trunk == "" || trunk == remote { + return "", false + } + _, err = run(ctx, clone, "git", "merge-base", "--is-ancestor", commit, remote) + return trunk, err == nil } // orNoTree is why a build has no source fingerprint, for its log. diff --git a/internal/builder/check.go b/internal/builder/check.go index 17a01b4..5857363 100644 --- a/internal/builder/check.go +++ b/internal/builder/check.go @@ -1,11 +1,13 @@ package builder import ( + "bufio" "bytes" "context" "encoding/json" "errors" "fmt" + "io" "net" "os" "os/exec" @@ -18,27 +20,40 @@ import ( "github.com/novox/mesh-controller/internal/facts" ) -// A pull request's merge check, run on the build seat (novox/hq to-be 45 §9). +// A pull request's merge check, run on the build seat (novox/hq to-be 45 §9, ADR 0237). // // **The build machine already has what a check needs**: the repositories, a container runtime, the // artifact store where the controller keeps the facts snapshot, and a Go toolchain. So a check is one // more kind of work on the build seat's queue rather than a CI the mesh would have to run beside itself. // +// **Two layers, each its own status on the pull request** (ADR 0237 as amended, 2026-10-06): +// +// - **the gate** (`mesh/merge-gate`) runs when the change touches a module of the mesh's graph — the +// controller, which holds the graph, says which (Modules) and which directories it adds a module in +// (New). The touched manifests through `module check`; every machine of the facts snapshot composed +// with the change and validated by the node-engine's own validator; then mesh-lab's replays. The +// judge is the controller the mesh runs — or, for a change to the controller, the change's own +// controller, and for a change to the node-engine, the running controller with the change's validator +// in place of the one it vendors. The graph decides whether this runs, never the repository. +// - **the repository's own check** (`mesh/repo-check`): its merge-check.sh, its unit tests and code +// quality, run when present in the toolchain it declares (`# mesh-check-toolchain: go|typescript` +// among its first lines; go when it declares none). A repository that reaches the build seat with +// none is said as a warning: it is the mesh's, and nothing of its own is tested before it merges. +// // One check: // // 1. clones the repository at the pull request's head, and beside it the repositories its check -// reads — the controller the mesh runs, the catalogue, the host — each at the ref asked; +// reads — the controller the mesh runs, the catalogue, the host, the lab — each at the ref asked; // 2. reads the facts snapshot the controller keeps, and with it **the versions the mesh runs**: the // store a check's tests stand on is the store's own release, and the bus the bus's; // 3. raises a throwaway PostgreSQL and a throwaway bus of those versions, labelled with the ask so a // kill or a crash leaves nothing behind; -// 4. builds the judge — the controller the mesh runs, or its main while the running one predates the -// merge gate — and runs the repository's own merge-check.sh, or the merge gate alone for a -// repository that declares none. **In the mesh's Go toolchain, in a container of its own**, never in -// the build machine's: a pull request is code nobody has approved yet, and the build machine holds -// the container runtime's socket; the check's container holds none, and reaches only the -// throwaway store and bus on loopback; -// 5. answers pass, warning or fail from what ran, and error — never pass — when it could not run. +// 4. runs the gate, then the repository's script — **each in a toolchain container of its own**, never +// in the build machine: a pull request is code nobody has approved yet, and the build machine holds +// the container runtime's socket; the check's container holds none, and reaches only the throwaway +// store and bus on loopback. Only the replays — mesh-lab's main, reviewed code — get the socket; +// 5. answers each layer pass, warning or fail from what ran, and error — never pass — when it could +// not run. // CheckSpec is one check, as the controller asks it. type CheckSpec struct { @@ -51,18 +66,40 @@ type CheckSpec struct { Paths []string // Beside are the repositories cloned next to it, by the directory they are found under. Beside map[string]Beside + // Modules are the modules of the mesh's graph the change touches, New the directories it adds a + // module in, and Manifests the manifests among them in the change's tree: the gate runs when Modules + // or New is not empty. + Modules []string + New []string + Manifests []string + // Base is the branch the pull request merges into: what the gate compares the change against. + Base string + // Judge is who judges the gate: "" the controller the mesh runs, "self" the change's own, "validator" + // the running one with the change's validator. + Judge string // Toolchain is the image a check's Go runs in: the mesh's own Go toolchain, as it holds it. - Toolchain string + // Toolchains is every toolchain the mesh holds, by language, for a script that declares another. + Toolchain string + Toolchains map[string]string } +// Gated is whether the change touches the mesh's graph, and so whether the gate runs. +func (s CheckSpec) Gated() bool { return len(s.Modules)+len(s.New) > 0 } + // ToolchainOf is the Go toolchain image among what the mesh holds, empty when it holds none. func ToolchainOf(held map[string]string) string { + return ToolchainsOf(held)["go"] +} + +// ToolchainsOf is every toolchain image the mesh holds, by language. +func ToolchainsOf(held map[string]string) map[string]string { + out := map[string]string{} for _, chain := range toolchains { - if chain.Language == "go" { - return held[chain.Base+"/"+chain.Artifact] + if image := held[chain.Base+"/"+chain.Artifact]; image != "" { + out[chain.Language] = image } } - return "" + return out } // Beside is one repository cloned next to the one checked. @@ -71,15 +108,24 @@ type Beside struct { Ref string } -// CheckVerdict is what came of one check. +// CheckVerdict is what came of one check. Verdict and Summary are the gate's; Gate and Repo each layer. type CheckVerdict struct { Verdict string Summary string Report string Took time.Duration + Gate *Layer + Repo *Layer } -// CheckScript is what a repository declares its merge check as: run from its root, with the +// Layer is one layer of a check, judged. +type Layer struct { + Verdict string + Summary string + Modules []string +} + +// CheckScript is what a repository declares its own merge check as: run from its root, with the // environment below. const CheckScript = "merge-check.sh" @@ -94,6 +140,7 @@ const ( EnvChanged = "MESH_CHECK_CHANGED" EnvVerdict = "MESH_CHECK_VERDICT" EnvBeside = "MESH_CHECK_BESIDE" + EnvModules = "MESH_CHECK_MODULES" ) // CheckTimeout bounds one check; a check that runs past it is an error, not a pass. @@ -102,6 +149,28 @@ var CheckTimeout = 45 * time.Minute // reportLines is how much of what a check printed travels in its verdict. const reportLines = 200 +// noModule is the gate's word for a change that touches nothing of the mesh's graph: a fact, not a +// missing check, so a pass. +const noModule = "the change touches no module of the mesh's graph" + +// noScript is the repository layer's word for a repository with no merge-check.sh of its own. +const noScript = "the repository declares no " + CheckScript + ": none of its own tests run before it merges" + +// toolchainLine is how a merge-check.sh declares the toolchain it runs in. +var toolchainLine = regexp.MustCompile(`^#\s*mesh-check-toolchain:\s*([a-z0-9-]+)\s*$`) + +// ScriptToolchain is the language a merge-check.sh declares it runs in, among its first twenty lines; +// go when it declares none. +func ScriptToolchain(script []byte) string { + lines := bufio.NewScanner(bytes.NewReader(script)) + for i := 0; i < 20 && lines.Scan(); i++ { + if m := toolchainLine.FindStringSubmatch(strings.TrimSpace(lines.Text())); m != nil { + return m[1] + } + } + return "go" +} + // Check runs one merge check. An error is that it could not run; the verdict is then "error". func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry string, forge GitCredential, log Log) (CheckVerdict, error) { @@ -137,13 +206,24 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry return nil } name := spec.Repo - if name == "" { + if name == "" || !safeName.MatchString(name) { name = "checked" } say("check", "%s/%s#%d at %s", spec.Owner, spec.Repo, spec.Number, short(spec.Ref)) if err := clone(spec.Repository, spec.Ref, name); err != nil { return CheckVerdict{}, err } + tree := filepath.Join(root, name) + script, scriptErr := os.ReadFile(filepath.Join(tree, CheckScript)) + hasScript := scriptErr == nil + gated := spec.Gated() + if !gated && !hasScript { + // Nothing to run: said, never passed silently. + v := CheckVerdict{Verdict: "pass", Summary: noModule, Took: time.Since(began), + Gate: &Layer{Verdict: "pass", Summary: noModule}, Repo: &Layer{Verdict: "warning", Summary: noScript}} + say("check", "%s; %s", noModule, noScript) + return v, nil + } for dir, b := range spec.Beside { if dir == name || !safeName.MatchString(dir) { continue @@ -204,102 +284,358 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry if spec.Toolchain == "" { return CheckVerdict{}, errors.New("the mesh holds no Go toolchain to run a check in") } - inToolchain := func(dir string, env []string, command ...string) []string { + in := func(image, dir string, env []string, command ...string) []string { // As the builder itself: what a check writes into the workspace is the builder's to remove. args := []string{"run", "--rm", "--network", "host", "--volume", workspace + ":" + workspace, "--workdir", dir, "--user", fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()), "--env", "HOME=" + workspace} for _, e := range env { args = append(args, "--env", e) } - return append(append(args, spec.Toolchain), command...) + return append(append(args, image), command...) } - - // The judge: the controller the mesh runs, or its main while the running one has no merge gate. - gate := "" - if name != "mesh-controller" { - gate, err = judge(ctx, labelled, run, root, inToolchain, say) - if err != nil { - return CheckVerdict{}, err - } + inToolchain := func(dir string, env []string, command ...string) []string { + return in(spec.Toolchain, dir, env, command...) } - - verdictFile := filepath.Join(root, "verdict.json") - env := append([]string{}, - EnvFacts+"="+factsFile, EnvGate+"="+gate, EnvGateStore+"="+storeURL, EnvTestStore+"="+storeURL, - EnvTestBus+"=nats://"+bus, EnvRepository+"="+spec.Owner+"/"+spec.Repo, - EnvChanged+"="+strings.Join(spec.Paths, ","), EnvVerdict+"="+verdictFile, EnvBeside+"="+root, - "GOCACHE="+filepath.Join(workspace, "go-cache"), "GOMODCACHE="+filepath.Join(workspace, "go-modules")) - tree := filepath.Join(root, name) - var command []string - if _, err := os.Stat(filepath.Join(tree, CheckScript)); err == nil { - say("check", "running its %s in the mesh's Go toolchain", CheckScript) - command = []string{"sh", CheckScript} - } else { - if gate == "" { - return CheckVerdict{}, fmt.Errorf("%s declares no %s and there is no judge to run", name, CheckScript) - } - say("check", "it declares no %s: the merge gate alone", CheckScript) - command = []string{"sh", "-c", `"$MESH_GATE" merge-gate --facts "$MESH_FACTS" --store "$MESH_GATE_POSTGRES" ` + - `--repository "$MESH_CHECK_REPOSITORY" --tree . --changed "$MESH_CHECK_CHANGED" --json > "$MESH_CHECK_VERDICT"`} - } - cmd := exec.CommandContext(ctx, "docker", LabelledArgs("docker", inToolchain(tree, env, command...), spec.ID)...) - inItsOwnGroup(cmd) var out tail - cmd.Stdout, cmd.Stderr = &out, &out - runErr := cmd.Run() + // running runs one container of the check, its output into the report, in a process group of its own. + running := func(args []string) error { + cmd := exec.CommandContext(ctx, "docker", LabelledArgs("docker", args, spec.ID)...) + inItsOwnGroup(cmd) + cmd.Stdout, cmd.Stderr = &out, &out + return cmd.Run() + } - // **And the replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed - // code, so given the container runtime the resolver replay raises containers with — against the bus - // of the release the mesh runs and the change's own catalogue when the change is to the catalogue. - var replayErr error - if lab := filepath.Join(root, "mesh-lab", "replays"); runErr == nil && ctx.Err() == nil { + // The judge. Its failing to build is the change's fault when the change is the judge or its validator, + // and the check's when it is the controller the mesh runs. + gate, judgeFault, err := judgeFor(ctx, labelled, run, spec, root, tree, inToolchain, say) + if err != nil { + return CheckVerdict{}, err + } + verdictFile := filepath.Join(root, "verdict.json") + env := []string{EnvFacts + "=" + factsFile, EnvGate + "=" + gate, EnvGateStore + "=" + storeURL, + EnvTestStore + "=" + storeURL, EnvTestBus + "=nats://" + bus, EnvRepository + "=" + spec.Owner + "/" + spec.Repo, + EnvChanged + "=" + strings.Join(spec.Paths, ","), EnvBeside + "=" + root, + EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","), + "GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")} + + v := CheckVerdict{} + modules := append(append([]string{}, spec.Modules...), prefixed("new:", spec.New)...) + timedOut := func() bool { return errors.Is(ctx.Err(), context.DeadlineExceeded) } + + // **The gate**, when the graph says the change touches it. + if !gated { + v.Gate = &Layer{Verdict: "pass", Summary: noModule} + } else { + v.Gate = &Layer{Modules: modules} + switch { + case judgeFault != "": + v.Gate.Verdict, v.Gate.Summary = "fail", judgeFault + default: + say("check", "the gate: %d module(s) of the graph touched — %s", len(modules), strings.Join(modules, ", ")) + fmt.Fprintf(&out, "--- the gate: %s\n", strings.Join(modules, ", ")) + v.Gate.Verdict, v.Gate.Summary = gateLayer(ctx, spec, tree, root, gate, verdictFile, env, inToolchain, + running, &out, bus, workspace, name, say) + } + if timedOut() { + v.Gate.Verdict, v.Gate.Summary = "error", fmt.Sprintf("the check ran past %s and was ended", CheckTimeout) + } + } + if ctx.Err() != nil && !timedOut() { + return v, ctx.Err() + } + + // **The repository's own check**, in the toolchain it declares. + switch { + case !hasScript: + v.Repo = &Layer{Verdict: "warning", Summary: noScript} + case timedOut(): + v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("the check ran past %s before its %s ran", CheckTimeout, CheckScript)} + default: + language := ScriptToolchain(script) + image := spec.Toolchains[language] + if language == "go" && image == "" { + image = spec.Toolchain + } + if image == "" { + v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s runs in the %s toolchain, which the mesh does "+ + "not hold", CheckScript, language)} + break + } + say("check", "running its %s in the mesh's %s toolchain", CheckScript, language) + fmt.Fprintf(&out, "--- its %s (%s toolchain)\n", CheckScript, language) + var own tail + cmd := exec.CommandContext(ctx, "docker", LabelledArgs("docker", in(image, tree, env, "sh", CheckScript), spec.ID)...) + inItsOwnGroup(cmd) + w := io.MultiWriter(&out, &own) + cmd.Stdout, cmd.Stderr = w, w + switch err := cmd.Run(); { + case timedOut(): + v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", CheckScript, CheckTimeout)} + case ctx.Err() != nil: + return v, ctx.Err() + case err != nil: + v.Repo = &Layer{Verdict: "fail", Summary: "its " + CheckScript + " failed: " + lastLine(own.String())} + default: + v.Repo = &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed"} + } + } + + v.Verdict, v.Summary = v.Gate.Verdict, v.Gate.Summary + v.Report, v.Took = out.String(), time.Since(began) + say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary, + strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second)) + return v, nil +} + +// gateLayer runs the gate: the touched manifests through `module check`, every machine composed with the +// change, and the replays of what the mesh runs. It answers the gate's verdict and summary. +func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFile string, env []string, + inToolchain func(string, []string, ...string) []string, running func([]string) error, out *tail, bus, workspace, + name string, say func(step, format string, args ...any)) (string, string) { + // 1. The manifests the change touches, as the judge reads them: a manifest it cannot read, or one with a + // problem the change brings, fails here. **What was already so on the base branch is said and fails + // nothing** — the gate's own rule: a module whose manifest the running controller already finds fault + // with would otherwise fail every pull request that touches it, for a fault none of them made. + var manifests []string + for _, m := range spec.Manifests { + if _, err := os.Stat(filepath.Join(tree, m)); err == nil { + manifests = append(manifests, m) + } + } + if len(manifests) > 0 { + checked := func(dir string) (string, error) { + var own tail + cmd := exec.CommandContext(ctx, "docker", LabelledArgs("docker", + inToolchain(dir, env, append([]string{gate, "module", "check"}, manifests...)...), spec.ID)...) + inItsOwnGroup(cmd) + w := io.MultiWriter(out, &own) + cmd.Stdout, cmd.Stderr = w, w + err := cmd.Run() + return own.String(), err + } + said, err := checked(tree) + if err != nil { + if ctx.Err() != nil { + return "error", "the check was ended during the module check" + } + // The same manifests as the base branch has them, beside the change. + was := "" + if spec.Base != "" { + base := filepath.Join(root, "base-manifests") + for _, m := range manifests { + body, err := gitShow(ctx, tree, "origin/"+spec.Base, m) + if err != nil { + continue // new on this branch: nothing was so before it + } + if err := os.MkdirAll(filepath.Dir(filepath.Join(base, m)), 0o755); err == nil { + _ = os.WriteFile(filepath.Join(base, m), body, 0o644) + } + } + fmt.Fprintf(out, "--- the same manifests on %s\n", spec.Base) + if _, err := os.Stat(base); err == nil { + was, _ = checked(base) + } + } + brought := newProblems(said, was) + if len(brought) > 0 { + return "fail", "a manifest the change touches fails the module check: " + brought[0] + } + fmt.Fprintf(out, "the module check's problems were all so on %s already: said, not the change's\n", spec.Base) + } + } + + // 2. Every machine composed with the change. + if err := running(inToolchain(tree, append(env, EnvVerdict+"="+verdictFile), "sh", "-c", + `"$MESH_GATE" merge-gate --facts "$MESH_FACTS" --store "$MESH_GATE_POSTGRES" `+ + `--repository "$MESH_CHECK_REPOSITORY" --tree . --changed "$MESH_CHECK_CHANGED" --json > "$MESH_CHECK_VERDICT"`)); err != nil { + if ctx.Err() != nil { + return "error", "the check was ended during the merge gate" + } + var said struct { + Verdict string `json:"verdict"` + Summary string `json:"summary"` + } + if raw, err := os.ReadFile(verdictFile); err == nil && json.Unmarshal(raw, &said) == nil && said.Verdict == "fail" { + return "fail", said.Summary + } + // The gate could not judge: not the change's fault, and never a pass. + return "error", "the merge gate could not judge the change: " + lastLine(out.String()) + } + var said struct { + Verdict string `json:"verdict"` + Summary string `json:"summary"` + } + raw, err := os.ReadFile(verdictFile) + if err != nil || json.Unmarshal(raw, &said) != nil || said.Verdict == "" { + return "error", "the merge gate said no verdict" + } + + // 3. **The replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed code, + // so given the container runtime the resolver replay raises containers with — against the bus of the + // release the mesh runs and the change's own catalogue when the change is to the catalogue. + if lab := filepath.Join(root, "mesh-lab", "replays"); ctx.Err() == nil { if _, err := os.Stat(lab); err == nil { catalogue := filepath.Join(root, "mesh-catalog") if name == "mesh-catalog" { catalogue = tree } say("check", "the replays of what the mesh runs, from mesh-lab") - fmt.Fprintln(&out, "--- the replays (mesh-lab replays/)") + fmt.Fprintln(out, "--- the replays (mesh-lab replays/)") args := inToolchain(lab, []string{EnvTestBus + "=nats://" + bus, "MESH_REPLAY_CATALOGUE=" + catalogue, "GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")}, "go", "test", "-count=1", "./...") - // The socket goes to the replays alone, never to the change's own script above. + // The socket goes to the replays alone, never to the change's own code above. args = append([]string{args[0], "--volume", "/var/run/docker.sock:/var/run/docker.sock"}, args[1:]...) - replays := exec.CommandContext(ctx, "docker", LabelledArgs("docker", args, spec.ID)...) - inItsOwnGroup(replays) - replays.Stdout, replays.Stderr = &out, &out - replayErr = replays.Run() + if err := running(args); err != nil { + if ctx.Err() != nil { + return "error", "the check was ended during the replays" + } + return "fail", "a replay of a core incident fails with this change: " + lastLine(out.String()) + } } } - v := CheckVerdict{Report: out.String(), Took: time.Since(began)} - var gateSaid struct { - Verdict string `json:"verdict"` - Summary string `json:"summary"` + if said.Verdict == "pass" || said.Verdict == "warning" || said.Verdict == "fail" { + return said.Verdict, said.Summary } - if raw, err := os.ReadFile(verdictFile); err == nil { - _ = json.Unmarshal(raw, &gateSaid) - } - switch { - case errors.Is(ctx.Err(), context.DeadlineExceeded): - v.Verdict, v.Summary = "error", fmt.Sprintf("the check ran past %s and was ended", CheckTimeout) - case ctx.Err() != nil: - return v, ctx.Err() - case runErr != nil: - v.Verdict = "fail" - v.Summary = gateSaid.Summary - if v.Summary == "" || gateSaid.Verdict != "fail" { - v.Summary = "the merge check failed: " + lastLine(out.String()) + return "error", "the merge gate said " + said.Verdict +} + +// judgeFor builds the judge of a check: the change's own controller (a change to the controller), the +// running controller built with the change's validator (a change to the node-engine), or the controller +// the mesh runs. It answers the judge's path; or, when the change makes its own judge unbuildable, why — +// the change's fault, a failing gate; or an error when the check cannot build a judge at all. A check +// whose gate does not run builds a judge only to hand its scripts one, and goes on without when it cannot. +func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, tree string, + inToolchain func(string, []string, ...string) []string, say func(step, format string, args ...any)) (string, string, error) { + gated := spec.Gated() + switch spec.Judge { + case "self": + bin := filepath.Join(root, "bin", "judge-of-itself") + if _, err := labelled(ctx, root, "docker", inToolchain(tree, + []string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")}, + "go", "build", "-o", bin, "./cmd/mesh-controller")...); err != nil { + return "", "the change's controller, which judges itself, does not build: " + firstLine(err.Error()), nil } - case replayErr != nil: - v.Verdict, v.Summary = "fail", "a replay of a core incident fails with this change: "+lastLine(out.String()) - default: - v.Verdict, v.Summary = "pass", "the merge check passed" - if gateSaid.Verdict == "warning" || gateSaid.Verdict == "pass" { - v.Verdict, v.Summary = gateSaid.Verdict, gateSaid.Summary + say("check", "judged by the change's own controller") + return bin, "", nil + } + bin, judgeTree, err := judge(ctx, labelled, run, root, inToolchain, say) + if err != nil { + if !gated { + say("check", "no judge for its script: %v", err) + return "", "", nil + } + return "", "", err + } + if spec.Judge != "validator" { + return bin, "", nil + } + // The node-engine's change: its validator in place of the one the judge vendors. + vendored := filepath.Join(root, judgeTree, "vendor", "github.com", "novox", "mesh-host") + for _, pkg := range []string{"validate", filepath.Join("internal", "declaration")} { + if err := replaceGoFiles(filepath.Join(tree, pkg), filepath.Join(vendored, pkg)); err != nil { + return "", "", fmt.Errorf("the change's validator could not be put in the judge: %w", err) } } - say("check", "%s — %s (%s)", strings.ToUpper(v.Verdict), v.Summary, v.Took.Round(time.Second)) - return v, nil + withValidator := filepath.Join(root, "bin", "judge-with-this-validator") + if _, err := labelled(ctx, root, "docker", inToolchain(filepath.Join(root, judgeTree), + []string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")}, + "go", "build", "-o", withValidator, "./cmd/mesh-controller")...); err != nil { + return "", "the controller the mesh runs does not build with this change's validator: " + firstLine(err.Error()), nil + } + say("check", "judged by the controller the mesh runs, with this change's validator") + return withValidator, "", nil +} + +// replaceGoFiles puts a package's Go files — never its tests — in place of another copy's. +func replaceGoFiles(from, into string) error { + old, err := filepath.Glob(filepath.Join(into, "*.go")) + if err != nil { + return err + } + for _, f := range old { + if err := os.Remove(f); err != nil { + return err + } + } + if err := os.MkdirAll(into, 0o755); err != nil { + return err + } + files, err := filepath.Glob(filepath.Join(from, "*.go")) + if err != nil { + return err + } + if len(files) == 0 { + return fmt.Errorf("%s holds no Go files", from) + } + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + body, err := os.ReadFile(f) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(into, filepath.Base(f)), body, 0o644); err != nil { + return err + } + } + return nil +} + +// problemLines are the lines of a module check's output that name a problem: not a manifest found ok, not +// the count, not the closing note. +func problemLines(s string) []string { + var out []string + for _, line := range strings.Split(s, "\n") { + line = strings.TrimSpace(line) + switch { + case line == "", strings.Contains(line, ": ok"), strings.Contains(line, "problem(s) in"), + strings.Contains(line, "manifest(s) checked"): + continue + } + out = append(out, line) + } + return out +} + +// newProblems are the problems a module check said of the change that it did not say of the base. +func newProblems(change, base string) []string { + was := map[string]bool{} + for _, p := range problemLines(base) { + was[p] = true + } + var out []string + for _, p := range problemLines(change) { + if !was[p] { + out = append(out, p) + } + } + if len(out) == 0 && len(problemLines(change)) == 0 { + // It failed and named nothing: not readable as already so. + out = append(out, lastLine(change)) + } + return out +} + +// gitShow is a file as a ref has it. +func gitShow(ctx context.Context, dir, ref, file string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "git", "show", ref+":"+filepath.ToSlash(file)) + cmd.Dir = dir + return cmd.Output() +} + +func prefixed(prefix string, items []string) []string { + out := make([]string, 0, len(items)) + for _, i := range items { + out = append(out, prefix+i) + } + return out +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") + return line } // safeName is a directory a repository beside a check may be cloned under. @@ -377,9 +713,10 @@ func dialable(ctx context.Context, address string) error { } // judge builds the controller that judges a change: the one the mesh runs, beside the check as -// mesh-controller — or, when that one predates the merge gate, the controller's main, said. +// mesh-controller — or, when that one predates the merge gate, the controller's main, said. It answers +// the binary and the directory it was built from. func judge(ctx context.Context, run, plain Runner, root string, inToolchain func(string, []string, ...string) []string, - say func(step, format string, args ...any)) (string, error) { + say func(step, format string, args ...any)) (string, string, error) { bin := filepath.Join(root, "bin", "mesh-controller") build := func(dir string) error { _, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir), @@ -394,21 +731,21 @@ func judge(ctx context.Context, run, plain Runner, root string, inToolchain func } if _, err := os.Stat(filepath.Join(root, "mesh-controller")); err == nil { if err := build("mesh-controller"); err != nil { - return "", fmt.Errorf("the controller the mesh runs does not build: %w", err) + return "", "", fmt.Errorf("the controller the mesh runs does not build: %w", err) } if hasGate() { say("check", "judged by the controller the mesh runs") - return bin, nil + return bin, "mesh-controller", nil } } if _, err := os.Stat(filepath.Join(root, "mesh-controller-main")); err != nil { - return "", errors.New("no controller beside the check to judge it with") + return "", "", errors.New("no controller beside the check to judge it with") } if err := build("mesh-controller-main"); err != nil { - return "", fmt.Errorf("the controller's main does not build: %w", err) + return "", "", fmt.Errorf("the controller's main does not build: %w", err) } say("check", "judged by the controller's main: the one the mesh runs predates the merge gate") - return bin, nil + return bin, "mesh-controller-main", nil } // tail keeps the last lines written to it. diff --git a/internal/builder/check_test.go b/internal/builder/check_test.go index d3911f1..768c4f4 100644 --- a/internal/builder/check_test.go +++ b/internal/builder/check_test.go @@ -50,6 +50,9 @@ func aCheckedRepository(t *testing.T, files map[string]string) (string, string) } git("init", "--quiet", "-b", "main") for name, body := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil { + t.Fatal(err) + } if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil { t.Fatal(err) } @@ -88,28 +91,31 @@ func labelled(id string) []string { func TestACheckRunsTheRepositorysOwnScriptBesideTheMeshsVersionsAndLeavesNothing(t *testing.T) { registry := checkEnvironment(t) - // The script proves what it was given: the facts, a store that answers, a bus that answers, and - // writes the gate's verdict where it is told to. + // The script proves what it was given: the facts, a store that answers, a bus that answers — and no + // container runtime socket. script := `set -e test -s "$MESH_FACTS" grep -q '"bus": "2.11.17"' "$MESH_FACTS" || grep -q '"bus":"2.11.17"' "$MESH_FACTS" case "$MESH_TEST_POSTGRES" in postgres://*127.0.0.1:*) ;; *) echo "no store: $MESH_TEST_POSTGRES"; exit 1;; esac case "$MESH_TEST_NATS" in nats://127.0.0.1:*) ;; *) echo "no bus: $MESH_TEST_NATS"; exit 1;; esac -test "$MESH_CHECK_REPOSITORY" = "novox/mesh-controller" +test "$MESH_CHECK_REPOSITORY" = "novox/hq" test "$MESH_CHECK_CHANGED" = "a.go,b.go" test ! -S /var/run/docker.sock || { echo "the check holds the container runtime's socket"; exit 1; } -echo '{"verdict":"warning","summary":"a merge rebuilds 14 module(s)"}' > "$MESH_CHECK_VERDICT" echo checked ` repo, head := aCheckedRepository(t, map[string]string{CheckScript: script}) id := fmt.Sprintf("check-test-%d", time.Now().UnixNano()) v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", - Repo: "mesh-controller", Number: 7, Paths: []string{"a.go", "b.go"}, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + Repo: "hq", Number: 7, Paths: []string{"a.go", "b.go"}, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) if err != nil { t.Fatal(err) } - if v.Verdict != "warning" || v.Summary != "a merge rebuilds 14 module(s)" || !strings.Contains(v.Report, "checked") { - t.Fatalf("the check answered %+v", v) + // Nothing of the graph touched: the gate a pass that says so, the repository's own check run. + if v.Gate == nil || v.Gate.Verdict != "pass" || v.Gate.Summary != noModule { + t.Errorf("the gate answered %+v", v.Gate) + } + if v.Repo == nil || v.Repo.Verdict != "pass" || !strings.Contains(v.Report, "checked") { + t.Fatalf("the repository's check answered %+v\n%s", v.Repo, v.Report) } if left := labelled(id); len(left) > 0 { t.Errorf("the check left %d container(s) behind", len(left)) @@ -120,12 +126,20 @@ func TestAFailingCheckFailsAndOneThatCannotRunIsNeverAPass(t *testing.T) { registry := checkEnvironment(t) repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo 'resource \"x.service\": refused'; exit 3\n"}) v, err := Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-fail-%d", time.Now().UnixNano()), - Repository: repo, Ref: head, Owner: "novox", Repo: "mesh-controller", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) if err != nil { t.Fatal(err) } - if v.Verdict != "fail" || !strings.Contains(v.Summary, "refused") { - t.Fatalf("a failing script answered %+v", v) + if v.Repo == nil || v.Repo.Verdict != "fail" || !strings.Contains(v.Repo.Summary, "refused") { + t.Fatalf("a failing script answered %+v", v.Repo) + } + + // A script in a toolchain the mesh does not hold: an error, never a pass. + repo, head = aCheckedRepository(t, map[string]string{CheckScript: "# mesh-check-toolchain: cobol\nexit 0\n"}) + v, err = Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-cobol-%d", time.Now().UnixNano()), + Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil || v.Repo == nil || v.Repo.Verdict != "error" { + t.Fatalf("a script in a toolchain nobody holds answered %+v, %v", v.Repo, err) } // Past its bound: an error, not a pass. @@ -134,19 +148,229 @@ func TestAFailingCheckFailsAndOneThatCannotRunIsNeverAPass(t *testing.T) { t.Cleanup(func() { CheckTimeout = was }) repo, head = aCheckedRepository(t, map[string]string{CheckScript: "sleep 120\n"}) v, err = Check(context.Background(), Command, CheckSpec{ID: fmt.Sprintf("check-slow-%d", time.Now().UnixNano()), - Repository: repo, Ref: head, Owner: "novox", Repo: "mesh-controller", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err == nil && v.Verdict == "pass" { - t.Fatalf("a check past its bound passed: %+v", v) - } - if err == nil && v.Verdict != "error" { - t.Fatalf("a check past its bound answered %+v", v) + Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err == nil && (v.Repo == nil || v.Repo.Verdict != "error") { + t.Fatalf("a check past its bound answered %+v", v.Repo) } // No facts: it cannot run, and says so. repo, head = aCheckedRepository(t, map[string]string{CheckScript: "exit 0\n"}) _, err = Check(t.Context(), Command, CheckSpec{ID: "check-nofacts", Repository: repo, Ref: head, Owner: "novox", - Repo: "mesh-controller", Toolchain: goToolchain}, t.TempDir(), "127.0.0.1:1", GitCredential{}, nil) + Repo: "hq", Toolchain: goToolchain}, t.TempDir(), "127.0.0.1:1", GitCredential{}, nil) if err == nil || !strings.Contains(err.Error(), "facts snapshot") { t.Fatalf("a check with no facts said %v", err) } } + +// A repository that touches nothing of the graph and has no script of its own runs nothing, and says +// both: the gate a pass that names why, the repository a warning — never silent. +func TestACheckWithNothingToRunSaysSo(t *testing.T) { + repo, head := aCheckedRepository(t, map[string]string{"README.md": "x"}) + v, err := Check(t.Context(), Command, CheckSpec{ID: "check-nothing", Repository: repo, Ref: head, Owner: "novox", + Repo: "hq"}, t.TempDir(), "", GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Gate == nil || v.Gate.Verdict != "pass" || v.Repo == nil || v.Repo.Verdict != "warning" || + !strings.Contains(v.Repo.Summary, CheckScript) { + t.Fatalf("a check with nothing to run said %+v / %+v", v.Gate, v.Repo) + } + // A gated change never takes that path: with no store to read the facts from, it cannot run. + _, err = Check(t.Context(), Command, CheckSpec{ID: "check-gated", Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-catalog", Modules: []string{"gitea"}}, t.TempDir(), "", GitCredential{}, nil) + if err == nil { + t.Fatal("a change touching a module ran nothing and was not refused") + } +} + +// A script declares its toolchain among its first lines; go when it declares none. +func TestAScriptDeclaresItsToolchain(t *testing.T) { + for script, want := range map[string]string{ + "#!/bin/sh\nset -eu\n": "go", + "#!/bin/sh\n# mesh-check-toolchain: typescript\nnpm test\n": "typescript", + "#!/bin/sh\n#mesh-check-toolchain:go\n": "go", + "#!/bin/sh\necho '# mesh-check-toolchain: python'\n": "go", + } { + if got := ScriptToolchain([]byte(script)); got != want { + t.Errorf("%q declares %q, read as %q", script, want, got) + } + } + held := map[string]string{"mesh-tools/build": "reg/mesh-tools-build@sha256:a", "mesh-tools-go/build": "reg/go@sha256:b"} + chains := ToolchainsOf(held) + if chains["typescript"] != "reg/mesh-tools-build@sha256:a" || chains["go"] != "reg/go@sha256:b" || ToolchainOf(held) != chains["go"] { + t.Fatalf("the toolchains held read as %v", chains) + } + if _, held := chains["python"]; held { + t.Error("a toolchain the mesh does not hold read as held") + } +} + +// A node-engine change's validator is put in place of the one the judge vendors: its Go files, never its tests. +func TestTheChangesValidatorReplacesTheVendoredOne(t *testing.T) { + from, into := t.TempDir(), t.TempDir() + for name, body := range map[string]string{"v.go": "package validate // new", "v_test.go": "package validate"} { + if err := os.WriteFile(filepath.Join(from, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(into, "old.go"), []byte("package validate // old"), 0o644); err != nil { + t.Fatal(err) + } + if err := replaceGoFiles(from, into); err != nil { + t.Fatal(err) + } + got, _ := filepath.Glob(filepath.Join(into, "*.go")) + if len(got) != 1 || filepath.Base(got[0]) != "v.go" { + t.Fatalf("the vendored validator holds %v", got) + } +} + +// aJudge is a controller that judges as told: `merge-gate` answers a warning, `module check` refuses a +// manifest that says it is broken. What the gate layer is tested against without building the real one. +var aJudge = map[string]string{ + "go.mod": "module example.org/judge\n\ngo 1.22\n", + "cmd/mesh-controller/main.go": `package main + +import ( + "fmt" + "os" + "strings" +) + +func main() { + switch { + case len(os.Args) == 2 && os.Args[1] == "merge-gate": + fmt.Println("usage: merge-gate --facts --store ") + os.Exit(2) + case len(os.Args) > 2 && os.Args[1] == "module": + for _, m := range os.Args[3:] { + body, _ := os.ReadFile(m) + if strings.Contains(string(body), "broken") { + fmt.Println(m + ": refused, it says it is broken") + os.Exit(1) + } + fmt.Println(m + ": ok") + } + case os.Args[1] == "merge-gate": + fmt.Println(` + "`" + `{"verdict":"warning","summary":"a merge rebuilds 14 module(s)"}` + "`" + `) + } +} +`, +} + +func TestTheGateRunsWhenTheGraphIsTouchedBesideTheRepositorysOwnCheck(t *testing.T) { + registry := checkEnvironment(t) + judgeRepo, judgeHead := aCheckedRepository(t, aJudge) + beside := map[string]Beside{"mesh-controller": {Repository: judgeRepo, Ref: judgeHead}} + check := func(files map[string]string, judge string) CheckVerdict { + t.Helper() + repo, head := aCheckedRepository(t, files) + id := fmt.Sprintf("check-gate-%d", time.Now().UnixNano()) + v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-catalog", Number: 9, Paths: []string{"modules/gitea/index.ts"}, Beside: beside, + Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Judge: judge, + Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if left := labelled(id); len(left) > 0 { + t.Errorf("the check left %d container(s) behind", len(left)) + } + return v + } + v := check(map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, CheckScript: "echo own; exit 0\n"}, "") + if v.Gate == nil || v.Gate.Verdict != "warning" || v.Gate.Summary != "a merge rebuilds 14 module(s)" || + strings.Join(v.Gate.Modules, ",") != "gitea" || v.Verdict != "warning" { + t.Fatalf("the gate answered %+v\n%s", v.Gate, v.Report) + } + if v.Repo == nil || v.Repo.Verdict != "pass" { + t.Fatalf("its own check answered %+v", v.Repo) + } + + v = check(map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`}, "") + if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "module check") || v.Repo.Verdict != "warning" { + t.Fatalf("a manifest the judge refuses answered %+v / %+v\n%s", v.Gate, v.Repo, v.Report) + } + + // A fault the base branch already had is said and fails nothing; one the change brings fails. + repo, _ := aCheckedRepository(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`}) + git := func(args ...string) string { + cmd := exec.Command("git", args...) + cmd.Dir = repo + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git("checkout", "--quiet", "-b", "change") + if err := os.WriteFile(filepath.Join(repo, "modules/gitea/index.ts"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + git("add", "-A") + git("commit", "--quiet", "-m", "y") + id := fmt.Sprintf("check-base-%d", time.Now().UnixNano()) + v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: git("rev-parse", "HEAD"), Owner: "novox", + Repo: "mesh-catalog", Base: "main", Paths: []string{"modules/gitea/index.ts"}, Beside: beside, + Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Toolchain: goToolchain}, + t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Gate.Verdict != "warning" || !strings.Contains(v.Report, "already") { + t.Fatalf("a fault the base already had failed the change: %+v\n%s", v.Gate, v.Report) + } + + // A change to the controller judges itself: one that does not build fails its own gate. + v = check(map[string]string{"go.mod": "module x\n\ngo 1.22\n", "cmd/mesh-controller/main.go": "package main\nfunc main() { nope }\n", + "modules/gitea/module.json": "{}"}, "self") + if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "does not build") { + t.Fatalf("a controller that does not build judged itself %+v", v.Gate) + } +} + +// **Only a commit on the trunk is published** (novox/hq ADR 0238): the build seat reads, from the clone it +// just made, the branch the forge names its default and whether the commit built is on it. +func TestABuildSaysWhetherItsCommitIsOnTheTrunk(t *testing.T) { + repo, onMain := aCheckedRepository(t, map[string]string{"module.json": `{"module":"x","version":"1"}`}) + git := func(dir string, args ...string) string { + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git(repo, "checkout", "--quiet", "-b", "feature") + if err := os.WriteFile(filepath.Join(repo, "x"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + git(repo, "add", "-A") + git(repo, "commit", "--quiet", "-m", "off the trunk") + offMain := git(repo, "rev-parse", "HEAD") + git(repo, "checkout", "--quiet", "main") + + clone := filepath.Join(t.TempDir(), "clone") + git(filepath.Dir(clone), "clone", "--quiet", repo, clone) + if trunk, on := trunkOf(t.Context(), Command, clone, onMain); trunk != "main" || !on { + t.Errorf("a commit on main reads as on %q: %v", trunk, on) + } + if trunk, on := trunkOf(t.Context(), Command, clone, offMain); trunk != "main" || on { + t.Errorf("a feature branch's commit reads as on %q: %v", trunk, on) + } + if got := strings.Join(branchesHolding(t.Context(), Command, clone, offMain), ","); got != "feature" { + t.Errorf("the feature branch's commit is said to be on %q", got) + } + if got := strings.Join(branchesHolding(t.Context(), Command, clone, onMain), ","); got != "feature,main" { + t.Errorf("main's commit is said to be on %q", got) + } + // A tree that says no trunk is not known — never read as on it. + if trunk, on := trunkOf(t.Context(), Command, repo, onMain); trunk != "" || on { + t.Errorf("a repository with no origin reads as trunk %q, on %v", trunk, on) + } +} diff --git a/internal/link/build.go b/internal/link/build.go index f0cebbe..ca58beb 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -106,8 +106,31 @@ type CheckRequest struct { // controller the mesh runs, the catalogue it holds, the host it runs — keyed by the directory name the // check finds it under. Beside map[string]CheckedOut `json:"beside,omitempty"` + // Modules are the modules of the mesh's graph the change touches, and New the directories it adds a + // module in that the graph does not hold (novox/hq ADR 0237 as amended): **the graph decides whether + // the gate runs**, not the repository. With neither, only the repository's own merge-check.sh runs. + Modules []string `json:"modules,omitempty"` + New []string `json:"new,omitempty"` + // Dependents are the modules a merge would build after Modules because they stand on them: the + // planner's dependency walk, said on the pull request. + Dependents []string `json:"dependents,omitempty"` + // Plan is the change plan of the commit checked, computed by the controller and echoed with the outcome. + Plan *ChangePlan `json:"plan,omitempty"` + // Manifests are the touched modules' manifests in the change's tree, by path from its root: what the + // gate puts through `module check`. + Manifests []string `json:"manifests,omitempty"` + // Judge is who judges the gate: empty for the controller the mesh runs; JudgeSelf for a change to + // the controller, judged by itself; JudgeValidator for a change to the node-engine, judged by the + // running controller built with the change's validator in place of the one it vendors. + Judge string `json:"judge,omitempty"` } +// Who judges a merge check's gate. +const ( + JudgeSelf = "self" + JudgeValidator = "validator" +) + // CheckedOut is a repository cloned beside a check, at a ref. type CheckedOut struct { Repository string `json:"repository"` @@ -123,6 +146,9 @@ type CheckOutcome struct { Report string `json:"report,omitempty"` // Took is how long it ran. Took string `json:"took,omitempty"` + // Gate and RepoCheck are its two layers; Verdict and Summary above are the gate's. + Gate *CheckLayer `json:"gate,omitempty"` + RepoCheck *CheckLayer `json:"repo-check,omitempty"` } // SourceOnSeat names a repository by the seat whose holder serves it and its path there. @@ -179,6 +205,15 @@ type BuildResult struct { // manifest the mesh keeps says nothing about it (novox/hq 04-ISSUES/131). Read []ReadRepository `json:"read,omitempty"` + // Trunk is the repository's default branch at the build, and OnTrunk whether the commit built is on it + // (novox/hq ADR 0238): **only a commit on the trunk is published** — the controller refuses to register + // a build of one off it. Empty Trunk is a build seat that could not say, or predates the rule. + Trunk string `json:"trunk,omitempty"` + OnTrunk bool `json:"on-trunk,omitempty"` + // Branches are every branch the commit is on: the trunk of a module that follows a branch other than + // the repository's default is the branch it follows. + Branches []string `json:"branches,omitempty"` + // SourceFingerprint is what the build was made from, hashed (novox/hq issue 280): the module's // tree at the commit, the trees of the contexts it read, its bases and toolchains by digest. Two // builds with one fingerprint are one build, however their digests differ — an image is not diff --git a/internal/link/events.go b/internal/link/events.go index a3eb907..d70f62a 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -221,6 +221,19 @@ type PullUpdated struct { // Paths are the files the pull request changes; PathsTruncated says there were more. Paths []string `json:"paths,omitempty"` PathsTruncated bool `json:"paths_truncated,omitempty"` + // Removed are the files among Paths the change deletes: a module whose manifest is among them is one + // the merge would remove. + Removed []string `json:"removed,omitempty"` + // ModuleDirs are the directories above the changed files that hold a `module.json` at the head, read + // as the merge announcer reads them at a merge commit (issue 278); ModuleDirsSaid says it looked. + // What the controller finds a module the graph does not hold yet with: a directory the change adds a + // module.json in is a new module, checked before it merges. + ModuleDirs []string `json:"module_dirs,omitempty"` + ModuleDirsSaid bool `json:"module_dirs_said,omitempty"` + // MergeCheck says the head holds a merge-check.sh at its root — the repository's own tests, the + // check's second layer; MergeCheckSaid says the announcer looked. + MergeCheck bool `json:"merge_check,omitempty"` + MergeCheckSaid bool `json:"merge_check_said,omitempty"` } // Checked is a pull request's merge check, judged: what the controller says as `checked`. @@ -238,6 +251,60 @@ type Checked struct { // ID is the ask, and On the machine that ran it. ID string `json:"id"` On string `json:"on,omitempty"` + // Gate and RepoCheck are the check's two layers, each its own status on the pull request (novox/hq + // ADR 0237 as amended): the mesh's — the modules of the graph the change touches, every machine + // composed with it — as `mesh/merge-gate`, and the repository's own merge-check.sh as + // `mesh/repo-check`. Verdict and Summary above are the gate's, for a forge holder that reads no more. + // RepoCheck is nil when the repository is not the mesh's and touches nothing of it: nothing is said. + Gate *CheckLayer `json:"gate,omitempty"` + RepoCheck *CheckLayer `json:"repo-check,omitempty"` + // Plan is the change plan of the commit checked (novox/hq ADR 0238): what a merge of it would build + // and send, posted with the verdict. + Plan *ChangePlan `json:"plan,omitempty"` +} + +// ChangePlan is what a change does to the mesh, computed from its diffset — a repository, the branch it +// merges into and the commit at hand — by the planner (novox/hq ADR 0238): **one commit, one plan**, the +// object a pull request's check posts, the release follows and a person reads. +type ChangePlan struct { + Repository string `json:"repository"` + Base string `json:"base"` + Head string `json:"head"` + // Moved are the modules a merge moves itself, Dependents those built after them because they stand + // on them, New the directories it adds a module in, and Unread the changed files no build reads. + Moved []string `json:"moved,omitempty"` + Dependents []string `json:"dependents,omitempty"` + New []string `json:"new,omitempty"` + Unread []string `json:"unread,omitempty"` + // Tiers are the build plan's order: each tier built after the one before. + Tiers [][]string `json:"tiers,omitempty"` + // Machines are the deploy plan: what each machine is sent, in the build plan's order, and what waits + // there for a person. + Machines []MachinePlan `json:"machines,omitempty"` + // Steps are what is not an ordinary send: a planned bus step, a provider whose consumers are sent again, + // a module that waits for a person, a module that keeps data. + Steps []string `json:"steps,omitempty"` + // Summary is the plan in one line, for a commit status. + Summary string `json:"summary"` +} + +// MachinePlan is one machine's part of a change plan. +type MachinePlan struct { + Machine string `json:"machine"` + Receives []string `json:"receives,omitempty"` + Waits []string `json:"waits,omitempty"` +} + +// CheckLayer is one layer of a merge check, judged. +type CheckLayer struct { + // Verdict is pass, warning, fail or error; error is never a pass. + Verdict string `json:"verdict"` + Summary string `json:"summary"` + // Modules are, for the gate, the modules of the mesh's graph the change touches — `new:` for a + // module the graph does not hold yet; Dependents those a merge would build after them because they + // stand on them — the planner's own answer. + Modules []string `json:"modules,omitempty"` + Dependents []string `json:"dependents,omitempty"` } type Upgraded struct { diff --git a/merge-check.sh b/merge-check.sh index 0062bb5..296fe27 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -1,22 +1,22 @@ #!/bin/sh -# The merge check of the controller (novox/hq to-be 45 §9), run by the build seat on every pull request -# before it merges — and by hand: `MESH_FACTS=facts.json MESH_GATE_POSTGRES=… MESH_TEST_POSTGRES=… -# MESH_TEST_NATS=… sh merge-check.sh`. +# mesh-check-toolchain: go # -# The build seat clones this repository with the catalogue and the host beside it (the tests read them -# there), reads the facts snapshot the controller keeps, and raises a throwaway store and bus of the -# versions the mesh runs; this says what is judged with them: +# The controller's own check (novox/hq ADR 0237 as amended): the second layer of a pull request's merge +# check, `mesh/repo-check`, run by the build seat in the mesh's Go toolchain — and by hand: +# `MESH_TEST_POSTGRES=… sh merge-check.sh`. +# +# The gate — every machine composed with this change's controller, which judges itself — is the build +# seat's first layer (`mesh/merge-gate`), run because the mesh's module graph builds `mesh-controller` +# from this repository; it is not repeated here. This is the code's own quality and its suite: # # 1. formatted and vetted; -# 2. the merge gate, judged by THIS change's controller: every machine of the snapshot composed with -# it and validated by the node-engine's own validator, against the mesh as it is; -# 3. the whole suite, the replays among it, against that store and that bus, one package at a time -# because the live tests share one bus's fixed names. +# 2. the whole suite, packages in parallel, against the throwaway store the build seat raised at the +# release the mesh runs; every live test on a bus of its own (internal/testbus), so nothing is +# serialised. Under the race detector when the toolchain has a C compiler, and said when it has not. # -# Fails on the first that fails. The gate's verdict is written where MESH_CHECK_VERDICT says, so the -# pull request is told the gate's own words. +# Exit 0 is a pass; anything else fails `mesh/repo-check` with the last line printed. set -eu -export GOFLAGS=-mod=vendor GOPROXY=off CGO_ENABLED=0 +export GOFLAGS=-mod=vendor GOPROXY=off unformatted=$(gofmt -l cmd internal examples) if [ -n "$unformatted" ]; then @@ -24,12 +24,11 @@ if [ -n "$unformatted" ]; then echo "$unformatted" exit 1 fi -go vet ./... +CGO_ENABLED=0 go vet ./... -judge="${MESH_CHECK_BESIDE:-${TMPDIR:-/tmp}}/bin/judge" -go build -o "$judge" ./cmd/mesh-controller -"$judge" merge-gate --facts "$MESH_FACTS" --store "$MESH_GATE_POSTGRES" \ - --repository "${MESH_CHECK_REPOSITORY:-novox/mesh-controller}" --tree . \ - --changed "${MESH_CHECK_CHANGED:-}" --json > "${MESH_CHECK_VERDICT:-/dev/null}" - -go test -p 1 -timeout 25m ./... +if command -v gcc >/dev/null 2>&1; then + CGO_ENABLED=1 go test -race -count=1 -timeout 30m ./... +else + echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector" + CGO_ENABLED=0 go test -count=1 -timeout 30m ./... +fi diff --git a/mesh-controller b/mesh-controller index 721ae36..b8fa8cd 100755 Binary files a/mesh-controller and b/mesh-controller differ diff --git a/module.json b/module.json index 50b0eb4..27c0402 100644 --- a/module.json +++ b/module.json @@ -27,16 +27,43 @@ "prepares": true, "tools": [ "tools", + "calls", "status", "nodes", "node", "modules", "seats", "builds", + "plans", "plan", "assign", "unassign", + "pin", + "unpin", "push", + "rotate", + "issue", + "settings", + "command", + "queue", + "cancel", + "clear", + "rebuild", + "replay", + "kill", + "pause", + "resume", + "hand-act", + "hand-acts", + "durations", + "conditions", + "healers", + "doctor", + "upgrade", + "bus", + "retire", + "cleanup", + "data", "build" ], "resources": [