Let the module graph decide what a pull request's check runs, in two layers (hq ADR 0237)

Every pull request the forge announces is mapped onto the mesh's module graph by the
merge handler's rule (issue 278): touching a module — or adding one — runs the gate
(mesh/merge-gate), its judge chosen by the graph (the controller judges itself, the
node-engine by its validator); a repository of the mesh that touches none runs only its
own merge-check.sh (mesh/repo-check), a warning when it has none. Nothing is left pending:
a repository outside the mesh touching nothing is told so as a pass.

The gate moves out of the per-repository scripts into the build seat, so a script is the
repository's own tests and declares its toolchain (go or typescript). The controller's
manifest names every verb of its seat again (ADR 0132), held by a test.
This commit is contained in:
jochen
2026-10-06 22:34:56 +02:00
parent d0580a17e5
commit 14127d4878
10 changed files with 996 additions and 168 deletions
+11 -2
View File
@@ -249,7 +249,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
if err == nil {
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
Took: v.Took.Round(time.Second).String()}
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo)}
}
} else if err == nil {
// The package-registry credential is a build input, so it is resolved before the clone: a
@@ -334,13 +334,22 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
c := request.Check
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
Toolchain: builder.ToolchainOf(request.Held)}
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge,
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
for dir, b := range c.Beside {
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
}
return spec
}
// layerOf is one layer of a check as the outcome carries it.
func layerOf(l *builder.Layer) *link.CheckLayer {
if l == nil {
return nil
}
return &link.CheckLayer{Verdict: l.Verdict, Summary: l.Summary, Modules: l.Modules}
}
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
// (novox/hq ADR 0076, issue 053).
//
+11
View File
@@ -92,3 +92,14 @@ func TestRegistrationRefusesADefinitionNamingAnInstallation(t *testing.T) {
t.Fatalf("a name declared on purpose passes; got %v", err)
}
}
// **The controller's own manifest names every verb of its seat** (novox/hq ADR 0132): its tools lagged
// the seat's verbs for weeks, and `module check` — the gate's first step for a change touching it —
// refused it. Held here, so a verb added to the table without the manifest fails this repository's
// own suite rather than its next pull request's gate.
func TestTheControllersManifestServesEveryVerbOfItsSeat(t *testing.T) {
var out strings.Builder
if err := moduleCheck([]string{"../../module.json"}, &out); err != nil {
t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String())
}
}
+232 -35
View File
@@ -4,6 +4,9 @@ import (
"context"
"encoding/json"
"fmt"
"path"
"slices"
"sort"
"strings"
"time"
@@ -12,43 +15,189 @@ import (
"github.com/novox/mesh-controller/internal/link"
)
// A pull request's merge check (novox/hq to-be 45 §9): the forge announces a pull request's new head,
// the controller asks the build seat to check it, and says the verdict as `checked`, which the forge's
// holder sets as the pull request's status. **Before merge, never after**: every check the mesh had ran
// after a merge, on a machine.
// A pull request's merge check (novox/hq to-be 45 §9, ADR 0237 as amended 2026-10-06): the forge
// announces every pull request's new head, the controller decides what is checked, asks the build seat to
// check it, and says the verdict as `checked`, which the forge's holder sets as the pull request's
// statuses. **Before merge, never after**: every check the mesh had ran after a merge, on a machine.
//
// What is checked is decided here and run there. Here: whether the mesh builds anything from the
// repository into that branch — a repository it builds nothing from is not its to judge — and what the
// check reads beside it: the controller the mesh runs (its judge, for a catalogue change: a manifest
// that controller cannot read fails, which is version skew caught), the catalogue the mesh holds, the
// host it runs. There: the repository's own merge-check.sh, or the merge gate alone for a repository
// that declares none (internal/builder/check.go).
// **The mesh's module graph decides, not the repository.** The controller holds the graph — every module,
// the repository and directory it is built from — and maps the pull request's changed paths onto it by
// the rule the merge handler uses (issue 278): a changed file inside a directory holding a module.json at
// the head is that module's, held or not; a file in no such directory, in a repository modules are built
// from into that branch, is shared code, and touches every module built from it there. A directory the
// change adds a module in, which the graph does not hold yet, is a new module and is checked too.
//
// - touches a module: the build seat runs **the gate** — `mesh/merge-gate`, the touched manifests, every
// machine composed with the change, the replays — and the repository's own merge-check.sh beside it;
// - touches none, in a repository that is the mesh's (it sources a module on some branch, or shares the
// core's owner): the gate is a pass that says so — a fact, not a missing check — and the repository's
// own merge-check.sh runs as `mesh/repo-check`, a warning when it has none;
// - touches none, anywhere else: the gate is a pass that says so, and nothing more is said.
//
// What is checked is decided here and run there (internal/builder/check.go).
// checkTimeout is how long one check may run on the build seat. Said here so the ask's watchdog (S6)
// and the builder agree on what late means.
const checkTimeout = 45 * time.Minute
// PullUpdated asks for a pull request's merge check.
// noModuleTouched is the gate's word for a change that touches nothing of the graph.
const noModuleTouched = "the change touches no module of the mesh's graph"
// noMergeCheck is the repository layer's word for a repository of the mesh with no merge-check.sh.
const noMergeCheck = "the repository declares no merge-check.sh: none of its own tests run before it merges"
// coreModules are the modules whose repositories are the mesh's core, by the directory a check finds
// each beside it — and whose owner is the mesh's own.
var coreModules = map[string]string{"mesh-controller": "mesh-controller", "mesh-host": "mesh-host",
"node-tools": "mesh-tools", "nats": "mesh-catalog"}
// checkScope is what a pull request touches of the mesh's graph.
type checkScope struct {
// Modules are the graph's modules built from the repository into the pull request's base that the
// change touches, sorted; New the directories it adds a module in that the graph does not hold.
Modules []string
New []string
// Manifests are their manifests in the change's tree.
Manifests []string
// Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the
// core's, or the change adds a module to it.
Mesh bool
// Judge is who judges the gate (link.JudgeSelf, link.JudgeValidator, or the running controller).
Judge string
// From is a module built from the repository, for how the mesh clones it; nil when none is.
From *inventory.Entry
}
func (s checkScope) gated() bool { return len(s.Modules)+len(s.New) > 0 }
// pullScope maps a pull request onto the mesh's module graph.
func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][]inventory.ReadRepository) checkScope {
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL, Commit: p.Commit,
Paths: p.Paths, PathsTruncated: p.PathsTruncated, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid}
var s checkScope
var from []inventory.Entry
known := map[string]bool{}
owners := map[string]bool{}
for i, e := range entries {
if e.Provided {
continue
}
if _, core := coreModules[e.Manifest.Module]; core {
if owner := sourceOwner(e.Source.Repository); owner != "" {
owners[owner] = true
}
}
if !sameRepository(e.Source.Repository, m) {
continue
}
s.Mesh = true
known[strings.Trim(e.Source.Path, "/")] = true
if s.From == nil {
s.From = &entries[i]
}
if sourceIs(e.Source, m) {
from = append(from, e)
}
}
touched := map[string]inventory.Entry{}
if len(from) > 0 {
for _, e := range whatTheMergeTouched(from, entries, m) {
touched[e.Manifest.Module] = e
}
}
// A module whose build packages source from this repository's branch is touched by any change to it.
for _, e := range entries {
if !e.Provided && readsFrom(read[e.Manifest.Module], m) {
touched[e.Manifest.Module] = e
}
}
for name, e := range touched {
s.Modules = append(s.Modules, name)
if sameRepository(e.Source.Repository, m) {
s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), "module.json"))
}
switch name {
case "mesh-controller":
s.Judge = link.JudgeSelf
case "mesh-host":
if s.Judge == "" {
s.Judge = link.JudgeValidator
}
}
}
// A module the graph does not hold yet, in a directory the head says holds one — or at the root.
for _, d := range saidModuleDirs(m) {
if !known[d] {
s.New = append(s.New, d)
s.Manifests = append(s.Manifests, d+"/module.json")
}
}
// And, said or not, a manifest the change adds or moves in a directory the graph does not know.
for _, changed := range p.Paths {
changed = strings.Trim(changed, "/")
if path.Base(changed) != "module.json" {
continue
}
d := path.Dir(changed)
if d == "." {
d = ""
}
if !known[d] {
if d == "" {
d = "."
}
s.New = append(s.New, d)
s.Manifests = append(s.Manifests, changed)
}
}
sort.Strings(s.Modules)
sort.Strings(s.New)
s.New = slices.Compact(s.New)
sort.Strings(s.Manifests)
s.Manifests = slices.Compact(s.Manifests)
if owners[strings.ToLower(p.Owner)] || s.gated() {
s.Mesh = true
}
return s
}
// sourceOwner is the owner of a recorded repository, a path on the git seat or a URL: novox/mesh-host → novox.
func sourceOwner(repository string) string {
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
if len(parts) < 2 {
return ""
}
return strings.ToLower(parts[len(parts)-2])
}
// PullUpdated decides a pull request's merge check, and asks for it when there is something to run.
func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
inv := f.open.inventory
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
moved := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL}
var from *inventory.Entry
for i, e := range entries {
if !e.Provided && sourceIs(e.Source, moved) {
from = &entries[i]
break
}
read, err := inv.ReadRepositories(ctx)
if err != nil {
return err
}
if from == nil {
fmt.Printf("%s/%s#%d (%.8s): the mesh builds nothing from it into %s, so it is not the mesh's to check\n",
p.Owner, p.Repo, p.Number, p.Commit, p.Base)
scope := pullScope(p, entries, read)
direct := link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched,
Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}}
switch {
case !scope.gated() && !scope.Mesh:
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched)
sayChecked(ctx, direct)
return nil
case !scope.gated() && p.MergeCheckSaid && !p.MergeCheck:
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck}
fmt.Printf("%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck)
sayChecked(ctx, direct)
return nil
}
request, err := checkRequestFor(ctx, f.open, p, *from, entries)
request, err := checkRequestFor(ctx, f.open, p, scope, entries)
if err != nil {
return err
}
@@ -61,14 +210,18 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
if err := ask.Ask(ctx, request); err != nil {
return err
}
fmt.Printf("%s/%s#%d (%.8s): asked %s to check it before it merges, as %s\n", p.Owner, p.Repo, p.Number,
p.Commit, seat, request.ID)
what := "its own merge-check.sh alone: " + noModuleTouched
if scope.gated() {
what = "the gate over " + strings.Join(append(append([]string{}, scope.Modules...), scope.New...), ", ")
}
fmt.Printf("%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number,
p.Commit, seat, what, request.ID)
return nil
}
// checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head,
// and what is read beside it.
func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, from inventory.Entry,
func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope,
entries []inventory.Entry) (link.BuildRequest, error) {
shelf := map[string]catalogue.Manifest{}
for _, e := range entries {
@@ -84,7 +237,12 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, from
}
return clonedFromSeat(world, s.Seat, s.Repository)
}
repository, err := clone(from.Source)
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
source := inventory.Source{Seat: gitSeat, Repository: p.Owner + "/" + p.Repo}
if scope.From != nil {
source = scope.From.Source
}
repository, err := clone(source)
if err != nil {
return link.BuildRequest{}, err
}
@@ -94,10 +252,8 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, from
}
// Beside it, at what the mesh runs: each core repository by the module the mesh builds from it.
beside := map[string]link.CheckedOut{}
byModule := map[string]string{"mesh-controller": "mesh-controller", "mesh-host": "mesh-host",
"node-tools": "mesh-tools", "nats": "mesh-catalog"}
for _, e := range entries {
dir, core := byModule[e.Manifest.Module]
dir, core := coreModules[e.Manifest.Module]
if !core || e.Provided || e.Source.Repository == "" {
continue
}
@@ -130,9 +286,10 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, from
Ref: p.Commit,
Held: heldBy(ctx),
Seats: seatBases(ctx),
Source: sourceOnSeat(from.Source),
Source: sourceOnSeat(source),
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
Paths: p.Paths, Beside: beside},
Paths: p.Paths, Beside: beside, Modules: scope.Modules, New: scope.New, Manifests: scope.Manifests,
Judge: scope.Judge},
}, nil
}
@@ -162,6 +319,12 @@ const maxCheckReport = 60 << 10
// checked says a merge check's verdict as the controller's `checked`. Nothing is recorded or
// registered: a check builds nothing (issue 240's rule for a dry run, kept for a check).
func checked(ctx context.Context, result link.BuildResult) {
sayChecked(ctx, checkedOf(result))
}
// checkedOf is what a check's outcome says: each layer, and an error — never a pass — for a check that
// could not run.
func checkedOf(result link.BuildResult) link.Checked {
c := link.Checked{ID: result.ID, On: result.On, Commit: result.Ref}
if result.Checked != nil {
c.Owner, c.Repo, c.Number = result.Checked.Owner, result.Checked.Repo, result.Checked.Number
@@ -169,8 +332,13 @@ func checked(ctx context.Context, result link.BuildResult) {
switch {
case result.Check != nil:
c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report
c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck
if c.Gate == nil {
// A build seat from before the layers: its verdict is the gate's.
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
}
case result.Failed != "":
// The check could not run: an error, never read as a pass.
// The check could not run: an error, never read as a pass — on both layers it was asked for.
c.Verdict, c.Summary = "error", "the check could not run: "+firstLine(result.Failed)
default:
c.Verdict, c.Summary = "error", "the build seat answered the check with no verdict"
@@ -178,11 +346,40 @@ func checked(ctx context.Context, result link.BuildResult) {
if c.Verdict == "" {
c.Verdict = "error"
}
if result.Check == nil {
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
c.RepoCheck = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
}
if result.Checked != nil && c.Gate != nil && len(c.Gate.Modules) == 0 {
c.Gate.Modules = append(append([]string{}, result.Checked.Modules...), prefixedAll("new:", result.Checked.New)...)
}
for _, l := range []*link.CheckLayer{c.Gate, c.RepoCheck} {
if l != nil && l.Verdict == "" {
l.Verdict = "error"
}
}
if len(c.Report) > maxCheckReport {
c.Report = "…" + c.Report[len(c.Report)-maxCheckReport:]
}
fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: %s — %s\n", result.ID, c.Owner, c.Repo, c.Number, c.Commit,
orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary)
return c
}
func prefixedAll(prefix string, items []string) []string {
out := make([]string, 0, len(items))
for _, i := range items {
out = append(out, prefix+i)
}
return out
}
// sayChecked says a merge check's verdict on the bus, where the forge's holder hears it.
func sayChecked(ctx context.Context, c link.Checked) {
repo := "none"
if c.RepoCheck != nil {
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
}
fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
c.Commit, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
if checkEvents == nil {
return
}
@@ -193,6 +390,6 @@ func checked(ctx context.Context, result link.BuildResult) {
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer stop()
if err := checkEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyChecked, body); err != nil {
fmt.Printf("%s: the verdict could not be said, so the pull request is not told it: %v\n", result.ID, err)
fmt.Printf("%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err)
}
}
+114
View File
@@ -0,0 +1,114 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **The mesh's module graph decides what a pull request's check runs**, not the repository (novox/hq
// ADR 0237 as amended): a change is mapped onto the graph by the rule a merge is (issue 278), the gate
// runs when it touches a module — a new one included — and a repository that is the mesh's and touches
// none still has its own merge-check.sh run.
func TestThePullRequestIsMappedOntoTheModuleGraph(t *testing.T) {
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
const controller = "http://forge.internal:20000/novox/mesh-controller.git"
const host = "http://forge.internal:20000/novox/mesh-host.git"
photos := fromRepo("photos", "http://forge.internal:20000/novox/photos.git", "")
photos.Source.Ref = "nox-mesh"
snake := fromRepo("snake", "jschoubben/snake", "")
snake.Source.Seat = "git"
entries := []inventory.Entry{
fromRepo("gitea", catalogue, "modules/gitea"),
fromRepo("keycloak", catalogue, "modules/keycloak"),
fromRepo("nats", catalogue, "modules/nats"),
fromRepo("mesh-controller", controller, ""),
fromRepo("mesh-host", host, ""),
photos, snake,
}
pull := func(owner, repo, base string, paths []string, dirs ...string) link.PullUpdated {
return link.PullUpdated{Owner: owner, Repo: repo, Base: base, Commit: "abc", Paths: paths,
ModuleDirs: dirs, ModuleDirsSaid: true}
}
for _, c := range []struct {
what string
p link.PullUpdated
modules, new, manifest string
mesh bool
judge string
}{
{"one module's own files", pull("novox", "mesh-catalog", "main", []string{"modules/gitea/index.ts"}, "modules/gitea"),
"gitea", "", "modules/gitea/module.json", true, ""},
{"shared code touches every module built from the repository",
pull("novox", "mesh-catalog", "main", []string{"tsconfig.json"}), "gitea,keycloak,nats", "",
"modules/gitea/module.json,modules/keycloak/module.json,modules/nats/module.json", true, ""},
{"a new module, said by the head", pull("novox", "mesh-catalog", "main",
[]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, "modules/newmod"),
"", "modules/newmod", "modules/newmod/module.json", true, ""},
{"the controller judges itself", pull("novox", "mesh-controller", "main", []string{"cmd/x.go"}),
"mesh-controller", "", "module.json", true, "self"},
{"the node-engine is judged with its validator", pull("novox", "mesh-host", "main", []string{"validate/v.go"}),
"mesh-host", "", "module.json", true, "validator"},
{"the core's owner, no module: the mesh's, its own check alone", pull("novox", "hq", "main", []string{"README.md"}),
"", "", "", true, ""},
{"a branch nothing is built from: the mesh's repository, no module", pull("novox", "photos", "master",
[]string{"server/x.js"}), "", "", "", true, ""},
{"the branch a module is built from", pull("novox", "photos", "nox-mesh", []string{"server/x.js"}),
"photos", "", "module.json", true, ""},
{"a repository on the forge's seat", pull("jschoubben", "snake", "main", []string{"index.html"}),
"snake", "", "module.json", true, ""},
{"a repository of nobody's, touching nothing", pull("someone", "dotfiles", "main", []string{"x"}),
"", "", "", false, ""},
{"a repository adding a module at its root", pull("someone", "newapp", "main", []string{"module.json", "x.js"}),
"", ".", "module.json", true, ""},
} {
s := pullScope(c.p, entries, nil)
got := []string{strings.Join(s.Modules, ","), strings.Join(s.New, ","), strings.Join(s.Manifests, ",")}
want := []string{c.modules, c.new, c.manifest}
for i, what := range []string{"modules", "new", "manifests"} {
if got[i] != want[i] {
t.Errorf("%s: %s %q, wanted %q", c.what, what, got[i], want[i])
}
}
if s.Mesh != c.mesh || s.Judge != c.judge {
t.Errorf("%s: the mesh's %v judged by %q, wanted %v by %q", c.what, s.Mesh, s.Judge, c.mesh, c.judge)
}
if s.gated() != (c.modules != "" || c.new != "") {
t.Errorf("%s: gated %v", c.what, s.gated())
}
}
}
// A module whose build packages another repository's source is touched by a change to it.
func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) {
entries := []inventory.Entry{fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")}
read := map[string][]inventory.ReadRepository{"node-tools": {{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}}
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Paths: []string{"go/x.go"}}, entries, read)
if strings.Join(s.Modules, ",") != "node-tools" || len(s.Manifests) != 0 {
t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests)
}
}
// Each layer is said; a check that could not run is an error on both, never a pass; a build seat from
// before the layers is read as the gate.
func TestAChecksLayersAreEachSaidAndAnErrorIsNeverAPass(t *testing.T) {
asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3, Modules: []string{"gitea"}}
c := checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"})
if c.Verdict != "error" || c.Gate == nil || c.Gate.Verdict != "error" || c.RepoCheck == nil || c.RepoCheck.Verdict != "error" {
t.Fatalf("a check that could not run said %+v", c)
}
if strings.Join(c.Gate.Modules, ",") != "gitea" {
t.Errorf("the gate names %v", c.Gate.Modules)
}
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "warning", Summary: "wide"}})
if c.Gate == nil || c.Gate.Verdict != "warning" || c.RepoCheck != nil {
t.Fatalf("an outcome without layers said %+v", c)
}
c = checkedOf(link.BuildResult{ID: "b", Ref: "abc", Checked: asked, Check: &link.CheckOutcome{Verdict: "pass",
Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "its merge-check.sh failed"}}})
if c.RepoCheck.Verdict != "fail" || c.Gate.Verdict != "pass" {
t.Fatalf("the layers said %+v / %+v", c.Gate, c.RepoCheck)
}
}
+367 -93
View File
@@ -1,11 +1,13 @@
package builder
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"os"
"os/exec"
@@ -18,27 +20,40 @@ import (
"github.com/novox/mesh-controller/internal/facts"
)
// A pull request's merge check, run on the build seat (novox/hq to-be 45 §9).
// A pull request's merge check, run on the build seat (novox/hq to-be 45 §9, ADR 0237).
//
// **The build machine already has what a check needs**: the repositories, a container runtime, the
// artifact store where the controller keeps the facts snapshot, and a Go toolchain. So a check is one
// more kind of work on the build seat's queue rather than a CI the mesh would have to run beside itself.
//
// **Two layers, each its own status on the pull request** (ADR 0237 as amended, 2026-10-06):
//
// - **the gate** (`mesh/merge-gate`) runs when the change touches a module of the mesh's graph — the
// controller, which holds the graph, says which (Modules) and which directories it adds a module in
// (New). The touched manifests through `module check`; every machine of the facts snapshot composed
// with the change and validated by the node-engine's own validator; then mesh-lab's replays. The
// judge is the controller the mesh runs — or, for a change to the controller, the change's own
// controller, and for a change to the node-engine, the running controller with the change's validator
// in place of the one it vendors. The graph decides whether this runs, never the repository.
// - **the repository's own check** (`mesh/repo-check`): its merge-check.sh, its unit tests and code
// quality, run when present in the toolchain it declares (`# mesh-check-toolchain: go|typescript`
// among its first lines; go when it declares none). A repository that reaches the build seat with
// none is said as a warning: it is the mesh's, and nothing of its own is tested before it merges.
//
// One check:
//
// 1. clones the repository at the pull request's head, and beside it the repositories its check
// reads — the controller the mesh runs, the catalogue, the host — each at the ref asked;
// reads — the controller the mesh runs, the catalogue, the host, the lab — each at the ref asked;
// 2. reads the facts snapshot the controller keeps, and with it **the versions the mesh runs**: the
// store a check's tests stand on is the store's own release, and the bus the bus's;
// 3. raises a throwaway PostgreSQL and a throwaway bus of those versions, labelled with the ask so a
// kill or a crash leaves nothing behind;
// 4. builds the judge — the controller the mesh runs, or its main while the running one predates the
// merge gate — and runs the repository's own merge-check.sh, or the merge gate alone for a
// repository that declares none. **In the mesh's Go toolchain, in a container of its own**, never in
// the build machine's: a pull request is code nobody has approved yet, and the build machine holds
// the container runtime's socket; the check's container holds none, and reaches only the
// throwaway store and bus on loopback;
// 5. answers pass, warning or fail from what ran, and error — never pass — when it could not run.
// 4. runs the gate, then the repository's script — **each in a toolchain container of its own**, never
// in the build machine: a pull request is code nobody has approved yet, and the build machine holds
// the container runtime's socket; the check's container holds none, and reaches only the throwaway
// store and bus on loopback. Only the replays — mesh-lab's main, reviewed code — get the socket;
// 5. answers each layer pass, warning or fail from what ran, and error — never pass — when it could
// not run.
// CheckSpec is one check, as the controller asks it.
type CheckSpec struct {
@@ -51,18 +66,38 @@ type CheckSpec struct {
Paths []string
// Beside are the repositories cloned next to it, by the directory they are found under.
Beside map[string]Beside
// Modules are the modules of the mesh's graph the change touches, New the directories it adds a
// module in, and Manifests the manifests among them in the change's tree: the gate runs when Modules
// or New is not empty.
Modules []string
New []string
Manifests []string
// Judge is who judges the gate: "" the controller the mesh runs, "self" the change's own, "validator"
// the running one with the change's validator.
Judge string
// Toolchain is the image a check's Go runs in: the mesh's own Go toolchain, as it holds it.
Toolchain string
// Toolchains is every toolchain the mesh holds, by language, for a script that declares another.
Toolchain string
Toolchains map[string]string
}
// Gated is whether the change touches the mesh's graph, and so whether the gate runs.
func (s CheckSpec) Gated() bool { return len(s.Modules)+len(s.New) > 0 }
// ToolchainOf is the Go toolchain image among what the mesh holds, empty when it holds none.
func ToolchainOf(held map[string]string) string {
return ToolchainsOf(held)["go"]
}
// ToolchainsOf is every toolchain image the mesh holds, by language.
func ToolchainsOf(held map[string]string) map[string]string {
out := map[string]string{}
for _, chain := range toolchains {
if chain.Language == "go" {
return held[chain.Base+"/"+chain.Artifact]
if image := held[chain.Base+"/"+chain.Artifact]; image != "" {
out[chain.Language] = image
}
}
return ""
return out
}
// Beside is one repository cloned next to the one checked.
@@ -71,15 +106,24 @@ type Beside struct {
Ref string
}
// CheckVerdict is what came of one check.
// CheckVerdict is what came of one check. Verdict and Summary are the gate's; Gate and Repo each layer.
type CheckVerdict struct {
Verdict string
Summary string
Report string
Took time.Duration
Gate *Layer
Repo *Layer
}
// CheckScript is what a repository declares its merge check as: run from its root, with the
// Layer is one layer of a check, judged.
type Layer struct {
Verdict string
Summary string
Modules []string
}
// CheckScript is what a repository declares its own merge check as: run from its root, with the
// environment below.
const CheckScript = "merge-check.sh"
@@ -94,6 +138,7 @@ const (
EnvChanged = "MESH_CHECK_CHANGED"
EnvVerdict = "MESH_CHECK_VERDICT"
EnvBeside = "MESH_CHECK_BESIDE"
EnvModules = "MESH_CHECK_MODULES"
)
// CheckTimeout bounds one check; a check that runs past it is an error, not a pass.
@@ -102,6 +147,28 @@ var CheckTimeout = 45 * time.Minute
// reportLines is how much of what a check printed travels in its verdict.
const reportLines = 200
// noModule is the gate's word for a change that touches nothing of the mesh's graph: a fact, not a
// missing check, so a pass.
const noModule = "the change touches no module of the mesh's graph"
// noScript is the repository layer's word for a repository with no merge-check.sh of its own.
const noScript = "the repository declares no " + CheckScript + ": none of its own tests run before it merges"
// toolchainLine is how a merge-check.sh declares the toolchain it runs in.
var toolchainLine = regexp.MustCompile(`^#\s*mesh-check-toolchain:\s*([a-z0-9-]+)\s*$`)
// ScriptToolchain is the language a merge-check.sh declares it runs in, among its first twenty lines;
// go when it declares none.
func ScriptToolchain(script []byte) string {
lines := bufio.NewScanner(bytes.NewReader(script))
for i := 0; i < 20 && lines.Scan(); i++ {
if m := toolchainLine.FindStringSubmatch(strings.TrimSpace(lines.Text())); m != nil {
return m[1]
}
}
return "go"
}
// Check runs one merge check. An error is that it could not run; the verdict is then "error".
func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry string, forge GitCredential,
log Log) (CheckVerdict, error) {
@@ -137,13 +204,24 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
return nil
}
name := spec.Repo
if name == "" {
if name == "" || !safeName.MatchString(name) {
name = "checked"
}
say("check", "%s/%s#%d at %s", spec.Owner, spec.Repo, spec.Number, short(spec.Ref))
if err := clone(spec.Repository, spec.Ref, name); err != nil {
return CheckVerdict{}, err
}
tree := filepath.Join(root, name)
script, scriptErr := os.ReadFile(filepath.Join(tree, CheckScript))
hasScript := scriptErr == nil
gated := spec.Gated()
if !gated && !hasScript {
// Nothing to run: said, never passed silently.
v := CheckVerdict{Verdict: "pass", Summary: noModule, Took: time.Since(began),
Gate: &Layer{Verdict: "pass", Summary: noModule}, Repo: &Layer{Verdict: "warning", Summary: noScript}}
say("check", "%s; %s", noModule, noScript)
return v, nil
}
for dir, b := range spec.Beside {
if dir == name || !safeName.MatchString(dir) {
continue
@@ -204,102 +282,297 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
if spec.Toolchain == "" {
return CheckVerdict{}, errors.New("the mesh holds no Go toolchain to run a check in")
}
inToolchain := func(dir string, env []string, command ...string) []string {
in := func(image, dir string, env []string, command ...string) []string {
// As the builder itself: what a check writes into the workspace is the builder's to remove.
args := []string{"run", "--rm", "--network", "host", "--volume", workspace + ":" + workspace, "--workdir", dir,
"--user", fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()), "--env", "HOME=" + workspace}
for _, e := range env {
args = append(args, "--env", e)
}
return append(append(args, spec.Toolchain), command...)
return append(append(args, image), command...)
}
// The judge: the controller the mesh runs, or its main while the running one has no merge gate.
gate := ""
if name != "mesh-controller" {
gate, err = judge(ctx, labelled, run, root, inToolchain, say)
if err != nil {
return CheckVerdict{}, err
}
inToolchain := func(dir string, env []string, command ...string) []string {
return in(spec.Toolchain, dir, env, command...)
}
verdictFile := filepath.Join(root, "verdict.json")
env := append([]string{},
EnvFacts+"="+factsFile, EnvGate+"="+gate, EnvGateStore+"="+storeURL, EnvTestStore+"="+storeURL,
EnvTestBus+"=nats://"+bus, EnvRepository+"="+spec.Owner+"/"+spec.Repo,
EnvChanged+"="+strings.Join(spec.Paths, ","), EnvVerdict+"="+verdictFile, EnvBeside+"="+root,
"GOCACHE="+filepath.Join(workspace, "go-cache"), "GOMODCACHE="+filepath.Join(workspace, "go-modules"))
tree := filepath.Join(root, name)
var command []string
if _, err := os.Stat(filepath.Join(tree, CheckScript)); err == nil {
say("check", "running its %s in the mesh's Go toolchain", CheckScript)
command = []string{"sh", CheckScript}
} else {
if gate == "" {
return CheckVerdict{}, fmt.Errorf("%s declares no %s and there is no judge to run", name, CheckScript)
}
say("check", "it declares no %s: the merge gate alone", CheckScript)
command = []string{"sh", "-c", `"$MESH_GATE" merge-gate --facts "$MESH_FACTS" --store "$MESH_GATE_POSTGRES" ` +
`--repository "$MESH_CHECK_REPOSITORY" --tree . --changed "$MESH_CHECK_CHANGED" --json > "$MESH_CHECK_VERDICT"`}
}
cmd := exec.CommandContext(ctx, "docker", LabelledArgs("docker", inToolchain(tree, env, command...), spec.ID)...)
inItsOwnGroup(cmd)
var out tail
cmd.Stdout, cmd.Stderr = &out, &out
runErr := cmd.Run()
// running runs one container of the check, its output into the report, in a process group of its own.
running := func(args []string) error {
cmd := exec.CommandContext(ctx, "docker", LabelledArgs("docker", args, spec.ID)...)
inItsOwnGroup(cmd)
cmd.Stdout, cmd.Stderr = &out, &out
return cmd.Run()
}
// **And the replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed
// code, so given the container runtime the resolver replay raises containers with — against the bus
// of the release the mesh runs and the change's own catalogue when the change is to the catalogue.
var replayErr error
if lab := filepath.Join(root, "mesh-lab", "replays"); runErr == nil && ctx.Err() == nil {
// The judge. Its failing to build is the change's fault when the change is the judge or its validator,
// and the check's when it is the controller the mesh runs.
gate, judgeFault, err := judgeFor(ctx, labelled, run, spec, root, tree, inToolchain, say)
if err != nil {
return CheckVerdict{}, err
}
verdictFile := filepath.Join(root, "verdict.json")
env := []string{EnvFacts + "=" + factsFile, EnvGate + "=" + gate, EnvGateStore + "=" + storeURL,
EnvTestStore + "=" + storeURL, EnvTestBus + "=nats://" + bus, EnvRepository + "=" + spec.Owner + "/" + spec.Repo,
EnvChanged + "=" + strings.Join(spec.Paths, ","), EnvBeside + "=" + root,
EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","),
"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")}
v := CheckVerdict{}
modules := append(append([]string{}, spec.Modules...), prefixed("new:", spec.New)...)
timedOut := func() bool { return errors.Is(ctx.Err(), context.DeadlineExceeded) }
// **The gate**, when the graph says the change touches it.
if !gated {
v.Gate = &Layer{Verdict: "pass", Summary: noModule}
} else {
v.Gate = &Layer{Modules: modules}
switch {
case judgeFault != "":
v.Gate.Verdict, v.Gate.Summary = "fail", judgeFault
default:
say("check", "the gate: %d module(s) of the graph touched — %s", len(modules), strings.Join(modules, ", "))
fmt.Fprintf(&out, "--- the gate: %s\n", strings.Join(modules, ", "))
v.Gate.Verdict, v.Gate.Summary = gateLayer(ctx, spec, tree, root, gate, verdictFile, env, inToolchain,
running, &out, bus, workspace, name, say)
}
if timedOut() {
v.Gate.Verdict, v.Gate.Summary = "error", fmt.Sprintf("the check ran past %s and was ended", CheckTimeout)
}
}
if ctx.Err() != nil && !timedOut() {
return v, ctx.Err()
}
// **The repository's own check**, in the toolchain it declares.
switch {
case !hasScript:
v.Repo = &Layer{Verdict: "warning", Summary: noScript}
case timedOut():
v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("the check ran past %s before its %s ran", CheckTimeout, CheckScript)}
default:
language := ScriptToolchain(script)
image := spec.Toolchains[language]
if language == "go" && image == "" {
image = spec.Toolchain
}
if image == "" {
v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s runs in the %s toolchain, which the mesh does "+
"not hold", CheckScript, language)}
break
}
say("check", "running its %s in the mesh's %s toolchain", CheckScript, language)
fmt.Fprintf(&out, "--- its %s (%s toolchain)\n", CheckScript, language)
var own tail
cmd := exec.CommandContext(ctx, "docker", LabelledArgs("docker", in(image, tree, env, "sh", CheckScript), spec.ID)...)
inItsOwnGroup(cmd)
w := io.MultiWriter(&out, &own)
cmd.Stdout, cmd.Stderr = w, w
switch err := cmd.Run(); {
case timedOut():
v.Repo = &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", CheckScript, CheckTimeout)}
case ctx.Err() != nil:
return v, ctx.Err()
case err != nil:
v.Repo = &Layer{Verdict: "fail", Summary: "its " + CheckScript + " failed: " + lastLine(own.String())}
default:
v.Repo = &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed"}
}
}
v.Verdict, v.Summary = v.Gate.Verdict, v.Gate.Summary
v.Report, v.Took = out.String(), time.Since(began)
say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary,
strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second))
return v, nil
}
// gateLayer runs the gate: the touched manifests through `module check`, every machine composed with the
// change, and the replays of what the mesh runs. It answers the gate's verdict and summary.
func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFile string, env []string,
inToolchain func(string, []string, ...string) []string, running func([]string) error, out *tail, bus, workspace,
name string, say func(step, format string, args ...any)) (string, string) {
// 1. The manifests the change touches, as the judge reads them: a manifest it cannot read fails here.
var manifests []string
for _, m := range spec.Manifests {
if _, err := os.Stat(filepath.Join(tree, m)); err == nil {
manifests = append(manifests, m)
}
}
if len(manifests) > 0 {
var own tail
cmd := exec.CommandContext(ctx, "docker", LabelledArgs("docker",
inToolchain(tree, env, append([]string{gate, "module", "check"}, manifests...)...), spec.ID)...)
inItsOwnGroup(cmd)
w := io.MultiWriter(out, &own)
cmd.Stdout, cmd.Stderr = w, w
if err := cmd.Run(); err != nil {
if ctx.Err() != nil {
return "error", "the check was ended during the module check"
}
return "fail", "a manifest the change touches fails the module check: " + firstProblem(own.String())
}
}
// 2. Every machine composed with the change.
if err := running(inToolchain(tree, append(env, EnvVerdict+"="+verdictFile), "sh", "-c",
`"$MESH_GATE" merge-gate --facts "$MESH_FACTS" --store "$MESH_GATE_POSTGRES" `+
`--repository "$MESH_CHECK_REPOSITORY" --tree . --changed "$MESH_CHECK_CHANGED" --json > "$MESH_CHECK_VERDICT"`)); err != nil {
if ctx.Err() != nil {
return "error", "the check was ended during the merge gate"
}
var said struct {
Verdict string `json:"verdict"`
Summary string `json:"summary"`
}
if raw, err := os.ReadFile(verdictFile); err == nil && json.Unmarshal(raw, &said) == nil && said.Verdict == "fail" {
return "fail", said.Summary
}
// The gate could not judge: not the change's fault, and never a pass.
return "error", "the merge gate could not judge the change: " + lastLine(out.String())
}
var said struct {
Verdict string `json:"verdict"`
Summary string `json:"summary"`
}
raw, err := os.ReadFile(verdictFile)
if err != nil || json.Unmarshal(raw, &said) != nil || said.Verdict == "" {
return "error", "the merge gate said no verdict"
}
// 3. **The replays of what the mesh runs** (to-be 45 §9, M9): mesh-lab's, from its main — reviewed code,
// so given the container runtime the resolver replay raises containers with — against the bus of the
// release the mesh runs and the change's own catalogue when the change is to the catalogue.
if lab := filepath.Join(root, "mesh-lab", "replays"); ctx.Err() == nil {
if _, err := os.Stat(lab); err == nil {
catalogue := filepath.Join(root, "mesh-catalog")
if name == "mesh-catalog" {
catalogue = tree
}
say("check", "the replays of what the mesh runs, from mesh-lab")
fmt.Fprintln(&out, "--- the replays (mesh-lab replays/)")
fmt.Fprintln(out, "--- the replays (mesh-lab replays/)")
args := inToolchain(lab, []string{EnvTestBus + "=nats://" + bus, "MESH_REPLAY_CATALOGUE=" + catalogue,
"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")},
"go", "test", "-count=1", "./...")
// The socket goes to the replays alone, never to the change's own script above.
// The socket goes to the replays alone, never to the change's own code above.
args = append([]string{args[0], "--volume", "/var/run/docker.sock:/var/run/docker.sock"}, args[1:]...)
replays := exec.CommandContext(ctx, "docker", LabelledArgs("docker", args, spec.ID)...)
inItsOwnGroup(replays)
replays.Stdout, replays.Stderr = &out, &out
replayErr = replays.Run()
if err := running(args); err != nil {
if ctx.Err() != nil {
return "error", "the check was ended during the replays"
}
return "fail", "a replay of a core incident fails with this change: " + lastLine(out.String())
}
}
}
v := CheckVerdict{Report: out.String(), Took: time.Since(began)}
var gateSaid struct {
Verdict string `json:"verdict"`
Summary string `json:"summary"`
if said.Verdict == "pass" || said.Verdict == "warning" || said.Verdict == "fail" {
return said.Verdict, said.Summary
}
if raw, err := os.ReadFile(verdictFile); err == nil {
_ = json.Unmarshal(raw, &gateSaid)
}
switch {
case errors.Is(ctx.Err(), context.DeadlineExceeded):
v.Verdict, v.Summary = "error", fmt.Sprintf("the check ran past %s and was ended", CheckTimeout)
case ctx.Err() != nil:
return v, ctx.Err()
case runErr != nil:
v.Verdict = "fail"
v.Summary = gateSaid.Summary
if v.Summary == "" || gateSaid.Verdict != "fail" {
v.Summary = "the merge check failed: " + lastLine(out.String())
return "error", "the merge gate said " + said.Verdict
}
// judgeFor builds the judge of a check: the change's own controller (a change to the controller), the
// running controller built with the change's validator (a change to the node-engine), or the controller
// the mesh runs. It answers the judge's path; or, when the change makes its own judge unbuildable, why —
// the change's fault, a failing gate; or an error when the check cannot build a judge at all. A check
// whose gate does not run builds a judge only to hand its scripts one, and goes on without when it cannot.
func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, tree string,
inToolchain func(string, []string, ...string) []string, say func(step, format string, args ...any)) (string, string, error) {
gated := spec.Gated()
switch spec.Judge {
case "self":
bin := filepath.Join(root, "bin", "judge-of-itself")
if _, err := labelled(ctx, root, "docker", inToolchain(tree,
[]string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")},
"go", "build", "-o", bin, "./cmd/mesh-controller")...); err != nil {
return "", "the change's controller, which judges itself, does not build: " + firstLine(err.Error()), nil
}
case replayErr != nil:
v.Verdict, v.Summary = "fail", "a replay of a core incident fails with this change: "+lastLine(out.String())
default:
v.Verdict, v.Summary = "pass", "the merge check passed"
if gateSaid.Verdict == "warning" || gateSaid.Verdict == "pass" {
v.Verdict, v.Summary = gateSaid.Verdict, gateSaid.Summary
say("check", "judged by the change's own controller")
return bin, "", nil
}
bin, judgeTree, err := judge(ctx, labelled, run, root, inToolchain, say)
if err != nil {
if !gated {
say("check", "no judge for its script: %v", err)
return "", "", nil
}
return "", "", err
}
if spec.Judge != "validator" {
return bin, "", nil
}
// The node-engine's change: its validator in place of the one the judge vendors.
vendored := filepath.Join(root, judgeTree, "vendor", "github.com", "novox", "mesh-host")
for _, pkg := range []string{"validate", filepath.Join("internal", "declaration")} {
if err := replaceGoFiles(filepath.Join(tree, pkg), filepath.Join(vendored, pkg)); err != nil {
return "", "", fmt.Errorf("the change's validator could not be put in the judge: %w", err)
}
}
say("check", "%s — %s (%s)", strings.ToUpper(v.Verdict), v.Summary, v.Took.Round(time.Second))
return v, nil
withValidator := filepath.Join(root, "bin", "judge-with-this-validator")
if _, err := labelled(ctx, root, "docker", inToolchain(filepath.Join(root, judgeTree),
[]string{"CGO_ENABLED=0", "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOCACHE=" + filepath.Join(filepath.Dir(root), "go-cache")},
"go", "build", "-o", withValidator, "./cmd/mesh-controller")...); err != nil {
return "", "the controller the mesh runs does not build with this change's validator: " + firstLine(err.Error()), nil
}
say("check", "judged by the controller the mesh runs, with this change's validator")
return withValidator, "", nil
}
// replaceGoFiles puts a package's Go files — never its tests — in place of another copy's.
func replaceGoFiles(from, into string) error {
old, err := filepath.Glob(filepath.Join(into, "*.go"))
if err != nil {
return err
}
for _, f := range old {
if err := os.Remove(f); err != nil {
return err
}
}
if err := os.MkdirAll(into, 0o755); err != nil {
return err
}
files, err := filepath.Glob(filepath.Join(from, "*.go"))
if err != nil {
return err
}
if len(files) == 0 {
return fmt.Errorf("%s holds no Go files", from)
}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
body, err := os.ReadFile(f)
if err != nil {
return err
}
if err := os.WriteFile(filepath.Join(into, filepath.Base(f)), body, 0o644); err != nil {
return err
}
}
return nil
}
// firstProblem is the first line of a module check's output that names a problem.
func firstProblem(s string) string {
for _, line := range strings.Split(s, "\n") {
line = strings.TrimSpace(line)
if line != "" && !strings.HasSuffix(line, "tool(s)") && !strings.Contains(line, ": ok") {
return line
}
}
return lastLine(s)
}
func prefixed(prefix string, items []string) []string {
out := make([]string, 0, len(items))
for _, i := range items {
out = append(out, prefix+i)
}
return out
}
func firstLine(s string) string {
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
return line
}
// safeName is a directory a repository beside a check may be cloned under.
@@ -377,9 +650,10 @@ func dialable(ctx context.Context, address string) error {
}
// judge builds the controller that judges a change: the one the mesh runs, beside the check as
// mesh-controller — or, when that one predates the merge gate, the controller's main, said.
// mesh-controller — or, when that one predates the merge gate, the controller's main, said. It answers
// the binary and the directory it was built from.
func judge(ctx context.Context, run, plain Runner, root string, inToolchain func(string, []string, ...string) []string,
say func(step, format string, args ...any)) (string, error) {
say func(step, format string, args ...any)) (string, string, error) {
bin := filepath.Join(root, "bin", "mesh-controller")
build := func(dir string) error {
_, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir),
@@ -394,21 +668,21 @@ func judge(ctx context.Context, run, plain Runner, root string, inToolchain func
}
if _, err := os.Stat(filepath.Join(root, "mesh-controller")); err == nil {
if err := build("mesh-controller"); err != nil {
return "", fmt.Errorf("the controller the mesh runs does not build: %w", err)
return "", "", fmt.Errorf("the controller the mesh runs does not build: %w", err)
}
if hasGate() {
say("check", "judged by the controller the mesh runs")
return bin, nil
return bin, "mesh-controller", nil
}
}
if _, err := os.Stat(filepath.Join(root, "mesh-controller-main")); err != nil {
return "", errors.New("no controller beside the check to judge it with")
return "", "", errors.New("no controller beside the check to judge it with")
}
if err := build("mesh-controller-main"); err != nil {
return "", fmt.Errorf("the controller's main does not build: %w", err)
return "", "", fmt.Errorf("the controller's main does not build: %w", err)
}
say("check", "judged by the controller's main: the one the mesh runs predates the merge gate")
return bin, nil
return bin, "mesh-controller-main", nil
}
// tail keeps the last lines written to it.
+166 -17
View File
@@ -50,6 +50,9 @@ func aCheckedRepository(t *testing.T, files map[string]string) (string, string)
}
git("init", "--quiet", "-b", "main")
for name, body := range files {
if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil {
t.Fatal(err)
}
@@ -88,28 +91,31 @@ func labelled(id string) []string {
func TestACheckRunsTheRepositorysOwnScriptBesideTheMeshsVersionsAndLeavesNothing(t *testing.T) {
registry := checkEnvironment(t)
// The script proves what it was given: the facts, a store that answers, a bus that answers, and
// writes the gate's verdict where it is told to.
// The script proves what it was given: the facts, a store that answers, a bus that answers — and no
// container runtime socket.
script := `set -e
test -s "$MESH_FACTS"
grep -q '"bus": "2.11.17"' "$MESH_FACTS" || grep -q '"bus":"2.11.17"' "$MESH_FACTS"
case "$MESH_TEST_POSTGRES" in postgres://*127.0.0.1:*) ;; *) echo "no store: $MESH_TEST_POSTGRES"; exit 1;; esac
case "$MESH_TEST_NATS" in nats://127.0.0.1:*) ;; *) echo "no bus: $MESH_TEST_NATS"; exit 1;; esac
test "$MESH_CHECK_REPOSITORY" = "novox/mesh-controller"
test "$MESH_CHECK_REPOSITORY" = "novox/hq"
test "$MESH_CHECK_CHANGED" = "a.go,b.go"
test ! -S /var/run/docker.sock || { echo "the check holds the container runtime's socket"; exit 1; }
echo '{"verdict":"warning","summary":"a merge rebuilds 14 module(s)"}' > "$MESH_CHECK_VERDICT"
echo checked
`
repo, head := aCheckedRepository(t, map[string]string{CheckScript: script})
id := fmt.Sprintf("check-test-%d", time.Now().UnixNano())
v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox",
Repo: "mesh-controller", Number: 7, Paths: []string{"a.go", "b.go"}, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
Repo: "hq", Number: 7, Paths: []string{"a.go", "b.go"}, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Verdict != "warning" || v.Summary != "a merge rebuilds 14 module(s)" || !strings.Contains(v.Report, "checked") {
t.Fatalf("the check answered %+v", v)
// Nothing of the graph touched: the gate a pass that says so, the repository's own check run.
if v.Gate == nil || v.Gate.Verdict != "pass" || v.Gate.Summary != noModule {
t.Errorf("the gate answered %+v", v.Gate)
}
if v.Repo == nil || v.Repo.Verdict != "pass" || !strings.Contains(v.Report, "checked") {
t.Fatalf("the repository's check answered %+v\n%s", v.Repo, v.Report)
}
if left := labelled(id); len(left) > 0 {
t.Errorf("the check left %d container(s) behind", len(left))
@@ -120,12 +126,20 @@ func TestAFailingCheckFailsAndOneThatCannotRunIsNeverAPass(t *testing.T) {
registry := checkEnvironment(t)
repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo 'resource \"x.service\": refused'; exit 3\n"})
v, err := Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-fail-%d", time.Now().UnixNano()),
Repository: repo, Ref: head, Owner: "novox", Repo: "mesh-controller", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Verdict != "fail" || !strings.Contains(v.Summary, "refused") {
t.Fatalf("a failing script answered %+v", v)
if v.Repo == nil || v.Repo.Verdict != "fail" || !strings.Contains(v.Repo.Summary, "refused") {
t.Fatalf("a failing script answered %+v", v.Repo)
}
// A script in a toolchain the mesh does not hold: an error, never a pass.
repo, head = aCheckedRepository(t, map[string]string{CheckScript: "# mesh-check-toolchain: cobol\nexit 0\n"})
v, err = Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-cobol-%d", time.Now().UnixNano()),
Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
if err != nil || v.Repo == nil || v.Repo.Verdict != "error" {
t.Fatalf("a script in a toolchain nobody holds answered %+v, %v", v.Repo, err)
}
// Past its bound: an error, not a pass.
@@ -134,19 +148,154 @@ func TestAFailingCheckFailsAndOneThatCannotRunIsNeverAPass(t *testing.T) {
t.Cleanup(func() { CheckTimeout = was })
repo, head = aCheckedRepository(t, map[string]string{CheckScript: "sleep 120\n"})
v, err = Check(context.Background(), Command, CheckSpec{ID: fmt.Sprintf("check-slow-%d", time.Now().UnixNano()),
Repository: repo, Ref: head, Owner: "novox", Repo: "mesh-controller", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
if err == nil && v.Verdict == "pass" {
t.Fatalf("a check past its bound passed: %+v", v)
}
if err == nil && v.Verdict != "error" {
t.Fatalf("a check past its bound answered %+v", v)
Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
if err == nil && (v.Repo == nil || v.Repo.Verdict != "error") {
t.Fatalf("a check past its bound answered %+v", v.Repo)
}
// No facts: it cannot run, and says so.
repo, head = aCheckedRepository(t, map[string]string{CheckScript: "exit 0\n"})
_, err = Check(t.Context(), Command, CheckSpec{ID: "check-nofacts", Repository: repo, Ref: head, Owner: "novox",
Repo: "mesh-controller", Toolchain: goToolchain}, t.TempDir(), "127.0.0.1:1", GitCredential{}, nil)
Repo: "hq", Toolchain: goToolchain}, t.TempDir(), "127.0.0.1:1", GitCredential{}, nil)
if err == nil || !strings.Contains(err.Error(), "facts snapshot") {
t.Fatalf("a check with no facts said %v", err)
}
}
// A repository that touches nothing of the graph and has no script of its own runs nothing, and says
// both: the gate a pass that names why, the repository a warning — never silent.
func TestACheckWithNothingToRunSaysSo(t *testing.T) {
repo, head := aCheckedRepository(t, map[string]string{"README.md": "x"})
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-nothing", Repository: repo, Ref: head, Owner: "novox",
Repo: "hq"}, t.TempDir(), "", GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Gate == nil || v.Gate.Verdict != "pass" || v.Repo == nil || v.Repo.Verdict != "warning" ||
!strings.Contains(v.Repo.Summary, CheckScript) {
t.Fatalf("a check with nothing to run said %+v / %+v", v.Gate, v.Repo)
}
// A gated change never takes that path: with no store to read the facts from, it cannot run.
_, err = Check(t.Context(), Command, CheckSpec{ID: "check-gated", Repository: repo, Ref: head, Owner: "novox",
Repo: "mesh-catalog", Modules: []string{"gitea"}}, t.TempDir(), "", GitCredential{}, nil)
if err == nil {
t.Fatal("a change touching a module ran nothing and was not refused")
}
}
// A script declares its toolchain among its first lines; go when it declares none.
func TestAScriptDeclaresItsToolchain(t *testing.T) {
for script, want := range map[string]string{
"#!/bin/sh\nset -eu\n": "go",
"#!/bin/sh\n# mesh-check-toolchain: typescript\nnpm test\n": "typescript",
"#!/bin/sh\n#mesh-check-toolchain:go\n": "go",
"#!/bin/sh\necho '# mesh-check-toolchain: python'\n": "go",
} {
if got := ScriptToolchain([]byte(script)); got != want {
t.Errorf("%q declares %q, read as %q", script, want, got)
}
}
held := map[string]string{"mesh-tools/build": "reg/mesh-tools-build@sha256:a", "mesh-tools-go/build": "reg/go@sha256:b"}
chains := ToolchainsOf(held)
if chains["typescript"] != "reg/mesh-tools-build@sha256:a" || chains["go"] != "reg/go@sha256:b" || ToolchainOf(held) != chains["go"] {
t.Fatalf("the toolchains held read as %v", chains)
}
if _, held := chains["python"]; held {
t.Error("a toolchain the mesh does not hold read as held")
}
}
// A node-engine change's validator is put in place of the one the judge vendors: its Go files, never its tests.
func TestTheChangesValidatorReplacesTheVendoredOne(t *testing.T) {
from, into := t.TempDir(), t.TempDir()
for name, body := range map[string]string{"v.go": "package validate // new", "v_test.go": "package validate"} {
if err := os.WriteFile(filepath.Join(from, name), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(into, "old.go"), []byte("package validate // old"), 0o644); err != nil {
t.Fatal(err)
}
if err := replaceGoFiles(from, into); err != nil {
t.Fatal(err)
}
got, _ := filepath.Glob(filepath.Join(into, "*.go"))
if len(got) != 1 || filepath.Base(got[0]) != "v.go" {
t.Fatalf("the vendored validator holds %v", got)
}
}
// aJudge is a controller that judges as told: `merge-gate` answers a warning, `module check` refuses a
// manifest that says it is broken. What the gate layer is tested against without building the real one.
var aJudge = map[string]string{
"go.mod": "module example.org/judge\n\ngo 1.22\n",
"cmd/mesh-controller/main.go": `package main
import (
"fmt"
"os"
"strings"
)
func main() {
switch {
case len(os.Args) == 2 && os.Args[1] == "merge-gate":
fmt.Println("usage: merge-gate --facts <file> --store <postgres>")
os.Exit(2)
case len(os.Args) > 2 && os.Args[1] == "module":
for _, m := range os.Args[3:] {
body, _ := os.ReadFile(m)
if strings.Contains(string(body), "broken") {
fmt.Println(m + ": refused, it says it is broken")
os.Exit(1)
}
fmt.Println(m + ": ok")
}
case os.Args[1] == "merge-gate":
fmt.Println(` + "`" + `{"verdict":"warning","summary":"a merge rebuilds 14 module(s)"}` + "`" + `)
}
}
`,
}
func TestTheGateRunsWhenTheGraphIsTouchedBesideTheRepositorysOwnCheck(t *testing.T) {
registry := checkEnvironment(t)
judgeRepo, judgeHead := aCheckedRepository(t, aJudge)
beside := map[string]Beside{"mesh-controller": {Repository: judgeRepo, Ref: judgeHead}}
check := func(files map[string]string, judge string) CheckVerdict {
t.Helper()
repo, head := aCheckedRepository(t, files)
id := fmt.Sprintf("check-gate-%d", time.Now().UnixNano())
v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox",
Repo: "mesh-catalog", Number: 9, Paths: []string{"modules/gitea/index.ts"}, Beside: beside,
Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Judge: judge,
Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if left := labelled(id); len(left) > 0 {
t.Errorf("the check left %d container(s) behind", len(left))
}
return v
}
v := check(map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, CheckScript: "echo own; exit 0\n"}, "")
if v.Gate == nil || v.Gate.Verdict != "warning" || v.Gate.Summary != "a merge rebuilds 14 module(s)" ||
strings.Join(v.Gate.Modules, ",") != "gitea" || v.Verdict != "warning" {
t.Fatalf("the gate answered %+v\n%s", v.Gate, v.Report)
}
if v.Repo == nil || v.Repo.Verdict != "pass" {
t.Fatalf("its own check answered %+v", v.Repo)
}
v = check(map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`}, "")
if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "module check") || v.Repo.Verdict != "warning" {
t.Fatalf("a manifest the judge refuses answered %+v / %+v\n%s", v.Gate, v.Repo, v.Report)
}
// A change to the controller judges itself: one that does not build fails its own gate.
v = check(map[string]string{"go.mod": "module x\n\ngo 1.22\n", "cmd/mesh-controller/main.go": "package main\nfunc main() { nope }\n",
"modules/gitea/module.json": "{}"}, "self")
if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "does not build") {
t.Fatalf("a controller that does not build judged itself %+v", v.Gate)
}
}
+21
View File
@@ -106,8 +106,26 @@ type CheckRequest struct {
// controller the mesh runs, the catalogue it holds, the host it runs — keyed by the directory name the
// check finds it under.
Beside map[string]CheckedOut `json:"beside,omitempty"`
// Modules are the modules of the mesh's graph the change touches, and New the directories it adds a
// module in that the graph does not hold (novox/hq ADR 0237 as amended): **the graph decides whether
// the gate runs**, not the repository. With neither, only the repository's own merge-check.sh runs.
Modules []string `json:"modules,omitempty"`
New []string `json:"new,omitempty"`
// Manifests are the touched modules' manifests in the change's tree, by path from its root: what the
// gate puts through `module check`.
Manifests []string `json:"manifests,omitempty"`
// Judge is who judges the gate: empty for the controller the mesh runs; JudgeSelf for a change to
// the controller, judged by itself; JudgeValidator for a change to the node-engine, judged by the
// running controller built with the change's validator in place of the one it vendors.
Judge string `json:"judge,omitempty"`
}
// Who judges a merge check's gate.
const (
JudgeSelf = "self"
JudgeValidator = "validator"
)
// CheckedOut is a repository cloned beside a check, at a ref.
type CheckedOut struct {
Repository string `json:"repository"`
@@ -123,6 +141,9 @@ type CheckOutcome struct {
Report string `json:"report,omitempty"`
// Took is how long it ran.
Took string `json:"took,omitempty"`
// Gate and RepoCheck are its two layers; Verdict and Summary above are the gate's.
Gate *CheckLayer `json:"gate,omitempty"`
RepoCheck *CheckLayer `json:"repo-check,omitempty"`
}
// SourceOnSeat names a repository by the seat whose holder serves it and its path there.
+27
View File
@@ -221,6 +221,16 @@ type PullUpdated struct {
// Paths are the files the pull request changes; PathsTruncated says there were more.
Paths []string `json:"paths,omitempty"`
PathsTruncated bool `json:"paths_truncated,omitempty"`
// ModuleDirs are the directories above the changed files that hold a `module.json` at the head, read
// as the merge announcer reads them at a merge commit (issue 278); ModuleDirsSaid says it looked.
// What the controller maps a pull request onto the mesh's module graph with: a file inside one is
// that module's, held or not, and one inside none, in a repository modules are built from, is shared.
ModuleDirs []string `json:"module_dirs,omitempty"`
ModuleDirsSaid bool `json:"module_dirs_said,omitempty"`
// MergeCheck says the head holds a merge-check.sh at its root — the repository's own tests, the
// check's second layer; MergeCheckSaid says the announcer looked.
MergeCheck bool `json:"merge_check,omitempty"`
MergeCheckSaid bool `json:"merge_check_said,omitempty"`
}
// Checked is a pull request's merge check, judged: what the controller says as `checked`.
@@ -238,6 +248,23 @@ type Checked struct {
// ID is the ask, and On the machine that ran it.
ID string `json:"id"`
On string `json:"on,omitempty"`
// Gate and RepoCheck are the check's two layers, each its own status on the pull request (novox/hq
// ADR 0237 as amended): the mesh's — the modules of the graph the change touches, every machine
// composed with it — as `mesh/merge-gate`, and the repository's own merge-check.sh as
// `mesh/repo-check`. Verdict and Summary above are the gate's, for a forge holder that reads no more.
// RepoCheck is nil when the repository is not the mesh's and touches nothing of it: nothing is said.
Gate *CheckLayer `json:"gate,omitempty"`
RepoCheck *CheckLayer `json:"repo-check,omitempty"`
}
// CheckLayer is one layer of a merge check, judged.
type CheckLayer struct {
// Verdict is pass, warning, fail or error; error is never a pass.
Verdict string `json:"verdict"`
Summary string `json:"summary"`
// Modules are, for the gate, the modules of the mesh's graph the change touches — `new:<dir>` for a
// module the graph does not hold yet.
Modules []string `json:"modules,omitempty"`
}
type Upgraded struct {
+20 -21
View File
@@ -1,22 +1,22 @@
#!/bin/sh
# The merge check of the controller (novox/hq to-be 45 §9), run by the build seat on every pull request
# before it merges — and by hand: `MESH_FACTS=facts.json MESH_GATE_POSTGRES=… MESH_TEST_POSTGRES=…
# MESH_TEST_NATS=… sh merge-check.sh`.
# mesh-check-toolchain: go
#
# The build seat clones this repository with the catalogue and the host beside it (the tests read them
# there), reads the facts snapshot the controller keeps, and raises a throwaway store and bus of the
# versions the mesh runs; this says what is judged with them:
# The controller's own check (novox/hq ADR 0237 as amended): the second layer of a pull request's merge
# check, `mesh/repo-check`, run by the build seat in the mesh's Go toolchain — and by hand:
# `MESH_TEST_POSTGRES=… sh merge-check.sh`.
#
# The gate — every machine composed with this change's controller, which judges itself — is the build
# seat's first layer (`mesh/merge-gate`), run because the mesh's module graph builds `mesh-controller`
# from this repository; it is not repeated here. This is the code's own quality and its suite:
#
# 1. formatted and vetted;
# 2. the merge gate, judged by THIS change's controller: every machine of the snapshot composed with
# it and validated by the node-engine's own validator, against the mesh as it is;
# 3. the whole suite, the replays among it, against that store and that bus, one package at a time
# because the live tests share one bus's fixed names.
# 2. the whole suite, packages in parallel, against the throwaway store the build seat raised at the
# release the mesh runs; every live test on a bus of its own (internal/testbus), so nothing is
# serialised. Under the race detector when the toolchain has a C compiler, and said when it has not.
#
# Fails on the first that fails. The gate's verdict is written where MESH_CHECK_VERDICT says, so the
# pull request is told the gate's own words.
# Exit 0 is a pass; anything else fails `mesh/repo-check` with the last line printed.
set -eu
export GOFLAGS=-mod=vendor GOPROXY=off CGO_ENABLED=0
export GOFLAGS=-mod=vendor GOPROXY=off
unformatted=$(gofmt -l cmd internal examples)
if [ -n "$unformatted" ]; then
@@ -24,12 +24,11 @@ if [ -n "$unformatted" ]; then
echo "$unformatted"
exit 1
fi
go vet ./...
CGO_ENABLED=0 go vet ./...
judge="${MESH_CHECK_BESIDE:-${TMPDIR:-/tmp}}/bin/judge"
go build -o "$judge" ./cmd/mesh-controller
"$judge" merge-gate --facts "$MESH_FACTS" --store "$MESH_GATE_POSTGRES" \
--repository "${MESH_CHECK_REPOSITORY:-novox/mesh-controller}" --tree . \
--changed "${MESH_CHECK_CHANGED:-}" --json > "${MESH_CHECK_VERDICT:-/dev/null}"
go test -p 1 -timeout 25m ./...
if command -v gcc >/dev/null 2>&1; then
CGO_ENABLED=1 go test -race -count=1 -timeout 30m ./...
else
echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector"
CGO_ENABLED=0 go test -count=1 -timeout 30m ./...
fi
+27
View File
@@ -27,16 +27,43 @@
"prepares": true,
"tools": [
"tools",
"calls",
"status",
"nodes",
"node",
"modules",
"seats",
"builds",
"plans",
"plan",
"assign",
"unassign",
"pin",
"unpin",
"push",
"rotate",
"issue",
"settings",
"command",
"queue",
"cancel",
"clear",
"rebuild",
"replay",
"kill",
"pause",
"resume",
"hand-act",
"hand-acts",
"durations",
"conditions",
"healers",
"doctor",
"upgrade",
"bus",
"retire",
"cleanup",
"data",
"build"
],
"resources": [