Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e9ce56e838 | ||
|
|
ee176b76ea | ||
|
|
cf071270a0 | ||
|
|
29f4a366c2 | ||
|
|
7ec760bca2 | ||
|
|
22f7a78503 | ||
|
|
3ff6a8b502 | ||
|
|
24cbc8961e | ||
|
|
547f4ee27d | ||
|
|
47dbf87e33 | ||
|
|
15378cfba7 | ||
|
|
539a31103b | ||
|
|
3ca48ff230 |
+10
-10
@@ -699,9 +699,9 @@ export class GiteaClient {
|
||||
};
|
||||
}
|
||||
|
||||
/** Protect a branch (novox/hq ADR 0237): the rule named for the branch is edited — its required statuses taken
|
||||
* off (ADR 0534 §5), everything else it says kept unless asked — or created, refusing direct pushes unless asked
|
||||
* otherwise. Answers the rule as it is now. */
|
||||
/** Make a branch require these commit statuses to merge (novox/hq ADR 0237): the rule named for the
|
||||
* branch is edited — its required statuses replaced by these, everything else it says kept unless
|
||||
* asked — or created, refusing direct pushes unless asked otherwise. Answers the rule as it is now. */
|
||||
async setBranchProtection(owner: string, repo: string, branch: string, want: ProtectionWanted): Promise<{ created: boolean; rule: BranchProtection }> {
|
||||
const had = await this.branchProtection(owner, repo, branch);
|
||||
const body = protectionBody(want, !had, branch);
|
||||
@@ -932,9 +932,9 @@ export class GiteaAdmin {
|
||||
return Array.isArray(res.body) ? res.body : [];
|
||||
}
|
||||
|
||||
/** Put main's rule right in every repository of an organisation (novox/hq ADR 0398 §2, issue 948): no push, only
|
||||
* the tools' account merges, and no status required (ADR 0534 §1). The rule there is edited; a repository with no
|
||||
* rule for main gets one. Answers the repositories it touched. With the administrator's login:
|
||||
/** Put main's rule right in every repository of an organisation (novox/hq ADR 0398 §2, issue 948): no push, and
|
||||
* only the tools' account merges. The rule there is edited, its statuses left as they are; a repository with no
|
||||
* rule for main gets one that requires none. Answers the repositories it touched. With the administrator's login:
|
||||
* the tools' account is no repository's administrator, so no tool can do this (ADR 0365 §2). */
|
||||
async ensureMainMerges(org: string): Promise<string[]> {
|
||||
const repos: string[] = [];
|
||||
@@ -954,11 +954,11 @@ export class GiteaAdmin {
|
||||
if (patch.status !== 200) GiteaAdmin.fail(rule, patch);
|
||||
continue;
|
||||
}
|
||||
// A rule named exactly main is matched before any glob rule, so it would hide the approvals and administrator
|
||||
// limit a glob rule covering main asks: they are copied into it (issue 948, review round 1). Its statuses are
|
||||
// not: main requires none (ADR 0534 §1).
|
||||
// A rule named exactly main is matched before any glob rule, so it would hide the statuses, approvals and
|
||||
// administrator limit a glob rule covering main asks: they are copied into it (issue 948, review round 1).
|
||||
const covering = rules.filter((r) => ruleCovers(String(r.rule_name ?? r.branch_name ?? ""), MAIN_BRANCH));
|
||||
const copied: Record<string, unknown> = {};
|
||||
const checks = [...new Set(covering.filter((r) => r.enable_status_check).flatMap((r) => (r.status_check_contexts ?? []).filter(Boolean)))];
|
||||
const copied: Record<string, unknown> = { enable_status_check: checks.length > 0, status_check_contexts: checks };
|
||||
const approvals = Math.max(0, ...covering.map((r) => Number(r.required_approvals ?? 0)));
|
||||
if (approvals > 0) copied.required_approvals = approvals;
|
||||
if (covering.some((r) => r.block_admin_merge_override === true)) copied.block_admin_merge_override = true;
|
||||
|
||||
@@ -1,135 +1,134 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The tests of ADR 0534 §1, §2 (the operator, 2026-10-23): the forge requires no status to merge. Main's rule keeps no
|
||||
// push, no push whitelist, the tools' account alone in its merge whitelist and no administrator merging past it, and
|
||||
// requires no status: the program writes it so on every repository it lists, and takes every required status off a
|
||||
// rule covering main in every repository of the organisation it does not list. Nothing else of an unlisted
|
||||
// repository's rule is sent.
|
||||
// The tests of issue 1301 (ADR 0447 §1, §5; ADR 0503 §3, §5; lead decisions L4 and L11): main requires `mesh/check`
|
||||
// alone in place of `mesh/merge-gate` and `mesh/repo-check`; a main that requires `mesh/repo-check` without
|
||||
// `mesh/merge-gate` (hq) keeps it; every repository of the organisation whose main requires `mesh/merge-gate` is
|
||||
// switched, listed in the value or not; no other status is dropped.
|
||||
|
||||
func narrowedRule(contexts ...any) map[string]any {
|
||||
return map[string]any{
|
||||
"enable_push": false, "enable_push_whitelist": false, "push_whitelist_usernames": []any{},
|
||||
"enable_merge_whitelist": true, "merge_whitelist_usernames": []any{tools},
|
||||
"enable_status_check": len(contexts) > 0, "status_check_contexts": contexts,
|
||||
"enable_status_check": true, "status_check_contexts": contexts,
|
||||
"block_admin_merge_override": true,
|
||||
}
|
||||
}
|
||||
|
||||
// requiresNone is whether a rule as the fake forge keeps it requires no status.
|
||||
func requiresNone(rule map[string]any) bool {
|
||||
return rule["enable_status_check"] != true && len(strs(rule["status_check_contexts"])) == 0
|
||||
}
|
||||
|
||||
// 0534-1: a listed repository whose main requires statuses has them all taken off, the rest of the rule kept; a
|
||||
// second pass changes nothing.
|
||||
func TestListedRepositoryRequiresNoStatus(t *testing.T) {
|
||||
// 1301-S1: a listed repository whose main requires the two old statuses is put right to `mesh/check` in their place,
|
||||
// every other status kept; a second pass changes nothing.
|
||||
func TestListedRepositorySwapsTheTwoStatusesForCheck(t *testing.T) {
|
||||
f := newFake()
|
||||
repo := f.add("novox/mesh-delivery", map[string]map[string]any{mainRule: narrowedRule("mesh/merge-gate", "mesh/repo-check", "other/x")}, "main")
|
||||
checked := f.add("novox/checked", map[string]map[string]any{mainRule: narrowedRule("mesh/check")}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{"mesh-delivery": {}, "checked": {}})
|
||||
for name, r := range map[string]*fRepo{"mesh-delivery": repo, "checked": checked} {
|
||||
rule := r.Rules[mainRule]
|
||||
if !requiresNone(rule) {
|
||||
t.Errorf("%s's main still requires %v", name, rule["status_check_contexts"])
|
||||
}
|
||||
if rule["enable_push"] != false || rule["enable_merge_whitelist"] != true ||
|
||||
strings.Join(strs(rule["merge_whitelist_usernames"]), ",") != tools || rule["block_admin_merge_override"] != true {
|
||||
t.Errorf("%s's main lost the rest of its rule: %v", name, rule)
|
||||
}
|
||||
if r := result(o, name); r.Did != DidPutRight {
|
||||
t.Errorf("%s outcome %+v, want put right", name, r)
|
||||
}
|
||||
o := run(t, s, Value{"mesh-delivery": {}})
|
||||
got := strs(repo.Rules[mainRule]["status_check_contexts"])
|
||||
if strings.Join(got, ",") != "other/x,mesh/check" {
|
||||
t.Errorf("main requires %v, want other/x and mesh/check", got)
|
||||
}
|
||||
if r := result(o, "mesh-delivery"); r.Did != DidPutRight {
|
||||
t.Errorf("outcome %+v, want put right", r)
|
||||
}
|
||||
before := len(f.writes())
|
||||
o = run(t, s, Value{"mesh-delivery": {}, "checked": {}})
|
||||
run(t, s, Value{"mesh-delivery": {}})
|
||||
if w := f.writes(); len(w) != before {
|
||||
t.Errorf("a second pass wrote %v", w[before:])
|
||||
}
|
||||
if result(o, "checked").Did != DidUnchanged {
|
||||
t.Errorf("second outcome %+v, want unchanged", result(o, "checked"))
|
||||
}
|
||||
}
|
||||
|
||||
// 0534-1: a repository the program makes requires no status from its first rule in the organisation on.
|
||||
func TestNewRepositoryRequiresNoStatus(t *testing.T) {
|
||||
// 1301-S1: a repository the program makes requires `mesh/check` alone from its first rule in the organisation on.
|
||||
func TestNewRepositoryRequiresCheckAlone(t *testing.T) {
|
||||
f := newFake()
|
||||
s, _ := syncer(t, f)
|
||||
run(t, s, Value{"fresh": {Description: "d"}})
|
||||
rule := f.repos["novox/fresh"].Rules[mainRule]
|
||||
if !requiresNone(rule) {
|
||||
t.Errorf("a new repository's main requires %v, want none", rule["status_check_contexts"])
|
||||
if got := strs(rule["status_check_contexts"]); rule["enable_status_check"] != true || strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("a new repository's main requires %v, want %s alone", got, CheckStatus)
|
||||
}
|
||||
if readme := f.repos["novox/fresh"].Files["README.md"]; strings.Contains(readme, "mesh/merge-gate") || strings.Contains(readme, "mesh/repo-check") {
|
||||
if readme := f.repos["novox/fresh"].Files["README.md"]; !strings.Contains(readme, CheckStatus) ||
|
||||
strings.Contains(readme, "mesh/merge-gate") || strings.Contains(readme, "mesh/repo-check") {
|
||||
t.Errorf("the README names the statuses of before: %q", readme)
|
||||
}
|
||||
}
|
||||
|
||||
// 0534-1: a listed repository whose main requires nothing stays so, and is not written when the rest is right.
|
||||
func TestListedRuleRequiringNothingIsRight(t *testing.T) {
|
||||
// 1301-S1: mesh/check is never dropped, and a rule that requires it beside a retired status loses the retired one.
|
||||
func TestCheckIsKeptAndARetiredStatusBesideItGoes(t *testing.T) {
|
||||
f := newFake()
|
||||
none := f.add("novox/none", map[string]map[string]any{mainRule: narrowedRule()}, "main")
|
||||
both := f.add("novox/both", map[string]map[string]any{mainRule: narrowedRule("mesh/check", "mesh/repo-check")}, "main")
|
||||
right := f.add("novox/right", map[string]map[string]any{mainRule: narrowedRule("mesh/check")}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{"none": {}})
|
||||
if !requiresNone(none.Rules[mainRule]) {
|
||||
t.Errorf("none requires %v", none.Rules[mainRule]["status_check_contexts"])
|
||||
o := run(t, s, Value{"both": {}, "right": {}})
|
||||
if got := strs(both.Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("both requires %v, want %s alone", got, CheckStatus)
|
||||
}
|
||||
if result(o, "none").Did != DidUnchanged {
|
||||
t.Errorf("outcome %+v, want unchanged", result(o, "none"))
|
||||
if got := strs(right.Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("right requires %v", got)
|
||||
}
|
||||
for _, w := range f.writes() {
|
||||
t.Errorf("a right rule was written: %s", w)
|
||||
if result(o, "right").Did != DidUnchanged || result(o, "both").Did != DidPutRight {
|
||||
t.Errorf("outcome %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// 0534-1: hq, whose main required mesh/repo-check alone, requires no status either, listed or not.
|
||||
func TestOwnCheckAloneGoesToo(t *testing.T) {
|
||||
// 1301-S1: a listed repository whose main requires nothing is given mesh/check, not the old two.
|
||||
func TestListedRuleRequiringNothingGetsCheck(t *testing.T) {
|
||||
f := newFake()
|
||||
none := f.add("novox/none", map[string]map[string]any{mainRule: {"enable_push": false}}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
run(t, s, Value{"none": {}})
|
||||
if got := strs(none.Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("none requires %v, want %s", got, CheckStatus)
|
||||
}
|
||||
}
|
||||
|
||||
// 1301-S2 (ADR 0503 §5): a main that requires mesh/repo-check without mesh/merge-gate is judged by its own script
|
||||
// alone (hq): it keeps mesh/repo-check and is not given mesh/check, listed or not.
|
||||
func TestOwnCheckAloneIsKept(t *testing.T) {
|
||||
f := newFake()
|
||||
hq := f.add("novox/hq", map[string]map[string]any{mainRule: narrowedRule("mesh/repo-check")}, "main")
|
||||
lab := f.add("novox/mesh-lab", map[string]map[string]any{mainRule: narrowedRule("mesh/repo-check")}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{"hq": {}})
|
||||
for name, repo := range map[string]*fRepo{"hq": hq, "mesh-lab": lab} {
|
||||
if !requiresNone(repo.Rules[mainRule]) {
|
||||
t.Errorf("%s requires %v, want none", name, repo.Rules[mainRule]["status_check_contexts"])
|
||||
}
|
||||
if r := result(o, name); r.Did != DidPutRight {
|
||||
t.Errorf("%s outcome %+v, want put right", name, r)
|
||||
if got := strs(repo.Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != "mesh/repo-check" {
|
||||
t.Errorf("%s requires %v, want mesh/repo-check alone", name, got)
|
||||
}
|
||||
}
|
||||
if result(o, "hq").Did != DidUnchanged {
|
||||
t.Errorf("hq outcome %+v, want unchanged", result(o, "hq"))
|
||||
}
|
||||
for _, w := range f.writes() {
|
||||
t.Errorf("a repository judged by its own check alone was written: %s", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 0534-1: a repository the value does not list whose rule covering main requires any status has them taken off, and
|
||||
// only its statuses: nothing else of its rule is sent. One that requires none is not written to.
|
||||
func TestUnlistedRepositoryRequiringAStatusHasItTakenOff(t *testing.T) {
|
||||
// 1301-S3 (L4, L11): a repository the value does not list whose main requires mesh/merge-gate has the two old
|
||||
// statuses swapped for mesh/check, and only its statuses: nothing else of its rule is sent. One that requires
|
||||
// neither, or mesh/repo-check alone, is not written to.
|
||||
func TestUnlistedRepositoryRequiringTheGateIsSwitched(t *testing.T) {
|
||||
f := newFake()
|
||||
catalog := f.add("novox/mesh-catalog", map[string]map[string]any{mainRule: {
|
||||
"enable_push": true, "enable_status_check": true, "status_check_contexts": []any{"mesh/check"},
|
||||
"enable_push": true, "enable_status_check": true, "status_check_contexts": []any{"mesh/merge-gate", "mesh/repo-check"},
|
||||
}}, "main")
|
||||
toolsGo := f.add("novox/mesh-tools-go", map[string]map[string]any{mainRule: narrowedRule("mesh/merge-gate")}, "main")
|
||||
f.add("novox/plain", map[string]map[string]any{mainRule: {"enable_push": false}}, "main")
|
||||
f.add("novox/unticked", map[string]map[string]any{mainRule: {"enable_push": false, "enable_status_check": false,
|
||||
"status_check_contexts": []any{"mesh/check"}}}, "main")
|
||||
f.add("novox/hq", map[string]map[string]any{mainRule: narrowedRule("mesh/repo-check")}, "main")
|
||||
f.add("novox/norule", nil, "main")
|
||||
f.add("mesh-admin/elsewhere", map[string]map[string]any{mainRule: narrowedRule("mesh/merge-gate")}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{})
|
||||
|
||||
if !requiresNone(catalog.Rules[mainRule]) {
|
||||
t.Errorf("mesh-catalog requires %v, want none", catalog.Rules[mainRule]["status_check_contexts"])
|
||||
if got := strs(catalog.Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("mesh-catalog requires %v, want %s", got, CheckStatus)
|
||||
}
|
||||
if catalog.Rules[mainRule]["enable_push"] != true {
|
||||
t.Errorf("an unlisted repository's rule was changed beyond its statuses: %v", catalog.Rules[mainRule])
|
||||
t.Errorf("an unlisted repository's rule was narrowed beyond its statuses: %v", catalog.Rules[mainRule])
|
||||
}
|
||||
if !requiresNone(toolsGo.Rules[mainRule]) {
|
||||
t.Errorf("mesh-tools-go requires %v, want none", toolsGo.Rules[mainRule]["status_check_contexts"])
|
||||
if got := strs(toolsGo.Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("mesh-tools-go requires %v, want %s", got, CheckStatus)
|
||||
}
|
||||
want := []string{
|
||||
"PATCH /api/v1/repos/novox/mesh-catalog/branch_protections/main",
|
||||
@@ -143,7 +142,7 @@ func TestUnlistedRepositoryRequiringAStatusHasItTakenOff(t *testing.T) {
|
||||
t.Errorf("%s outcome %+v, want put right", n, r)
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"plain", "unticked", "norule", "elsewhere"} {
|
||||
for _, n := range []string{"plain", "hq", "norule", "elsewhere"} {
|
||||
if r := result(o, n); r.Name != "" {
|
||||
t.Errorf("%s is in the outcome though nothing was done: %+v", n, r)
|
||||
}
|
||||
@@ -155,27 +154,27 @@ func TestUnlistedRepositoryRequiringAStatusHasItTakenOff(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 0534-1: the organisation's repositories are read page by page, so one past the first page is put right too.
|
||||
// 1301-S3: the organisation's repositories are read page by page, so one past the first page is switched too.
|
||||
func TestUnlistedRepositoriesAreReadPastTheFirstPage(t *testing.T) {
|
||||
f := newFake()
|
||||
f.pageSize = 2
|
||||
for _, n := range []string{"a", "b", "c", "d", "e"} {
|
||||
f.add("novox/"+n, map[string]map[string]any{mainRule: narrowedRule("mesh/check")}, "main")
|
||||
f.add("novox/"+n, map[string]map[string]any{mainRule: narrowedRule("mesh/merge-gate", "mesh/repo-check")}, "main")
|
||||
}
|
||||
s, _ := syncer(t, f)
|
||||
run(t, s, Value{})
|
||||
for _, n := range []string{"a", "b", "c", "d", "e"} {
|
||||
if rule := f.repos["novox/"+n].Rules[mainRule]; !requiresNone(rule) {
|
||||
t.Errorf("%s requires %v", n, rule["status_check_contexts"])
|
||||
if got := strs(f.repos["novox/"+n].Rules[mainRule]["status_check_contexts"]); strings.Join(got, ",") != CheckStatus {
|
||||
t.Errorf("%s requires %v", n, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 0534-1: a refused removal is said with the repository and its step, and the pass goes on to the next.
|
||||
func TestRefusedRemovalIsSaid(t *testing.T) {
|
||||
// 1301-S3: a refused swap is said with the repository and its step, and the pass goes on to the next.
|
||||
func TestRefusedSwapIsSaid(t *testing.T) {
|
||||
f := newFake()
|
||||
f.add("novox/a", map[string]map[string]any{mainRule: narrowedRule("mesh/check")}, "main")
|
||||
f.add("novox/b", map[string]map[string]any{mainRule: narrowedRule("mesh/check")}, "main")
|
||||
f.add("novox/a", map[string]map[string]any{mainRule: narrowedRule("mesh/merge-gate")}, "main")
|
||||
f.add("novox/b", map[string]map[string]any{mainRule: narrowedRule("mesh/merge-gate")}, "main")
|
||||
f.refuse = func(method, path string) (int, string) {
|
||||
if method == "PATCH" && strings.Contains(path, "/novox/a/") {
|
||||
return 403, "no"
|
||||
@@ -184,8 +183,8 @@ func TestRefusedRemovalIsSaid(t *testing.T) {
|
||||
}
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{})
|
||||
if r := result(o, "a"); r.Did != DidRefused || r.Step != StepNoStatus {
|
||||
t.Errorf("a: %+v, want refused at %s", r, StepNoStatus)
|
||||
if r := result(o, "a"); r.Did != DidRefused || r.Step != StepSwap {
|
||||
t.Errorf("a: %+v, want refused at %s", r, StepSwap)
|
||||
}
|
||||
if r := result(o, "b"); r.Did != DidPutRight {
|
||||
t.Errorf("b: %+v, want put right", r)
|
||||
@@ -214,50 +213,3 @@ func TestOrganisationListRefusedIsSaid(t *testing.T) {
|
||||
t.Errorf("the refused list is not in the outcome: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// 0534-2 (the operator, 2026-10-23 ~12:05): the operator's own forge login stays on main's merge whitelist beside the
|
||||
// tools' account, so the operator can always merge by hand at the forge; a rule naming the tools' account alone is put
|
||||
// right, and a second pass changes nothing.
|
||||
func TestTheOperatorsLoginStaysBesideTheTools(t *testing.T) {
|
||||
f := newFake()
|
||||
repo := f.add("novox/mesh-delivery", map[string]map[string]any{mainRule: narrowedRule()}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
s.Mergers = []string{"jschoubben", tools, "jschoubben"}
|
||||
o := run(t, s, Value{"mesh-delivery": {}, "fresh": {}})
|
||||
for name, rule := range map[string]map[string]any{"mesh-delivery": repo.Rules[mainRule], "fresh": f.repos["novox/fresh"].Rules[mainRule]} {
|
||||
if got := strings.Join(strs(rule["merge_whitelist_usernames"]), ","); got != tools+",jschoubben" {
|
||||
t.Errorf("%s's merge whitelist is %s, want %s and jschoubben", name, got, tools)
|
||||
}
|
||||
if !requiresNone(rule) || rule["enable_push"] != false {
|
||||
t.Errorf("%s's rule: %v", name, rule)
|
||||
}
|
||||
}
|
||||
if r := result(o, "mesh-delivery"); r.Did != DidPutRight {
|
||||
t.Errorf("outcome %+v, want put right", r)
|
||||
}
|
||||
before := len(f.writes())
|
||||
run(t, s, Value{"mesh-delivery": {}, "fresh": {}})
|
||||
if w := f.writes(); len(w) != before {
|
||||
t.Errorf("a second pass wrote %v", w[before:])
|
||||
}
|
||||
}
|
||||
|
||||
// 0534-2: the mergers are read from the module's runtime config file; a missing key is none; a name that is no forge login
|
||||
// is refused.
|
||||
func TestReadMergers(t *testing.T) {
|
||||
p := filepath.Join(t.TempDir(), "r.json")
|
||||
_ = os.WriteFile(p, []byte(`{"url":"","main-mergers":["jschoubben"]}`), 0o600)
|
||||
if m, err := ReadMergers(p); err != nil || strings.Join(m, ",") != "jschoubben" {
|
||||
t.Fatalf("%v %v", m, err)
|
||||
}
|
||||
_ = os.WriteFile(p, []byte(`{"url":""}`), 0o600)
|
||||
if m, err := ReadMergers(p); err != nil || len(m) != 0 {
|
||||
t.Fatalf("%v %v", m, err)
|
||||
}
|
||||
for _, bad := range []string{`["a b"]`, `["../x"]`, `[""]`, `"jschoubben"`} {
|
||||
_ = os.WriteFile(p, []byte(`{"main-mergers":`+bad+`}`), 0o600)
|
||||
if _, err := ReadMergers(p); err == nil {
|
||||
t.Errorf("%s was accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,8 +126,7 @@ func lets(rule map[string]any, login string) bool {
|
||||
// and in the organisation the tools' team does (it is on every repository there); nobody else here.
|
||||
func reaches(full, login string) bool {
|
||||
owner := strings.SplitN(full, "/", 2)[0]
|
||||
// The operator's own login is an owner of the organisation, so it reaches every repository there (ADR 0534 §2).
|
||||
return owner == login || (owner == org && (login == tools || login == "jschoubben"))
|
||||
return owner == login || (owner == org && login == tools)
|
||||
}
|
||||
|
||||
// keepReaching drops from a rule's merge whitelist every user without access to the repository, as gitea's
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
// again whenever the value changes: it reads the file the mesh renders from that value and, with the setup login,
|
||||
// makes every listed repository that does not exist under that login's own account, protects its main, writes
|
||||
// main's first commit, narrows main's rule and only then moves it into the organisation; one that exists only has
|
||||
// its main rule put right, requiring no status (novox/hq ADR 0534 §1). A repository the value does not list is
|
||||
// touched only to take the required statuses off a rule covering its main. It keeps what it did, per
|
||||
// its main rule put right, requiring mesh/check in place of the two statuses it replaces (issue 1301). A repository
|
||||
// the value does not list is touched only to swap those two for mesh/check when its main requires mesh/merge-gate. It keeps what it did, per
|
||||
// repository, in the outcome file, says each change on stdout in one line and never a secret, and exits 1 when a
|
||||
// step was refused, its last stderr lines naming each (outcome.go Report): the node-engine's gate on a run-once
|
||||
// step (ADR 0136) reports it and runs it again on every apply until it passes.
|
||||
@@ -30,8 +30,7 @@ func main() {
|
||||
func runSync() int {
|
||||
forge, missing := forgehttp.FromEnv()
|
||||
file, outcome := os.Getenv("MESH_REPOSITORIES_FILE"), os.Getenv("MESH_REPOSITORIES_OUTCOME")
|
||||
for k, v := range map[string]string{"MESH_REPOSITORIES_FILE": file, "MESH_REPOSITORIES_OUTCOME": outcome,
|
||||
"MESH_GITEA_CONFIG_FILE": os.Getenv("MESH_GITEA_CONFIG_FILE")} {
|
||||
for k, v := range map[string]string{"MESH_REPOSITORIES_FILE": file, "MESH_REPOSITORIES_OUTCOME": outcome} {
|
||||
if v == "" {
|
||||
missing = append(missing, k)
|
||||
}
|
||||
@@ -57,11 +56,6 @@ func runSync() int {
|
||||
_ = WriteOutcome(outcome, o)
|
||||
return Report(o, os.Stderr)
|
||||
}
|
||||
if s.Mergers, err = ReadMergers(os.Getenv("MESH_GITEA_CONFIG_FILE")); err != nil {
|
||||
o := Outcome{At: time.Now().UTC(), Org: s.Org, Failed: err.Error()}
|
||||
_ = WriteOutcome(outcome, o)
|
||||
return Report(o, os.Stderr)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
o, _ := s.Sync(ctx, value) // a forge not reachable is in o.Failed
|
||||
|
||||
@@ -64,7 +64,6 @@ func TestManifestDeclaresTheValueAndItsTrigger(t *testing.T) {
|
||||
if v, err := ReadValue(filled); err != nil || len(v) != 1 || v["mesh-delivery"].Description != "d" {
|
||||
t.Errorf("the filled template reads as %v, %v; want mesh-delivery", v, err)
|
||||
}
|
||||
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "process" && has(strs(r["restart-on"]), file["id"].(string)) {
|
||||
process = r
|
||||
@@ -106,61 +105,3 @@ func TestManifestDeclaresTheValueAndItsTrigger(t *testing.T) {
|
||||
t.Errorf("repositories is declared unlike agent-read-teams: %v vs %v", v, teams)
|
||||
}
|
||||
}
|
||||
|
||||
// 0534-2: the manifest declares main-mergers as a trusted list naming the operator's login by default, merges it into
|
||||
// the module's runtime config file (with url, which the TypeScript part reads), and the repositories program is given
|
||||
// that file and runs again when it changes.
|
||||
func TestManifestDeclaresTheMainMergers(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Config map[string]map[string]any `json:"config"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := m.Config[MergersKey]
|
||||
if v == nil || v["type"] != "list" || v["trusted"] != true || strings.Join(strs(v["default"]), ",") != "jschoubben" {
|
||||
t.Fatalf("config %s = %v, want a trusted list defaulting to the operator's login", MergersKey, v)
|
||||
}
|
||||
var cfg, process map[string]any
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == "runtime-config" {
|
||||
cfg = r
|
||||
}
|
||||
if r["name"] == "gitea-repositories" {
|
||||
process = r
|
||||
}
|
||||
}
|
||||
if cfg == nil || !has(strs(cfg["takes-config"]), MergersKey) || cfg["merge"] != "json" {
|
||||
t.Fatalf("runtime-config does not take %s: %v", MergersKey, cfg)
|
||||
}
|
||||
text, err := os.ReadFile("../../" + cfg["template"].(string))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file := filepath.Join(t.TempDir(), "config.json")
|
||||
_ = os.WriteFile(file, text, 0o600)
|
||||
if got, err := ReadMergers(file); err != nil || strings.Join(got, ",") != "jschoubben" {
|
||||
t.Errorf("the template's mergers read as %v, %v; want the default", got, err)
|
||||
}
|
||||
env, _ := process["env"].(map[string]any)
|
||||
if env["MESH_GITEA_CONFIG_FILE"] != cfg["path"] {
|
||||
t.Errorf("the repositories program reads %v, want %v", env["MESH_GITEA_CONFIG_FILE"], cfg["path"])
|
||||
}
|
||||
if !has(strs(process["restart-on"]), "runtime-config") {
|
||||
t.Errorf("the repositories program does not run again when the mergers change: %v", process["restart-on"])
|
||||
}
|
||||
}
|
||||
|
||||
func has(list []string, want string) bool {
|
||||
for _, s := range list {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1141/AC1: a repository the program makes gets main's rule with an administrator's merge past it refused,
|
||||
// 1141/AC1: a repository the program makes gets main's rule with an administrator's merge past a status refused,
|
||||
// as every repository's main has had since 2026-10-07.
|
||||
func TestNewRepositoryRefusesTheAdministratorsOverride(t *testing.T) {
|
||||
f := newFake()
|
||||
@@ -18,7 +18,7 @@ func TestNewRepositoryRefusesTheAdministratorsOverride(t *testing.T) {
|
||||
}
|
||||
|
||||
// 1141/AC2: a narrowed main rule that still lets an administrator override is not right: it is put right, its
|
||||
// statuses taken off (ADR 0534 §1), and a second pass changes nothing.
|
||||
// statuses kept (the two old ones as mesh/check, issue 1301), and a second pass changes nothing.
|
||||
func TestNarrowedRuleLettingTheAdministratorOverrideIsPutRight(t *testing.T) {
|
||||
f := newFake()
|
||||
repo := f.add("novox/mesh-delivery", map[string]map[string]any{mainRule: {
|
||||
@@ -33,8 +33,8 @@ func TestNarrowedRuleLettingTheAdministratorOverrideIsPutRight(t *testing.T) {
|
||||
if rule["block_admin_merge_override"] != true {
|
||||
t.Errorf("the administrator's override is still allowed: %v", rule)
|
||||
}
|
||||
if !requiresNone(rule) {
|
||||
t.Errorf("statuses %v, want none (ADR 0534 §1)", rule["status_check_contexts"])
|
||||
if c := strs(rule["status_check_contexts"]); strings.Join(c, ",") != "other/x,mesh/check" {
|
||||
t.Errorf("statuses %v, want other/x kept and mesh/check for the two it replaces (issue 1301)", c)
|
||||
}
|
||||
if r := result(o, "mesh-delivery"); r.Did != DidPutRight {
|
||||
t.Errorf("outcome %+v, want put right", r)
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -27,11 +26,46 @@ const (
|
||||
MainBranch = "main"
|
||||
)
|
||||
|
||||
// noStatus is what main's rule says of statuses: none is required to merge (novox/hq ADR 0534 §1, the operator,
|
||||
// 2026-10-23). The checks still run and report; mesh-delivery asks the operator's Approve on a passing verdict and the
|
||||
// merge tool judges it at the merge (ADR 0534 §3, §4). The forge enforces none of them.
|
||||
func noStatus() map[string]any {
|
||||
return map[string]any{"enable_status_check": false, "status_check_contexts": []string{}}
|
||||
// CheckStatus is the one status main requires (ADR 0447 §1, issue 1301): the checker's verdict on a pull request's
|
||||
// head merged onto its trunk's head. It replaces GateStatus and OwnCheckStatus, and is never dropped.
|
||||
const CheckStatus = "mesh/check"
|
||||
|
||||
// GateStatus and OwnCheckStatus are the two statuses CheckStatus replaces: the controller's merge gate and the
|
||||
// repository's own merge-check.sh (ADR 0238 §4, ADR 0428 §5 before ADR 0447 §1).
|
||||
const (
|
||||
GateStatus = "mesh/merge-gate"
|
||||
OwnCheckStatus = "mesh/repo-check"
|
||||
)
|
||||
|
||||
// wanted is what a main rule requiring have should require: have with GateStatus and OwnCheckStatus replaced by
|
||||
// CheckStatus, every other status kept in its order (ADR 0447 §1). A rule requiring OwnCheckStatus with neither
|
||||
// GateStatus nor CheckStatus is judged by its own script alone, as hq is until issue 1296 settles how hq is
|
||||
// checked (ADR 0503 §5): the checker writes no status there (ADR 0484 §8), so it is left as it is. This reads the
|
||||
// rule, not a list of names (lead decision L11 of R4e).
|
||||
func wanted(have []string) []string {
|
||||
if !has(have, GateStatus) && !has(have, CheckStatus) && has(have, OwnCheckStatus) {
|
||||
return have
|
||||
}
|
||||
out := []string{}
|
||||
for _, s := range have {
|
||||
if s != GateStatus && s != OwnCheckStatus && s != CheckStatus {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return append(out, CheckStatus)
|
||||
}
|
||||
|
||||
func has(list []string, want string) bool {
|
||||
for _, s := range list {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func sameList(a, b []string) bool {
|
||||
return strings.Join(a, "\x00") == strings.Join(b, "\x00")
|
||||
}
|
||||
|
||||
// The steps of one repository, as an outcome names a refused one (ADR 0428 §6).
|
||||
@@ -48,8 +82,8 @@ const (
|
||||
StepRuleAfterMove = "rule-after-move"
|
||||
// StepList reads the organisation's repositories, for those the value does not list (issue 1301).
|
||||
StepList = "list"
|
||||
// StepNoStatus takes the required statuses off main's rule of a repository the value does not list (ADR 0534 §1).
|
||||
StepNoStatus = "no-status"
|
||||
// StepSwap swaps the two old statuses for CheckStatus on a repository the value does not list (issue 1301).
|
||||
StepSwap = "swap"
|
||||
)
|
||||
|
||||
// What a pass did with one repository.
|
||||
@@ -72,37 +106,6 @@ type Repo struct {
|
||||
// Value is the config value: each repository of the organisation by name.
|
||||
type Value map[string]Repo
|
||||
|
||||
// MergersKey is the config value naming the forge logins that merge into main beside the tools' account (ADR 0534 §2).
|
||||
const MergersKey = "main-mergers"
|
||||
|
||||
// loginName is a forge login: what a merge whitelist may name.
|
||||
var loginName = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,39}$`)
|
||||
|
||||
// ReadMergers reads the main-mergers value from the module's runtime config file, which the mesh merges the value into
|
||||
// (module.json runtime-config); a file without the key names none.
|
||||
func ReadMergers(file string) ([]string, error) {
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the module's config file cannot be read (%s): %w", file, err)
|
||||
}
|
||||
var whole map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &whole); err != nil {
|
||||
return nil, fmt.Errorf("the module's config file %s is not JSON: %w", file, err)
|
||||
}
|
||||
var out []string
|
||||
if r, ok := whole[MergersKey]; ok && string(r) != "null" {
|
||||
if err := json.Unmarshal(r, &out); err != nil {
|
||||
return nil, fmt.Errorf("%s in %s is not a list of forge logins: %w", MergersKey, file, err)
|
||||
}
|
||||
}
|
||||
for _, n := range out {
|
||||
if !loginName.MatchString(n) || strings.Contains(n, "..") {
|
||||
return nil, fmt.Errorf("%s in %s lists %q, which is not a forge login", MergersKey, file, n)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ReadValue reads the file the mesh renders from the config value. A file without the key is the empty value,
|
||||
// which makes nothing and touches nothing.
|
||||
func ReadValue(file string) (Value, error) {
|
||||
@@ -163,11 +166,8 @@ type Syncer struct {
|
||||
Forge forgehttp.Client // the setup login's
|
||||
Org string // the organisation, novox
|
||||
Admin string // the setup login, under whose own account a repository is made
|
||||
Tools string // the tools' account, which merges into main for mesh-delivery
|
||||
// Mergers are the forge logins that merge into main beside the tools' account: the operator's own, so the operator
|
||||
// can always merge by hand at the forge (ADR 0534 §2; the gitea module's config value main-mergers).
|
||||
Mergers []string
|
||||
Log func(format string, a ...any)
|
||||
Tools string // the tools' account, the only one that merges into main
|
||||
Log func(format string, a ...any)
|
||||
// WaitFor is how long to wait for the forge to answer; none when zero.
|
||||
WaitFor, Poll time.Duration
|
||||
}
|
||||
@@ -203,7 +203,7 @@ func (s *Syncer) Sync(ctx context.Context, v Value) (Outcome, error) {
|
||||
if r.Did == DidRefused {
|
||||
s.say("refused: %s/%s at the step %s: %s", s.Org, r.Name, r.Step, r.Said)
|
||||
} else {
|
||||
s.say("%s: %s/%s (main requires no status)", r.Did, s.Org, r.Name)
|
||||
s.say("%s: %s/%s (%s in place of %s and %s)", r.Did, s.Org, r.Name, CheckStatus, GateStatus, OwnCheckStatus)
|
||||
}
|
||||
out.Repositories = append(out.Repositories, r)
|
||||
}
|
||||
@@ -213,10 +213,12 @@ func (s *Syncer) Sync(ctx context.Context, v Value) (Outcome, error) {
|
||||
// orgPage is how many repositories one read of the organisation's list asks for.
|
||||
const orgPage = 50
|
||||
|
||||
// switchUnlisted takes every required status off a rule covering main in every repository of the organisation the
|
||||
// value does not list (ADR 0534 §1): the forge requires no status to merge anywhere, with no list kept by hand. Only
|
||||
// the statuses are sent; nothing else of a rule the value does not hold is changed (ADR 0428 §7). A main whose rules
|
||||
// require no status gets no call. It answers only what it did: a repository put right, or refused.
|
||||
// switchUnlisted swaps the two old statuses for CheckStatus on main's rule of every repository of the organisation
|
||||
// the value does not list whose main requires GateStatus (issue 1301, lead decisions L4 and L11 of R4e): the
|
||||
// repositories the merge gate judges are switched in the delivery that has the checker write mesh/check (ADR 0447
|
||||
// §5), with no list kept by hand. Only the statuses of a rule covering main are sent; nothing else of a rule the
|
||||
// value does not hold is changed (ADR 0428 §7 for all of it but the statuses the mesh itself requires). A main that
|
||||
// does not require GateStatus gets no call. It answers only what it did: a repository switched, or refused.
|
||||
func (s *Syncer) switchUnlisted(ctx context.Context, v Value) []Result {
|
||||
var names []string
|
||||
for page := 1; ; page++ {
|
||||
@@ -246,12 +248,13 @@ func (s *Syncer) switchUnlisted(ctx context.Context, v Value) []Result {
|
||||
}
|
||||
var did *Result
|
||||
for _, r := range rules {
|
||||
if ok, _ := path.Match(ruleName(r), MainBranch); !ok || len(required(r)) == 0 {
|
||||
if ok, _ := path.Match(ruleName(r), MainBranch); !ok || !has(required(r), GateStatus) {
|
||||
continue
|
||||
}
|
||||
body := map[string]any{"enable_status_check": true, "status_check_contexts": wanted(required(r))}
|
||||
if err := s.Forge.SendJSON(ctx, http.MethodPatch,
|
||||
repoAPI(s.Org, n)+"/branch_protections/"+url.PathEscape(ruleName(r)), noStatus(), nil); err != nil {
|
||||
did = &Result{Name: n, Did: DidRefused, Step: StepNoStatus, Said: err.Error()}
|
||||
repoAPI(s.Org, n)+"/branch_protections/"+url.PathEscape(ruleName(r)), body, nil); err != nil {
|
||||
did = &Result{Name: n, Did: DidRefused, Step: StepSwap, Said: err.Error()}
|
||||
break
|
||||
}
|
||||
did = &Result{Name: n, Did: DidPutRight}
|
||||
@@ -346,9 +349,9 @@ func (s *Syncer) one(ctx context.Context, name string, r Repo) Result {
|
||||
return refused(name, StepFirstCommit, err)
|
||||
}
|
||||
}
|
||||
if rule == nil || !isRight(rule, s.Tools, s.Mergers) {
|
||||
if rule == nil || !isRight(rule, s.Tools) {
|
||||
if err := s.Forge.SendJSON(ctx, http.MethodPatch, repoAPI(s.Admin, name)+"/branch_protections/"+MainBranch,
|
||||
narrowed(s.Tools, s.Mergers), nil); err != nil {
|
||||
narrowed(rule, s.Tools), nil); err != nil {
|
||||
return refused(name, StepNarrow, err)
|
||||
}
|
||||
}
|
||||
@@ -412,6 +415,24 @@ func strList(v any) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// statuses is what the rules require together, as wanted makes it.
|
||||
func statuses(rules ...rule) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
add := func(s string) {
|
||||
if !seen[s] {
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
for _, r := range rules {
|
||||
for _, s := range required(r) {
|
||||
add(s)
|
||||
}
|
||||
}
|
||||
return wanted(out)
|
||||
}
|
||||
|
||||
// required is what a rule requires: its statuses when it checks them, else none.
|
||||
func required(r rule) []string {
|
||||
if r == nil || r["enable_status_check"] != true {
|
||||
@@ -420,58 +441,46 @@ func required(r rule) []string {
|
||||
return strList(r["status_check_contexts"])
|
||||
}
|
||||
|
||||
// narrowed is main's rule of ADR 0398 §2 (protection.ts mainRuleBody): no push, only the tools' account merges, no
|
||||
// administrator merging past the rule, as every repository's main has been since 2026-10-07 (issue 1141), and no
|
||||
// status required (ADR 0534 §1). A field the forge keeps beside these is left as it is.
|
||||
func narrowed(toolsAccount string, mergers []string) map[string]any {
|
||||
body := map[string]any{
|
||||
// narrowed is main's rule of ADR 0398 §2 (protection.ts mainRuleBody): no push, only the tools' account merges,
|
||||
// requiring the statuses (ADR 0428 §5), and no administrator merging past them, as every repository's main has been
|
||||
// since 2026-10-07 (issue 1141). A field the forge keeps beside these is left as it is.
|
||||
func narrowed(r rule, toolsAccount string, also ...rule) map[string]any {
|
||||
return map[string]any{
|
||||
"enable_push": false, "enable_push_whitelist": false, "push_whitelist_usernames": []string{},
|
||||
"enable_merge_whitelist": true, "merge_whitelist_usernames": whitelist(toolsAccount, mergers),
|
||||
"enable_merge_whitelist": true, "merge_whitelist_usernames": []string{toolsAccount},
|
||||
"enable_status_check": true, "status_check_contexts": statuses(append([]rule{r}, also...)...),
|
||||
"block_admin_merge_override": true,
|
||||
}
|
||||
for k, v := range noStatus() {
|
||||
body[k] = v
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// whitelist is main's merge whitelist: the tools' account first, then each other merger once, in the value's order.
|
||||
func whitelist(toolsAccount string, mergers []string) []string {
|
||||
out := []string{toolsAccount}
|
||||
for _, m := range mergers {
|
||||
if !slices.Contains(out, m) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// isRight says whether a rule already is narrowed and requires no status.
|
||||
func isRight(r rule, toolsAccount string, mergers []string) bool {
|
||||
// isRight says whether a rule already is narrowed and requires what wanted makes of it.
|
||||
func isRight(r rule, toolsAccount string) bool {
|
||||
if r["enable_push"] != false || r["enable_push_whitelist"] == true || r["enable_merge_whitelist"] != true {
|
||||
return false
|
||||
}
|
||||
if m := strList(r["merge_whitelist_usernames"]); !slices.Equal(m, whitelist(toolsAccount, mergers)) {
|
||||
if m := strList(r["merge_whitelist_usernames"]); len(m) != 1 || m[0] != toolsAccount {
|
||||
return false
|
||||
}
|
||||
return r["enable_status_check"] != true && len(strList(r["status_check_contexts"])) == 0 &&
|
||||
have := strList(r["status_check_contexts"])
|
||||
return r["enable_status_check"] == true && len(have) > 0 && sameList(have, wanted(have)) &&
|
||||
r["block_admin_merge_override"] == true
|
||||
}
|
||||
|
||||
// putRight makes main's rule in an organisation's repository the narrowed one, requiring no status (ADR 0534 §1).
|
||||
// A repository with no rule named main gets one, carrying the approvals a glob rule covering main asks (client.ts
|
||||
// ensureMainMerges, issue 948). It never unprotects (ADR 0428 §7).
|
||||
// putRight makes main's rule in an organisation's repository the narrowed one, keeping every status it requires but
|
||||
// the two CheckStatus replaces (ADR 0447 §1).
|
||||
// A repository with no rule named main gets one, carrying what a glob rule covering main asks (client.ts
|
||||
// ensureMainMerges, issue 948). It takes away no other status and never unprotects (ADR 0428 §5, §7).
|
||||
func (s *Syncer) putRight(ctx context.Context, owner, name string) (bool, error) {
|
||||
rules, err := s.rules(ctx, owner, name)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if own := ruleNamed(rules, MainBranch); own != nil {
|
||||
if isRight(own, s.Tools, s.Mergers) {
|
||||
if isRight(own, s.Tools) {
|
||||
return false, nil
|
||||
}
|
||||
return true, s.Forge.SendJSON(ctx, http.MethodPatch, repoAPI(owner, name)+"/branch_protections/"+MainBranch,
|
||||
narrowed(s.Tools, s.Mergers), nil)
|
||||
narrowed(own, s.Tools), nil)
|
||||
}
|
||||
var covering []rule
|
||||
for _, r := range rules {
|
||||
@@ -479,7 +488,7 @@ func (s *Syncer) putRight(ctx context.Context, owner, name string) (bool, error)
|
||||
covering = append(covering, r)
|
||||
}
|
||||
}
|
||||
body := narrowed(s.Tools, s.Mergers)
|
||||
body := narrowed(nil, s.Tools, covering...)
|
||||
body["rule_name"] = MainBranch
|
||||
approvals := 0.0
|
||||
for _, r := range covering {
|
||||
|
||||
@@ -19,7 +19,7 @@ const mainRule = "main"
|
||||
// 1124/AC2: a listed repository that does not exist is made under the setup account, given main's rule letting that
|
||||
// login alone push, given its first commit, narrowed to no push and the tools' merge requiring mesh/check (1124/AC5
|
||||
// as issue 1301 changes it), and only then moved into the organisation. Nothing reaches the organisation before the
|
||||
// narrowed rule, and nothing else is pushed to main. Main requires no status (ADR 0534 §1).
|
||||
// narrowed rule, and nothing else is pushed to main.
|
||||
func TestNewRepositoryIsMadeInOrder(t *testing.T) {
|
||||
f := newFake()
|
||||
s, _ := syncer(t, f)
|
||||
@@ -66,8 +66,9 @@ func TestNewRepositoryIsMadeInOrder(t *testing.T) {
|
||||
if rule["enable_merge_whitelist"] != true || strings.Join(strs(rule["merge_whitelist_usernames"]), ",") != tools {
|
||||
t.Errorf("main's merge whitelist is not the tools' account alone: %v", rule)
|
||||
}
|
||||
if !requiresNone(rule) {
|
||||
t.Errorf("main requires a status (ADR 0534 §1): %v", rule)
|
||||
checks := strs(rule["status_check_contexts"])
|
||||
if rule["enable_status_check"] != true || strings.Join(checks, ",") != CheckStatus {
|
||||
t.Errorf("main does not require mesh/check alone (1124/AC5, issue 1301): %v", rule)
|
||||
}
|
||||
if r := result(o, "mesh-delivery"); r.Did != DidMade || r.Step != "" {
|
||||
t.Errorf("outcome %+v, want made", r)
|
||||
@@ -86,7 +87,7 @@ func TestNewRepositoryIsMadeInOrder(t *testing.T) {
|
||||
|
||||
// 1124/AC2: the forge drops from a merge whitelist a user without access to the repository, and under the setup
|
||||
// account the tools' account has none; so once the repository is in the organisation its main rule is read back and
|
||||
// put right: the tools' account the only one that merges, no status required (ADR 0534 §1).
|
||||
// put right: the tools' account the only one that merges, mesh/check required.
|
||||
func TestOrganisationRuleListsTheToolsAfterTheMove(t *testing.T) {
|
||||
f := newFake()
|
||||
s, _ := syncer(t, f)
|
||||
@@ -102,8 +103,9 @@ func TestOrganisationRuleListsTheToolsAfterTheMove(t *testing.T) {
|
||||
if rule["enable_push"] != false {
|
||||
t.Errorf("after the move main takes a push: %v", rule)
|
||||
}
|
||||
if !requiresNone(rule) {
|
||||
t.Errorf("after the move main requires a status (ADR 0534 §1): %v", rule)
|
||||
checks := strs(rule["status_check_contexts"])
|
||||
if rule["enable_status_check"] != true || strings.Join(checks, ",") != CheckStatus {
|
||||
t.Errorf("after the move main does not require mesh/check alone: %v", rule)
|
||||
}
|
||||
if r := result(o, "mesh-delivery"); r.Did != DidMade {
|
||||
t.Errorf("outcome %+v, want made", r)
|
||||
@@ -134,7 +136,8 @@ func TestAdminRuleLetsTheSetupLoginAlonePush(t *testing.T) {
|
||||
}
|
||||
|
||||
// 1124/AC3: a listed repository that exists in the organisation is not made again, only its main rule put right;
|
||||
// a rule still letting the setup login push is narrowed, and requires no status (ADR 0534 §1).
|
||||
// a rule still letting the setup login push is narrowed, its statuses kept but the two mesh/check replaces (1124/AC5,
|
||||
// issue 1301).
|
||||
func TestExistingOrganisationRepositoryIsOnlyPutRight(t *testing.T) {
|
||||
f := newFake()
|
||||
hq := f.add("novox/hq", map[string]map[string]any{mainRule: {
|
||||
@@ -154,11 +157,12 @@ func TestExistingOrganisationRepositoryIsOnlyPutRight(t *testing.T) {
|
||||
if rule["enable_push"] != false || strings.Join(strs(rule["merge_whitelist_usernames"]), ",") != tools {
|
||||
t.Errorf("hq's rule not narrowed: %v", rule)
|
||||
}
|
||||
if !requiresNone(rule) {
|
||||
t.Errorf("hq's rule requires %v, want none (ADR 0534 §1)", rule["status_check_contexts"])
|
||||
checks := strs(rule["status_check_contexts"])
|
||||
if strings.Join(checks, ",") != "other/x,mesh/check" {
|
||||
t.Errorf("hq's rule requires %v, want other/x and mesh/check", checks)
|
||||
}
|
||||
made := bare.Rules[mainRule]
|
||||
if made == nil || made["enable_push"] != false || !requiresNone(made) {
|
||||
if made == nil || made["enable_push"] != false || strings.Join(strs(made["status_check_contexts"]), ",") != CheckStatus {
|
||||
t.Errorf("bare's rule: %v", made)
|
||||
}
|
||||
if hq.Commits != 0 || bare.Commits != 0 {
|
||||
@@ -208,33 +212,33 @@ func TestRepositoryUnderTheSetupAccountIsFinished(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0534 §1: putting a rule right takes every required status off it, whichever they are, and a rule already right
|
||||
// is not written.
|
||||
func TestStatusesAreTakenOff(t *testing.T) {
|
||||
// 1124/AC5 as issue 1301 changes it: putting a rule right adds mesh/check when the rule requires mesh/merge-gate,
|
||||
// and keeps every other status there; a rule requiring mesh/repo-check alone keeps it (ADR 0503 §5).
|
||||
func TestStatusesAreAddedAndNeverDropped(t *testing.T) {
|
||||
f := newFake()
|
||||
only := f.add("novox/only", map[string]map[string]any{mainRule: {
|
||||
"enable_push": false, "enable_merge_whitelist": true, "merge_whitelist_usernames": []any{tools},
|
||||
"enable_status_check": true, "status_check_contexts": []any{"mesh/repo-check"},
|
||||
}}, "main")
|
||||
right := f.add("novox/right", map[string]map[string]any{mainRule: {
|
||||
more := f.add("novox/more", map[string]map[string]any{mainRule: {
|
||||
"enable_push": false, "enable_merge_whitelist": true, "merge_whitelist_usernames": []any{tools},
|
||||
"enable_status_check": false, "status_check_contexts": []any{},
|
||||
"enable_status_check": true, "status_check_contexts": []any{"mesh/delivery", "mesh/check"},
|
||||
"block_admin_merge_override": true, // right since issue 1141
|
||||
}}, "main")
|
||||
s, _ := syncer(t, f)
|
||||
o := run(t, s, Value{"only": {}, "right": {}})
|
||||
if !requiresNone(only.Rules[mainRule]) {
|
||||
t.Errorf("only: %v", only.Rules[mainRule]["status_check_contexts"])
|
||||
o := run(t, s, Value{"only": {}, "more": {}})
|
||||
if c := strs(only.Rules[mainRule]["status_check_contexts"]); strings.Join(c, ",") != "mesh/repo-check" {
|
||||
t.Errorf("only: %v", c)
|
||||
}
|
||||
if !requiresNone(right.Rules[mainRule]) {
|
||||
t.Errorf("right: %v", right.Rules[mainRule]["status_check_contexts"])
|
||||
if c := strs(more.Rules[mainRule]["status_check_contexts"]); len(c) != 2 || !has(c, "mesh/delivery") {
|
||||
t.Errorf("more lost a status: %v", c)
|
||||
}
|
||||
for _, w := range f.writes() {
|
||||
if strings.Contains(w, "/right/") {
|
||||
if strings.Contains(w, "/more/") {
|
||||
t.Errorf("a right rule was written: %s", w)
|
||||
}
|
||||
}
|
||||
if result(o, "right").Did != DidUnchanged || result(o, "only").Did != DidPutRight {
|
||||
if result(o, "more").Did != DidUnchanged || result(o, "only").Did != DidPutRight {
|
||||
t.Errorf("outcome %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"main-mergers": ["jschoubben"]}
|
||||
{}
|
||||
|
||||
+18
-16
@@ -10,16 +10,15 @@
|
||||
// Everything else — administration, protection, settings, collaborators, teams, hooks, keys, tokens,
|
||||
// statuses, a merge, a repository deleted or moved — has its own tool or is the operator's, at the
|
||||
// forge as themselves;
|
||||
// - `gitea_branch_protection_set` lets no person push and no administrator merge past a rule, and on main keeps
|
||||
// the tools' account the only one that merges; it requires no status, as no rule of the mesh does (ADR 0534 §5);
|
||||
// - `gitea_merge_pull_request` merges only into a branch that refuses its push, only a head whose mesh/check passed
|
||||
// against the trunk's head, approved by the operator where the branch requires no status (ADR 0534 §4), and only
|
||||
// that head;
|
||||
// - `gitea_branch_protection_set` only makes protection stronger;
|
||||
// - `gitea_merge_pull_request` merges only a head whose every required status succeeded, and only that
|
||||
// head;
|
||||
// - `gitea_delete_repo` refuses a repository any protection rule guards.
|
||||
//
|
||||
// Pure, so it is tested without a forge; the tools do the asking (tools/index.ts).
|
||||
|
||||
import { MAIN_BRANCH, mainMergers, type BranchProtection, type ProtectionWanted } from "./protection.js";
|
||||
import { MAIN_BRANCH, type BranchProtection, type ProtectionWanted } from "./protection.js";
|
||||
import { TOOLS_ACCOUNT } from "./tools-account.js";
|
||||
|
||||
/** A refusal of policy, in the words the caller sees. Never an error of the forge's. */
|
||||
export class Refused extends Error {
|
||||
@@ -164,7 +163,7 @@ export function judgeApiCall(method: string, rawPath: string, body: unknown): Ap
|
||||
`that is not the default branch and that no protection rule names. The forge's administration, a ` +
|
||||
`branch's protection, a repository's settings, collaborators, teams, hooks, keys, tokens and statuses ` +
|
||||
`are the operator's at the forge, or have their own tool (gitea_merge_pull_request, gitea_commit_status, ` +
|
||||
`gitea_branch_protection_set, which never lets a person push). Nothing was sent`,
|
||||
`gitea_branch_protection_set, which only strengthens). Nothing was sent`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -288,7 +287,7 @@ export function judgeContentsWrite(target: string, defaultBranch: string, rules:
|
||||
* Refuses a protection change that weakens what guards a branch (novox/hq ADR 0266): the branch's own rule, or —
|
||||
* when it has none — every glob rule that covers it, since a rule named for the branch takes the glob's place.
|
||||
* A rule made new takes the forge's defaults for what it does not say: pushes refused, an administrator's
|
||||
* override allowed. A required status taken away is no weakening: no rule of the mesh requires one (ADR 0534 §5).
|
||||
* override allowed.
|
||||
*/
|
||||
export function judgeProtectionChange(before: BranchProtection | null, want: ProtectionWanted,
|
||||
covering: BranchProtection[] = [], branch = ""): string | null {
|
||||
@@ -306,34 +305,37 @@ export function judgeProtectionChange(before: BranchProtection | null, want: Pro
|
||||
}
|
||||
return null; // A new rule protects what nothing protected.
|
||||
}
|
||||
const had = before.enable_status_check ? [...new Set((before.status_check_contexts ?? []).filter(Boolean))] : [];
|
||||
const wanted = new Set(want.statusChecks.map((c) => c.trim()).filter(Boolean));
|
||||
const dropped = had.filter((c) => !wanted.has(c));
|
||||
const why: string[] = [];
|
||||
if (dropped.length) why.push(`it drops the required status ${dropped.join(", ")}`);
|
||||
if (want.push === true && before.enable_push === false) why.push("it lets a person push to the branch directly");
|
||||
if (want.blockAdminOverride === false && before.block_admin_merge_override === true) {
|
||||
why.push("it lets an administrator merge past what the rule asks");
|
||||
why.push("it lets an administrator merge past a status that has not succeeded");
|
||||
}
|
||||
if (!why.length) return null;
|
||||
return `the protection of ${before.rule_name ?? before.branch_name} is only made stronger through a tool: ` +
|
||||
`${why.join("; ")}. Whoever may call a tool includes agents, and a branch's protection is what keeps a ` +
|
||||
`change from landing but through a pull request the operator approved (novox/hq ADR 0266, ADR 0534) — weakening it is the ` +
|
||||
`change from merging without its checks and the operator (novox/hq ADR 0266) — weakening it is the ` +
|
||||
`operator's, at the forge as themselves. Nothing was changed`;
|
||||
}
|
||||
|
||||
/** Main's rule takes no push and lets only the tools' account and main-mergers merge (novox/hq ADR 0398 §2, issue 948,
|
||||
* ADR 0534 §2): a change that enables push, drops the merge whitelist or widens it past them is refused, whatever the
|
||||
* rule was. */
|
||||
/** Main's rule takes no push and lets only the tools' account merge (novox/hq ADR 0398 §2, issue 948): a change
|
||||
* that enables push, drops the merge whitelist or widens it past that account is refused, whatever the rule was. */
|
||||
function judgeMain(before: BranchProtection | null, want: ProtectionWanted): string | null {
|
||||
const why: string[] = [];
|
||||
if (want.push === true) why.push("it lets a person push to the branch directly");
|
||||
if (want.mergeWhitelist !== undefined) {
|
||||
const names = want.mergeWhitelist ?? [];
|
||||
if (names.length === 0) why.push("it drops the merge whitelist of main");
|
||||
else if (names.some((n) => !mainMergers().includes(n))) {
|
||||
why.push(`it widens who may merge into main beyond ${mainMergers().join(", ")}`);
|
||||
else if (names.some((n) => n !== TOOLS_ACCOUNT)) {
|
||||
why.push(`it widens who may merge into main beyond ${TOOLS_ACCOUNT}`);
|
||||
}
|
||||
}
|
||||
if (!why.length) return null;
|
||||
return `the protection of ${before?.rule_name ?? before?.branch_name ?? "main"} is only made stronger through a tool: ` +
|
||||
`${why.join("; ")}. Main takes no push, and only ${mainMergers().join(" and ")} merge into it (novox/hq ADR 0398 §2, ADR 0534 §2) — ` +
|
||||
`${why.join("; ")}. Main takes no push, and only ${TOOLS_ACCOUNT} merges into it (novox/hq ADR 0398 §2) — ` +
|
||||
`weakening that is the operator's, at the forge as themselves. Nothing was changed`;
|
||||
}
|
||||
|
||||
|
||||
+16
-31
@@ -5,12 +5,10 @@
|
||||
// puts its statuses back to pending through that writer, so where the check runs is said and no second check is asked.
|
||||
// Each bundle the runtime serves is its own process; from the tools' bundle the pending bypassed the writer's chain.
|
||||
//
|
||||
// **It judges mesh/check whether or not the base requires it** (issue 1301; ADR 0447 §1, §4; ADR 0534 §4): the checker's
|
||||
// verdict, whose line names the trunk it judged ("pass <head12> on <trunk12>"), must be a pass against the trunk's head.
|
||||
// The forge requires no status to merge (ADR 0534 §1); where a base still requires one of the controller's old statuses,
|
||||
// until the forge's holder takes it off, that status judges, as before. A mesh/check of another trunk is refused and
|
||||
// nothing is asked: the checker sets it out of date on the trunk's move and checks it again. Where the base requires no
|
||||
// status, only a head the operator's Approve names (approved_heads) merges (ADR 0534 §4, ADR 0421 §9).
|
||||
// **It judges on the status the base requires** (issue 1301; ADR 0447 §1, §4; ADR 0503 §5): mesh/check, the checker's,
|
||||
// whose line names the trunk it judged ("pass <head12> on <trunk12>"); where a base does not require it, the controller's
|
||||
// status it still requires (hq's mesh/repo-check), as before. A mesh/check of another trunk is refused and nothing is
|
||||
// asked: the checker sets it out of date on the trunk's move and checks it again.
|
||||
|
||||
import { announce } from "./announce.js";
|
||||
import type { ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
@@ -38,10 +36,9 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
||||
now: given.now ?? (() => new Date()) };
|
||||
return {
|
||||
name: "gitea_merge_pull_request",
|
||||
description: "Merge a pull request at the head you name — the commit you reviewed — when that is still its head, " +
|
||||
"its base refuses this account's push, its mesh/check passed judged against the trunk's head as it is now, and, " +
|
||||
"where the base requires no status (novox/hq ADR 0534), the operator's Approve names it in approved_heads; " +
|
||||
"optionally deleting the source branch afterwards. Refused " +
|
||||
description: "Merge a pull request at the head you name — the commit you reviewed — when that is still its head " +
|
||||
"and every status its base branch's protection requires succeeded there (mesh/check; hq's mesh/repo-check), " +
|
||||
"judged against the trunk's head as it is now, optionally deleting the source branch afterwards. Refused " +
|
||||
"otherwise, with nothing merged: no tool merges past a check, and none merges a head that moved after it was " +
|
||||
"reviewed. A verdict judged against an earlier trunk commit is refused, naming both, and the head's check against " +
|
||||
"the trunk's head is asked once, with how long the last check of the repository took: ask again at the same head " +
|
||||
@@ -103,11 +100,11 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
||||
`approved (novox/hq ADR 0421). Nothing was merged`);
|
||||
}
|
||||
const rule = await gitea.appliedProtection(owner, repo, pull.base);
|
||||
// Only into a branch whose protection refuses this account's push: a pull request whose base was moved to a
|
||||
// branch nothing guards would land where anyone pushes (novox/hq ADR 0266, ADR 0534 §4).
|
||||
if (!rule || rule.enable_push !== false) {
|
||||
throw new Refused(`${owner}/${repo}#${number} merges into ${pull.base}, which no protection rule refusing ` +
|
||||
`a push guards: a tool merges only into such a branch (novox/hq ADR 0266, ADR 0534). Nothing was merged`);
|
||||
// Only into a branch whose protection requires the checks: a pull request whose base was moved to a
|
||||
// branch nothing guards would merge past every check (novox/hq ADR 0266).
|
||||
if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) {
|
||||
throw new Refused(`${owner}/${repo}#${number} merges into ${pull.base}, which no protection rule requiring ` +
|
||||
`the checks guards: a tool merges only into such a branch (novox/hq ADR 0266). Nothing was merged`);
|
||||
}
|
||||
// **The verdict must be judged against the trunk's head** (novox/hq ADR 0323 §5, issue 341): a check judges the
|
||||
// head merged into a named trunk commit, and a merge of another trunk is not the one that check judged. Compared
|
||||
@@ -117,16 +114,7 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
||||
const trunkHead = await gitea.branchHead(owner, repo, pull.base);
|
||||
const checks = await gitea.headChecks(owner, repo, pull.head_sha);
|
||||
const required = requiredOf(rule) ?? [];
|
||||
// **The forge requires no status; the operator's Approve and mesh/check do** (ADR 0534 §4): where the base requires
|
||||
// no status, only a head the Approve names merges, and mesh/check judges it as where it was required.
|
||||
if (required.length === 0 && !approved) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: ${pull.base} requires no status, so a merge into it ` +
|
||||
`lands only a head the operator's Approve names: give approved_heads with ${owner}/${repo}@<full sha> ` +
|
||||
`(novox/hq ADR 0534, ADR 0421 §9). Nothing was merged`);
|
||||
}
|
||||
const kept = maintained(required);
|
||||
const byCheck = required.includes(CHECK_STATUS) || kept.length === 0;
|
||||
if (byCheck) {
|
||||
if (required.includes(CHECK_STATUS)) {
|
||||
// **mesh/check is the checker's** (ADR 0447 §4): it is judged here against the trunk's head, and refused when it
|
||||
// is not current; nothing is written or asked — the checker checks a head again when its trunk moves.
|
||||
const check = checks[CHECK_STATUS];
|
||||
@@ -136,10 +124,6 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
||||
`${check ? `pending (${check.description ?? ""})` : "not set yet"}; the checker answers it. Ask again at ` +
|
||||
`${twelve(pull.head_sha)} once it passes against ${pull.base}'s head ${twelve(trunkHead)}. Nothing was merged`);
|
||||
}
|
||||
if (check.state !== "success") {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its ${CHECK_STATUS} is ${check.state} ` +
|
||||
`(${check.description ?? ""}). A pull request merges once its check passes (novox/hq ADR 0534). Nothing was merged`);
|
||||
}
|
||||
if (!at.current) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its ${CHECK_STATUS} ` +
|
||||
`${at.judged ? `was judged on trunk ${twelve(at.judged)}` : "names no trunk commit it was judged on"}, and ` +
|
||||
@@ -149,6 +133,7 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
||||
}
|
||||
// Where the base still requires the controller's statuses (hq's mesh/repo-check; mesh/merge-gate only on a
|
||||
// revert), the first it requires judges, as before (ADR 0323 §5).
|
||||
const kept = maintained(required);
|
||||
const judge = kept[0] ?? CHECK_CONTEXT;
|
||||
const at = kept.length ? verdictAtTrunk(checks, trunkHead, judge) : { current: true, onItsWay: false };
|
||||
if (!at.current) {
|
||||
@@ -212,9 +197,9 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
||||
// of them is no verdict to merge on, never read as none.
|
||||
// Under mesh/check (issue 1301) the group is the checker's one verdict on every member's head (ADR 0447 §2) and the
|
||||
// Approve's heads above; a record the controller's verdicts left before the switch judges nothing there.
|
||||
const heard = byCheck ? undefined : readSiblings(siblingsFile(deps.stateDir))[memberKey(owner, repo, number)];
|
||||
const heard = required.includes(CHECK_STATUS) ? undefined : readSiblings(siblingsFile(deps.stateDir))[memberKey(owner, repo, number)];
|
||||
const siblings = heard && sameCommit(heard.head, pull.head_sha) ? heard.siblings : [];
|
||||
if (!siblings.length && !byCheck &&
|
||||
if (!siblings.length && !required.includes(CHECK_STATUS) &&
|
||||
namesSiblings(statuses.find((s) => s.context === CHECK_CONTEXT)?.description)) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict read siblings, and which ones is not held ` +
|
||||
`here, so the group cannot be weighed. Ask its check again at ${twelve(pull.head_sha)}. Nothing was merged`);
|
||||
|
||||
@@ -172,15 +172,9 @@
|
||||
},
|
||||
"repositories": {
|
||||
"type": "object",
|
||||
"description": "The repositories of the organisation novox the mesh holds (ADR 0428): each by name, {\"<name>\": {\"private\": true, \"description\": \"...\"}}. One that does not exist is made by the module's setup login under its own account, its main protected and given a first commit (a README naming the description and a merge-check.sh), its rule narrowed to no push, the tools' account and main-mergers alone merging, and no status required (ADR 0534), and only then moved into the organisation; one that exists only has main's rule put right so. Every other repository has the statuses its main requires taken off; nothing else of a repository out of the value is touched.",
|
||||
"description": "The repositories of the organisation novox the mesh holds (ADR 0428): each by name, {\"<name>\": {\"private\": true, \"description\": \"...\"}}. One that does not exist is made by the module's setup login under its own account, its main protected and given a first commit (a README naming the description and a merge-check.sh), its rule narrowed to no push and the tools' merge requiring mesh/check, and only then moved into the organisation; one that exists only has main's rule put right, mesh/check in place of mesh/merge-gate and mesh/repo-check and its other statuses kept (a main requiring mesh/repo-check alone keeps it). Every other repository whose main requires mesh/merge-gate has the two swapped for mesh/check; nothing else of a repository out of the value is touched.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"main-mergers": {
|
||||
"type": "list",
|
||||
"description": "The forge logins that may merge into main beside the tools' account, on every repository of the organisation (ADR 0534 §2): the operator's own, so the operator can always merge by hand at the forge. The tools' account is always on main's merge whitelist; no agent's account belongs here.",
|
||||
"trusted": true,
|
||||
"default": ["jschoubben"]
|
||||
}
|
||||
},
|
||||
"facts": {
|
||||
@@ -309,8 +303,7 @@
|
||||
"template": "files/config.json",
|
||||
"merge": "json",
|
||||
"takes-config": [
|
||||
"url",
|
||||
"main-mergers"
|
||||
"url"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -373,12 +366,10 @@
|
||||
"MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret",
|
||||
"MESH_GITEA_TOOLS_USER": "mesh-tools",
|
||||
"MESH_REPOSITORIES_FILE": "${dir:mesh-state}/repositories.json",
|
||||
"MESH_REPOSITORIES_OUTCOME": "${dir:runtime-state}/repositories-outcome.json",
|
||||
"MESH_GITEA_CONFIG_FILE": "${dir:mesh-state}/config.json"
|
||||
"MESH_REPOSITORIES_OUTCOME": "${dir:runtime-state}/repositories-outcome.json"
|
||||
},
|
||||
"restart-on": [
|
||||
"repositories-file",
|
||||
"runtime-config"
|
||||
"repositories-file"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+11
-37
@@ -1,9 +1,7 @@
|
||||
// A branch's protection (novox/hq ADR 0237): no direct push, and on main only the tools' account merges (ADR 0398 §2).
|
||||
// The forge requires no status to merge (ADR 0534 §1): the checks still run and report, and mesh-delivery and the
|
||||
// merge tool judge them (ADR 0534 §3, §4), never the forge. Pure, so it is tested without a forge; the client does the
|
||||
// asking (client.ts).
|
||||
// A branch's protection (novox/hq ADR 0237): what makes a pull request wait for the mesh's check — `mesh/check`,
|
||||
// the checker's verdict on the head merged onto its trunk (ADR 0447 §1; hq's `mesh/repo-check` until issue 1296) —
|
||||
// before it may merge. Pure, so it is tested without a forge; the client does the asking (client.ts).
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { TOOLS_ACCOUNT } from "./tools-account.js";
|
||||
|
||||
/** The branch whose rule takes no push and lets only the tools' account merge (novox/hq ADR 0398 §2, issue 948). */
|
||||
@@ -26,50 +24,26 @@ export interface BranchProtection {
|
||||
/** What a branch's protection is set to. A field not given is left as the rule has it (or the forge's
|
||||
* default on a new rule), except pushes, which a new rule refuses unless `push` says otherwise. */
|
||||
export interface ProtectionWanted {
|
||||
/** The statuses a pull request must have succeeded before it merges, e.g. mesh/check. Empty: none. */
|
||||
statusChecks: string[];
|
||||
/** Whether a person may push to the branch directly. */
|
||||
push?: boolean;
|
||||
/** Whether an administrator is stopped from merging past what the rule asks. */
|
||||
/** Whether an administrator is stopped from merging past a status that has not succeeded. */
|
||||
blockAdminOverride?: boolean;
|
||||
/** Who may merge into the branch; for main only ever the tools' account (ADR 0398 §2). Not given: left as it is.
|
||||
* Empty or null asks for no whitelist, which the guard refuses for main; the body for main ignores it. */
|
||||
mergeWhitelist?: string[] | null;
|
||||
}
|
||||
|
||||
/** What every rule the mesh writes says of statuses: none is required to merge (novox/hq ADR 0534 §1). */
|
||||
export function noStatusBody(): Record<string, unknown> {
|
||||
return { enable_status_check: false, status_check_contexts: [] };
|
||||
}
|
||||
|
||||
/** A forge login a merge whitelist may name. */
|
||||
const LOGIN = /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,39}$/;
|
||||
|
||||
/** Who merges into main: the tools' account, for mesh-delivery, then each login of the module's config value
|
||||
* main-mergers once, the operator's own, so the operator can always merge by hand at the forge (novox/hq ADR 0534 §2).
|
||||
* Read from the module's runtime config file (MESH_GITEA_CONFIG_FILE); a file not there, or a name that is no login,
|
||||
* leaves the tools' account alone, the narrowest whitelist. */
|
||||
export function mainMergers(env: NodeJS.ProcessEnv = process.env): string[] {
|
||||
const out = [TOOLS_ACCOUNT];
|
||||
let given: unknown = [];
|
||||
try {
|
||||
if (env.MESH_GITEA_CONFIG_FILE) given = JSON.parse(readFileSync(env.MESH_GITEA_CONFIG_FILE, "utf8"))["main-mergers"] ?? [];
|
||||
} catch {
|
||||
given = [];
|
||||
}
|
||||
for (const name of Array.isArray(given) ? given : []) {
|
||||
if (typeof name === "string" && LOGIN.test(name) && !name.includes("..") && !out.includes(name)) out.push(name);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** What main's rule always says: no push, only the tools' account and main-mergers merge (novox/hq ADR 0398 §2,
|
||||
* issue 948, ADR 0534 §2), and no status required (ADR 0534 §1). */
|
||||
export function mainRuleBody(mergers: string[] = mainMergers()): Record<string, unknown> {
|
||||
return { enable_push: false, enable_merge_whitelist: true, merge_whitelist_usernames: mergers, ...noStatusBody() };
|
||||
/** What main's rule always says: no push, and only the tools' account merges (novox/hq ADR 0398 §2, issue 948). */
|
||||
export function mainRuleBody(): Record<string, unknown> {
|
||||
return { enable_push: false, enable_merge_whitelist: true, merge_whitelist_usernames: [TOOLS_ACCOUNT] };
|
||||
}
|
||||
|
||||
/** The forge's body for a wanted protection. */
|
||||
export function protectionBody(want: ProtectionWanted, creating: boolean, branch = ""): Record<string, unknown> {
|
||||
const body: Record<string, unknown> = noStatusBody();
|
||||
const checks = [...new Set(want.statusChecks.map((c) => c.trim()).filter(Boolean))];
|
||||
const body: Record<string, unknown> = { enable_status_check: checks.length > 0, status_check_contexts: checks };
|
||||
if (want.push !== undefined) body.enable_push = want.push;
|
||||
else if (creating) body.enable_push = false;
|
||||
if (want.blockAdminOverride !== undefined) body.block_admin_merge_override = want.blockAdminOverride;
|
||||
|
||||
@@ -109,7 +109,7 @@ const server = createServer(async (req, res) => {
|
||||
const rule = (state.rules as any[]).find((r) => ruleCovers(String(r.rule_name ?? r.branch_name), branch));
|
||||
return json(res, 200, { name: branch, commit: { id: branch === "main" ? state.trunk : "c0ffee" }, protected: !!rule,
|
||||
...(rule ? { effective_branch_protection_name: rule.rule_name ?? rule.branch_name, enable_status_check: !!rule.enable_status_check,
|
||||
status_check_contexts: rule.status_check_contexts ?? [], required_approvals: rule.required_approvals ?? 0, user_can_push: !!rule.enable_push } : {}) });
|
||||
status_check_contexts: rule.status_check_contexts ?? [], required_approvals: rule.required_approvals ?? 0, user_can_push: false } : {}) });
|
||||
}
|
||||
if (method === "GET" && path === `${repo}/pulls/7/files`) {
|
||||
return json(res, 200, url.searchParams.get("page") === "1" ? [{ filename: "modules/gitea/x.ts", status: "modified" }] : []);
|
||||
@@ -237,24 +237,18 @@ test("gitea_api still writes what an author writes", async () => {
|
||||
assert.deepEqual(writes(), ["POST /repos/novox/mesh-catalog/issues/3/comments", "POST /repos/novox/mesh-catalog/pulls/7/reviews"]);
|
||||
});
|
||||
|
||||
// **The forge requires no status to merge** (novox/hq ADR 0534 §5): the tool writes no required status and takes no
|
||||
// status argument; taking a required status away is no weakening it refuses, and a push or an administrator's merge
|
||||
// past the rule still is.
|
||||
test("gitea_branch_protection_set writes no required status, and refuses a push or an administrator's override", async () => {
|
||||
test("gitea_branch_protection_set only strengthens", async () => {
|
||||
state.admin = true; // a repository administrator's token: the tools' account is refused the rules by the forge
|
||||
const set = (args: Record<string, unknown>) =>
|
||||
tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-catalog", branch: "main", ...args });
|
||||
assert.equal((tool("gitea_branch_protection_set").input as Record<string, unknown>).status_checks, undefined,
|
||||
"the tool takes no status argument");
|
||||
await assert.rejects(set({ push: true }), /push to the branch directly/);
|
||||
await assert.rejects(set({ block_admin_override: false }), /merge past/);
|
||||
await assert.rejects(set({ status_checks: "" }), /drops the required status mesh\/merge-gate, mesh\/repo-check/);
|
||||
await assert.rejects(set({ status_checks: "mesh/merge-gate" }), /drops the required status mesh\/repo-check/);
|
||||
await assert.rejects(set({ status_checks: "mesh/merge-gate,mesh/repo-check", push: true }), /push to the branch directly/);
|
||||
await assert.rejects(set({ status_checks: "mesh/merge-gate,mesh/repo-check", block_admin_override: false }),
|
||||
/merge past a status/);
|
||||
assert.deepEqual(writes(), []);
|
||||
await set({ status_checks: "mesh/check" }); // an argument it no longer takes is not read
|
||||
await set({ status_checks: "mesh/merge-gate,mesh/repo-check,mesh/delivery" });
|
||||
assert.deepEqual(writes(), ["PATCH /repos/novox/mesh-catalog/branch_protections/main"]);
|
||||
const sent = state.sent[0].body;
|
||||
assert.equal(sent.enable_status_check, false);
|
||||
assert.deepEqual(sent.status_check_contexts, []);
|
||||
assert.equal(sent.enable_push, false);
|
||||
});
|
||||
|
||||
test("gitea_merge_pull_request merges only a head whose required statuses all succeeded, and only that head", async () => {
|
||||
@@ -480,62 +474,26 @@ test("gitea_api sends the path it judged, and refuses moving a pull request's ba
|
||||
assert.equal(writes().length, 2);
|
||||
});
|
||||
|
||||
test("gitea_merge_pull_request merges only into a branch whose protection refuses its push", async () => {
|
||||
const approved_heads = [`novox/mesh-catalog@${REVIEWED}`];
|
||||
state.statuses = [{ context: "mesh/check", state: "success", description: `pass ${REVIEWED.slice(0, 12)} on ${TRUNK.slice(0, 12)}` }];
|
||||
test("gitea_merge_pull_request merges only into a branch whose protection requires the checks", async () => {
|
||||
state.rules = [];
|
||||
await assert.rejects(tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED,
|
||||
approved_heads }), /no protection rule refusing/);
|
||||
state.rules = [{ rule_name: "main", enable_push: true, enable_status_check: false, status_check_contexts: [] }];
|
||||
await assert.rejects(tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED,
|
||||
approved_heads }), /no protection rule refusing/);
|
||||
await assert.rejects(tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED }),
|
||||
/no protection rule requiring the checks guards/);
|
||||
state.rules = [{ rule_name: "main", enable_push: false, enable_status_check: false, status_check_contexts: [] }];
|
||||
await assert.rejects(tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED }),
|
||||
/no protection rule requiring the checks guards/);
|
||||
assert.deepEqual(writes(), []);
|
||||
});
|
||||
|
||||
// **The forge requires no status to merge; the merge tool still judges mesh/check, and asks the operator's Approve**
|
||||
// (novox/hq ADR 0534 §4): on a main whose rule takes no push and requires no status, a head merges only with
|
||||
// approved_heads naming it and a mesh/check that passed against the trunk's head at the merge.
|
||||
test("gitea_merge_pull_request judges mesh/check on a main requiring no status, and merges only an approved head", async () => {
|
||||
const { keepSiblings, siblingsFile } = await import("../dist/siblings.js");
|
||||
keepSiblings(siblingsFile(stateDir), {});
|
||||
const approved_heads = [`novox/mesh-catalog@${REVIEWED}`];
|
||||
const merge = (args: Record<string, unknown> = { approved_heads }) =>
|
||||
tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED, ...args });
|
||||
state.rules = [{ rule_name: "main", enable_push: false, enable_status_check: false, status_check_contexts: [],
|
||||
block_admin_merge_override: true, enable_merge_whitelist: true, merge_whitelist_usernames: ["mesh-tools"] }];
|
||||
const line = (trunk: string, s = "success", words = "pass") =>
|
||||
[{ context: "mesh/check", state: s, description: `${words} ${REVIEWED.slice(0, 12)} on ${trunk.slice(0, 12)} | unit 12 run` }];
|
||||
const before = state.emitted.length;
|
||||
|
||||
state.statuses = line(TRUNK);
|
||||
await assert.rejects(merge({}), /Approve/);
|
||||
state.statuses = [];
|
||||
await assert.rejects(merge(), /mesh\/check is not set yet/);
|
||||
state.statuses = line(TRUNK, "pending", "checking");
|
||||
await assert.rejects(merge(), /mesh\/check is pending/);
|
||||
state.statuses = line(TRUNK, "failure", "fail");
|
||||
await assert.rejects(merge(), /mesh\/check is failure/);
|
||||
state.statuses = line(MOVED);
|
||||
await assert.rejects(merge(), new RegExp(`${MOVED.slice(0, 12)}.*${TRUNK.slice(0, 12)}`));
|
||||
state.statuses = judgedAt(TRUNK); // the old two, current: not mesh/check
|
||||
await assert.rejects(merge(), /mesh\/check/);
|
||||
assert.deepEqual(writes(), [], "nothing is written: mesh/check is the checker's");
|
||||
assert.equal(state.emitted.length, before, "no check is asked");
|
||||
assert.equal(state.merged, false);
|
||||
|
||||
state.statuses = line(TRUNK);
|
||||
assert.equal(((await merge()) as Record<string, unknown>).merged, true);
|
||||
});
|
||||
|
||||
test("a rule named for a branch is not made weaker than the glob rule that guards it now", async () => {
|
||||
state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365)
|
||||
state.rules.push({ rule_name: "release/*", enable_push: false, enable_status_check: true,
|
||||
status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true });
|
||||
const set = (args: Record<string, unknown>) =>
|
||||
tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-catalog", branch: "release/1", ...args });
|
||||
await assert.rejects(set({}), /merge past/);
|
||||
await assert.rejects(set({ status_checks: "" }), /drops the required status mesh\/merge-gate/);
|
||||
await assert.rejects(set({ status_checks: "mesh/merge-gate" }), /merge past a status/);
|
||||
assert.deepEqual(writes(), []);
|
||||
await set({ block_admin_override: true }); // the glob's required status is no longer asked (ADR 0534 §5)
|
||||
await set({ status_checks: "mesh/merge-gate", block_admin_override: true });
|
||||
assert.deepEqual(writes(), ["POST /repos/novox/mesh-catalog/branch_protections"]);
|
||||
});
|
||||
|
||||
@@ -553,25 +511,26 @@ test("the judges, without a forge", () => {
|
||||
assert.ok("refused" in judgeApiCall("POST", "/repos/a/b/branch_protections", {}));
|
||||
assert.ok("contents" in judgeApiCall("POST", "/repos/a/b/contents", { new_branch: "x" }));
|
||||
assert.ok(ruleCovers("release/**", "release/a/b") && ruleCovers("feat/*", "feat/a") && !ruleCovers("feat/*", "feat/a/b"));
|
||||
assert.equal(judgeProtectionChange(null, {}), null);
|
||||
assert.equal(judgeProtectionChange(protectedMain(), {}), null, "taking its required statuses off is no weakening (ADR 0534 §5)");
|
||||
assert.equal(judgeProtectionChange(null, { statusChecks: [] }), null);
|
||||
assert.equal(judgeProtectionChange(protectedMain(), { statusChecks: ["mesh/repo-check", "mesh/merge-gate"] }), null);
|
||||
});
|
||||
|
||||
test("main's protection is never weakened in its push or its merge whitelist (issue 948, ADR 0398 §2)", () => {
|
||||
const strong = () => ({ ...protectedMain(), enable_merge_whitelist: true, merge_whitelist_usernames: ["mesh-tools"] });
|
||||
assert.equal(judgeProtectionChange(strong(), {}, [], "main"), null);
|
||||
assert.equal(judgeProtectionChange(strong(), { mergeWhitelist: ["mesh-tools"] }, [], "main"), null);
|
||||
assert.match(judgeProtectionChange(strong(), { mergeWhitelist: [] }, [], "main") ?? "", /merge whitelist/);
|
||||
assert.match(judgeProtectionChange(strong(), { mergeWhitelist: null }, [], "main") ?? "", /merge whitelist/);
|
||||
assert.match(judgeProtectionChange(strong(), { mergeWhitelist: ["mesh-tools", "agent-ace"] }, [], "main") ?? "", /widens/);
|
||||
assert.match(judgeProtectionChange(strong(), { mergeWhitelist: ["agent-ace"] }, [], "main") ?? "", /widens/);
|
||||
assert.match(judgeProtectionChange(strong(), { push: true }, [], "main") ?? "", /push/);
|
||||
const checks = ["mesh/merge-gate", "mesh/repo-check"];
|
||||
assert.equal(judgeProtectionChange(strong(), { statusChecks: checks }, [], "main"), null);
|
||||
assert.equal(judgeProtectionChange(strong(), { statusChecks: checks, mergeWhitelist: ["mesh-tools"] }, [], "main"), null);
|
||||
assert.match(judgeProtectionChange(strong(), { statusChecks: checks, mergeWhitelist: [] }, [], "main") ?? "", /merge whitelist/);
|
||||
assert.match(judgeProtectionChange(strong(), { statusChecks: checks, mergeWhitelist: null }, [], "main") ?? "", /merge whitelist/);
|
||||
assert.match(judgeProtectionChange(strong(), { statusChecks: checks, mergeWhitelist: ["mesh-tools", "agent-ace"] }, [], "main") ?? "", /widens/);
|
||||
assert.match(judgeProtectionChange(strong(), { statusChecks: checks, mergeWhitelist: ["agent-ace"] }, [], "main") ?? "", /widens/);
|
||||
assert.match(judgeProtectionChange(strong(), { statusChecks: checks, push: true }, [], "main") ?? "", /push/);
|
||||
// Even a rule that lets a person push already, or has no rule yet, is not given a push through a tool.
|
||||
assert.match(judgeProtectionChange({ ...strong(), enable_push: true }, { push: true }, [], "main") ?? "", /push/);
|
||||
assert.match(judgeProtectionChange(null, { push: true }, [], "main") ?? "", /push/);
|
||||
assert.match(judgeProtectionChange(null, { mergeWhitelist: ["agent-ace"] }, [], "main") ?? "", /widens/);
|
||||
assert.match(judgeProtectionChange({ ...strong(), enable_push: true }, { statusChecks: checks, push: true }, [], "main") ?? "", /push/);
|
||||
assert.match(judgeProtectionChange(null, { statusChecks: checks, push: true }, [], "main") ?? "", /push/);
|
||||
assert.match(judgeProtectionChange(null, { statusChecks: checks, mergeWhitelist: ["agent-ace"] }, [], "main") ?? "", /widens/);
|
||||
// Another branch is judged as before.
|
||||
assert.equal(judgeProtectionChange(null, { push: true }, [], "feat/x"), null);
|
||||
assert.equal(judgeProtectionChange(null, { statusChecks: checks, push: true }, [], "feat/x"), null);
|
||||
});
|
||||
|
||||
test("gitea_api never acts as another user: sudo, in any case, and any query key it does not send are refused", async () => {
|
||||
|
||||
@@ -52,7 +52,7 @@ test("gitea_branch_protection_get answers the rule for a branch, null when it ha
|
||||
|
||||
test("gitea_branch_protection_set still gets the forge's refusal, as the tools' account", async () => {
|
||||
await assert.rejects(
|
||||
tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-controller", branch: "main" }),
|
||||
tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-controller", branch: "main", status_checks: "mesh/merge-gate" }),
|
||||
/403/,
|
||||
);
|
||||
assert.ok(!seen.some((s) => /^(PATCH|POST|PUT|DELETE) .*branch_protections.* as admin$/.test(s)), seen.join("\n"));
|
||||
|
||||
@@ -4,33 +4,32 @@ import { test } from "node:test";
|
||||
// A branch's protection (novox/hq ADR 0237): what the operator's agent sets so a pull request waits for the
|
||||
// mesh's merge check before it merges.
|
||||
|
||||
test("a rule's body requires no status, whatever the rule required (ADR 0534 §5)", async () => {
|
||||
test("the statuses asked for replace the rule's, and an empty list requires none", async () => {
|
||||
const { protectionBody } = await import("../dist/protection.js");
|
||||
assert.deepEqual(protectionBody({}, false), { enable_status_check: false, status_check_contexts: [] },
|
||||
"an edited rule keeps its pushes as they are, and requires no status");
|
||||
assert.deepEqual(protectionBody({ statusChecks: ["mesh/merge-gate", " mesh/repo-check", "", "mesh/merge-gate"] }, false),
|
||||
{ enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"] },
|
||||
"an edited rule keeps its pushes as they are, and each status once");
|
||||
assert.deepEqual(protectionBody({ statusChecks: [] }, false), { enable_status_check: false, status_check_contexts: [] });
|
||||
});
|
||||
|
||||
test("a new rule refuses direct pushes unless asked; an administrator's override only when said", async () => {
|
||||
const { protectionBody } = await import("../dist/protection.js");
|
||||
assert.equal(protectionBody({}, true).enable_push, false);
|
||||
assert.equal(protectionBody({ push: true }, true).enable_push, true);
|
||||
assert.equal(protectionBody({}, true).block_admin_merge_override, undefined);
|
||||
assert.equal(protectionBody({ blockAdminOverride: true }, false).block_admin_merge_override, true);
|
||||
assert.equal(protectionBody({ statusChecks: ["mesh/merge-gate"] }, true).enable_push, false);
|
||||
assert.equal(protectionBody({ statusChecks: ["mesh/merge-gate"], push: true }, true).enable_push, true);
|
||||
assert.equal(protectionBody({ statusChecks: ["mesh/merge-gate"] }, true).block_admin_merge_override, undefined);
|
||||
assert.equal(protectionBody({ statusChecks: ["mesh/merge-gate"], blockAdminOverride: true }, false).block_admin_merge_override, true);
|
||||
});
|
||||
|
||||
// Issue 948 (novox/hq ADR 0398 §2): main takes no push, and only the tools' account merges; it requires no status
|
||||
// (ADR 0534 §1).
|
||||
// Issue 948 (novox/hq ADR 0398 §2): main takes no push, and only the tools' account merges.
|
||||
test("main's body always refuses pushes and lets only the tools' account merge, whatever is asked", async () => {
|
||||
const { protectionBody } = await import("../dist/protection.js");
|
||||
const { TOOLS_ACCOUNT } = await import("../dist/tools-account.js");
|
||||
for (const creating of [true, false]) {
|
||||
for (const want of [{}, { push: true }, { push: false }]) {
|
||||
for (const want of [{ statusChecks: ["mesh/merge-gate"] }, { statusChecks: [], push: true }, { statusChecks: [], push: false }]) {
|
||||
const body = protectionBody(want, creating, "main");
|
||||
assert.equal(body.enable_push, false, "main takes no push, an edit and a push=true included");
|
||||
assert.equal(body.enable_merge_whitelist, true);
|
||||
assert.deepEqual(body.merge_whitelist_usernames, [TOOLS_ACCOUNT]);
|
||||
assert.equal(body.enable_status_check, false);
|
||||
assert.deepEqual(body.status_check_contexts, []);
|
||||
}
|
||||
}
|
||||
assert.deepEqual(TOOLS_ACCOUNT, "mesh-tools");
|
||||
@@ -38,28 +37,13 @@ test("main's body always refuses pushes and lets only the tools' account merge,
|
||||
|
||||
test("another branch's body carries no merge whitelist", async () => {
|
||||
const { protectionBody } = await import("../dist/protection.js");
|
||||
const body = protectionBody({}, false, "release/1");
|
||||
const body = protectionBody({ statusChecks: ["mesh/merge-gate"] }, false, "release/1");
|
||||
assert.equal(body.enable_merge_whitelist, undefined);
|
||||
assert.equal(body.merge_whitelist_usernames, undefined);
|
||||
assert.equal(protectionBody({ statusChecks: [] }, false).enable_merge_whitelist, undefined);
|
||||
});
|
||||
|
||||
test("the rule main is put right in the forge's own words: no push, the tools' account merges, no status required", async () => {
|
||||
test("the rule main is put right in the forge's own words: the body a setup sends carries no status checks of its own", async () => {
|
||||
const { mainRuleBody } = await import("../dist/protection.js");
|
||||
assert.deepEqual(mainRuleBody(), { enable_push: false, enable_merge_whitelist: true, merge_whitelist_usernames: ["mesh-tools"],
|
||||
enable_status_check: false, status_check_contexts: [] });
|
||||
});
|
||||
|
||||
// The operator's own login stays on main's merge whitelist beside the tools' account (novox/hq ADR 0534 §2): read from
|
||||
// the module's config value main-mergers, in the runtime config file; nothing there leaves the tools' account alone.
|
||||
test("main's merge whitelist is the tools' account, then the configured mergers once each", async () => {
|
||||
const { mainMergers, mainRuleBody } = await import("../dist/protection.js");
|
||||
const { mkdtempSync, writeFileSync } = await import("node:fs");
|
||||
const { join } = await import("node:path");
|
||||
const { tmpdir } = await import("node:os");
|
||||
const file = join(mkdtempSync(join(tmpdir(), "gitea-mergers-")), "config.json");
|
||||
writeFileSync(file, JSON.stringify({ url: "", "main-mergers": ["jschoubben", "mesh-tools", "jschoubben", "../x", "a b"] }));
|
||||
assert.deepEqual(mainMergers({ MESH_GITEA_CONFIG_FILE: file }), ["mesh-tools", "jschoubben"]);
|
||||
assert.deepEqual(mainMergers({}), ["mesh-tools"]);
|
||||
assert.deepEqual(mainMergers({ MESH_GITEA_CONFIG_FILE: join(tmpdir(), "no-such-gitea-config.json") }), ["mesh-tools"]);
|
||||
assert.deepEqual(mainRuleBody(["mesh-tools", "jschoubben"]).merge_whitelist_usernames, ["mesh-tools", "jschoubben"]);
|
||||
assert.deepEqual(mainRuleBody(), { enable_push: false, enable_merge_whitelist: true, merge_whitelist_usernames: ["mesh-tools"] });
|
||||
});
|
||||
|
||||
@@ -235,7 +235,7 @@ test("with the tools' token, the forge writes a status and refuses a protection
|
||||
const client = fresh(env);
|
||||
await client.setCommitStatus("novox", "hq", "abc123def4567890abc123def4567890abc12345", { state: "success", context: "mesh/merge-gate", description: "ok" } as any);
|
||||
await assert.rejects(
|
||||
client.setBranchProtection("novox", "hq", "main", {}),
|
||||
client.setBranchProtection("novox", "hq", "main", { statusChecks: ["mesh/merge-gate"] }),
|
||||
/403/,
|
||||
);
|
||||
});
|
||||
@@ -350,7 +350,7 @@ test("a team another process made between the look and the create is found, not
|
||||
assert.ok(team?.members.has(TOOLS_ACCOUNT));
|
||||
});
|
||||
|
||||
test("setting the tools' account up puts main's rule right in every repository: no push, only the tools' account merges, no status (issue 948, ADR 0534 §1)", async () => {
|
||||
test("setting the tools' account up puts main's rule right in every repository: no push, only the tools' account merges (issue 948)", async () => {
|
||||
const env = await environment();
|
||||
const from = forge.calls.length;
|
||||
await fresh(env).listRepos();
|
||||
@@ -358,8 +358,7 @@ test("setting the tools' account up puts main's rule right in every repository:
|
||||
assert.equal(hq.enable_push, false);
|
||||
assert.equal(hq.enable_merge_whitelist, true);
|
||||
assert.deepEqual(hq.merge_whitelist_usernames, [TOOLS_ACCOUNT]);
|
||||
assert.deepEqual(hq.status_check_contexts, [], "a rule found has its required statuses taken off (ADR 0534 §1)");
|
||||
assert.equal(hq.enable_status_check, false);
|
||||
assert.deepEqual(hq.status_check_contexts, ["mesh/merge-gate"], "a rule found keeps the statuses it requires");
|
||||
const catalog = forge.mainRules.get("mesh-catalog");
|
||||
assert.ok(catalog, "a repository with no rule for main is given one");
|
||||
assert.equal(catalog.rule_name, "main");
|
||||
@@ -369,13 +368,13 @@ test("setting the tools' account up puts main's rule right in every repository:
|
||||
"only the administrator's login touches a rule: the tools' account is no repository's administrator");
|
||||
});
|
||||
|
||||
test("a glob rule covering main gives its approvals and override, not its statuses, when main is given a rule of its own name (issue 948, ADR 0534 §1)", async () => {
|
||||
test("a glob rule covering main keeps its statuses when main is given a rule of its own name (issue 948, review round 1)", async () => {
|
||||
const env = await environment();
|
||||
await fresh(env).listRepos();
|
||||
const rule = forge.mainRules.get("globbed");
|
||||
assert.ok(rule, "main is given its own rule");
|
||||
assert.deepEqual(rule.status_check_contexts, []);
|
||||
assert.equal(rule.enable_status_check, false);
|
||||
assert.deepEqual(rule.status_check_contexts, ["mesh/merge-gate", "mesh/repo-check"]);
|
||||
assert.equal(rule.enable_status_check, true);
|
||||
assert.equal(rule.block_admin_merge_override, true);
|
||||
assert.equal(rule.required_approvals, 1);
|
||||
assert.equal(rule.enable_push, false);
|
||||
|
||||
@@ -429,13 +429,14 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
},
|
||||
{
|
||||
name: "gitea_branch_protection_set",
|
||||
description: "Protect a branch. Never weaker (novox/hq ADR 0266): a change that lets a person push directly or lets an administrator merge past the rule is refused, and on main so is a change that drops the merge whitelist (only mesh-tools merges there) or widens it (novox/hq ADR 0398 §2, issue 948) — that is the operator's, at the forge. The forge requires no status to merge (ADR 0534): the rule written requires none, and one the rule required is taken off; the checks are judged by mesh-delivery and the merge tool. Edits the branch's rule (everything else kept unless given) or creates one, which refuses direct pushes unless push=true. Answers the rule before and after.",
|
||||
description: "Only ever stronger (novox/hq ADR 0266): a change that drops a required status, lets a person push directly or lets an administrator merge past a status is refused, and on main so is a change that drops the merge whitelist (only mesh-tools merges there) or widens it (novox/hq ADR 0398 §2, issue 948) — that is the operator's, at the forge. Make a branch require commit statuses before a pull request merges into it — the mesh's checker sets `mesh/check` (ADR 0447), and the repositories program requires it on every main. Edits the branch's rule (its required statuses replaced by these, everything else kept unless given) or creates one, which refuses direct pushes unless push=true. Answers the rule before and after.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
branch: { type: "string", description: "the branch to protect, e.g. main" },
|
||||
status_checks: { type: "string", description: "comma-separated statuses required to merge, e.g. mesh/check; empty requires none" },
|
||||
push: { type: "boolean", description: "whether a person may push to the branch directly (a new rule refuses it when not given)" },
|
||||
block_admin_override: { type: "boolean", description: "stop an administrator merging past what the rule asks (left as it is when not given)" },
|
||||
block_admin_override: { type: "boolean", description: "stop an administrator merging past a status that has not succeeded (left as it is when not given)" },
|
||||
},
|
||||
run: async (args) => {
|
||||
const owner = String(args.owner), repo = String(args.repo), branch = String(args.branch ?? "").trim();
|
||||
@@ -444,6 +445,7 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
|
||||
const before = await gitea.branchProtection(owner, repo, branch);
|
||||
const want = {
|
||||
statusChecks: String(args.status_checks ?? "").split(","),
|
||||
push: args.push === undefined ? undefined : Boolean(args.push),
|
||||
blockAdminOverride: args.block_admin_override === undefined ? undefined : Boolean(args.block_admin_override),
|
||||
};
|
||||
|
||||
@@ -37,8 +37,17 @@ Three read tools answer what the module stored in its own states, without the gu
|
||||
forge-account provision's ssh name, so the account's own ssh files and git rewrite are used) into a workspace of the
|
||||
work directory, then runs `go mod download` and `go mod verify` in every directory with a `go.mod` that does not vendor,
|
||||
with `GOTOOLCHAIN=local`, `GOFLAGS=-mod=mod` and the module's cache directory. Every command runs with an
|
||||
environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. Workspaces older than a day
|
||||
are removed.
|
||||
environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes.
|
||||
|
||||
## Room to work and the workspace's end
|
||||
|
||||
Every step is refused, ending `refused` with `refused: <node> has N GiB free under the floor of 20 GiB (on <dir>)`, while
|
||||
the work or the cache directory's filesystem has less than 20 GiB free (build-agent's floor, issue 1839); the health is
|
||||
then unhealthy with the same text, so the builder chooses another node and the controller raises the condition. A
|
||||
workspace is held while a step runs on it. `build_go_release {workspace}` removes it at once (no guard, no floor; one
|
||||
already gone is no error, one a running step holds is refused); its caller calls it once its last step answered. A
|
||||
failed fetch removes its own workspace. A workspace no caller released is removed two hours after its last use (ADR
|
||||
0509), and the removal is said in the log as the caller's fault.
|
||||
|
||||
A pin on a pull request's head (issue 1148): when `go mod download` fails on a `git.novox.be/novox/*` pseudo-version, the
|
||||
fetch looks for that commit among the forge's `refs/pull/*/head`, places the repository at it under the workspace's
|
||||
@@ -96,6 +105,16 @@ fingerprint once, as the version the build nodes agree on (ADR 0477 §3), and th
|
||||
version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors
|
||||
is listed and tested from its vendor directory (`-mod=vendor`).
|
||||
|
||||
## A change to this module is judged by its candidate
|
||||
|
||||
`build_go_digest` and `build_go_test` take `runner`: `installed` (the default) runs the step as this program;
|
||||
`workspace` compiles the workspace's own `modules/mesh-build-go` (`go build` with `GOPROXY=off`, `GOTOOLCHAIN=local`, in a
|
||||
run step shut off) into the workspace's `candidate/mesh-build-go`, once per workspace, and runs the step as that program:
|
||||
`<candidate> level <spec>` in one bubblewrap, the spec naming the step (`digest`, `unit` or `integration`). This program
|
||||
keeps the fetch, the forge key and the bus login (ADR 0535 §1). A candidate that does not compile fails the step with the
|
||||
compiler's first error, and nothing runs (§3). Every digest and level answer says `judged_by`: `installed`, or
|
||||
`candidate <commit of the tree>`. The checker asks `workspace` when a merge changes `modules/mesh-build-go/` (issue 1898).
|
||||
|
||||
## Build
|
||||
|
||||
`build_go_build {workspace, module, artifact}` builds one Go bundle artifact of a module of the workspace with the build
|
||||
|
||||
@@ -11,37 +11,72 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The lead's decision L5 (the checker keeps a verdict of a core repository): the fetch writes the siblings it placed
|
||||
// beside the workspace, and the digest folds each one's repository and commit into every package's digest and echoes
|
||||
// them, so a verdict is kept only for the siblings it was judged with.
|
||||
func TestTheSiblingsCommitsAreInEveryDigestAndEchoed(t *testing.T) {
|
||||
// writeSiblings records the siblings the fetch placed, as the fetch does.
|
||||
func writeSiblings(t *testing.T, m *Module, placed []Sibling) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(placed)
|
||||
if err := os.WriteFile(filepath.Join(m.Work, "ws1", besideRecord), raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1896/AC1 and AC3 (ADR 0536 §1): a sibling's commit changes no digest of a package whose tests reach no file
|
||||
// naming MESH_CHECK_BESIDE; the answer still echoes every placed sibling, in form 4.
|
||||
func TestASiblingsCommitLeavesAPackageThatDoesNotReadTheSiblingsAsItWas(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, _ := digestFixture(t)
|
||||
alone := digests(t, m, "unit")
|
||||
if len(alone.Beside) != 0 {
|
||||
t.Errorf("a workspace without siblings echoes %v", alone.Beside)
|
||||
}
|
||||
write := func(placed []Sibling) {
|
||||
raw, _ := json.Marshal(placed)
|
||||
if err := os.WriteFile(filepath.Join(m.Work, "ws1", besideRecord), raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
other := strings.Repeat("b", 40)
|
||||
write([]Sibling{{Repository: "novox/mesh-sdk", Commit: commitSHA, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}})
|
||||
writeSiblings(t, m, []Sibling{{Repository: "novox/mesh-sdk", Commit: commitSHA, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}})
|
||||
with := digests(t, m, "unit")
|
||||
want := []string{"novox/mesh-catalog " + commitSHA, "novox/mesh-sdk " + commitSHA}
|
||||
if !slices.Equal(with.Beside, want) {
|
||||
t.Errorf("echoed %v, want %v", with.Beside, want)
|
||||
}
|
||||
write([]Sibling{{Repository: "novox/mesh-sdk", Commit: other, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}})
|
||||
writeSiblings(t, m, []Sibling{{Repository: "novox/mesh-sdk", Commit: other, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}})
|
||||
moved := digests(t, m, "unit")
|
||||
for p, d := range alone.Packages {
|
||||
if with.Packages[p] == d {
|
||||
t.Errorf("%s: the siblings left the digest as it was without them", p)
|
||||
if with.Packages[p] != d || moved.Packages[p] != d {
|
||||
t.Errorf("%s: a sibling left its digest changed though no test of it reads the siblings", p)
|
||||
}
|
||||
if moved.Packages[p] == with.Packages[p] {
|
||||
t.Errorf("%s: a sibling's new commit left the digest as it was", p)
|
||||
}
|
||||
if !slices.Equal(moved.Beside, []string{"novox/mesh-catalog " + commitSHA, "novox/mesh-sdk " + other}) {
|
||||
t.Errorf("echoed %v after the move", moved.Beside)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1896/AC2: a package whose test imports an in-tree package naming MESH_CHECK_BESIDE is judged again when a
|
||||
// sibling moves; a package beside it that does not import it is not.
|
||||
func TestASiblingsCommitChangesTheDigestOfAPackageWhoseTestsReachAReaderOfTheSiblings(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, tree := digestFixture(t)
|
||||
for name, text := range map[string]string{
|
||||
"f/f.go": "package f\n\n// Beside is where the siblings lie.\nconst Beside = \"MESH_CHECK_BESIDE\"\n",
|
||||
"e/e_test.go": "package e\n\nimport (\n\t\"testing\"\n\n\t\"fixture/digest/f\"\n)\n\nfunc TestE(t *testing.T) {\n\tif f.Beside == \"\" {\n\t\tt.Fatal(\"e\")\n\t}\n}\n",
|
||||
"e/e.go": "package e\n",
|
||||
"g/g_test.go": "package g\n\nimport \"testing\"\n\nfunc TestG(t *testing.T) {\n\t// MESH_CHECK_BESIDE named in the test's own file\n}\n",
|
||||
"g/g.go": "package g\n",
|
||||
} {
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(tree, name)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
edit(t, tree, name, text)
|
||||
}
|
||||
writeSiblings(t, m, []Sibling{{Repository: "novox/mesh-sdk", Commit: commitSHA, Dir: "mesh-sdk"}})
|
||||
before := digests(t, m, "unit")
|
||||
writeSiblings(t, m, []Sibling{{Repository: "novox/mesh-sdk", Commit: strings.Repeat("b", 40), Dir: "mesh-sdk"}})
|
||||
after := digests(t, m, "unit")
|
||||
for _, p := range []string{"fixture/digest/e", "fixture/digest/g"} {
|
||||
if len(before.Packages[p]) != 64 || before.Packages[p] == after.Packages[p] {
|
||||
t.Errorf("%s reads the siblings and its digest stayed (%q, %q)", p, before.Packages[p], after.Packages[p])
|
||||
}
|
||||
}
|
||||
for _, p := range []string{"fixture/digest/a", "fixture/digest/c"} {
|
||||
if before.Packages[p] != after.Packages[p] {
|
||||
t.Errorf("%s does not read the siblings and its digest changed", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -81,7 +116,7 @@ func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) {
|
||||
t.Errorf("the integration digest listed %q", s.command)
|
||||
}
|
||||
}
|
||||
if DigestVersion != "3" {
|
||||
if DigestVersion != "4" {
|
||||
t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
// A change to this module is judged by its own candidate (novox/hq ADR 0535; issue 1898): asked with the runner
|
||||
// workspace, the digest and test steps compile the program of mesh-build-go from the workspace's merged tree in a run
|
||||
// step, shut off, and run the step as that program in one shut-off process (`<candidate> level <spec>`, level.go), where
|
||||
// this program would run the step's commands. This program keeps the fetch, the forge key and the bus login; the
|
||||
// candidate's program holds none of them. Each answer says which program ran it (judged_by).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// The runners of a digest or test step: this installed program, or the candidate's compiled from the workspace.
|
||||
const (
|
||||
RunnerInstalled = "installed"
|
||||
RunnerWorkspace = "workspace"
|
||||
)
|
||||
|
||||
// runnerProperty is the runner argument as the digest and test tools announce it: a checker reads it there, and an
|
||||
// installed module that does not announce it judges as it always did (ADR 0535 §2).
|
||||
var runnerProperty = map[string]any{"type": "string", "enum": []string{RunnerInstalled, RunnerWorkspace},
|
||||
"description": "optional: installed (the default) runs the step as this installed program; workspace compiles the " +
|
||||
"workspace's own mesh-build-go (modules/mesh-build-go of the merged tree) shut off and runs the step as it, for a " +
|
||||
"change to this module (ADR 0535); the answer's judged_by names the program that ran"}
|
||||
|
||||
// candidateModule is the directory of this module's go.mod in the catalogue's tree, and candidateMain its program's
|
||||
// package there.
|
||||
const (
|
||||
candidateModule = "modules/mesh-build-go"
|
||||
candidateMain = "./cmd/mesh-build-go"
|
||||
)
|
||||
|
||||
// runnerArg reads the runner argument: "" (the installed program), installed or workspace.
|
||||
func runnerArg(v any) (string, error) {
|
||||
if v == nil {
|
||||
return "", nil
|
||||
}
|
||||
s, ok := v.(string)
|
||||
if !ok || (s != "" && s != RunnerInstalled && s != RunnerWorkspace) {
|
||||
return "", fmt.Errorf("the runner %v is neither %s nor %s", v, RunnerInstalled, RunnerWorkspace)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// workspaceThere refuses a workspace id that is not one, or a workspace this node never fetched.
|
||||
func (m *Module) workspaceThere(workspace string) error {
|
||||
if !workspaceOK.MatchString(workspace) {
|
||||
return fmt.Errorf("%q is not a workspace id", workspace)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.Work, workspace, "tree")); err != nil {
|
||||
return fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// candidateLocks hold one mutex per workspace, so its candidate is compiled once.
|
||||
var candidateLocks sync.Map
|
||||
|
||||
// candidate is the program of mesh-build-go compiled from the workspace's tree, in the workspace beside its tree
|
||||
// (candidate/mesh-build-go), compiled once per workspace, in a run step shut off with the caches read-only and nothing
|
||||
// fetched (GOPROXY=off: the fetch step downloaded every module the tree's go.mod files name). A candidate that does not
|
||||
// compile is the step's failure, naming the compiler's first error (ADR 0535 §3).
|
||||
func (m *Module) candidate(ctx context.Context, workspace string) (string, error) {
|
||||
exe := filepath.Join(m.Work, workspace, "candidate", "mesh-build-go")
|
||||
mu, _ := candidateLocks.LoadOrStore(exe, &sync.Mutex{})
|
||||
mu.(*sync.Mutex).Lock()
|
||||
defer mu.(*sync.Mutex).Unlock()
|
||||
if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() {
|
||||
return exe, nil
|
||||
}
|
||||
dir := filepath.Join(m.Work, workspace, "tree", candidateModule)
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil {
|
||||
return "", fmt.Errorf("the workspace holds no %s/go.mod, so it has no candidate mesh-build-go to run", candidateModule)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(exe), 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
run, err := m.stepRunner(workspace, false)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
env := append(m.goEnvFor(dir), "GOPROXY=off", "GOTOOLCHAIN=local")
|
||||
r := run(ctx, env, "go", "-C", dir, "build", "-o", exe, candidateMain)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("the candidate's mesh-build-go does not compile: %s", compileError(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
return exe, nil
|
||||
}
|
||||
|
||||
// compileError is the first line of the compiler's output that says an error: not the `# package` heading go build
|
||||
// prints above a package's errors, nor Go's progress.
|
||||
func compileError(out string) string {
|
||||
for _, l := range strings.Split(out, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") && !strings.HasPrefix(l, "go: downloading ") {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return firstLine(out)
|
||||
}
|
||||
|
||||
// mergeCommit is the commit the workspace's tree holds: the merge the fetch made, or the head it fetched.
|
||||
func (m *Module) mergeCommit(ctx context.Context, workspace string) (string, error) {
|
||||
r := m.Run(ctx, nil, "git", "-C", filepath.Join(m.Work, workspace, "tree"), "rev-parse", "HEAD")
|
||||
commit := strings.TrimSpace(r.Stdout)
|
||||
if r.Status != 0 || r.Err != "" || commit == "" {
|
||||
return "", fmt.Errorf("the workspace's commit: git rev-parse: %s", firstLine(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
return commit, nil
|
||||
}
|
||||
|
||||
// asCandidate runs the spec's step as the candidate's program, shut off, and answers what it said and the judged_by of
|
||||
// the answer: "candidate <merge commit>".
|
||||
func (m *Module) asCandidate(ctx context.Context, spec levelSpec) (levelAnswer, string, error) {
|
||||
commit, err := m.mergeCommit(ctx, spec.Workspace)
|
||||
if err != nil {
|
||||
return levelAnswer{}, "", err
|
||||
}
|
||||
exe, err := m.candidate(ctx, spec.Workspace)
|
||||
if err != nil {
|
||||
return levelAnswer{}, "", err
|
||||
}
|
||||
a, err := m.shutOff(ctx, exe, spec)
|
||||
if err != nil {
|
||||
return levelAnswer{}, "", fmt.Errorf("the candidate's mesh-build-go: %w", err)
|
||||
}
|
||||
return a, "candidate " + commit, nil
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// candidateMainSource is a candidate's program that answers a mark: asked `level <spec>`, it tries what a run step must
|
||||
// not reach (the account's key and login, a name, a socket of the machine) and answers a passing level of 42 tests whose
|
||||
// why is "mark" and what it reached.
|
||||
const candidateMainSource = `package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 3 || os.Args[1] != "level" {
|
||||
os.Exit(2)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
_, err := os.ReadFile(%q)
|
||||
seen["key"] = err == nil
|
||||
_, err = os.ReadFile(%q)
|
||||
seen["login"] = err == nil
|
||||
_, err = net.LookupHost("git.novox.be")
|
||||
seen["resolved"] = err == nil
|
||||
c, err := net.DialTimeout("tcp", %q, 2*time.Second)
|
||||
if err == nil {
|
||||
c.Close()
|
||||
}
|
||||
seen["tcp"] = err == nil
|
||||
raw, _ := json.Marshal(seen)
|
||||
var spec struct {
|
||||
Step string ` + "`json:\"step\"`" + `
|
||||
}
|
||||
json.Unmarshal([]byte(os.Args[2]), &spec)
|
||||
out, _ := json.Marshal(map[string]any{"result": map[string]any{"level": spec.Step, "passed": true, "tests": 42,
|
||||
"packages": []any{}, "why": "mark " + string(raw)}})
|
||||
fmt.Println(string(out))
|
||||
}
|
||||
`
|
||||
|
||||
// 1898/AC4 (ADR 0535 §1): a tree whose mesh-build-go program answers a mark: the unit level with the runner workspace
|
||||
// compiles that program and runs it shut off, answers its mark judged by the candidate at the tree's commit, and the
|
||||
// program reached no key, no login, no name and no socket of the machine.
|
||||
func TestTheUnitLevelUnderTheWorkspaceRunnerRunsTheTreesProgramShutOff(t *testing.T) {
|
||||
needTools(t, "the candidate's program shut off", "go", "git", "bwrap")
|
||||
home := t.TempDir()
|
||||
key, login := filepath.Join(home, ".ssh", "mesh_ed25519"), filepath.Join(home, "broker")
|
||||
for _, f := range []string{key, login} {
|
||||
if err := os.MkdirAll(filepath.Dir(f), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(f, []byte("secret"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
tcp, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tcp.Close()
|
||||
|
||||
work, err := os.MkdirTemp("", "w")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(work) })
|
||||
tree := filepath.Join(work, "ws1", "tree")
|
||||
dir := filepath.Join(tree, candidateModule)
|
||||
if err := os.MkdirAll(filepath.Join(dir, "cmd", "mesh-build-go"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module meshbuildgo\n\ngo 1.25.0\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src := fmt.Sprintf(candidateMainSource, key, login, tcp.Addr().String())
|
||||
if err := os.WriteFile(filepath.Join(dir, "cmd", "mesh-build-go", "main.go"), []byte(src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range [][]string{{"init", "-q"}, {"add", "-A"}, {"-c", "user.email=a@b", "-c", "user.name=x", "commit", "-qm", "x"}} {
|
||||
if out, err := exec.Command("git", append([]string{"-C", tree}, c...)...).CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v %s", c, err, out)
|
||||
}
|
||||
}
|
||||
head, err := exec.Command("git", "-C", tree, "rev-parse", "HEAD").Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()}
|
||||
m.Run = StepRunner(home)
|
||||
out, err := m.test(context.Background(), map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res := out.(*TestResult)
|
||||
if res.Tests != 42 || !strings.HasPrefix(res.Why, "mark ") {
|
||||
t.Fatalf("result = %+v, want the candidate's mark", res)
|
||||
}
|
||||
if want := "candidate " + strings.TrimSpace(string(head)); res.JudgedBy != want {
|
||||
t.Errorf("judged by %q, want %q", res.JudgedBy, want)
|
||||
}
|
||||
var seen map[string]bool
|
||||
if err := json.Unmarshal([]byte(strings.TrimPrefix(res.Why, "mark ")), &seen); err != nil || len(seen) != 4 {
|
||||
t.Fatalf("the program said %q: %v", res.Why, err)
|
||||
}
|
||||
for what, reached := range seen {
|
||||
if reached {
|
||||
t.Errorf("the candidate's program reached %s", what)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Issue 1898 K1, K2 (ADR 0535): asked with the runner workspace, the digest and test steps compile the workspace's own
|
||||
// mesh-build-go shut off and run the step as it, in one shut-off process; the answer says which program ran.
|
||||
|
||||
const candidateSHA = "fedcba9876543210fedcba9876543210fedcba98"
|
||||
|
||||
// candidateRig is a module over a workspace whose tree holds modules/mesh-build-go, with a runner standing for git, go
|
||||
// and the candidate's program; what ran shut off is in seen, every command in ran.
|
||||
type candidateRig struct {
|
||||
m *Module
|
||||
states *keptStates
|
||||
seen []shutOff
|
||||
ran []string
|
||||
compile forgekey.Ran // what go build answers; Status 0 writes the program
|
||||
answer func(spec levelSpec) levelAnswer // what the candidate's program answers
|
||||
specs []levelSpec
|
||||
compiled int
|
||||
}
|
||||
|
||||
func newCandidateRig(t *testing.T) *candidateRig {
|
||||
t.Helper()
|
||||
cr := &candidateRig{states: newStates()}
|
||||
cr.m = moduleWith(cr.states, "go1.27.1")
|
||||
cr.m.Home, cr.m.Work, cr.m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
|
||||
cr.m.Executable = "/opt/bundle/mesh-build-go"
|
||||
dir := filepath.Join(cr.m.Work, "ws1", "tree", candidateModule)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module meshbuildgo\n\ngo 1.26.0\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cr.answer = func(spec levelSpec) levelAnswer {
|
||||
if spec.Step == stepDigest {
|
||||
return levelAnswer{Digest: &DigestResult{Level: spec.Level, Packages: map[string]string{"meshbuildgo/cmd/mesh-build-go": "mark"}}}
|
||||
}
|
||||
return levelAnswer{Result: &TestResult{Level: spec.Step, Passed: true, Tests: 42, Packages: []PackageResult{}, Why: "mark"}}
|
||||
}
|
||||
exe := filepath.Join(cr.m.Work, "ws1", "candidate", "mesh-build-go")
|
||||
cr.m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
cr.ran = append(cr.ran, line)
|
||||
switch {
|
||||
case name == "git" && slices.Contains(args, "rev-parse"):
|
||||
return forgekey.Ran{Stdout: candidateSHA + "\n"}
|
||||
case name == "git" && slices.Contains(args, "ls-files"):
|
||||
return forgekey.Ran{Stdout: "go.mod\n"}
|
||||
case name == "postgres":
|
||||
return forgekey.Ran{Stdout: "postgres (PostgreSQL) 18.6\n"}
|
||||
case name == "go" && slices.Contains(args, "build"):
|
||||
cr.compiled++
|
||||
for _, want := range []string{"GOPROXY=off", "GOTOOLCHAIN=local", "GOFLAGS=-mod=mod"} {
|
||||
if !slices.Contains(env, want) {
|
||||
t.Errorf("the candidate compiled without %s: %v", want, env)
|
||||
}
|
||||
}
|
||||
if cr.compile.Status == 0 && cr.compile.Err == "" {
|
||||
if err := os.WriteFile(args[slices.Index(args, "-o")+1], []byte("program"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return cr.compile
|
||||
case name == exe && len(args) == 2 && args[0] == levelArg:
|
||||
var spec levelSpec
|
||||
if err := json.Unmarshal([]byte(args[1]), &spec); err != nil {
|
||||
t.Errorf("the spec %s: %v", args[1], err)
|
||||
}
|
||||
cr.specs = append(cr.specs, spec)
|
||||
raw, _ := json.Marshal(cr.answer(spec))
|
||||
return forgekey.Ran{Stdout: string(raw) + "\n"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1, Err: "not expected here: " + line}
|
||||
}, &cr.seen)
|
||||
return cr
|
||||
}
|
||||
|
||||
func (cr *candidateRig) test(t *testing.T, args map[string]any) (TestResult, delivery.Step) {
|
||||
t.Helper()
|
||||
if _, err := stepTool(t, cr.m, "build_go_test")(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := cr.states.waitEnd(t)
|
||||
var res TestResult
|
||||
if st.State == delivery.StepPassed {
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatalf("result %s: %v", st.Result, err)
|
||||
}
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
|
||||
func TestTheDigestAndTestToolsAnnounceTheRunner(t *testing.T) { // 1898 K1, ADR 0535 §2
|
||||
m := moduleWith(newStates(), "go1.27.1")
|
||||
for _, s := range m.StepTools() {
|
||||
if s.Name != "build_go_digest" && s.Name != "build_go_test" {
|
||||
continue
|
||||
}
|
||||
p, _ := s.Input["properties"].(map[string]any)["runner"].(map[string]any)
|
||||
if p == nil || !slices.Equal(p["enum"].([]string), []string{RunnerInstalled, RunnerWorkspace}) {
|
||||
t.Errorf("%s announces the runner %v, want installed and workspace", s.Name, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheUnitLevelUnderTheWorkspaceRunnerRunsTheCandidateShutOff(t *testing.T) { // 1898 K1, K2 (AC4's unit part)
|
||||
cr := newCandidateRig(t)
|
||||
res, st := cr.test(t, map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace, "packages": []any{"./..."}})
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if res.Tests != 42 || res.Why != "mark" || res.JudgedBy != "candidate "+candidateSHA {
|
||||
t.Errorf("result = %+v, want the candidate's mark judged by candidate %s", res, candidateSHA)
|
||||
}
|
||||
if len(cr.specs) != 1 || cr.specs[0].Step != stepUnit || cr.specs[0].Workspace != "ws1" || !slices.Equal(cr.specs[0].Packages, []string{"./..."}) || cr.specs[0].Bound != UnitBound {
|
||||
t.Errorf("the candidate was asked %+v, want the unit level of ws1 with its packages and bound", cr.specs)
|
||||
}
|
||||
for _, l := range cr.ran {
|
||||
if strings.HasPrefix(l, "go ") && strings.Contains(l, " test ") {
|
||||
t.Errorf("the installed program ran %q: the level is the candidate's to run", l)
|
||||
}
|
||||
}
|
||||
// Two commands shut off: the compile, then the candidate's process; each as a run step is (ADR 0448 §2).
|
||||
if len(cr.seen) != 2 || !strings.Contains(cr.seen[0].command, " build ") || !strings.Contains(cr.seen[0].command, "-C "+filepath.Join(cr.m.Work, "ws1", "tree", candidateModule)) ||
|
||||
!strings.HasPrefix(cr.seen[1].command, filepath.Join(cr.m.Work, "ws1", "candidate", "mesh-build-go")+" level ") {
|
||||
t.Fatalf("shut off %+v, want the compile and the candidate's process", cr.seen)
|
||||
}
|
||||
for _, s := range cr.seen {
|
||||
checkShutOff(t, cr.m, s, false)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheIntegrationLevelUnderTheWorkspaceRunnerRunsTheCandidate(t *testing.T) { // 1898 K2
|
||||
cr := newCandidateRig(t)
|
||||
res, st := cr.test(t, map[string]any{"workspace": "ws1", "level": LevelIntegration, "runner": RunnerWorkspace})
|
||||
if st.State != delivery.StepPassed || res.JudgedBy != "candidate "+candidateSHA {
|
||||
t.Fatalf("the step ended %s (%s): %+v", st.State, st.Reason, res)
|
||||
}
|
||||
if len(cr.specs) != 1 || cr.specs[0].Step != stepIntegration || cr.specs[0].Bound != IntegrationRun || cr.specs[0].Setup != IntegrationSetup {
|
||||
t.Errorf("the candidate was asked %+v, want the integration level with its bounds", cr.specs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheDigestUnderTheWorkspaceRunnerIsTheCandidatesAndTheCandidateIsCompiledOnce(t *testing.T) { // 1898 K1, K2
|
||||
cr := newCandidateRig(t)
|
||||
out, err := stepTool(t, cr.m, "build_go_digest")(map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := out.(DigestResult)
|
||||
if d.Packages["meshbuildgo/cmd/mesh-build-go"] != "mark" || d.JudgedBy != "candidate "+candidateSHA {
|
||||
t.Errorf("digest = %+v, want the candidate's, judged by candidate %s", d, candidateSHA)
|
||||
}
|
||||
if len(cr.specs) != 1 || cr.specs[0].Step != stepDigest || cr.specs[0].Level != LevelUnit {
|
||||
t.Errorf("the candidate was asked %+v", cr.specs)
|
||||
}
|
||||
cr.test(t, map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace})
|
||||
if cr.compiled != 1 {
|
||||
t.Errorf("compiled the candidate %d times for two steps of one workspace, want once", cr.compiled)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACandidateThatDoesNotCompileFailsTheStepWithTheCompilersFirstErrorAndRunsNoLevel(t *testing.T) { // 1898/AC5
|
||||
cr := newCandidateRig(t)
|
||||
cr.compile = forgekey.Ran{Status: 1, Stderr: "# meshbuildgo/cmd/mesh-build-go\ncmd/mesh-build-go/digest.go:12:2: undefined: x\ncmd/mesh-build-go/digest.go:13:2: undefined: y\n"}
|
||||
_, st := cr.test(t, map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace})
|
||||
if st.State == delivery.StepPassed || !strings.Contains(st.Reason, "does not compile: cmd/mesh-build-go/digest.go:12:2: undefined: x") || strings.Contains(st.Reason, "undefined: y") {
|
||||
t.Fatalf("the step ended %s: %q, want it failed naming the first compile error", st.State, st.Reason)
|
||||
}
|
||||
if len(cr.specs) != 0 {
|
||||
t.Errorf("the candidate ran %+v after it failed to compile", cr.specs)
|
||||
}
|
||||
for _, l := range cr.ran {
|
||||
if strings.Contains(l, " test ") {
|
||||
t.Errorf("a level ran: %q", l)
|
||||
}
|
||||
}
|
||||
if _, err := stepTool(t, cr.m, "build_go_digest")(map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": RunnerWorkspace}); err == nil ||
|
||||
!strings.Contains(err.Error(), "undefined: x") {
|
||||
t.Errorf("the digest answered %v, want the compile error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheInstalledProgramSaysItJudged(t *testing.T) { // 1898 K1, ADR 0535 §5
|
||||
m, states, ws, _ := workspaceFor(t, "slow")
|
||||
for _, runner := range []any{nil, RunnerInstalled} {
|
||||
args := map[string]any{"workspace": ws, "level": LevelUnit}
|
||||
if runner != nil {
|
||||
args["runner"] = runner
|
||||
}
|
||||
res, st := runUnitWith(t, m, states, args)
|
||||
if st.State != delivery.StepPassed || res.JudgedBy != RunnerInstalled {
|
||||
t.Errorf("runner %v: the step ended %s, judged by %q, want installed", runner, st.State, res.JudgedBy)
|
||||
}
|
||||
}
|
||||
dm, _ := digestFixture(t)
|
||||
if d := digests(t, dm, LevelUnit); d.JudgedBy != RunnerInstalled {
|
||||
t.Errorf("the installed digest is judged by %q, want installed", d.JudgedBy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnknownRunnerIsRefused(t *testing.T) { // 1898 K1
|
||||
cr := newCandidateRig(t)
|
||||
if _, err := cr.m.digest(context.Background(), map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": "elsewhere"}); err == nil {
|
||||
t.Error("the digest took the runner elsewhere")
|
||||
}
|
||||
if _, err := cr.m.test(context.Background(), map[string]any{"workspace": "ws1", "level": LevelUnit, "runner": "elsewhere"}); err == nil {
|
||||
t.Error("the test step took the runner elsewhere")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheShutOffProcessRunsTheStepItIsAsked(t *testing.T) { // 1898 K2: the candidate's side of the spec
|
||||
m, _, ws, calls := workspaceFor(t, "slow")
|
||||
m.Inside = true
|
||||
a := m.answer(context.Background(), levelSpec{Step: stepUnit, Workspace: ws, Packages: []string{"./..."}, Bound: UnitBound})
|
||||
if a.Error != "" || a.Result == nil || a.Result.Level != LevelUnit || a.Result.Tests == 0 {
|
||||
t.Fatalf("the unit step answered %+v", a)
|
||||
}
|
||||
ranTest := false
|
||||
for _, c := range *calls {
|
||||
ranTest = ranTest || strings.HasPrefix(c.line, "go -C ") && strings.Contains(c.line, " test -json ")
|
||||
}
|
||||
if !ranTest {
|
||||
t.Errorf("the process ran no go test: %+v", *calls)
|
||||
}
|
||||
dm, _ := digestFixture(t)
|
||||
dm.Inside = true
|
||||
if a := dm.answer(context.Background(), levelSpec{Step: stepDigest, Level: LevelUnit, Workspace: "ws1"}); a.Error != "" || a.Digest == nil || len(a.Digest.Packages) == 0 {
|
||||
t.Errorf("the digest step answered %+v", a)
|
||||
}
|
||||
if a := m.answer(context.Background(), levelSpec{Step: "deploy", Workspace: ws}); a.Error == "" {
|
||||
t.Error("the process took a step that is not one")
|
||||
}
|
||||
}
|
||||
|
||||
func runUnitWith(t *testing.T, m *Module, states *keptStates, args map[string]any) (TestResult, delivery.Step) {
|
||||
t.Helper()
|
||||
if _, err := stepTool(t, m, "build_go_test")(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res TestResult
|
||||
if st.State == delivery.StepPassed {
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatalf("result %s: %v", st.Result, err)
|
||||
}
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
@@ -5,6 +5,7 @@ package main
|
||||
// digest is unchanged is not run again. The tool answers in its call (it is short, ADR 0469 §2).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
@@ -22,7 +23,12 @@ import (
|
||||
// DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind.
|
||||
// 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3). 3: the siblings'
|
||||
// commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement).
|
||||
const DigestVersion = "3"
|
||||
// 4: the siblings' commits enter only the digest of a package whose tests reach a file naming MESH_CHECK_BESIDE (ADR 0536 §1).
|
||||
const DigestVersion = "4"
|
||||
|
||||
// besideVariable is the name a file bears when it reads where the siblings lie; a package whose tests reach such a file
|
||||
// depends on the siblings' commits.
|
||||
const besideVariable = "MESH_CHECK_BESIDE"
|
||||
|
||||
// integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation
|
||||
// for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run.
|
||||
@@ -83,6 +89,9 @@ type DigestResult struct {
|
||||
// only when they are the siblings it gave the fetch.
|
||||
Beside []string `json:"beside,omitempty"`
|
||||
WithoutTests []string `json:"without_tests,omitempty"`
|
||||
// JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
|
||||
// runner workspace.
|
||||
JudgedBy string `json:"judged_by,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Module) digestTool() StepTool {
|
||||
@@ -97,6 +106,7 @@ func (m *Module) digestTool() StepTool {
|
||||
"properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
|
||||
"level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
|
||||
"runner": runnerProperty,
|
||||
},
|
||||
"required": []string{"workspace", "level"},
|
||||
},
|
||||
@@ -107,12 +117,49 @@ func (m *Module) digestTool() StepTool {
|
||||
func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
level, _ := args["level"].(string)
|
||||
runner, err := runnerArg(args["runner"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if runner == RunnerWorkspace {
|
||||
if err := m.workspaceThere(workspace); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch level {
|
||||
case LevelUnit:
|
||||
case LevelIntegration:
|
||||
if _, err := m.checkPostgres(ctx); err != nil {
|
||||
return nil, err // a *buildstep.Refusal, as the installed program's digest says it
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
|
||||
}
|
||||
a, judged, err := m.asCandidate(ctx, levelSpec{Step: stepDigest, Level: level, Workspace: workspace})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.Digest == nil {
|
||||
return nil, fmt.Errorf("the candidate's mesh-build-go answered no digest")
|
||||
}
|
||||
a.Digest.JudgedBy = judged
|
||||
return *a.Digest, nil
|
||||
}
|
||||
res, err := m.digestHere(ctx, workspace, level)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.JudgedBy = RunnerInstalled
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// digestHere is the digest step as this program runs it.
|
||||
func (m *Module) digestHere(ctx context.Context, workspace, level string) (DigestResult, error) {
|
||||
if !workspaceOK.MatchString(workspace) {
|
||||
return nil, fmt.Errorf("%q is not a workspace id", workspace)
|
||||
return DigestResult{}, fmt.Errorf("%q is not a workspace id", workspace)
|
||||
}
|
||||
tree := filepath.Join(m.Work, workspace, "tree")
|
||||
if _, err := os.Stat(tree); err != nil {
|
||||
return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
|
||||
return DigestResult{}, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
|
||||
}
|
||||
var tags []string
|
||||
header := []string{"digest " + DigestVersion, "level " + level}
|
||||
@@ -120,34 +167,35 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
|
||||
case LevelUnit:
|
||||
case LevelIntegration:
|
||||
if _, err := m.checkPostgres(ctx); err != nil {
|
||||
return nil, err
|
||||
return DigestResult{}, err
|
||||
}
|
||||
tags = integrationTags
|
||||
header = append(header, "postgres "+strconv.Itoa(m.Recipe.PostgresMajor), "nats-server "+m.Recipe.NatsServer)
|
||||
default:
|
||||
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
|
||||
return DigestResult{}, fmt.Errorf("the level %q is neither unit nor integration", level)
|
||||
}
|
||||
header = append(header, "tags "+strings.Join(tags, ","))
|
||||
beside, err := m.besideLines(workspace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return DigestResult{}, err
|
||||
}
|
||||
besideHeader := make([]string, 0, len(beside))
|
||||
for _, b := range beside {
|
||||
header = append(header, "beside "+b)
|
||||
besideHeader = append(besideHeader, "beside "+b)
|
||||
}
|
||||
|
||||
dirs, err := m.goModDirs(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return DigestResult{}, err
|
||||
}
|
||||
res := DigestResult{Level: level, Packages: map[string]string{}, Beside: beside}
|
||||
for _, d := range dirs {
|
||||
pkgs, err := m.listPackages(ctx, workspace, tree, d, tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return DigestResult{}, err
|
||||
}
|
||||
if err := digestModule(&res, pkgs, tree, header); err != nil {
|
||||
return nil, err
|
||||
if err := digestModule(&res, pkgs, tree, header, besideHeader); err != nil {
|
||||
return DigestResult{}, err
|
||||
}
|
||||
}
|
||||
sort.Strings(res.WithoutTests)
|
||||
@@ -190,7 +238,7 @@ func parsePackages(out string) (map[string]*goPackage, error) {
|
||||
|
||||
// digestModule adds to res the digest of each root package of one go.mod directory. A root is a package of the main
|
||||
// module that is listed, not only a dependency, and not a variant of another.
|
||||
func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header []string) error {
|
||||
func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header, beside []string) error {
|
||||
roots := []string{}
|
||||
for path, p := range pkgs {
|
||||
if p.ForTest == "" && !p.DepOnly && !p.Standard && !strings.HasSuffix(path, ".test") && p.Module != nil && p.Module.Main {
|
||||
@@ -208,7 +256,7 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he
|
||||
if testMain == nil {
|
||||
return fmt.Errorf("go list gave no test binary for %s", path)
|
||||
}
|
||||
d, err := packageDigest(root, testMain, pkgs, tree, header)
|
||||
d, err := packageDigest(root, testMain, pkgs, tree, header, beside)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
@@ -219,8 +267,9 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he
|
||||
|
||||
// packageDigest is the digest of one package: a hash over a sorted text of the header, the files of every package its
|
||||
// tests reach in the module's own tree (by their content, under their path in the tree), the version and checksum of
|
||||
// every module outside it, and the package's own test data. A package of the standard library is the Go version's.
|
||||
func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header []string) (string, error) {
|
||||
// every module outside it, and the package's own test data. The beside lines (the siblings' commits) join the header
|
||||
// only when a file the tests reach names MESH_CHECK_BESIDE. A package of the standard library is the Go version's.
|
||||
func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header, beside []string) (string, error) {
|
||||
reached := map[string]*goPackage{root.ImportPath: root}
|
||||
for _, dep := range testMain.Deps {
|
||||
if p := pkgs[dep]; p != nil {
|
||||
@@ -228,6 +277,7 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s
|
||||
}
|
||||
}
|
||||
lines := []string{}
|
||||
readsBeside := false
|
||||
for _, p := range reached {
|
||||
if p.Standard || p.Module == nil {
|
||||
continue
|
||||
@@ -246,6 +296,11 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s
|
||||
return "", err
|
||||
}
|
||||
lines = append(lines, l)
|
||||
if !readsBeside {
|
||||
if raw, err := os.ReadFile(filepath.Join(p.Dir, f)); err == nil && bytes.Contains(raw, []byte(besideVariable)) {
|
||||
readsBeside = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
data := filepath.Join(root.Dir, "testdata")
|
||||
@@ -263,6 +318,9 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s
|
||||
}
|
||||
}
|
||||
sort.Strings(lines)
|
||||
if readsBeside {
|
||||
header = append(append([]string{}, header...), beside...)
|
||||
}
|
||||
h := sha256.New()
|
||||
for _, l := range append(append([]string{}, header...), lines...) {
|
||||
h.Write([]byte(l + "\n"))
|
||||
|
||||
@@ -173,7 +173,7 @@ func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) {
|
||||
testMain := &goPackage{ImportPath: "x/r.test", Deps: []string{ext.ImportPath}}
|
||||
pkgs[root.ImportPath] = root
|
||||
digest := func() string {
|
||||
d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"})
|
||||
d, err := packageDigest(root, testMain, pkgs, dir, []string{"digest 1"}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -6,12 +6,15 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
@@ -23,11 +26,6 @@ var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
|
||||
const ForgeAlias = "mesh-forge"
|
||||
|
||||
// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it
|
||||
// fetches and every round it checks its toolchain. A workspace lies in the module's work directory, never in its
|
||||
// home (ADR 0466 §1).
|
||||
const WorkspaceAge = 24 * time.Hour
|
||||
|
||||
var (
|
||||
repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
workspaceOK = regexp.MustCompile(`^[a-z0-9]+$`)
|
||||
@@ -82,27 +80,44 @@ func (m *Module) fetchTool() StepTool {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
// fetch makes a workspace and fills it. A workspace lies in the module's work directory, never in its home (ADR 0466
|
||||
// §1); it is held while the fetch runs, and removed when the fetch fails, since nobody learns its id to release it
|
||||
// (issue 1839).
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
|
||||
// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
|
||||
// named paths, the workspace is narrowed to them (buildstep.Narrow).
|
||||
req, err := buildstep.ParseFetchRequest(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
||||
siblings, err := siblingsArg(args["beside"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
id := buildstep.NewID()
|
||||
done := m.spaces().Hold(id)
|
||||
defer func() {
|
||||
done()
|
||||
if err != nil {
|
||||
if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
|
||||
m.Say("the failed fetch's workspace %s: %v", id, rerr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return m.fetchInto(ctx, req, id, siblings)
|
||||
}
|
||||
|
||||
func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id string, siblings []siblingArg) (any, error) {
|
||||
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
||||
var err error
|
||||
root := filepath.Join(m.Work, id)
|
||||
tree := filepath.Join(root, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remote := m.remote(repository)
|
||||
if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.FetchOptions...); err != nil {
|
||||
if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.fetchOptions(ctx, repository, commit, ref)...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := FetchResult{Workspace: id, Commit: commit}
|
||||
@@ -111,6 +126,9 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := m.dissociate(ctx, tree); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if res.Beside, err = m.placeSiblings(ctx, root, siblings); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -131,11 +149,27 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
if base != "" && res.Merge == "" {
|
||||
return res, nil // a conflict: the tree is at the trunk, nothing of the head is built or tested
|
||||
}
|
||||
for _, d := range dirs {
|
||||
// The directories download side by side, up to the bound (ADR 0536 §2); the first failure in directory order is the
|
||||
// step's. Resolving a pin places a repository beside the workspace, which two directories may share: one at a time.
|
||||
var pins sync.Mutex
|
||||
errs := make([]error, len(dirs))
|
||||
slots := make(chan struct{}, m.parallel())
|
||||
var wg sync.WaitGroup
|
||||
for i, d := range dirs {
|
||||
if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil {
|
||||
continue // a module that vendors resolves nothing
|
||||
}
|
||||
if err := m.downloadModules(ctx, root, filepath.Join(tree, d), d); err != nil {
|
||||
wg.Add(1)
|
||||
slots <- struct{}{}
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-slots }()
|
||||
errs[i] = m.downloadModules(ctx, root, filepath.Join(tree, d), d, &pins)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
for _, err := range errs {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
@@ -145,13 +179,15 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
// downloadModules downloads and verifies the Go modules of the directory dir (d in the tree). A pin of a forge module
|
||||
// that Go cannot fetch is looked for among the forge's pull request heads and handed to Go by a modfile (pins.go); a
|
||||
// failure that names no such pin is the first failure.
|
||||
func (m *Module) downloadModules(ctx context.Context, root, dir, d string) error {
|
||||
func (m *Module) downloadModules(ctx context.Context, root, dir, d string, pins *sync.Mutex) error {
|
||||
for {
|
||||
failure := m.downloadAndVerify(ctx, dir, d)
|
||||
if failure == nil {
|
||||
return nil
|
||||
}
|
||||
pins.Lock()
|
||||
progress, err := m.resolvePins(ctx, root, dir, failure.Error())
|
||||
pins.Unlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -161,8 +197,41 @@ func (m *Module) downloadModules(ctx context.Context, root, dir, d string) error
|
||||
}
|
||||
}
|
||||
|
||||
// verifyLocks hold one mutex per digest of a go.mod and go.sum, so two directories with the same pair verify once.
|
||||
var verifyLocks sync.Map
|
||||
|
||||
// verifiedMark is the file naming a pass of go mod verify for a go.mod and go.sum pair, "" when a pass is not kept:
|
||||
// the module cache is only read by steps that are shut off with it read-only (ADR 0448 §3), so the pass holds only then,
|
||||
// and a directory with a pin modfile verifies another file.
|
||||
func (m *Module) verifiedMark(dir string) string {
|
||||
if m.Cache == "" || m.Bubblewrap == nil {
|
||||
return ""
|
||||
}
|
||||
if _, ok := m.pinModfile(dir); ok {
|
||||
return ""
|
||||
}
|
||||
h := sha256.New()
|
||||
for _, name := range []string{"go.mod", "go.sum"} {
|
||||
raw, _ := os.ReadFile(filepath.Join(dir, name))
|
||||
fmt.Fprintf(h, "%s %d\n", name, len(raw))
|
||||
h.Write(raw)
|
||||
}
|
||||
return filepath.Join(m.Cache, "verified", hex.EncodeToString(h.Sum(nil)))
|
||||
}
|
||||
|
||||
func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error {
|
||||
for _, verb := range []string{"download", "verify"} {
|
||||
mark := ""
|
||||
if verb == "verify" {
|
||||
if mark = m.verifiedMark(dir); mark != "" {
|
||||
mu, _ := verifyLocks.LoadOrStore(mark, &sync.Mutex{})
|
||||
mu.(*sync.Mutex).Lock()
|
||||
defer mu.(*sync.Mutex).Unlock()
|
||||
if _, err := os.Stat(mark); err == nil {
|
||||
continue // this go.mod and go.sum passed before
|
||||
}
|
||||
}
|
||||
}
|
||||
cmd, env := []string{"-C", dir, "mod", verb}, m.goEnvFor(dir)
|
||||
if modfile, ok := m.pinModfile(dir); ok && verb == "verify" {
|
||||
// go mod verify takes no -modfile: it is run on the modfile itself, which names the placed repositories.
|
||||
@@ -172,6 +241,12 @@ func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error {
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("go mod %s in %s: %s", verb, d, causeLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr))
|
||||
}
|
||||
if mark != "" {
|
||||
// Said after the pass; a mark that cannot be written only means the next fetch verifies again.
|
||||
if os.MkdirAll(filepath.Dir(mark), 0o700) == nil {
|
||||
os.WriteFile(mark, nil, 0o600)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -236,21 +311,10 @@ func (m *Module) goEnvFor(dir string) []string {
|
||||
return env
|
||||
}
|
||||
|
||||
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
|
||||
// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
|
||||
// net for a caller that never released one (issue 1839, ADR 0509).
|
||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||
entries, err := os.ReadDir(m.Work)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
|
||||
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
|
||||
}
|
||||
}
|
||||
m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
|
||||
}
|
||||
|
||||
// clock is the time now: Now when a test set it, else the wall clock.
|
||||
|
||||
@@ -30,6 +30,7 @@ type fakeStep struct {
|
||||
gomods string
|
||||
gate chan struct{} // when set, the first git command waits for it
|
||||
goRule func(line string) forgekey.Ran
|
||||
files map[string]string // files (relative to the tree) written at checkout: go.mod and go.sum of the fixture
|
||||
vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout
|
||||
rule func(line string) (forgekey.Ran, bool) // answers a command before the defaults do
|
||||
}
|
||||
@@ -55,6 +56,10 @@ func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...s
|
||||
case name == "git" && strings.Contains(line, " checkout "):
|
||||
for i, a := range args {
|
||||
if a == "-C" {
|
||||
for name, text := range f.files {
|
||||
os.MkdirAll(filepath.Dir(filepath.Join(args[i+1], name)), 0o755)
|
||||
os.WriteFile(filepath.Join(args[i+1], name), []byte(text), 0o644)
|
||||
}
|
||||
for _, v := range f.vendors {
|
||||
os.MkdirAll(filepath.Join(args[i+1], v, "vendor"), 0o755)
|
||||
os.WriteFile(filepath.Join(args[i+1], v, "vendor", "modules.txt"), nil, 0o644)
|
||||
@@ -265,10 +270,11 @@ func TestARepositoryThatIsNotOwnerSlashNameIsRefusedBeforeAnythingRuns(t *testin
|
||||
}
|
||||
}
|
||||
|
||||
// Workspaces older than a day are removed; younger ones, and anything that is not a workspace, stay.
|
||||
func TestWorkspacesOlderThanADayAreRemoved(t *testing.T) {
|
||||
// Unheld workspaces last used more than two hours ago are removed (issue 1839, ADR 0509); younger ones, and anything
|
||||
// that is not a workspace, stay.
|
||||
func TestUnheldWorkspacesPastTwoHoursAreRemoved(t *testing.T) {
|
||||
m := &Module{Work: t.TempDir()}
|
||||
for name, age := range map[string]time.Duration{"old1": 25 * time.Hour, "young": time.Hour, "Keep-This": 48 * time.Hour} {
|
||||
for name, age := range map[string]time.Duration{"old1": 3 * time.Hour, "young": time.Hour, "Keep-This": 48 * time.Hour} {
|
||||
dir := filepath.Join(m.Work, name)
|
||||
os.MkdirAll(filepath.Join(dir, "tree"), 0o755)
|
||||
when := time.Now().Add(-age)
|
||||
|
||||
@@ -75,8 +75,16 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa
|
||||
env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...)
|
||||
// Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run
|
||||
// below finds them built. Whether they compile is for the run to say; its failure is not read here.
|
||||
// Only the directories the selected packages lie in compile (ADR 0536 §3); no selection is every directory.
|
||||
byDir, err := packagesByDir(tree, dirs, packages)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, d := range dirs {
|
||||
m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1", "./...")
|
||||
if len(byDir[d]) == 0 {
|
||||
continue
|
||||
}
|
||||
m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", append([]string{"-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1"}, byDir[d]...)...)
|
||||
}
|
||||
setup := m.clock().Sub(began)
|
||||
|
||||
|
||||
@@ -31,6 +31,10 @@ type integFake struct {
|
||||
busDown bool
|
||||
runDirs []string // run directories seen when the store was started, to be gone at the end
|
||||
panicOn string // a phase that panics
|
||||
// gomods is what git ls-files answers ("go.mod\n" when empty); onTest is called, unlocked, as each go test -json
|
||||
// command runs, with its line.
|
||||
gomods string
|
||||
onTest func(line string)
|
||||
}
|
||||
|
||||
func newIntegFake(t *testing.T, captured string) *integFake {
|
||||
@@ -63,10 +67,16 @@ func (f *integFake) run(ctx context.Context, env []string, name string, args ...
|
||||
case name == "postgres":
|
||||
return forgekey.Ran{Stdout: f.postgres + "\n"}
|
||||
case name == "git" && len(args) > 2 && args[2] == "ls-files":
|
||||
if f.gomods != "" {
|
||||
return forgekey.Ran{Stdout: f.gomods}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "go.mod\n"}
|
||||
case name == "go" && strings.Contains(line, "-run ^$"):
|
||||
f.tick("compile")
|
||||
case name == "go":
|
||||
if f.onTest != nil {
|
||||
f.onTest(line)
|
||||
}
|
||||
f.tick("test")
|
||||
status := 0
|
||||
if strings.Contains(f.captured, `"Action":"fail"`) {
|
||||
|
||||
@@ -18,11 +18,22 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// levelArg is the first argument that makes this binary an integration level's process.
|
||||
// levelArg is the first argument that makes this binary a step's shut-off process: the integration level's, and, when
|
||||
// the merge changes this module, the candidate's program running a digest or a level (ADR 0535 §1, candidate.go).
|
||||
const levelArg = "level"
|
||||
|
||||
// levelSpec is what the shut-off process is asked: the workspace, the packages and the -run pattern, the level's bounds.
|
||||
// The steps a shut-off process runs (levelSpec.Step); empty is the integration level, as a spec said before the runner.
|
||||
const (
|
||||
stepDigest = "digest"
|
||||
stepUnit = LevelUnit
|
||||
stepIntegration = LevelIntegration
|
||||
)
|
||||
|
||||
// levelSpec is what the shut-off process is asked: the step, the workspace, the packages and the -run pattern, the
|
||||
// level's bounds; for a digest, its level.
|
||||
type levelSpec struct {
|
||||
Step string `json:"step,omitempty"`
|
||||
Level string `json:"level,omitempty"`
|
||||
Work string `json:"work"`
|
||||
Cache string `json:"cache"`
|
||||
Workspace string `json:"workspace"`
|
||||
@@ -32,10 +43,11 @@ type levelSpec struct {
|
||||
Bound time.Duration `json:"bound"`
|
||||
}
|
||||
|
||||
// levelAnswer is the one line the process prints: the level's result, or why it could not run.
|
||||
// levelAnswer is the one line the process prints: the level's result or the digest, or why it could not run.
|
||||
type levelAnswer struct {
|
||||
Result *TestResult `json:"result,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Result *TestResult `json:"result,omitempty"`
|
||||
Digest *DigestResult `json:"digest,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// executable is this program's path, bound read-only into the step: Executable when a test set it.
|
||||
@@ -50,35 +62,63 @@ func (m *Module) executable() (string, error) {
|
||||
return filepath.EvalSymlinks(exe)
|
||||
}
|
||||
|
||||
// integrationShutOff runs the integration level in one shut-off process and reads its answer.
|
||||
// integrationShutOff runs the integration level in one shut-off process of this program and reads its answer.
|
||||
func (m *Module) integrationShutOff(ctx context.Context, workspace string, packages []string, pattern string) (*TestResult, error) {
|
||||
exe, err := m.executable()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the integration level's program: %w", err)
|
||||
}
|
||||
b := m.integrationBounds()
|
||||
spec, err := json.Marshal(levelSpec{Work: m.Work, Cache: m.Cache, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run})
|
||||
a, err := m.shutOff(ctx, exe, levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.Result == nil {
|
||||
return nil, fmt.Errorf("the integration level's shut-off process answered no result")
|
||||
}
|
||||
return a.Result, nil
|
||||
}
|
||||
|
||||
// shutOff runs the program exe as one shut-off process of the workspace (`exe level <spec>`) and reads its answer: the
|
||||
// workspace read-write, the program alone read-only, the shared caches beneath a throwaway layer, the Node build
|
||||
// module's toolchains read-only for the integration level.
|
||||
func (m *Module) shutOff(ctx context.Context, exe string, spec levelSpec) (levelAnswer, error) {
|
||||
spec.Work, spec.Cache = m.Work, m.Cache
|
||||
raw, err := json.Marshal(spec)
|
||||
if err != nil {
|
||||
return levelAnswer{}, err
|
||||
}
|
||||
for _, c := range m.caches() {
|
||||
if err := os.MkdirAll(c, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("the shared cache %s: %w", c, err)
|
||||
return levelAnswer{}, fmt.Errorf("the shared cache %s: %w", c, err)
|
||||
}
|
||||
}
|
||||
iso := m.isolation(workspace, false)
|
||||
iso := m.isolation(spec.Workspace, false)
|
||||
iso.ReadOnly = append(iso.ReadOnly, exe) // the program alone, not the directory it lies in
|
||||
iso.ReadOnly = append(iso.ReadOnly, m.nodeToolchains()...)
|
||||
r := buildstep.Isolated(m.Run, iso)(ctx, nil, exe, levelArg, string(spec))
|
||||
if spec.Step == stepIntegration || spec.Step == "" {
|
||||
iso.ReadOnly = append(iso.ReadOnly, m.nodeToolchains()...)
|
||||
}
|
||||
r := buildstep.Isolated(m.Run, iso)(ctx, nil, exe, levelArg, string(raw))
|
||||
var answer levelAnswer
|
||||
line := lastLine(r.Stdout)
|
||||
if err := json.Unmarshal([]byte(line), &answer); err != nil || (answer.Result == nil && answer.Error == "") {
|
||||
return nil, fmt.Errorf("the integration level's shut-off process answered nothing it says (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout))
|
||||
if err := json.Unmarshal([]byte(line), &answer); err != nil || (answer.Result == nil && answer.Digest == nil && answer.Error == "") {
|
||||
return levelAnswer{}, fmt.Errorf("the %s step's shut-off process answered nothing it says (status %d %s): %s", stepName(spec.Step), r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return nil, fmt.Errorf("%s", answer.Error)
|
||||
return levelAnswer{}, fmt.Errorf("%s", answer.Error)
|
||||
}
|
||||
return answer.Result, nil
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// stepName names a spec's step in words.
|
||||
func stepName(step string) string {
|
||||
switch step {
|
||||
case stepDigest:
|
||||
return "digest"
|
||||
case stepUnit:
|
||||
return "unit level's"
|
||||
}
|
||||
return "integration level's"
|
||||
}
|
||||
|
||||
// NodeToolchains is the Node build module's toolchains directory on this node: its cache directory (module.json
|
||||
@@ -124,18 +164,40 @@ func runLevelProcess(args []string) int {
|
||||
if !workspaceOK.MatchString(spec.Workspace) {
|
||||
return say(levelAnswer{Error: fmt.Sprintf("%q is not a workspace id", spec.Workspace)})
|
||||
}
|
||||
m := &Module{Inside: true, Recipe: Current(), Work: spec.Work, Cache: spec.Cache, Home: os.Getenv("HOME"),
|
||||
m := &Module{Inside: true, Recipe: Current(), Work: spec.Work, Cache: spec.Cache, Home: os.Getenv("HOME"), UnitBound: spec.Bound,
|
||||
IntegrationBounds: Bounds{Setup: spec.Setup, Run: spec.Bound}, Say: func(string, ...any) {}}
|
||||
m.Run = StepRunner(m.Home)
|
||||
ctx := context.Background()
|
||||
major, err := m.checkPostgres(ctx)
|
||||
if err != nil {
|
||||
return say(levelAnswer{Error: err.Error()})
|
||||
return say(m.answer(context.Background(), spec))
|
||||
}
|
||||
|
||||
// answer runs the spec's step in this process, already shut off: a digest of a level, the unit level or the integration
|
||||
// level of the workspace.
|
||||
func (m *Module) answer(ctx context.Context, spec levelSpec) levelAnswer {
|
||||
tree := filepath.Join(m.Work, spec.Workspace, "tree")
|
||||
switch spec.Step {
|
||||
case stepDigest:
|
||||
d, err := m.digestHere(ctx, spec.Workspace, spec.Level)
|
||||
if err != nil {
|
||||
return levelAnswer{Error: err.Error()}
|
||||
}
|
||||
return levelAnswer{Digest: &d}
|
||||
case stepUnit:
|
||||
res, err := m.runLevel(ctx, tree, levelOpts{workspace: spec.Workspace, level: LevelUnit, packages: spec.Packages, run: spec.Run,
|
||||
bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(spec.Workspace)})
|
||||
if err != nil {
|
||||
return levelAnswer{Error: err.Error()}
|
||||
}
|
||||
return levelAnswer{Result: res}
|
||||
case stepIntegration, "":
|
||||
major, err := m.checkPostgres(ctx)
|
||||
if err != nil {
|
||||
return levelAnswer{Error: err.Error()}
|
||||
}
|
||||
res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major)
|
||||
if err != nil {
|
||||
return levelAnswer{Error: err.Error()}
|
||||
}
|
||||
return levelAnswer{Result: res}
|
||||
}
|
||||
tree := filepath.Join(spec.Work, spec.Workspace, "tree")
|
||||
res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major)
|
||||
if err != nil {
|
||||
return say(levelAnswer{Error: err.Error()})
|
||||
}
|
||||
return say(levelAnswer{Result: res})
|
||||
return levelAnswer{Error: fmt.Sprintf("the step %q is not digest, unit or integration", spec.Step)}
|
||||
}
|
||||
|
||||
@@ -83,6 +83,9 @@ type TestResult struct {
|
||||
OverWhich string `json:"over_which,omitempty"`
|
||||
Store string `json:"store,omitempty"`
|
||||
Bus string `json:"bus,omitempty"`
|
||||
// JudgedBy is the program that ran the level (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
|
||||
// runner workspace.
|
||||
JudgedBy string `json:"judged_by,omitempty"`
|
||||
}
|
||||
|
||||
// event is one line of `go test -json` (test2json, and the build events go test adds).
|
||||
|
||||
@@ -46,6 +46,7 @@ func main() {
|
||||
Work: WorkDir,
|
||||
Cache: CacheDir,
|
||||
}
|
||||
m.Floor = floorOf(m.Node)
|
||||
if u, err := user.Current(); err == nil {
|
||||
m.Home = u.HomeDir
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// mirrorLocks hold one mutex per mirror of this process; the lock file beside the mirror holds off another process.
|
||||
var mirrorLocks sync.Map
|
||||
|
||||
// mirrorDir is where the bare mirror of a repository lies: one per repository in the module's cache (ADR 0536 §2).
|
||||
func (m *Module) mirrorDir(repository string) string {
|
||||
return filepath.Join(m.Cache, "mirror", filepath.FromSlash(repository)+".git")
|
||||
}
|
||||
|
||||
// fetchOptions are the options a fetch of repository runs with: the module's own, and the mirror of the repository
|
||||
// brought to the commit, so git transfers only the objects the mirror lacks. A mirror that cannot be brought to the
|
||||
// commit is said and left out: the fetch is then the full one, as it was.
|
||||
func (m *Module) fetchOptions(ctx context.Context, repository, commit, ref string) []buildstep.Option {
|
||||
opts := m.FetchOptions
|
||||
if m.Cache == "" || !repositoryOK.MatchString(repository) {
|
||||
return opts
|
||||
}
|
||||
mirror := m.mirrorDir(repository)
|
||||
if err := m.updateMirror(ctx, mirror, m.remote(repository), commit, ref); err != nil {
|
||||
if m.Say != nil {
|
||||
m.Say("the mirror of %s is not used: %v", repository, err)
|
||||
}
|
||||
return opts
|
||||
}
|
||||
return append(append([]buildstep.Option{}, opts...), buildstep.WithMirror(mirror))
|
||||
}
|
||||
|
||||
// updateMirror brings the mirror to the commit under the mirror's lock: a mutex for the checks of this process, a lock
|
||||
// file for another process of the node.
|
||||
func (m *Module) updateMirror(ctx context.Context, mirror, remote, commit, ref string) error {
|
||||
mu, _ := mirrorLocks.LoadOrStore(mirror, &sync.Mutex{})
|
||||
mu.(*sync.Mutex).Lock()
|
||||
defer mu.(*sync.Mutex).Unlock()
|
||||
if err := os.MkdirAll(mirror, 0o700); err != nil { // git init --bare takes an empty directory
|
||||
return err
|
||||
}
|
||||
lock, err := os.OpenFile(mirror+".lock", os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer lock.Close()
|
||||
if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
|
||||
return fmt.Errorf("lock %s: %w", mirror, err)
|
||||
}
|
||||
defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)
|
||||
return buildstep.UpdateMirror(ctx, m.Run, mirror, remote, commit, ref, m.FetchOptions...)
|
||||
}
|
||||
|
||||
// dissociate makes the repository in dir self-contained again, as git clone --dissociate does: the objects it borrowed
|
||||
// from the mirror are packed into it and its alternates file goes, so a step shut off with the workspace alone (the
|
||||
// mirror is not bound into it) finds every object. The network saving stays: the borrowing was for the transfer.
|
||||
func (m *Module) dissociate(ctx context.Context, dir string) error {
|
||||
alternates := filepath.Join(dir, ".git", "objects", "info", "alternates")
|
||||
if _, err := os.Stat(alternates); err != nil {
|
||||
return nil // no mirror was used
|
||||
}
|
||||
if r := m.Run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-C", dir, "repack", "-a", "-d", "-q"); r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("git repack in %s: %s", dir, firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
return os.Remove(alternates)
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func fetchOnce(t *testing.T, m *Module, states *keptStates, args map[string]any) FetchResult {
|
||||
t.Helper()
|
||||
if _, err := fetchTool(t, m)(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
var res FetchResult
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Issue 890/AC4 (ADR 0536 §2): two fetches of one repository on one module bring the same mirror to their commit, and
|
||||
// each workspace's alternates file names the mirror's objects.
|
||||
func TestTwoFetchesOfARepositoryUseOneMirrorAndReferenceIt(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n"}
|
||||
m, states := fetchModule(t, f)
|
||||
mirror := filepath.Join(m.Cache, "mirror", "novox", "mesh-sdk.git")
|
||||
var workspaces []string
|
||||
for i := 0; i < 2; i++ {
|
||||
workspaces = append(workspaces, fetchOnce(t, m, states, fetchArgs).Workspace)
|
||||
}
|
||||
updates := 0
|
||||
for _, l := range f.lines() {
|
||||
if strings.Contains(l, "-C "+mirror+" fetch --no-tags origin "+commitSHA) {
|
||||
updates++
|
||||
}
|
||||
if strings.Contains(l, "mirror") && !strings.Contains(l, mirror) {
|
||||
t.Errorf("a command names another mirror: %s", l)
|
||||
}
|
||||
}
|
||||
if updates != 2 {
|
||||
t.Errorf("the mirror %s was updated %d times, want 2:\n%s", mirror, updates, strings.Join(f.lines(), "\n"))
|
||||
}
|
||||
// The fake git writes nothing, so the alternates file the SDK wrote stands until the fetch dissociates: it is gone
|
||||
// at the end and git repack -a -d ran in each tree.
|
||||
for _, ws := range workspaces {
|
||||
tree := filepath.Join(m.Work, ws, "tree")
|
||||
if _, err := os.Stat(filepath.Join(tree, ".git", "objects", "info", "alternates")); err == nil {
|
||||
t.Errorf("the workspace %s still borrows from the mirror", ws)
|
||||
}
|
||||
if !strings.Contains(strings.Join(f.lines(), "\n"), "-C "+tree+" repack -a -d") {
|
||||
t.Errorf("git repack -a -d did not run in %s", tree)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A sibling is placed through its own repository's mirror.
|
||||
func TestASiblingIsPlacedThroughItsOwnMirror(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n"}
|
||||
m, states := fetchModule(t, f)
|
||||
args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{
|
||||
map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}}}
|
||||
ws := fetchOnce(t, m, states, args).Workspace
|
||||
for _, repo := range []string{"mesh-controller", "mesh-sdk"} {
|
||||
mirror := filepath.Join(m.Cache, "mirror", "novox", repo+".git")
|
||||
if !strings.Contains(strings.Join(f.lines(), "\n"), "-C "+mirror+" fetch --no-tags origin "+commitSHA) {
|
||||
t.Errorf("the mirror of %s was not updated", repo)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.Work, ws, "beside", "mesh-sdk", ".git", "objects", "info", "alternates")); err == nil {
|
||||
t.Errorf("the sibling still borrows from the mirror")
|
||||
}
|
||||
}
|
||||
|
||||
// Two checks at once on one node update one mirror one after the other.
|
||||
func TestTwoUpdatesOfOneMirrorNeverRunAtOnce(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
inside, most := 0, 0
|
||||
f := &fakeStep{gomods: "go.mod\n"}
|
||||
f.rule = func(line string) (r runAnswer, ok bool) {
|
||||
if !strings.Contains(line, "fetch --no-tags origin") || strings.Contains(line, "--depth") {
|
||||
return r, false
|
||||
}
|
||||
mu.Lock()
|
||||
inside++
|
||||
if inside > most {
|
||||
most = inside
|
||||
}
|
||||
mu.Unlock()
|
||||
sleepABit()
|
||||
mu.Lock()
|
||||
inside--
|
||||
mu.Unlock()
|
||||
return r, true
|
||||
}
|
||||
m, states := fetchModule(t, f)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 2; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
fetchTool(t, m)(fetchArgs)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
states.waitEnd(t)
|
||||
states.waitEnd(t)
|
||||
if most != 1 {
|
||||
t.Errorf("%d updates of the mirror ran at once", most)
|
||||
}
|
||||
}
|
||||
|
||||
// A mirror that cannot be updated does not fail the fetch: the fetch goes on without it.
|
||||
func TestAMirrorThatCannotBeUpdatedLetsTheFetchGoOnWithoutIt(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n"}
|
||||
f.rule = func(line string) (r runAnswer, ok bool) {
|
||||
if strings.Contains(line, "mirror") && strings.Contains(line, " fetch ") {
|
||||
return runAnswer{Status: 1, Stderr: "fatal: no space left\n"}, true
|
||||
}
|
||||
return r, false
|
||||
}
|
||||
m, states := fetchModule(t, f)
|
||||
ws := fetchOnce(t, m, states, fetchArgs).Workspace
|
||||
if _, err := os.Stat(filepath.Join(m.Work, ws, "tree", ".git", "objects", "info", "alternates")); err == nil {
|
||||
t.Errorf("the workspace borrows from a mirror that is not there")
|
||||
}
|
||||
}
|
||||
|
||||
func sleepABit() { time.Sleep(30 * time.Millisecond) }
|
||||
|
||||
// The workspace is self-contained after the fetch: no alternates file, and git finds the commit with the mirror gone,
|
||||
// as a step shut off with the workspace alone needs. Real git, a local repository as the forge.
|
||||
func TestTheWorkspaceIsSelfContainedAfterAFetchThroughTheMirror(t *testing.T) {
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("no git")
|
||||
}
|
||||
forge := t.TempDir()
|
||||
src := filepath.Join(forge, "novox", "mesh-sdk.git")
|
||||
if err := os.MkdirAll(src, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
git := func(dir string, args ...string) string {
|
||||
cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.org", "-c", "commit.gpgsign=false"}, args...)...)
|
||||
cmd.Env = append(os.Environ(), "GIT_CONFIG_GLOBAL=/dev/null")
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v %s", args, err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
git(src, "init", "-q")
|
||||
if err := os.WriteFile(filepath.Join(src, "f.txt"), []byte("one\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
git(src, "add", ".")
|
||||
git(src, "commit", "-q", "-m", "one")
|
||||
commit := git(src, "rev-parse", "HEAD")
|
||||
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.27.1")
|
||||
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
|
||||
m.Run = StepRunner(m.Home)
|
||||
m.RemoteBase = "file://" + forge + "/"
|
||||
m.FetchOptions = []buildstep.Option{buildstep.AllowFileRemote()}
|
||||
ws := fetchOnce(t, m, states, map[string]any{"repository": "novox/mesh-sdk", "commit": commit}).Workspace
|
||||
tree := filepath.Join(m.Work, ws, "tree")
|
||||
if _, err := os.Stat(filepath.Join(tree, ".git", "objects", "info", "alternates")); err == nil {
|
||||
t.Errorf("the workspace still has an alternates file")
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Cache, "mirror")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out, err := exec.Command("git", "-C", tree, "cat-file", "-e", commit+"^{commit}").CombinedOutput(); err != nil {
|
||||
t.Errorf("git does not find the commit with the mirror gone: %v %s", err, out)
|
||||
}
|
||||
if out, err := exec.Command("git", "-C", tree, "fsck", "--connectivity-only").CombinedOutput(); err != nil {
|
||||
t.Errorf("the workspace is not whole: %v %s", err, out)
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
@@ -53,6 +54,9 @@ type Module struct {
|
||||
// Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is
|
||||
// unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing.
|
||||
Bubblewrap func(ctx context.Context) (string, error)
|
||||
// Parallel is how many go.mod directories a level runs at once; zero is half the processors, and at least one
|
||||
// (ADR 0536 §3: a node runs two checks at once).
|
||||
Parallel int
|
||||
// Inside is the integration level's own process, started shut off in bubblewrap by a test step (level.go): its
|
||||
// commands run as they are, never shut off a second time.
|
||||
Inside bool
|
||||
@@ -68,6 +72,12 @@ type Module struct {
|
||||
// StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own
|
||||
// program, StartBus in bus.go, when nil).
|
||||
StartBus func(ctx context.Context, dir string) (url string, stop func(), err error)
|
||||
// Floor refuses every step while the work or cache filesystem has less free than it (issue 1839); main sets
|
||||
// floorOf(node), and a zero Floor checks nothing.
|
||||
Floor buildstep.Floor
|
||||
|
||||
spacesOnce sync.Once
|
||||
workspaces *buildstep.Workspaces
|
||||
}
|
||||
|
||||
// guard is the SDK guard for the version the probe just read: the module names no Go version (ADR 0477 §2), so the
|
||||
@@ -116,6 +126,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
|
||||
if err := m.checkBubblewrap(ctx); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("%s installed, the version the module builds with (recipe %s)", installed, m.Recipe.Version()))
|
||||
}
|
||||
|
||||
@@ -128,6 +141,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string
|
||||
if err := m.Check(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer m.holdArg(args)()
|
||||
return run(ctx, args)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -144,6 +161,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any,
|
||||
if err := m.Check(m.Ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer m.holdArg(args)()
|
||||
return run(m.Ctx, args)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,7 +115,7 @@ func TestAGoThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {
|
||||
{Name: "short_probe", Run: run},
|
||||
}, m.StepTools()...)
|
||||
for _, tool := range m.Tools(steps) {
|
||||
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool {
|
||||
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool {
|
||||
continue
|
||||
}
|
||||
var reason string
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -28,6 +29,7 @@ type pinHarness struct {
|
||||
pulls string // what git ls-remote answers
|
||||
onTrunk bool // the pin is fetchable as it is: go mod download never says unknown revision
|
||||
cmds []cmdRecord
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newPinHarness(t *testing.T, gomod string, vendor bool, pulls string) (*pinHarness, *keptStates) {
|
||||
@@ -36,7 +38,9 @@ func newPinHarness(t *testing.T, gomod string, vendor bool, pulls string) (*pinH
|
||||
h := &pinHarness{t: t, m: m, f: f, pulls: pulls}
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
h.mu.Lock()
|
||||
h.cmds = append(h.cmds, cmdRecord{env: env, line: line})
|
||||
h.mu.Unlock()
|
||||
switch {
|
||||
case name == "git" && strings.Contains(line, " checkout ") && !strings.Contains(line, "pinned"):
|
||||
for i, a := range args {
|
||||
|
||||
@@ -96,7 +96,10 @@ func (m *Module) placeSiblings(ctx context.Context, root string, siblings []sibl
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, s.ref, m.FetchOptions...); err != nil {
|
||||
if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, s.ref, m.fetchOptions(ctx, s.repository, s.commit, s.ref)...); err != nil {
|
||||
return nil, fmt.Errorf("beside %s: %w", s.repository, err)
|
||||
}
|
||||
if err := m.dissociate(ctx, dir); err != nil {
|
||||
return nil, fmt.Errorf("beside %s: %w", s.repository, err)
|
||||
}
|
||||
if link := firstSymlink(dir); link != "" {
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every step is refused while the work
|
||||
// or cache filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases
|
||||
// it, and swept unheld past buildstep.UnheldBound.
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// ReleaseTool removes a workspace whose caller's steps have ended.
|
||||
const ReleaseTool = "build_go_release"
|
||||
|
||||
// floorOf is the floor the module keeps on node: build-agent's.
|
||||
func floorOf(node string) buildstep.Floor {
|
||||
return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor}
|
||||
}
|
||||
|
||||
// room refuses while the work or cache filesystem is under the floor.
|
||||
func (m *Module) room() error { return m.Floor.Check(m.Work, m.Cache) }
|
||||
|
||||
// spaces are the module's workspaces under Work.
|
||||
func (m *Module) spaces() *buildstep.Workspaces {
|
||||
m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} })
|
||||
return m.workspaces
|
||||
}
|
||||
|
||||
// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none.
|
||||
func (m *Module) holdArg(args map[string]any) func() {
|
||||
ws, _ := args["workspace"].(string)
|
||||
if ws == "" {
|
||||
return func() {}
|
||||
}
|
||||
return m.spaces().Hold(ws)
|
||||
}
|
||||
|
||||
// releaseTool is build_go_release: no guard and no floor, since it only frees room.
|
||||
func (m *Module) releaseTool() stdio.Tool {
|
||||
return stdio.Tool{Name: ReleaseTool,
|
||||
Description: "Remove a workspace whose steps have ended (fetch, test, build): its caller says so once its last step " +
|
||||
"answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " +
|
||||
"is removed two hours after its last use.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"},
|
||||
}, "required": []string{"workspace"}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ws, _ := args["workspace"].(string)
|
||||
if err := m.spaces().Release(ws); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"released": ws}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509).
|
||||
func (m *Module) sweepSaid(removed []string) {
|
||||
if len(removed) > 0 && m.Say != nil {
|
||||
m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
|
||||
|
||||
func underFloor(node string) buildstep.Floor {
|
||||
f := floorOf(node)
|
||||
f.Free = func(string) (uint64, error) { return 4 << 30, nil }
|
||||
return f
|
||||
}
|
||||
|
||||
// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
|
||||
func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
|
||||
if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
|
||||
t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC1: under the floor every step tool is refused, naming the node, what is free and the floor, no step runs, and
|
||||
// the module is unhealthy saying so, so the builder chooses another node.
|
||||
func TestUnderTheFloorEveryStepIsRefusedAndTheModuleIsUnhealthy(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.27.1")
|
||||
m.Work, m.Cache, m.Floor = t.TempDir(), t.TempDir(), underFloor("ace")
|
||||
ran := false
|
||||
run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil }
|
||||
steps := []StepTool{{Name: "long_probe", Long: true, Run: run}, {Name: "short_probe", Run: run}}
|
||||
for _, tool := range m.Tools(steps) {
|
||||
switch tool.Name {
|
||||
case "long_probe":
|
||||
if _, err := tool.Run(map[string]any{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
|
||||
t.Errorf("the long step ended %s: %q, want refused: %q", st.State, st.Reason, underTheFloor)
|
||||
}
|
||||
case "short_probe":
|
||||
if _, err := tool.Run(map[string]any{}); err == nil || !strings.HasPrefix(err.Error(), underTheFloor) {
|
||||
t.Errorf("the short step answered %v, want %q", err, underTheFloor)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ran {
|
||||
t.Errorf("a step ran under the floor")
|
||||
}
|
||||
if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
|
||||
t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
|
||||
}
|
||||
}
|
||||
|
||||
func releaseTool(t *testing.T, m *Module) func(map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == ReleaseTool {
|
||||
return tool.Run
|
||||
}
|
||||
}
|
||||
t.Fatalf("no tool %s", ReleaseTool)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1839/AC2: build_go_release removes the workspace at once, even under the floor; a second release is no error.
|
||||
func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
|
||||
m, _ := fetchModule(t, &fakeStep{})
|
||||
m.Floor = underFloor("ace")
|
||||
ws := filepath.Join(m.Work, "abc1")
|
||||
os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, err := releaseTool(t, m)(map[string]any{"workspace": "abc1"}); err != nil {
|
||||
t.Fatalf("release %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(ws); !os.IsNotExist(err) {
|
||||
t.Errorf("the workspace is still there after its release")
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
|
||||
func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n", goRule: func(string) forgekey.Ran { return forgekey.Ran{Status: 1, Stderr: "boom"} }}
|
||||
m, states := fetchModule(t, f)
|
||||
if _, err := fetchTool(t, m)(fetchArgs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepFailed {
|
||||
t.Fatalf("the step ended %s, want failed", st.State)
|
||||
}
|
||||
if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
|
||||
t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is.
|
||||
func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
|
||||
m := &Module{Work: t.TempDir()}
|
||||
for _, name := range []string{"held1", "lost1"} {
|
||||
dir := filepath.Join(m.Work, name)
|
||||
os.MkdirAll(filepath.Join(dir, "tree"), 0o755)
|
||||
}
|
||||
done := m.spaces().Hold("held1")
|
||||
defer done()
|
||||
when := time.Now().Add(-3 * time.Hour)
|
||||
for _, name := range []string{"held1", "lost1"} {
|
||||
os.Chtimes(filepath.Join(m.Work, name), when, when)
|
||||
}
|
||||
m.CleanWorkspaces(time.Now())
|
||||
if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
|
||||
t.Errorf("the held workspace was swept")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
|
||||
t.Errorf("the unheld workspace three hours old was kept")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// threeModules gives the workspace three go.mod directories, a, b and c, whose modules are fixture/a, fixture/b and
|
||||
// fixture/c.
|
||||
func threeModules(t *testing.T, f *integFake) (*Module, *keptStates) {
|
||||
t.Helper()
|
||||
m, states := integModule(t, f)
|
||||
f.gomods = "a/go.mod\nb/go.mod\nc/go.mod\n"
|
||||
for _, d := range []string{"a", "b", "c"} {
|
||||
dir := filepath.Join(m.Work, "ws1", "tree", d)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module fixture/"+d+"\n\ngo 1.25.0\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return m, states
|
||||
}
|
||||
|
||||
func runIntegrationOf(t *testing.T, m *Module, states *keptStates, packages ...any) (TestResult, delivery.Step) {
|
||||
t.Helper()
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name != "build_go_test" {
|
||||
continue
|
||||
}
|
||||
args := map[string]any{"workspace": "ws1", "level": "integration"}
|
||||
if len(packages) > 0 {
|
||||
args["packages"] = packages
|
||||
}
|
||||
if _, err := tool.Run(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res TestResult
|
||||
if st.State == delivery.StepPassed {
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
|
||||
// Issue 890/AC1 (ADR 0536 §3): with packages selected in one of three go.mod directories, the integration level
|
||||
// compiles and runs go test in that directory only.
|
||||
func TestOnlyTheSelectedDirectoryCompilesAndRuns(t *testing.T) {
|
||||
f := newIntegFake(t, "slow")
|
||||
m, states := threeModules(t, f)
|
||||
if _, st := runIntegrationOf(t, m, states, "fixture/b/x"); st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
tree := filepath.Join(m.Work, "ws1", "tree")
|
||||
compiled, ran := 0, 0
|
||||
for _, l := range strings.Split(f.lines(), "\n") {
|
||||
if !strings.HasPrefix(l, "go ") || !strings.Contains(l, " test ") {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(l, "go -C "+filepath.Join(tree, "b")+" test ") {
|
||||
t.Errorf("go test ran outside the selected directory: %s", l)
|
||||
}
|
||||
if strings.Contains(l, "-run ^$") {
|
||||
compiled++
|
||||
} else {
|
||||
ran++
|
||||
}
|
||||
}
|
||||
if compiled != 1 || ran != 1 {
|
||||
t.Errorf("compiled %d times and ran %d times, want once each:\n%s", compiled, ran, f.lines())
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 890/AC1: no selection keeps every directory, compiled and run.
|
||||
func TestWithoutASelectionEveryDirectoryCompilesAndRuns(t *testing.T) {
|
||||
f := newIntegFake(t, "slow")
|
||||
m, states := threeModules(t, f)
|
||||
runIntegrationOf(t, m, states)
|
||||
compiled, ran := 0, 0
|
||||
for _, l := range strings.Split(f.lines(), "\n") {
|
||||
if strings.HasPrefix(l, "go ") && strings.Contains(l, " test ") {
|
||||
if strings.Contains(l, "-run ^$") {
|
||||
compiled++
|
||||
} else {
|
||||
ran++
|
||||
}
|
||||
}
|
||||
}
|
||||
if compiled != 3 || ran != 3 {
|
||||
t.Errorf("compiled %d and ran %d, want 3 and 3:\n%s", compiled, ran, f.lines())
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 890/AC2: with two directories selected and a runner that blocks, both go test commands run at once; the
|
||||
// result lists the packages in directory order.
|
||||
func TestSelectedDirectoriesRunAtOnceAndAnswerInDirectoryOrder(t *testing.T) {
|
||||
f := newIntegFake(t, "slow")
|
||||
var mu sync.Mutex
|
||||
running, most := 0, 0
|
||||
release := make(chan struct{})
|
||||
f.onTest = func(line string) {
|
||||
mu.Lock()
|
||||
running++
|
||||
if running > most {
|
||||
most = running
|
||||
}
|
||||
if running == 2 {
|
||||
close(release)
|
||||
}
|
||||
mu.Unlock()
|
||||
select {
|
||||
case <-release:
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
mu.Lock()
|
||||
running--
|
||||
mu.Unlock()
|
||||
}
|
||||
m, states := threeModules(t, f)
|
||||
m.Parallel = 3
|
||||
res, st := runIntegrationOf(t, m, states, "fixture/c/x", "fixture/a/x")
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if most != 2 {
|
||||
t.Errorf("at most %d go test commands ran at once, want 2", most)
|
||||
}
|
||||
dirs := []string{}
|
||||
for _, p := range res.Packages {
|
||||
dirs = append(dirs, p.Dir)
|
||||
}
|
||||
if got := fmt.Sprint(dirs); !strings.HasPrefix(got, "[a ") || !strings.HasSuffix(got, " c]") {
|
||||
t.Errorf("the packages lie in directories %s, want a's before c's", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound holds: with one directory at a time, no two go test commands run at once.
|
||||
func TestTheDirectoriesAtOnceNeverPassTheBound(t *testing.T) {
|
||||
f := newIntegFake(t, "slow")
|
||||
var mu sync.Mutex
|
||||
running, most := 0, 0
|
||||
f.onTest = func(string) {
|
||||
mu.Lock()
|
||||
running++
|
||||
if running > most {
|
||||
most = running
|
||||
}
|
||||
mu.Unlock()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
mu.Lock()
|
||||
running--
|
||||
mu.Unlock()
|
||||
}
|
||||
m, states := threeModules(t, f)
|
||||
m.Parallel = 1
|
||||
runIntegrationOf(t, m, states)
|
||||
if most != 1 {
|
||||
t.Errorf("%d go test commands ran at once with a bound of 1", most)
|
||||
}
|
||||
}
|
||||
@@ -10,8 +10,12 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// testTimeout is go test's own timeout for a level: a bound on a test that never ends, far above any level's bound.
|
||||
@@ -33,7 +37,8 @@ func (m *Module) testTool() StepTool {
|
||||
"level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
|
||||
"packages": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "package patterns of each module, ./... when none"},
|
||||
"run": map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"},
|
||||
"run": map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"},
|
||||
"runner": runnerProperty,
|
||||
},
|
||||
"required": []string{"workspace", "level"},
|
||||
},
|
||||
@@ -68,13 +73,48 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
||||
if _, err := os.Stat(tree); err != nil {
|
||||
return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
|
||||
}
|
||||
switch level {
|
||||
case LevelUnit:
|
||||
return m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)})
|
||||
case LevelIntegration:
|
||||
return m.integration(ctx, workspace, tree, packages, pattern)
|
||||
runner, err := runnerArg(args["runner"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
|
||||
var res *TestResult
|
||||
switch {
|
||||
case runner == RunnerWorkspace && (level == LevelUnit || level == LevelIntegration):
|
||||
return m.testAsCandidate(ctx, workspace, level, packages, pattern)
|
||||
case level == LevelUnit:
|
||||
res, err = m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)})
|
||||
case level == LevelIntegration:
|
||||
res, err = m.integration(ctx, workspace, tree, packages, pattern)
|
||||
default:
|
||||
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.JudgedBy = RunnerInstalled
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// testAsCandidate runs a level as the candidate's program (ADR 0535 §1): the integration level is refused first as the
|
||||
// installed program would refuse it (no postgres), then the whole level runs in the candidate's one shut-off process.
|
||||
func (m *Module) testAsCandidate(ctx context.Context, workspace, level string, packages []string, pattern string) (*TestResult, error) {
|
||||
spec := levelSpec{Step: stepUnit, Workspace: workspace, Packages: packages, Run: pattern, Bound: m.unitBound()}
|
||||
if level == LevelIntegration {
|
||||
if _, err := m.checkPostgres(ctx); err != nil {
|
||||
return nil, err // a *buildstep.Refusal: this level only
|
||||
}
|
||||
b := m.integrationBounds()
|
||||
spec = levelSpec{Step: stepIntegration, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run}
|
||||
}
|
||||
a, judged, err := m.asCandidate(ctx, spec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.Result == nil {
|
||||
return nil, fmt.Errorf("the candidate's mesh-build-go answered no result of the %s level", level)
|
||||
}
|
||||
a.Result.JudgedBy = judged
|
||||
return a.Result, nil
|
||||
}
|
||||
|
||||
// besideEnv is MESH_CHECK_BESIDE, naming the directory build_go_fetch placed the workspace's siblings in (ADR 0484
|
||||
@@ -143,26 +183,71 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
|
||||
return nil, err
|
||||
}
|
||||
began := m.clock()
|
||||
// The directories that hold a selected package run side by side, up to the bound, each into a result of its own;
|
||||
// the results join in directory order, so the answer does not depend on which ended first (ADR 0536 §3).
|
||||
type dirRun struct {
|
||||
res *TestResult
|
||||
rd *reader
|
||||
r runAnswer
|
||||
noise []string
|
||||
}
|
||||
var todo []string
|
||||
for _, d := range dirs {
|
||||
packages := byDir[d]
|
||||
if len(packages) == 0 {
|
||||
continue // no package asked for lies in this module
|
||||
if len(byDir[d]) > 0 { // no package asked for lies in the others
|
||||
todo = append(todo, d)
|
||||
}
|
||||
args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout}
|
||||
if len(o.tags) > 0 {
|
||||
args = append(args, "-tags", strings.Join(o.tags, ","))
|
||||
}
|
||||
if o.run != "" {
|
||||
args = append(args, "-run", o.run)
|
||||
}
|
||||
args = append(args, packages...)
|
||||
r := run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
|
||||
rd := newReader(res, d)
|
||||
noise := rd.read(r.Stdout)
|
||||
rd.finish()
|
||||
if (r.Status != 0 || r.Err != "") && len(rd.pkgs) == 0 {
|
||||
}
|
||||
runs := make([]dirRun, len(todo))
|
||||
slots := make(chan struct{}, m.parallel())
|
||||
var wg sync.WaitGroup
|
||||
var crash any // a panic of a run goes on in the caller, which cleans up after it
|
||||
var crashMu sync.Mutex
|
||||
for i, d := range todo {
|
||||
wg.Add(1)
|
||||
slots <- struct{}{}
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-slots }()
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
crashMu.Lock()
|
||||
crash = p
|
||||
crashMu.Unlock()
|
||||
}
|
||||
}()
|
||||
args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout}
|
||||
if len(o.tags) > 0 {
|
||||
args = append(args, "-tags", strings.Join(o.tags, ","))
|
||||
}
|
||||
if o.run != "" {
|
||||
args = append(args, "-run", o.run)
|
||||
}
|
||||
args = append(args, byDir[d]...)
|
||||
dr := &runs[i]
|
||||
dr.res = &TestResult{}
|
||||
dr.r = run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
|
||||
dr.rd = newReader(dr.res, d)
|
||||
dr.noise = dr.rd.read(dr.r.Stdout)
|
||||
dr.rd.finish()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if crash != nil {
|
||||
panic(crash)
|
||||
}
|
||||
for i, d := range todo {
|
||||
dr := runs[i]
|
||||
if (dr.r.Status != 0 || dr.r.Err != "") && len(dr.rd.pkgs) == 0 {
|
||||
// go test said nothing a test said: it could not run (a go.mod it cannot read, a toolchain it cannot find).
|
||||
return nil, fmt.Errorf("go test in %s: %s", d, firstLine(r.Err+"\n"+strings.Join(noise, "\n")+"\n"+r.Stderr))
|
||||
return nil, fmt.Errorf("go test in %s: %s", d, firstLine(dr.r.Err+"\n"+strings.Join(dr.noise, "\n")+"\n"+dr.r.Stderr))
|
||||
}
|
||||
res.Tests += dr.res.Tests
|
||||
res.Skipped += dr.res.Skipped
|
||||
res.Failed = append(res.Failed, dr.res.Failed...)
|
||||
res.Packages = append(res.Packages, dr.res.Packages...)
|
||||
res.Slowest = append(res.Slowest, dr.res.Slowest...)
|
||||
if dr.res.Elapsed > res.Elapsed {
|
||||
res.Elapsed = dr.res.Elapsed
|
||||
}
|
||||
}
|
||||
if o.wall {
|
||||
@@ -172,6 +257,17 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// runAnswer is what a step runner answers.
|
||||
type runAnswer = forgekey.Ran
|
||||
|
||||
// parallel is how many directories a level runs at once.
|
||||
func (m *Module) parallel() int {
|
||||
if m.Parallel > 0 {
|
||||
return m.Parallel
|
||||
}
|
||||
return max(1, runtime.NumCPU()/2)
|
||||
}
|
||||
|
||||
// packagesByDir says which patterns run in which go.mod directory. A relative pattern (./...) runs in every directory, as
|
||||
// it always did; an import path, as build_go_digest keys its packages, runs only in the directory whose module holds it
|
||||
// (the longest module path that is the path or its prefix), as another module's path would fail the step. No pattern
|
||||
|
||||
@@ -40,6 +40,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
||||
tools = append(tools, t)
|
||||
}
|
||||
tools = append(tools, m.ReadTools()...)
|
||||
tools = append(tools, m.releaseTool())
|
||||
return append(tools,
|
||||
stdio.Tool{Name: StatusTool,
|
||||
Description: "What the Go build module is on this machine: the Go version its recipe builds with and the one installed, " +
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
func shutOffModule(t *testing.T, f *fakeStep) (*Module, *keptStates) {
|
||||
t.Helper()
|
||||
m, states := fetchModule(t, f)
|
||||
// A module with bubblewrap runs its steps shut off, with the module cache read-only: a pass of go mod verify holds.
|
||||
m.Bubblewrap = func(context.Context) (string, error) { return "bubblewrap 0.13.0", nil }
|
||||
return m, states
|
||||
}
|
||||
|
||||
func countVerifies(f *fakeStep) int {
|
||||
n := 0
|
||||
for _, l := range f.lines() {
|
||||
if strings.HasPrefix(l, "go -C ") && strings.HasSuffix(l, " mod verify") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Issue 890/AC5 (ADR 0536 §2): two directories with the same go.mod and go.sum verify once, and a later fetch with the
|
||||
// same pair verifies no time; a different go.mod verifies again.
|
||||
func TestGoModVerifyRunsOncePerGoModAndGoSumDigest(t *testing.T) {
|
||||
same := map[string]string{"a/go.mod": "module a\n", "a/go.sum": "x\n", "b/go.mod": "module a\n", "b/go.sum": "x\n"}
|
||||
f := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: same}
|
||||
m, states := shutOffModule(t, f)
|
||||
fetchOnce(t, m, states, fetchArgs)
|
||||
if n := countVerifies(f); n != 1 {
|
||||
t.Errorf("two directories with one go.mod and go.sum verified %d times, want 1", n)
|
||||
}
|
||||
fetchOnce(t, m, states, fetchArgs)
|
||||
if n := countVerifies(f); n != 1 {
|
||||
t.Errorf("a later fetch with the same pair verified again (%d in all)", n)
|
||||
}
|
||||
|
||||
g := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: map[string]string{"a/go.mod": "module a\n", "a/go.sum": "x\n", "b/go.mod": "module b\n", "b/go.sum": "x\n"}}
|
||||
m, states = shutOffModule(t, g)
|
||||
fetchOnce(t, m, states, fetchArgs)
|
||||
if n := countVerifies(g); n != 2 {
|
||||
t.Errorf("two different go.mod verified %d times, want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Without bubblewrap the module cache is not read-only to a step: a pass is never reused.
|
||||
func TestAPassOfVerifyIsNotReusedWhenStepsAreNotShutOff(t *testing.T) {
|
||||
f := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: map[string]string{"a/go.mod": "m\n", "b/go.mod": "m\n"}}
|
||||
m, states := fetchModule(t, f) // no Bubblewrap
|
||||
fetchOnce(t, m, states, fetchArgs)
|
||||
fetchOnce(t, m, states, fetchArgs)
|
||||
if n := countVerifies(f); n != 4 {
|
||||
t.Errorf("verified %d times, want every directory every fetch (4)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A verify that fails leaves no mark: the next fetch verifies again.
|
||||
func TestAFailedVerifyLeavesNoMark(t *testing.T) {
|
||||
fail := true
|
||||
f := &fakeStep{gomods: "go.mod\n", files: map[string]string{"go.mod": "m\n"}}
|
||||
f.goRule = func(line string) forgekey.Ran {
|
||||
if strings.HasSuffix(line, " mod verify") && fail {
|
||||
return forgekey.Ran{Status: 1, Stdout: "x v1: dir has been modified\n"}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
m, states := shutOffModule(t, f)
|
||||
fetchTool(t, m)(fetchArgs)
|
||||
states.waitEnd(t)
|
||||
fail = false
|
||||
fetchOnce(t, m, states, fetchArgs)
|
||||
if n := countVerifies(f); n != 2 {
|
||||
t.Errorf("verified %d times, want 2 (the failed one and the retry)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 890 (ADR 0536 §2): the downloads of the directories run side by side, never more at once than the bound.
|
||||
func TestDownloadsRunSideBySideUpToTheBound(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
running, most := 0, 0
|
||||
f := &fakeStep{gomods: "a/go.mod\nb/go.mod\nc/go.mod\nd/go.mod\n", files: map[string]string{
|
||||
"a/go.mod": "a\n", "b/go.mod": "b\n", "c/go.mod": "c\n", "d/go.mod": "d\n"}}
|
||||
f.goRule = func(line string) forgekey.Ran {
|
||||
if strings.HasSuffix(line, " mod download") {
|
||||
mu.Lock()
|
||||
running++
|
||||
if running > most {
|
||||
most = running
|
||||
}
|
||||
mu.Unlock()
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
mu.Lock()
|
||||
running--
|
||||
mu.Unlock()
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
m, states := fetchModule(t, f)
|
||||
m.Parallel = 2
|
||||
fetchOnce(t, m, states, fetchArgs)
|
||||
if most != 2 {
|
||||
t.Errorf("at most %d downloads ran at once, want 2", most)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ module meshbuildgo
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023121526-2248b985bfac
|
||||
github.com/distribution/distribution/v3 v3.1.2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1 h1:MqP6mWWVxM+niUGPxSSh4Q8lduYkkQ7oTAkDeC9nyv4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023120306-780be2fae5a1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023121526-2248b985bfac h1:xx601bHwpO/aq5KyE08t4ymSXGHGnk+D9VHfmOq+5P8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023121526-2248b985bfac/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
"build_go_test",
|
||||
"build_go_digest",
|
||||
"build_go_build",
|
||||
"build_go_release",
|
||||
"build_go_step",
|
||||
"build_go_steps",
|
||||
"build_go_toolchains",
|
||||
|
||||
@@ -37,6 +37,12 @@ The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can re
|
||||
- It refuses, naming why, an image whose build context is a whole repository (`context`) or whose `build.on` names
|
||||
another module's artifact (`unsupportedImage` in `build.go`): those forms belong to the images build-agent builds
|
||||
until it is removed at R4g.
|
||||
- `build_image_release {workspace}` removes a workspace at once (no guard, no floor; one already gone is no error, one a
|
||||
running step holds is refused); its caller calls it once its last step answered. A failed fetch removes its own
|
||||
workspace, and one no caller released is removed two hours after its last use (ADR 0509).
|
||||
- A fetch and a build are refused, `refused: <node> has N GiB free under the floor of 20 GiB (on <dir>)`, and the health
|
||||
is unhealthy with that text, while the work directory's or the image store's filesystem has less than 20 GiB free
|
||||
(issue 1839). A copy writes nothing to the disk and is not refused.
|
||||
- `build_image_step`, `build_image_steps`, `build_image_status` and `build_image_health` read.
|
||||
|
||||
The registry's address is placed into `<home>/registry` as `${seat:mesh-artifact-store:reach}` (host:port): the provisioner's client fills a moved module's placement, and the artifact store has not moved, so a `${bound:artifact-store:...}` is refused at composition (issue 1703).
|
||||
|
||||
@@ -26,11 +26,6 @@ var (
|
||||
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
|
||||
const ForgeAlias = "mesh-forge"
|
||||
|
||||
// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it
|
||||
// fetches and every round it checks itself. A workspace lies in the module's work directory, never in its home
|
||||
// (ADR 0466 §1).
|
||||
const WorkspaceAge = 24 * time.Hour
|
||||
|
||||
// FetchResult is what a passed fetch step says: the workspace it made and the commit in it, and, only when the call
|
||||
// gave a base, the merge preview of ADR 0484 clause 3 (the trunk commit, the merge commit of the head onto it, the
|
||||
// files in conflict, the paths the merge changes and the tree of each module directory). A head that does not
|
||||
@@ -66,7 +61,9 @@ func (m *Module) fetchTool() StepTool {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
// fetch makes a workspace and fills it. The workspace is held while the fetch runs, and removed when the fetch fails,
|
||||
// since nobody learns its id to release it (issue 1839).
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
commit, _ := args["commit"].(string)
|
||||
ref, _ := args["ref"].(string)
|
||||
@@ -82,6 +79,19 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
}
|
||||
m.CleanWorkspaces(m.clock())
|
||||
id := buildstep.NewID()
|
||||
done := m.spaces().Hold(id)
|
||||
defer func() {
|
||||
done()
|
||||
if err != nil {
|
||||
if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
|
||||
m.Say("the failed fetch's workspace %s: %v", id, rerr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return m.fetchInto(ctx, repository, commit, ref, base, id)
|
||||
}
|
||||
|
||||
func (m *Module) fetchInto(ctx context.Context, repository, commit, ref, base, id string) (any, error) {
|
||||
tree := filepath.Join(m.Work, id, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
return nil, err
|
||||
@@ -124,21 +134,10 @@ func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base
|
||||
return err
|
||||
}
|
||||
|
||||
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
|
||||
// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
|
||||
// net for a caller that never released one (issue 1839, ADR 0509).
|
||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||
entries, err := os.ReadDir(m.Work)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
|
||||
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
|
||||
}
|
||||
}
|
||||
m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
|
||||
}
|
||||
|
||||
// clock is the time now: Now when a test set it, else the wall clock.
|
||||
|
||||
@@ -53,6 +53,11 @@ func realModule(t *testing.T) (*Module, *keptStates) {
|
||||
m := &Module{Node: "ace", Account: u.Username, UID: u.Uid, Home: home, Work: t.TempDir(), Store: t.TempDir(),
|
||||
Steps: states.putStep, Ctx: context.Background(), Say: t.Logf, Run: StepRunner(home),
|
||||
FetchOptions: []buildstep.Option{buildstep.AllowFileRemote()}}
|
||||
// buildah being installed is not enough: inside the check's sandbox rootless buildah cannot make its user
|
||||
// namespace, so the module's own guard decides, as it does before every step (issue 1894).
|
||||
if err := m.Check(context.Background()); err != nil {
|
||||
t.Skipf("NOT RUN: this machine cannot build images where the check runs (%v); proven live on a build node", err)
|
||||
}
|
||||
return m, states
|
||||
}
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ func main() {
|
||||
Work: WorkDir,
|
||||
Store: StoreDir,
|
||||
}
|
||||
m.Floor = floorOf(m.Node)
|
||||
if u, err := user.Current(); err == nil {
|
||||
m.Home, m.Account, m.UID = u.HomeDir, u.Username, u.Uid
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
@@ -45,6 +46,12 @@ type Module struct {
|
||||
RegistryPause time.Duration
|
||||
// Now is the clock; time.Now when nil.
|
||||
Now func() time.Time
|
||||
// Floor refuses every fetch and build while the work or the image store's filesystem has less free than it (issue
|
||||
// 1839); main sets floorOf(node), and a zero Floor checks nothing.
|
||||
Floor buildstep.Floor
|
||||
|
||||
spacesOnce sync.Once
|
||||
workspaces *buildstep.Workspaces
|
||||
}
|
||||
|
||||
// Look is what the module found out about its machine.
|
||||
@@ -104,6 +111,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
|
||||
if err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made", l.Buildah, l.Account, l.UIDs, l.GIDs))
|
||||
}
|
||||
|
||||
@@ -126,6 +136,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string
|
||||
if err := m.Check(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer m.holdArg(args)()
|
||||
return run(ctx, args)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -142,6 +156,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any,
|
||||
if err := m.Check(m.Ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer m.holdArg(args)()
|
||||
return run(m.Ctx, args)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every fetch and build is refused while
|
||||
// the work or image store's filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases
|
||||
// it, and swept unheld past buildstep.UnheldBound.
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// ReleaseTool removes a workspace whose caller's steps have ended.
|
||||
const ReleaseTool = "build_image_release"
|
||||
|
||||
// floorOf is the floor the module keeps on node: build-agent's.
|
||||
func floorOf(node string) buildstep.Floor {
|
||||
return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor}
|
||||
}
|
||||
|
||||
// room refuses while the work or the image store's filesystem is under the floor.
|
||||
func (m *Module) room() error { return m.Floor.Check(m.Work, m.Store) }
|
||||
|
||||
// spaces are the module's workspaces under Work.
|
||||
func (m *Module) spaces() *buildstep.Workspaces {
|
||||
m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} })
|
||||
return m.workspaces
|
||||
}
|
||||
|
||||
// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none.
|
||||
func (m *Module) holdArg(args map[string]any) func() {
|
||||
ws, _ := args["workspace"].(string)
|
||||
if ws == "" {
|
||||
return func() {}
|
||||
}
|
||||
return m.spaces().Hold(ws)
|
||||
}
|
||||
|
||||
// releaseTool is build_image_release: no guard and no floor, since it only frees room.
|
||||
func (m *Module) releaseTool() stdio.Tool {
|
||||
return stdio.Tool{Name: ReleaseTool,
|
||||
Description: "Remove a workspace whose steps have ended (fetch, build): its caller says so once its last step " +
|
||||
"answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " +
|
||||
"is removed two hours after its last use.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"},
|
||||
}, "required": []string{"workspace"}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ws, _ := args["workspace"].(string)
|
||||
if err := m.spaces().Release(ws); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"released": ws}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509).
|
||||
func (m *Module) sweepSaid(removed []string) {
|
||||
if len(removed) > 0 && m.Say != nil {
|
||||
m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
|
||||
|
||||
func underFloor(node string) buildstep.Floor {
|
||||
f := floorOf(node)
|
||||
f.Free = func(string) (uint64, error) { return 4 << 30, nil }
|
||||
return f
|
||||
}
|
||||
|
||||
// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
|
||||
func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
|
||||
if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
|
||||
t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC1: under the floor a fetch and a build are refused, naming the node, what is free and the floor, nothing runs,
|
||||
// and the module is unhealthy saying so, so the builder chooses another node.
|
||||
func TestUnderTheFloorAFetchAndABuildAreRefusedAndTheModuleIsUnhealthy(t *testing.T) {
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
m.Floor = underFloor("ace")
|
||||
for name, args := range map[string]map[string]any{
|
||||
FetchTool: {"repository": "novox/mesh-catalog", "commit": commitSHA},
|
||||
BuildTool: {"workspace": "abc1", "module": "modules/m", "artifact": "image"},
|
||||
} {
|
||||
if _, err := tool(t, m, name)(args); err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
|
||||
t.Errorf("%s ended %s: %q, want refused: %q", name, st.State, st.Reason, underTheFloor)
|
||||
}
|
||||
}
|
||||
for _, l := range f.lines() {
|
||||
if strings.HasPrefix(l, "git ") || strings.Contains(l, " bud ") {
|
||||
t.Errorf("a refused step ran %q", l)
|
||||
}
|
||||
}
|
||||
if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
|
||||
t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC2: build_image_release removes the workspace at once, even under the floor; a second release is no error.
|
||||
func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
|
||||
m := moduleWith(t, newFake(t), newStates())
|
||||
m.Floor = underFloor("ace")
|
||||
ws := filepath.Join(m.Work, "abc1")
|
||||
os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, err := tool(t, m, ReleaseTool)(map[string]any{"workspace": "abc1"}); err != nil {
|
||||
t.Fatalf("release %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(ws); !os.IsNotExist(err) {
|
||||
t.Errorf("the workspace is still there after its release")
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
|
||||
func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if name == "git" && strings.Contains(strings.Join(args, " "), " fetch ") {
|
||||
return forgekey.Ran{Status: 128, Stderr: "fatal: unreachable"}
|
||||
}
|
||||
return f.run(ctx, env, name, args...)
|
||||
}
|
||||
if _, err := tool(t, m, FetchTool)(map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepFailed {
|
||||
t.Fatalf("the step ended %s (%s), want failed", st.State, st.Reason)
|
||||
}
|
||||
if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
|
||||
t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is.
|
||||
func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
|
||||
m := &Module{Work: t.TempDir()}
|
||||
for _, name := range []string{"held1", "lost1"} {
|
||||
os.MkdirAll(filepath.Join(m.Work, name, "tree"), 0o755)
|
||||
}
|
||||
done := m.spaces().Hold("held1")
|
||||
defer done()
|
||||
when := time.Now().Add(-3 * time.Hour)
|
||||
for _, name := range []string{"held1", "lost1"} {
|
||||
os.Chtimes(filepath.Join(m.Work, name), when, when)
|
||||
}
|
||||
m.CleanWorkspaces(time.Now())
|
||||
if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
|
||||
t.Errorf("the held workspace was swept")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
|
||||
t.Errorf("the unheld workspace three hours old was kept")
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
||||
tools = append(tools, t)
|
||||
}
|
||||
tools = append(tools, m.ReadTools()...)
|
||||
tools = append(tools, m.releaseTool())
|
||||
return append(tools,
|
||||
stdio.Tool{Name: StatusTool,
|
||||
Description: "What the image build module is on this machine: the buildah installed, the account's subordinate " +
|
||||
|
||||
@@ -3,7 +3,7 @@ module meshbuildimage
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
"build_image_fetch",
|
||||
"build_image_build",
|
||||
"build_image_copy",
|
||||
"build_image_release",
|
||||
"build_image_step",
|
||||
"build_image_steps",
|
||||
"build_image_status",
|
||||
|
||||
@@ -57,7 +57,11 @@ toolchain install: a long step answers at once with its id and ends on `<node>.<
|
||||
removed). The mesh's registry reaches `npm ci` through a private config file (`--userconfig`), so
|
||||
other `@novox/` packages install. A module with packages and no lock does not fail the fetch: its result lists it
|
||||
under `refused` with the reason, and a build of that bundle fails naming it (issue 1459). Every command runs through `forgekey.ExecRunnerClean(0)` with PATH, HOME, the locale and its own
|
||||
words, nothing of this process's environment. Workspaces older than a day are removed at each fetch.
|
||||
words, nothing of this process's environment. A failed fetch removes its own workspace; `build_node_release {workspace}` removes one at once (no guard, no
|
||||
floor; its caller calls it once its last step answered), and one no caller released is removed two hours after its
|
||||
last use (ADR 0509), never while a step holds it. Every step is refused, `refused: <node> has N GiB free under the
|
||||
floor of 20 GiB (on <dir>)`, and the health is unhealthy with that text, while the work or the toolchain cache
|
||||
filesystem has less than 20 GiB free (issue 1839).
|
||||
The arguments are mesh-sdk's `buildstep.FetchRequest` (ADR 0499 §4): with `dir` and `paths` (from the builder, never
|
||||
from a manifest) the workspace is narrowed to those paths, the manifest and the lock files before anything else. A
|
||||
member is built against the SDK version its own lock names (the lead's decision L10 of R4e, ADR 0500 §5-§6): the
|
||||
|
||||
@@ -33,9 +33,6 @@ const ForgeAlias = "mesh-forge"
|
||||
// sdkPackage is the package the toolchain carries and a module's own dependencies never supply.
|
||||
const sdkPackage = "@novox/mesh-sdk"
|
||||
|
||||
// WorkspaceAge is how long a workspace is kept.
|
||||
const WorkspaceAge = 24 * time.Hour
|
||||
|
||||
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
|
||||
var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
@@ -105,20 +102,36 @@ func (m *Module) fetchTool() StepTool {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
// fetch makes a workspace and fills it. The workspace is held while the fetch runs, and removed when the fetch fails,
|
||||
// since nobody learns its id to release it (issue 1839).
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
|
||||
// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
|
||||
// named paths, the workspace is narrowed to them (buildstep.Narrow).
|
||||
req, err := buildstep.ParseFetchRequest(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
||||
group, err := groupPackageArg(args["package"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
id := buildstep.NewID()
|
||||
done := m.spaces().Hold(id)
|
||||
defer func() {
|
||||
done()
|
||||
if err != nil {
|
||||
if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
|
||||
m.Say("the failed fetch's workspace %s: %v", id, rerr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return m.fetchInto(ctx, req, group, id)
|
||||
}
|
||||
|
||||
func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, group *groupPackage, id string) (any, error) {
|
||||
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
||||
var err error
|
||||
root := filepath.Join(m.Work, id)
|
||||
tree := filepath.Join(root, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
@@ -354,19 +367,8 @@ func removeSDKCopies(nm string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
|
||||
// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
|
||||
// net for a caller that never released one (issue 1839, ADR 0509).
|
||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||
entries, err := os.ReadDir(m.Work)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
|
||||
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
|
||||
}
|
||||
}
|
||||
m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ func main() {
|
||||
Bubblewrap: ProbeBubblewrap,
|
||||
RegistryFile: filepath.Join(home, "registry"),
|
||||
}
|
||||
m.Floor = floorOf(m.Node)
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
if m.Node != "" {
|
||||
go m.Start(ctx)
|
||||
|
||||
@@ -59,8 +59,14 @@ type Module struct {
|
||||
// unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing.
|
||||
Bubblewrap func(ctx context.Context) (string, error)
|
||||
|
||||
mu sync.Mutex // guards Recipe
|
||||
install sync.Mutex // one install at a time
|
||||
// Floor refuses every step while the work or toolchain cache filesystem has less free than it (issue 1839); main
|
||||
// sets floorOf(node), and a zero Floor checks nothing.
|
||||
Floor buildstep.Floor
|
||||
|
||||
spacesOnce sync.Once
|
||||
workspaces *buildstep.Workspaces
|
||||
mu sync.Mutex // guards Recipe
|
||||
install sync.Mutex // one install at a time
|
||||
}
|
||||
|
||||
func (m *Module) recipe() Recipe {
|
||||
@@ -122,6 +128,7 @@ func (m *Module) Start(ctx context.Context) {
|
||||
if err := m.Prepare(ctx); err != nil && m.Say != nil {
|
||||
m.Say("the toolchain: %v", err)
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
@@ -138,6 +145,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
|
||||
if err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("node %s installed, the version the module builds with (recipe %s)", installed, m.recipe().Version()))
|
||||
}
|
||||
|
||||
@@ -150,6 +160,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string
|
||||
if err := m.Prepare(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer m.holdArg(args)()
|
||||
return run(ctx, args)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -166,6 +180,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any,
|
||||
if err := m.Prepare(m.Ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer m.holdArg(args)()
|
||||
return run(m.Ctx, args)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,7 +117,7 @@ func TestANodeThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T)
|
||||
{Name: "short_probe", Run: run},
|
||||
}, (&Module{}).StepTools()...)
|
||||
for _, tool := range m.Tools(steps) {
|
||||
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool {
|
||||
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool {
|
||||
continue
|
||||
}
|
||||
var reason string
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every step is refused while the work
|
||||
// or toolchain cache filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases
|
||||
// it, and swept unheld past buildstep.UnheldBound.
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// ReleaseTool removes a workspace whose caller's steps have ended.
|
||||
const ReleaseTool = "build_node_release"
|
||||
|
||||
// floorOf is the floor the module keeps on node: build-agent's.
|
||||
func floorOf(node string) buildstep.Floor {
|
||||
return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor}
|
||||
}
|
||||
|
||||
// room refuses while the work or toolchain cache filesystem is under the floor.
|
||||
func (m *Module) room() error {
|
||||
paths := []string{m.Work}
|
||||
if m.Toolchain != nil {
|
||||
paths = append(paths, m.Toolchain.Paths.Cache)
|
||||
}
|
||||
return m.Floor.Check(paths...)
|
||||
}
|
||||
|
||||
// spaces are the module's workspaces under Work.
|
||||
func (m *Module) spaces() *buildstep.Workspaces {
|
||||
m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} })
|
||||
return m.workspaces
|
||||
}
|
||||
|
||||
// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none.
|
||||
func (m *Module) holdArg(args map[string]any) func() {
|
||||
ws, _ := args["workspace"].(string)
|
||||
if ws == "" {
|
||||
return func() {}
|
||||
}
|
||||
return m.spaces().Hold(ws)
|
||||
}
|
||||
|
||||
// releaseTool is build_node_release: no guard and no floor, since it only frees room.
|
||||
func (m *Module) releaseTool() stdio.Tool {
|
||||
return stdio.Tool{Name: ReleaseTool,
|
||||
Description: "Remove a workspace whose steps have ended (fetch, test, build): its caller says so once its last step " +
|
||||
"answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " +
|
||||
"is removed two hours after its last use.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"},
|
||||
}, "required": []string{"workspace"}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ws, _ := args["workspace"].(string)
|
||||
if err := m.spaces().Release(ws); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"released": ws}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509).
|
||||
func (m *Module) sweepSaid(removed []string) {
|
||||
if len(removed) > 0 && m.Say != nil {
|
||||
m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
|
||||
|
||||
func underFloor(node string) buildstep.Floor {
|
||||
f := floorOf(node)
|
||||
f.Free = func(string) (uint64, error) { return 4 << 30, nil }
|
||||
return f
|
||||
}
|
||||
|
||||
// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
|
||||
func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
|
||||
if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
|
||||
t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC1: under the floor every step tool is refused, naming the node, what is free and the floor, no step runs, and
|
||||
// the module is unhealthy saying so, so the builder chooses another node.
|
||||
func TestUnderTheFloorEveryStepIsRefusedAndTheModuleIsUnhealthy(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(states, "v26.10.0")
|
||||
m.Work, m.Floor = t.TempDir(), underFloor("ace")
|
||||
ran := false
|
||||
run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil }
|
||||
steps := []StepTool{{Name: "long_probe", Long: true, Run: run}, {Name: "short_probe", Run: run}}
|
||||
for _, tool := range m.Tools(steps) {
|
||||
switch tool.Name {
|
||||
case "long_probe":
|
||||
if _, err := tool.Run(map[string]any{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
|
||||
t.Errorf("the long step ended %s: %q, want refused: %q", st.State, st.Reason, underTheFloor)
|
||||
}
|
||||
case "short_probe":
|
||||
if _, err := tool.Run(map[string]any{}); err == nil || !strings.HasPrefix(err.Error(), underTheFloor) {
|
||||
t.Errorf("the short step answered %v, want %q", err, underTheFloor)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ran {
|
||||
t.Errorf("a step ran under the floor")
|
||||
}
|
||||
if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
|
||||
t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
|
||||
}
|
||||
}
|
||||
|
||||
func releaseTool(t *testing.T, m *Module) func(map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == ReleaseTool {
|
||||
return tool.Run
|
||||
}
|
||||
}
|
||||
t.Fatalf("no tool %s", ReleaseTool)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1839/AC2: build_node_release removes the workspace at once, even under the floor; a second release is no error.
|
||||
func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
|
||||
m, _, work := newMachine(t)
|
||||
m.Floor = underFloor("ace")
|
||||
ws := filepath.Join(work, "abc1")
|
||||
os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
|
||||
for i := 0; i < 2; i++ {
|
||||
if _, err := releaseTool(t, m)(map[string]any{"workspace": "abc1"}); err != nil {
|
||||
t.Fatalf("release %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(ws); !os.IsNotExist(err) {
|
||||
t.Errorf("the workspace is still there after its release")
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
|
||||
func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
|
||||
m, f := fetchMachine(t, typeScriptModule(`{}`), nil)
|
||||
f.answer["git fetch"] = forgekey.Ran{Status: 128, Stderr: "fatal: unreachable"}
|
||||
if _, err := m.fetch(context.Background(), fetchArgs); err == nil {
|
||||
t.Fatal("the fetch passed, want it failed")
|
||||
}
|
||||
if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
|
||||
t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is,
|
||||
// also at the module's own round and not only at a fetch.
|
||||
func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
|
||||
m := &Module{Work: t.TempDir()}
|
||||
for _, name := range []string{"held1", "lost1"} {
|
||||
os.MkdirAll(filepath.Join(m.Work, name, "tree"), 0o755)
|
||||
}
|
||||
done := m.spaces().Hold("held1")
|
||||
defer done()
|
||||
when := time.Now().Add(-3 * time.Hour)
|
||||
for _, name := range []string{"held1", "lost1"} {
|
||||
os.Chtimes(filepath.Join(m.Work, name), when, when)
|
||||
}
|
||||
m.CleanWorkspaces(time.Now())
|
||||
if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
|
||||
t.Errorf("the held workspace was swept")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
|
||||
t.Errorf("the unheld workspace three hours old was kept")
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
||||
tools = append(tools, t)
|
||||
}
|
||||
tools = append(tools, m.ReadTools()...)
|
||||
tools = append(tools, m.releaseTool())
|
||||
return append(tools,
|
||||
stdio.Tool{Name: StatusTool,
|
||||
Description: "What the Node build module is on this machine: the Node version its recipe builds with and the one installed, " +
|
||||
|
||||
@@ -3,7 +3,7 @@ module meshbuildnode
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467
|
||||
github.com/distribution/distribution/v3 v3.1.2
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
"build_node_fetch",
|
||||
"build_node_test",
|
||||
"build_node_build",
|
||||
"build_node_release",
|
||||
"build_node_step",
|
||||
"build_node_steps",
|
||||
"build_node_toolchains",
|
||||
|
||||
@@ -520,7 +520,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
|
||||
passed := b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", judgedAt("novox/app", 7, head, "pass", "t1t1t1t1t1t1"))
|
||||
|
||||
// No review names the head, so nothing is asked (ADR 0496, issue 1803): the delivery waits for a review.
|
||||
rt.until("the delivery to wait for a review", 5*time.Second, func() bool {
|
||||
rt.until("the delivery to wait for a review", eventWithin, func() bool {
|
||||
e, err := b.kvs["pulls"].Get(ctx, "novox.app.7")
|
||||
var k PullKey
|
||||
return err == nil && json.Unmarshal(e.Value(), &k) == nil && k.Verdict == "pass"
|
||||
@@ -537,7 +537,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
|
||||
"title": announced.Title, "head_sha": head, "base": "main", "body": "Reviewed-by: lead@shanks at " + head})
|
||||
_ = passed
|
||||
var key string
|
||||
rt.until("the ask to be written", 5*time.Second, func() bool { key = onlyAsk(b); return key != "" })
|
||||
rt.until("the ask to be written", eventWithin, func() bool { key = onlyAsk(b); return key != "" })
|
||||
ask := askAt(t, b, key)
|
||||
t.Logf("the ask was written %s after the edit's event", time.Since(edit))
|
||||
if !strings.Contains(ask.Ask.Explanation, "reviewed by lead on shanks") || ask.Ask.ID != sdk.AskID(key) {
|
||||
@@ -547,7 +547,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
|
||||
t.Fatalf("merged before the Approve")
|
||||
}
|
||||
// The pull request's key holds its head, the trunk commit it was asked against and the verdict.
|
||||
rt.until("the pull request's key to hold head, trunk and verdict", 5*time.Second, func() bool {
|
||||
rt.until("the pull request's key to hold head, trunk and verdict", eventWithin, func() bool {
|
||||
e, err := b.kvs["pulls"].Get(ctx, "novox.app.7")
|
||||
var k PullKey
|
||||
return err == nil && json.Unmarshal(e.Value(), &k) == nil && k == PullKey{Head: head, Trunk: "t1t1t1t1t1t1", Verdict: "pass"}
|
||||
@@ -559,7 +559,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
|
||||
t.Fatalf("the gate asked %s %d times: the bridge writes to the forge", tool, n)
|
||||
}
|
||||
}
|
||||
rt.until("the owed keys to be written", 5*time.Second, func() bool {
|
||||
rt.until("the owed keys to be written", eventWithin, func() bool {
|
||||
keys, err := b.kvs["owed"].Keys(ctx)
|
||||
return err == nil && len(keys) > 0
|
||||
})
|
||||
@@ -567,7 +567,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
|
||||
// seconds, with the Approve's heads, and the gate asks the forge nothing.
|
||||
approved := b.answer(t, key, OptionApprove)
|
||||
var handed sdk.Merge
|
||||
rt.until("the merge to be handed to the bridge", 5*time.Second, func() bool { handed = mergeAt(b, "fix/x"); return len(handed.Pulls) > 0 })
|
||||
rt.until("the merge to be handed to the bridge", eventWithin, func() bool { handed = mergeAt(b, "fix/x"); return len(handed.Pulls) > 0 })
|
||||
if len(handed.Pulls) != 1 || handed.Pulls[0].Head.Head != head || handed.Pulls[0].Number != 7 || handed.Pulls[0].State != sdk.MergeMerging ||
|
||||
handed.Approved[sdk.ApprovedKey("novox/app", 7)] != head || handed.Ask != key {
|
||||
t.Fatalf("the merge handed is %+v; wanted pull request 7 at the approved head, merging, under the ask %s", handed, key)
|
||||
@@ -579,7 +579,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
|
||||
// The bridge says it merged; the mesh's source then holds the merge, in one write.
|
||||
b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: "novox/app", Number: 7, Head: head, Merge: strings.Repeat("b", 40),
|
||||
Root: "r", Trees: map[string]string{"app": "t"}})
|
||||
rt.until("source.main to hold the merge", 5*time.Second, func() bool {
|
||||
rt.until("source.main to hold the merge", eventWithin, func() bool {
|
||||
e, err := b.kvs["source"].Get(ctx, sdk.SourceMainKey)
|
||||
if err != nil {
|
||||
return false
|
||||
@@ -743,7 +743,7 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe
|
||||
// One member is handed, with both heads approved, within seconds of the answer; the second only after the bridge says the
|
||||
// first merged.
|
||||
var first sdk.Merge
|
||||
rt.until("the first member to be handed", 5*time.Second, func() bool { first = mergeAt(b, "feat/x"); return len(first.Pulls) > 0 })
|
||||
rt.until("the first member to be handed", eventWithin, func() bool { first = mergeAt(b, "feat/x"); return len(first.Pulls) > 0 })
|
||||
if len(first.Pulls) != 1 || first.Pulls[0].State != sdk.MergeMerging || len(first.Approved) != 2 ||
|
||||
first.Approved[sdk.ApprovedKey("novox/mesh-controller", 1)] != ctlHead || first.Approved[sdk.ApprovedKey("novox/mesh-catalog", 2)] != catHead {
|
||||
t.Fatalf("the first merge handed is %+v", first)
|
||||
@@ -757,14 +757,14 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe
|
||||
b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: one.Repository, Number: one.Number, Head: one.Head.Head,
|
||||
Merge: strings.Repeat("a", 40), Root: "r", Trees: map[string]string{"x": "t"}})
|
||||
var both sdk.Merge
|
||||
rt.until("the second member to be handed", 5*time.Second, func() bool { both = mergeAt(b, "feat/x"); return len(both.Pulls) == 2 })
|
||||
rt.until("the second member to be handed", eventWithin, func() bool { both = mergeAt(b, "feat/x"); return len(both.Pulls) == 2 })
|
||||
if both.Pulls[0].State != sdk.MergeMerged || both.Pulls[1].State != sdk.MergeMerging || both.Pulls[1].Repository == one.Repository {
|
||||
t.Fatalf("the second merge handed is %+v", both.Pulls)
|
||||
}
|
||||
two := both.Pulls[1]
|
||||
b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: two.Repository, Number: two.Number, Head: two.Head.Head,
|
||||
Merge: strings.Repeat("b", 40), Root: "r", Trees: map[string]string{"x": "t"}})
|
||||
rt.until("source.main to hold both merges", 5*time.Second, func() bool {
|
||||
rt.until("source.main to hold both merges", eventWithin, func() bool {
|
||||
e, err := b.kvs["source"].Get(ctx, sdk.SourceMainKey)
|
||||
if err != nil {
|
||||
return false
|
||||
@@ -942,3 +942,9 @@ func mustDecode[T any](t *testing.T, raw []byte) T {
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// eventWithin is how long a test waits for one step of the gate's path over the real bus. On an idle machine each step
|
||||
// takes milliseconds (the tests log it), but a slow, loaded build node ran the ask step in 6.69 s against the old 5 s
|
||||
// (issue 1404). A failed first ask is asked again only after askAgainAfter (30 s, react.go), and 20 s deliberately does not
|
||||
// cover that: a failed first ask still fails the test.
|
||||
const eventWithin = 20 * time.Second
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// dockerVerdict decides what a test that starts containers does: run, skip or fail. found is the result of looking
|
||||
// the docker program up, probeErr the result of asking the daemon for its version (nil when it answered). The checker's
|
||||
// integration level runs shut off (no network, no /run), so on a build node the program is found and the daemon is not
|
||||
// reachable; that is a skip, not a failure, except where docker is required (MESH_REQUIRE_DOCKER=1).
|
||||
func dockerVerdict(found bool, lookErr, probeErr error, required bool) (action, reason string) {
|
||||
var why string
|
||||
switch {
|
||||
case !found:
|
||||
why = fmt.Sprintf("there is no docker in this toolchain (%v)", lookErr)
|
||||
case probeErr != nil:
|
||||
why = fmt.Sprintf("no docker daemon is reachable from here (%v)", probeErr)
|
||||
default:
|
||||
return "run", ""
|
||||
}
|
||||
if required {
|
||||
return "fail", "MESH_REQUIRE_DOCKER=1 and " + why
|
||||
}
|
||||
return "skip", why
|
||||
}
|
||||
|
||||
func TestDockerVerdict(t *testing.T) {
|
||||
boom := fmt.Errorf("cannot connect")
|
||||
cases := []struct {
|
||||
name string
|
||||
found bool
|
||||
look, probe error
|
||||
required bool
|
||||
want string
|
||||
}{
|
||||
{"answers", true, nil, nil, false, "run"},
|
||||
{"answers, required", true, nil, nil, true, "run"},
|
||||
{"no program", false, boom, nil, false, "skip"},
|
||||
{"no program, required", false, boom, nil, true, "fail"},
|
||||
{"program but no daemon", true, nil, boom, false, "skip"},
|
||||
{"program but no daemon, required", true, nil, boom, true, "fail"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, why := dockerVerdict(c.found, c.look, c.probe, c.required)
|
||||
if got != c.want {
|
||||
t.Errorf("%s: got %s, want %s", c.name, got, c.want)
|
||||
}
|
||||
if got != "run" && why == "" {
|
||||
t.Errorf("%s: no reason given", c.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -60,14 +60,24 @@ var snapshotGrants = struct{ publish, subscribe []string }{
|
||||
|
||||
func needDocker(t *testing.T) {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
// The Go toolchain container of the merge check holds no docker. Where docker is expected
|
||||
// (MESH_REQUIRE_DOCKER=1, which merge-check.sh sets for its docker step) its absence is a failure.
|
||||
if os.Getenv("MESH_REQUIRE_DOCKER") == "1" {
|
||||
t.Fatalf("MESH_REQUIRE_DOCKER=1 and there is no docker here: %v", err)
|
||||
_, lookErr := exec.LookPath("docker")
|
||||
var probeErr error
|
||||
if lookErr == nil {
|
||||
// The binary alone proves nothing: the checker's integration level runs shut off (no network, no /run), so on
|
||||
// a build node docker is found and its daemon is not reachable (issue 1823).
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if out, err := exec.CommandContext(ctx, "docker", "version", "--format", "{{.Server.Version}}").CombinedOutput(); err != nil {
|
||||
probeErr = fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
t.Skipf("NOT RUN: this test starts a throwaway nats container and there is no docker in this toolchain (%v); "+
|
||||
"it is proof only where merge-check.sh's docker step runs it (MESH_CHECK_DOCKER=1)", err)
|
||||
}
|
||||
// Where docker is expected (MESH_REQUIRE_DOCKER=1, which merge-check.sh sets for its docker step) its absence is a failure.
|
||||
switch action, why := dockerVerdict(lookErr == nil, lookErr, probeErr, os.Getenv("MESH_REQUIRE_DOCKER") == "1"); action {
|
||||
case "fail":
|
||||
t.Fatal(why)
|
||||
case "skip":
|
||||
t.Skipf("NOT RUN: this test starts a throwaway nats container and %s; "+
|
||||
"it is proof only where merge-check.sh's docker step runs it (MESH_CHECK_DOCKER=1)", why)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user