Compare commits
61
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c255d56b71 | ||
|
|
da29f77b10 | ||
|
|
5a4ac49913 | ||
|
|
94dab68635 | ||
|
|
76e4ec4dc6 | ||
|
|
26908a8950 | ||
|
|
27a7757b97 | ||
|
|
87fe2c45cf | ||
|
|
24b244bc88 | ||
|
|
5baf8ad1d7 | ||
|
|
8da33e43d4 | ||
|
|
52de4c320e | ||
|
|
5e516317e9 | ||
|
|
6b6d5cba71 | ||
|
|
8393e29621 | ||
|
|
61563abf2f | ||
|
|
071c6885d9 | ||
|
|
8c5f27627b | ||
|
|
82bd6f49ff | ||
|
|
ceb3aac9b3 | ||
|
|
98c66fb1e4 | ||
|
|
07ec186031 | ||
|
|
2aecbe8055 | ||
|
|
3def293d94 | ||
|
|
9a8e54d988 | ||
|
|
2da9329df4 | ||
|
|
ecbee53cfd | ||
|
|
dcdf4238c7 | ||
|
|
d3423f54ed | ||
|
|
0e322d087f | ||
|
|
0300a6fd99 | ||
|
|
d21b81b00a | ||
|
|
b1e610b213 | ||
|
|
0134697f97 | ||
|
|
21583d14ad | ||
|
|
a4f5df797b | ||
|
|
ac2c56f2a9 | ||
|
|
b4a7bb9dea | ||
|
|
64b9ded2db | ||
|
|
67d6a5a7c1 | ||
|
|
5fe5f26e7e | ||
|
|
b3221b6ab8 | ||
|
|
0e7e220d49 | ||
|
|
879600202a | ||
|
|
565d0a848a | ||
|
|
71e094b2de | ||
|
|
e9279ce3c2 | ||
|
|
c9414b1c98 | ||
|
|
ea53b2f920 | ||
|
|
99cc073e2a | ||
|
|
b94150fcdb | ||
|
|
e2136ec0c9 | ||
|
|
7627503d9d | ||
|
|
351c7c4bf4 | ||
|
|
55a26b22cf | ||
|
|
100e74f283 | ||
|
|
c8e1e2b2fa | ||
|
|
d0d00f922a | ||
|
|
c2bd7760b2 | ||
|
|
f0fd53281b | ||
|
|
fb039250f6 |
@@ -14,6 +14,7 @@ modules/claude-code/claude-code
|
|||||||
modules/mesh-watcher/mesh-watcher
|
modules/mesh-watcher/mesh-watcher
|
||||||
modules/dnsmasq/dnsmasq
|
modules/dnsmasq/dnsmasq
|
||||||
modules/mesh-issues/cmd/mesh-issues/mesh-issues
|
modules/mesh-issues/cmd/mesh-issues/mesh-issues
|
||||||
|
modules/mesh-issues/mesh-issues
|
||||||
modules/mounts/cmd/mounts/mounts
|
modules/mounts/cmd/mounts/mounts
|
||||||
modules/mesh-knowledge/cmd/mesh-knowledge/mesh-knowledge
|
modules/mesh-knowledge/cmd/mesh-knowledge/mesh-knowledge
|
||||||
modules/mesh-knowledge/mesh-knowledge
|
modules/mesh-knowledge/mesh-knowledge
|
||||||
@@ -34,3 +35,7 @@ modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image
|
|||||||
/checks/paths/paths
|
/checks/paths/paths
|
||||||
/modules/mesh-build-go/mesh-build-go
|
/modules/mesh-build-go/mesh-build-go
|
||||||
/modules/mesh-delivery-gate/mesh-delivery-gate
|
/modules/mesh-delivery-gate/mesh-delivery-gate
|
||||||
|
|
||||||
|
# Built by their modules' build step (issue 1577), never committed.
|
||||||
|
modules/forticlient/cmd/forticlient-tools/forticlient-tools
|
||||||
|
modules/networkmanager/uplink-tools
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
module git.novox.be/novox/mesh-catalog/checks/nobinaries
|
||||||
|
|
||||||
|
go 1.22
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
// Package nobinaries refuses a compiled program under modules/ (issue 1577): it is build output, made by the module's build step,
|
||||||
|
// and `go build` in a module's folder writes one beside go.mod where `git add -A` takes it.
|
||||||
|
package nobinaries
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Kind says what executable format the first bytes of a file are, or "" when they are none.
|
||||||
|
func Kind(head []byte) string {
|
||||||
|
switch {
|
||||||
|
case bytes.HasPrefix(head, []byte("\x7fELF")):
|
||||||
|
return "ELF"
|
||||||
|
case bytes.HasPrefix(head, []byte("MZ")):
|
||||||
|
return "PE"
|
||||||
|
case bytes.HasPrefix(head, []byte{0xcf, 0xfa, 0xed, 0xfe}), bytes.HasPrefix(head, []byte{0xce, 0xfa, 0xed, 0xfe}),
|
||||||
|
bytes.HasPrefix(head, []byte{0xfe, 0xed, 0xfa, 0xcf}), bytes.HasPrefix(head, []byte{0xfe, 0xed, 0xfa, 0xce}),
|
||||||
|
bytes.HasPrefix(head, []byte{0xca, 0xfe, 0xba, 0xbe}) && len(head) >= 8 && head[7] < 0x20 && head[6] == 0 && head[5] == 0 && head[4] == 0:
|
||||||
|
return "Mach-O"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Files are the files of the catalogue's root that the change holds under modules/: the tracked ones when the root is a git
|
||||||
|
// checkout (an ignored binary a local build left is not the change's), else every file there.
|
||||||
|
func Files(root string) ([]string, error) {
|
||||||
|
cmd := exec.Command("git", "-C", root, "ls-files", "-z", "--", "modules")
|
||||||
|
if out, err := cmd.Output(); err == nil && len(out) > 0 {
|
||||||
|
var files []string
|
||||||
|
for _, f := range strings.Split(strings.TrimRight(string(out), "\x00"), "\x00") {
|
||||||
|
if _, err := os.Lstat(filepath.Join(root, f)); err == nil { // a file the change deletes is not there
|
||||||
|
files = append(files, f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return files, nil
|
||||||
|
}
|
||||||
|
var files []string
|
||||||
|
err := filepath.WalkDir(filepath.Join(root, "modules"), func(p string, d os.DirEntry, err error) error {
|
||||||
|
if err != nil || d.IsDir() || !d.Type().IsRegular() {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rel, _ := filepath.Rel(root, p)
|
||||||
|
files = append(files, rel)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return files, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find names each file of files (paths from root) that is a compiled program, with its format.
|
||||||
|
func Find(root string, files []string) ([]string, error) {
|
||||||
|
var found []string
|
||||||
|
for _, f := range files {
|
||||||
|
p := filepath.Join(root, f)
|
||||||
|
if st, err := os.Lstat(p); err != nil || !st.Mode().IsRegular() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fh, err := os.Open(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
head := make([]byte, 8)
|
||||||
|
n, err := io.ReadFull(fh, head)
|
||||||
|
fh.Close()
|
||||||
|
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if k := Kind(head[:n]); k != "" {
|
||||||
|
found = append(found, fmt.Sprintf("%s (%s)", f, k))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
package nobinaries
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFindNamesEachCompiledProgramAndNothingElse(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
files := map[string][]byte{
|
||||||
|
"modules/a/a": []byte("\x7fELF\x02\x01\x01\x00rest"),
|
||||||
|
"modules/b/cmd/b/b.exe": []byte("MZ\x90\x00"),
|
||||||
|
"modules/c/c": {0xcf, 0xfa, 0xed, 0xfe, 7, 0, 0, 1},
|
||||||
|
"modules/d/main.go": []byte("package main\n"),
|
||||||
|
"modules/e/short": []byte("\x7f"),
|
||||||
|
"modules/f/empty": {},
|
||||||
|
"modules/g/Foo.class": {0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 52}, // Java, not Mach-O
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for f, b := range files {
|
||||||
|
p := filepath.Join(root, f)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(p, b, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
names = append(names, f)
|
||||||
|
}
|
||||||
|
got, err := Find(root, names)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 3 {
|
||||||
|
t.Fatalf("found %v; wanted modules/a/a, modules/b/cmd/b/b.exe and modules/c/c only", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The catalogue's own tree holds no compiled program: a compiled program is build output, so remove it and add it to .gitignore.
|
||||||
|
func TestTheCatalogueHoldsNoCompiledProgram(t *testing.T) {
|
||||||
|
root := filepath.Join("..", "..")
|
||||||
|
files, err := Files(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
found, err := Find(root, files)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, f := range found {
|
||||||
|
t.Errorf("%s: a compiled program is build output: remove it and add it to .gitignore (issue 1577)", f)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||||
Binary file not shown.
@@ -275,6 +275,53 @@ reported no key gets no entry, and its client says it waits. Keys titled `mesh:`
|
|||||||
a key added by hand never is. Every consumer is made again every ten minutes and at once when a key changes.
|
a key added by hand never is. Every consumer is made again every ten minutes and at once when a key changes.
|
||||||
`go test ./cmd/forge-account-provider ./internal/forgeaccount` runs it against a fake forge.
|
`go test ./cmd/forge-account-provider ./internal/forgeaccount` runs it against a fake forge.
|
||||||
|
|
||||||
|
## The mesh's accounts read every repository (the operator, 2026-10-26)
|
||||||
|
|
||||||
|
The tools act as the plain account `mesh-tools` (ADR 0365 §2, issue 469), which writes to the organisation `novox`
|
||||||
|
through its team; the build modules fetch over ssh as `nox-builder` (ADR 0466 §3), in a team that reads the
|
||||||
|
organisation's code; the machines' agents use `nox-agent` (ADR 0410 §1). Any other repository each saw only when public,
|
||||||
|
so a private repository of a user answered every tool 404 and every build's fetch "not authorized to read". `readall.ts`
|
||||||
|
makes each of these accounts (one list, `meshAccounts` in `tools-account.ts`, whose two Go names
|
||||||
|
`internal/forgeaccount`'s test holds to its own constants) a collaborator that **reads only** on each repository it
|
||||||
|
cannot read: one repository at a time, no organisation membership, the least the forge offers. An account the forge
|
||||||
|
does not hold yet (the forge-account provider makes `nox-builder` and `nox-agent` when their first consumer asks) is
|
||||||
|
named and asked nothing; the start's pass is run again ten minutes apart until every account was there once.
|
||||||
|
|
||||||
|
- **When:** one pass over every repository of every user and organisation at the start (a forge not answering yet is
|
||||||
|
asked again a minute apart until one pass ran), and on the forge's system webhook: `forge.repository.created` and
|
||||||
|
`forge.fork` at once, and the first event of any repository this process has not seen readable (the forge may not
|
||||||
|
send a creation for every owner). Nothing polls for it.
|
||||||
|
- **Not caught:** a repository made private after it was seen readable. The forge sends no event for a change of
|
||||||
|
visibility, and the start's pass already saw it readable, so its events ask nothing. It is granted at the module's
|
||||||
|
next start, or now by `gitea_grant_read`; `gitea_unreadable_repos` shows it meanwhile. A repository whose permission
|
||||||
|
the forge will not answer is named as refused (or, in the listing, with permission `unknown`) and the pass goes on.
|
||||||
|
- **With what:** the site administrator's login (`MESH_GITEA_ADMIN_PASSWORD_FILE`), because adding a collaborator asks a
|
||||||
|
repository's administration, which on another user's repository only the site administrator holds. It is used for
|
||||||
|
five calls alone: list every repository, read one, whether an account exists, read an account's permission on one,
|
||||||
|
add an account as a read collaborator. The tools' own calls keep their token.
|
||||||
|
- **Tools:** `gitea_unreadable_repos` (per account, every repository it cannot read, the permission the forge gives
|
||||||
|
it there and why, a count per account and the accounts the forge does not hold) and `gitea_grant_read` (the same
|
||||||
|
grant by hand, now, on every repository or on those named, for every account or those named; each grant read back
|
||||||
|
from the forge, a refusal answered in the forge's own words).
|
||||||
|
|
||||||
|
### The merging account writes the repositories the mesh delivers
|
||||||
|
|
||||||
|
`mesh-tools` sets every commit status and merges; outside the organisation it only reads (above), so a pull request of
|
||||||
|
a user's repository could be checked and asked but never merged (issue 1945: car-hunter #2's status refused 403 after
|
||||||
|
the operator's Approve). `writes.ts` makes it a collaborator that **writes** on each repository that builds a registered
|
||||||
|
module, and on no other (the operator, 2026-10-26): the build modules' and the agents' accounts keep read.
|
||||||
|
|
||||||
|
- **Which:** the indexer's state `mesh-delivery-indexer.modules` (in this module's `reads`), one key per registered
|
||||||
|
module, each naming its repository. A repository of `novox` needs nothing: the team writes it. Every other repository
|
||||||
|
of a user stays read-only.
|
||||||
|
- **When:** the state's watch hands every module at the start and each one registered after; a repository not yet
|
||||||
|
written is granted then, read back from the forge, and said on the bus as `gitea.repo.writable` {owner, repo,
|
||||||
|
account}, on which mesh-delivery-gate reads that repository again. Nothing polls.
|
||||||
|
- **Tools:** `gitea_unwritable_repos` (every delivered repository the account cannot write, its permission and why)
|
||||||
|
and `gitea_grant_write` (the same grant by hand, on every delivered repository or those named; what it granted, what
|
||||||
|
was written already, what the forge refused in its own words, and the repositories left read-only and why).
|
||||||
|
`gitea_commit_statuses` answers `merge.writes`, whether the account may write the repository at all.
|
||||||
|
|
||||||
## Files it does not know
|
## Files it does not know
|
||||||
|
|
||||||
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
||||||
|
|||||||
+116
-5
@@ -50,6 +50,9 @@ export interface GiteaRepo {
|
|||||||
default_branch?: string;
|
default_branch?: string;
|
||||||
/** When anything last moved in it — a push, and so a merge. */
|
/** When anything last moved in it — a push, and so a merge. */
|
||||||
updated_at?: string;
|
updated_at?: string;
|
||||||
|
/** Whether the account asking may write it (the forge's permissions.push or admin): a commit status and a merge need
|
||||||
|
* it. Absent when the forge did not say. */
|
||||||
|
writes?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** An issue, with its labels flattened to names. */
|
/** An issue, with its labels flattened to names. */
|
||||||
@@ -98,6 +101,10 @@ export interface GiteaPull {
|
|||||||
html_url: string;
|
html_url: string;
|
||||||
/** The description: where a pull request says it goes after another (`after: <repository>`, novox/hq ADR 0239). */
|
/** The description: where a pull request says it goes after another (`after: <repository>`, novox/hq ADR 0239). */
|
||||||
body?: string;
|
body?: string;
|
||||||
|
/** Whether the forge would merge it into its base now (gitea's `mergeable`): false while it is in conflict, while the
|
||||||
|
* forge is still working that out after its base moved, or while it is a work in progress; absent when the forge did not
|
||||||
|
* say. What a group's merge reads of every member before it merges any (novox/hq issue 1275). */
|
||||||
|
mergeable?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** An open pull request with what its base requires and what its head has: what gitea_open_pulls answers. */
|
/** An open pull request with what its base requires and what its head has: what gitea_open_pulls answers. */
|
||||||
@@ -789,14 +796,29 @@ export class GiteaClient {
|
|||||||
|
|
||||||
/** Merge a pull request — never forced past its checks, and, given the head that was checked, only that head:
|
/** Merge a pull request — never forced past its checks, and, given the head that was checked, only that head:
|
||||||
* the forge refuses the merge when the branch moved since (novox/hq ADR 0266). */
|
* the forge refuses the merge when the branch moved since (novox/hq ADR 0266). */
|
||||||
|
/** How long a merge waits before it asks again after the forge's 405 "Please try again later" (novox/hq issue 1275):
|
||||||
|
* the forge says so while it works out whether the pull request merges, for a few seconds after its base moved. */
|
||||||
|
static mergeRetryAfter: number[] = [1000, 2000, 4000];
|
||||||
|
|
||||||
async mergePullRequest(owner: string, repo: string, index: number, method = "merge", deleteBranch = false,
|
async mergePullRequest(owner: string, repo: string, index: number, method = "merge", deleteBranch = false,
|
||||||
headCommit?: string): Promise<void> {
|
headCommit?: string): Promise<void> {
|
||||||
indexOf(index);
|
indexOf(index);
|
||||||
await this.request(`${repoPath(owner, repo)}/pulls/${index}/merge`, {
|
for (let attempt = 0; ; attempt++) {
|
||||||
method: "POST",
|
try {
|
||||||
body: JSON.stringify({ Do: method, delete_branch_after_merge: deleteBranch, force_merge: false,
|
await this.request(`${repoPath(owner, repo)}/pulls/${index}/merge`, {
|
||||||
...(headCommit ? { head_commit_id: headCommit } : {}) }),
|
method: "POST",
|
||||||
});
|
body: JSON.stringify({ Do: method, delete_branch_after_merge: deleteBranch, force_merge: false,
|
||||||
|
...(headCommit ? { head_commit_id: headCommit } : {}) }),
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
} catch (err) {
|
||||||
|
// Only the forge's transient answer is asked again, briefly; every other refusal, and this one past the last
|
||||||
|
// wait, is the caller's to say. head_commit_id keeps a head that moved in between from landing.
|
||||||
|
const wait = GiteaClient.mergeRetryAfter[attempt];
|
||||||
|
if (wait === undefined || !tryAgainLater(err)) throw err;
|
||||||
|
await new Promise((r) => setTimeout(r, wait));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- Mappers: the wire shape is broad and unstable; the mesh sees only these fields. ----
|
// ---- Mappers: the wire shape is broad and unstable; the mesh sees only these fields. ----
|
||||||
@@ -812,6 +834,8 @@ export class GiteaClient {
|
|||||||
html_url: r.html_url,
|
html_url: r.html_url,
|
||||||
default_branch: r.default_branch,
|
default_branch: r.default_branch,
|
||||||
updated_at: r.updated_at ?? undefined,
|
updated_at: r.updated_at ?? undefined,
|
||||||
|
...(r.permissions && typeof r.permissions === "object"
|
||||||
|
? { writes: r.permissions.push === true || r.permissions.admin === true } : {}),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -842,6 +866,7 @@ export class GiteaClient {
|
|||||||
updated_at: p.updated_at ?? undefined,
|
updated_at: p.updated_at ?? undefined,
|
||||||
html_url: p.html_url,
|
html_url: p.html_url,
|
||||||
body: typeof p.body === "string" ? p.body : undefined,
|
body: typeof p.body === "string" ? p.body : undefined,
|
||||||
|
mergeable: typeof p.mergeable === "boolean" && !p.merged ? p.mergeable : undefined,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -853,6 +878,25 @@ interface AdminResponse {
|
|||||||
readonly body: any;
|
readonly body: any;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A repository as the site administrator sees it: what saying why an account cannot read it needs (readall.ts). */
|
||||||
|
export interface ForgeRepository {
|
||||||
|
full_name: string;
|
||||||
|
owner: string;
|
||||||
|
name: string;
|
||||||
|
private: boolean;
|
||||||
|
archived: boolean;
|
||||||
|
/** The owner's own visibility: public, limited or private. A public repository of an owner not public is hidden
|
||||||
|
* from whoever is no member. */
|
||||||
|
owner_visibility: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function forgeRepository(r: any): ForgeRepository {
|
||||||
|
const owner = String(r?.owner?.login ?? r?.owner?.username ?? "");
|
||||||
|
const name = String(r?.name ?? "");
|
||||||
|
return { full_name: String(r?.full_name ?? `${owner}/${name}`), owner, name, private: r?.private === true,
|
||||||
|
archived: r?.archived === true, owner_visibility: String(r?.owner?.visibility ?? "public") };
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The forge's admin client, over **basic auth** — gitea's own code, living in the module, used only
|
* The forge's admin client, over **basic auth** — gitea's own code, living in the module, used only
|
||||||
* by the provisioner (novox/hq ADR 0048/0076).
|
* by the provisioner (novox/hq ADR 0048/0076).
|
||||||
@@ -970,6 +1014,66 @@ export class GiteaAdmin {
|
|||||||
return repos;
|
return repos;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- The tools' account reads every repository (readall.ts) ----
|
||||||
|
|
||||||
|
/** Every repository on the forge, of every user and organisation: the search as the site administrator, whom the
|
||||||
|
* forge answers every repository, private ones and those of private owners included. Only a read. */
|
||||||
|
async everyRepository(): Promise<ForgeRepository[]> {
|
||||||
|
const all: ForgeRepository[] = [];
|
||||||
|
for (let page = 1; page <= 200; page++) {
|
||||||
|
const res = await this.request(`/repos/search?limit=50&page=${page}`);
|
||||||
|
if (res.status !== 200) GiteaAdmin.fail("/repos/search", res);
|
||||||
|
const list: any[] = Array.isArray(res.body?.data) ? res.body.data : [];
|
||||||
|
all.push(...list.map(forgeRepository));
|
||||||
|
if (list.length < 50) break;
|
||||||
|
}
|
||||||
|
return all;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One repository as the site administrator sees it, or null when the forge holds none by that name. */
|
||||||
|
async repository(owner: string, repo: string): Promise<ForgeRepository | null> {
|
||||||
|
const res = await this.request(repoPath(owner, repo));
|
||||||
|
if (res.status === 404) return null;
|
||||||
|
if (res.status !== 200) GiteaAdmin.fail(repoPath(owner, repo), res);
|
||||||
|
return forgeRepository(res.body);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether the forge holds an account of this name. */
|
||||||
|
async accountExists(user: string): Promise<boolean> {
|
||||||
|
const path = `/users/${encodeURIComponent(user)}`;
|
||||||
|
const res = await this.request(path);
|
||||||
|
if (res.status === 404) return false;
|
||||||
|
if (res.status !== 200) GiteaAdmin.fail(path, res);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** What the forge itself says an account may do on a repository: none, read, write, admin or owner. */
|
||||||
|
async repositoryPermission(owner: string, repo: string, user: string): Promise<string> {
|
||||||
|
const path = `${repoPath(owner, repo)}/collaborators/${encodeURIComponent(user)}/permission`;
|
||||||
|
const res = await this.request(path);
|
||||||
|
if (res.status !== 200) GiteaAdmin.fail(path, res);
|
||||||
|
return String(res.body?.permission ?? "none");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Make an account a collaborator of a repository that reads only. The forge asks a repository's administrator for
|
||||||
|
* it, which on a repository of another user only the site administrator is. Idempotent: the forge answers 204 again. */
|
||||||
|
async addReadCollaborator(owner: string, repo: string, user: string): Promise<void> {
|
||||||
|
await this.addCollaborator(owner, repo, user, "read");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Make an account a collaborator of a repository that writes: pushes a branch, sets a commit status, merges (writes.ts:
|
||||||
|
* only on a repository that builds a registered module). An existing read collaborator is changed to write. */
|
||||||
|
async addWriteCollaborator(owner: string, repo: string, user: string): Promise<void> {
|
||||||
|
await this.addCollaborator(owner, repo, user, "write");
|
||||||
|
}
|
||||||
|
|
||||||
|
private async addCollaborator(owner: string, repo: string, user: string, permission: "read" | "write"): Promise<void> {
|
||||||
|
const path = `${repoPath(owner, repo)}/collaborators/${encodeURIComponent(user)}`;
|
||||||
|
const res = await this.request(path, { method: "PUT", body: JSON.stringify({ permission }) });
|
||||||
|
if (res.status === 204 || res.status === 200) return;
|
||||||
|
GiteaAdmin.fail(path, res);
|
||||||
|
}
|
||||||
|
|
||||||
/** Ensure the npm-owner org exists. 201 created, 2xx/404-then-created, and 422/409 (a concurrent
|
/** Ensure the npm-owner org exists. 201 created, 2xx/404-then-created, and 422/409 (a concurrent
|
||||||
* create won the race) are all success. */
|
* create won the race) are all success. */
|
||||||
async ensureOrg(name: string): Promise<void> {
|
async ensureOrg(name: string): Promise<void> {
|
||||||
@@ -1160,3 +1264,10 @@ export function directoriesAbove(paths: string[]): string[] {
|
|||||||
function encodePath(path: string): string {
|
function encodePath(path: string): string {
|
||||||
return pathSegments("file path", path);
|
return pathSegments("file path", path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Whether an error is the forge's 405 "Please try again later" on a merge: its answer while the pull request's
|
||||||
|
* mergeability is being worked out (or it is in conflict, which a later ask says the same way). */
|
||||||
|
export function tryAgainLater(err: unknown): boolean {
|
||||||
|
const words = err instanceof Error ? err.message : String(err);
|
||||||
|
return / 405 /.test(words) && /try again later/i.test(words);
|
||||||
|
}
|
||||||
|
|||||||
+50
-3
@@ -58,8 +58,11 @@ import { CHECK_CONTEXT, CHECK_STATUS, REPO_CHECK_CONTEXT, commentFor, composedGa
|
|||||||
import { GateWriter } from "./gate.js";
|
import { GateWriter } from "./gate.js";
|
||||||
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
|
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
|
||||||
import { keepSiblings, noteComposed, noteVerdict, readSiblings, siblingsFile, siblingsOf, weighSiblings } from "./siblings.js";
|
import { keepSiblings, noteComposed, noteVerdict, readSiblings, siblingsFile, siblingsOf, weighSiblings } from "./siblings.js";
|
||||||
import { bindInBackground } from "./bind.js";
|
import { bindInBackground, bindUntilHeld } from "./bind.js";
|
||||||
import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
|
import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
|
||||||
|
import { ReadAll, readAllTools } from "./readall.js";
|
||||||
|
import { DeliveredWrites, INDEXER_MODULES, writeTools, type ModulesChange } from "./writes.js";
|
||||||
|
import { TOOLS_ACCOUNT, TOOLS_ORG, meshAccounts } from "./tools-account.js";
|
||||||
|
|
||||||
// Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
|
// Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
|
||||||
// nothing to watch; log and stay quiet rather than crash the runtime. With one, the first poll mints
|
// nothing to watch; log and stay quiet rather than crash the runtime. With one, the first poll mints
|
||||||
@@ -500,8 +503,21 @@ if (!gitea) pace.unheard("the forge is not watched: no way to a token", false);
|
|||||||
// Said with `emit`, not the SDK's `keeps`, which the toolchain's SDK does not have yet: switch to `keeps` once that SDK is 0.1.15 or later (issue 1852).
|
// Said with `emit`, not the SDK's `keeps`, which the toolchain's SDK does not have yet: switch to `keeps` once that SDK is 0.1.15 or later (issue 1852).
|
||||||
await emit("states", statesAnnouncement(process.env.MESH_MODULE || "gitea", process.env.MESH_NODE));
|
await emit("states", statesAnnouncement(process.env.MESH_MODULE || "gitea", process.env.MESH_NODE));
|
||||||
|
|
||||||
|
// **The mesh's accounts read every repository** (readall.ts; the operator, 2026-10-26): the tools', the build modules'
|
||||||
|
// and the machines' agents' (meshAccounts, tools-account.ts), granted with the site
|
||||||
|
// administrator's login, which the module holds for setting the forge up and uses here for that grant alone. Made on
|
||||||
|
// first use, so a module whose admin login is missing still serves the rest and the tools say why.
|
||||||
|
let readAllOnce: ReadAll | null = null;
|
||||||
|
const readAll = (): ReadAll => (readAllOnce ??= new ReadAll(GiteaAdmin.fromEnv(), meshAccounts(process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT)));
|
||||||
|
|
||||||
|
// **The merging account writes every repository that builds a registered module** (writes.ts; the operator,
|
||||||
|
// 2026-10-26; issue 1945): which those are is the indexer's state, watched below. Made on first use, as readAll.
|
||||||
|
let writesOnce: DeliveredWrites | null = null;
|
||||||
|
const writes = (): DeliveredWrites => (writesOnce ??= new DeliveredWrites(GiteaAdmin.fromEnv(),
|
||||||
|
process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT, TOOLS_ORG, (l) => console.log(`[gitea] ${l}`), (type, body) => emit(type, body)));
|
||||||
|
|
||||||
// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729).
|
// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729).
|
||||||
registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({
|
registerModuleTools("gitea", () => [eventsTool(pace), ...readAllTools(readAll), ...writeTools(writes), ...(gitea ? [commitStatusTool({
|
||||||
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
||||||
gates,
|
gates,
|
||||||
}), mergeTool(gitea, { gates })] : [])]);
|
}), mergeTool(gitea, { gates })] : [])]);
|
||||||
@@ -543,7 +559,12 @@ async function listenForForge(client: GiteaClient): Promise<boolean> {
|
|||||||
const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery),
|
const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery),
|
||||||
refused: (reason, from) => pace.refused(reason, from),
|
refused: (reason, from) => pace.refused(reason, from),
|
||||||
publish: (event) => emit(event.name, event.body),
|
publish: (event) => emit(event.name, event.body),
|
||||||
act: (event) => actOn(client, event) });
|
act: async (event) => {
|
||||||
|
// A repository made, or one not yet seen readable, is granted read before anything acts on it (readall.ts).
|
||||||
|
await Promise.resolve().then(() => readAll().onEvent(event)).catch((err) =>
|
||||||
|
console.error(`[gitea] whether the mesh's accounts read the repository of ${event.name} could not be settled — ${err instanceof Error ? err.message : err}`));
|
||||||
|
await actOn(client, event);
|
||||||
|
} });
|
||||||
await new Promise<void>((resolve, reject) => {
|
await new Promise<void>((resolve, reject) => {
|
||||||
server.once("error", reject);
|
server.once("error", reject);
|
||||||
server.listen(port, host, () => resolve());
|
server.listen(port, host, () => resolve());
|
||||||
@@ -594,6 +615,32 @@ if (gitea) {
|
|||||||
// of the node starts left gitea_commit_status, gitea_merge_pull_request and gitea_events served nowhere. Until it is
|
// of the node starts left gitea_commit_status, gitea_merge_pull_request and gitea_events served nowhere. Until it is
|
||||||
// listening, Pace keeps the polls at 30 s, so nothing is missed meanwhile; listenForForge says its own failure.
|
// listening, Pace keeps the polls at 30 s, so nothing is missed meanwhile; listenForForge says its own failure.
|
||||||
void listenForForge(client);
|
void listenForForge(client);
|
||||||
|
// The start's one pass over every repository (readall.ts), not awaited for the same reason. A forge not answering yet
|
||||||
|
// is asked again, a minute apart, until one pass has run; after that only the forge's events grant.
|
||||||
|
// An account of the list the forge does not hold yet (the forge-account provider makes nox-builder and nox-agent when
|
||||||
|
// their first consumer asks, and the forge sends no event for a user made) is looked for again ten minutes apart
|
||||||
|
// until every account was granted once; the line of each pass names it.
|
||||||
|
const readAllAtStart = (): void => void Promise.resolve().then(() => readAll().atStart()).then((r) => {
|
||||||
|
if (r.absent.length) setTimeout(readAllAtStart, 600_000);
|
||||||
|
}, (err) => {
|
||||||
|
console.error(`[gitea] whether the mesh's accounts read every repository could not be settled; asked again in a minute — ${err instanceof Error ? err.message : err}`);
|
||||||
|
setTimeout(readAllAtStart, 60_000);
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
readAll();
|
||||||
|
readAllAtStart();
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[gitea] the mesh's accounts are not given read on every repository: no site administrator's login — ${err instanceof Error ? err.message : err}`);
|
||||||
|
}
|
||||||
|
// The indexer's modules, watched: the watch hands every key's latest value at its start and then each change, so a
|
||||||
|
// repository that builds a registered module is made writable at the start and when its first module is registered
|
||||||
|
// (writes.ts). A read not granted yet is said and asked again until it holds.
|
||||||
|
try {
|
||||||
|
const w = writes();
|
||||||
|
void bindUntilHeld(INDEXER_MODULES, () => state<unknown>(INDEXER_MODULES).watch((change) => w.take(change as ModulesChange)));
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[gitea] the merging account is not given write on the delivered repositories: no site administrator's login — ${err instanceof Error ? err.message : err}`);
|
||||||
|
}
|
||||||
// Main takes no push and only the tools' account merges, in every repository, kept and not only when a token is
|
// Main takes no push and only the tools' account merges, in every repository, kept and not only when a token is
|
||||||
// minted: at the start and every ten minutes after, so a repository made later gets it too (issue 948, ADR 0398 §2).
|
// minted: at the start and every ten minutes after, so a repository made later gets it too (issue 948, ADR 0398 §2).
|
||||||
// Only while the setting keep-main-rules is true (mainrules.ts); false, the default, leaves every repository's rule as set.
|
// Only while the setting keep-main-rules is true (mainrules.ts); false, the default, leaves every repository's rule as set.
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
package forgeaccount
|
package forgeaccount
|
||||||
|
|
||||||
import "testing"
|
import (
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
func TestForgeUserOfAMachineAccountIsTheOneAgentUser(t *testing.T) {
|
func TestForgeUserOfAMachineAccountIsTheOneAgentUser(t *testing.T) {
|
||||||
if got := ForgeUser("g14", "machine"); got != "nox-agent" {
|
if got := ForgeUser("g14", "machine"); got != "nox-agent" {
|
||||||
@@ -15,6 +19,18 @@ func TestForgeUserOfAMachineAccountIsTheOneAgentUser(t *testing.T) {
|
|||||||
if got := NodeAccount("g14"); got != "agent-g14" {
|
if got := NodeAccount("g14"); got != "agent-g14" {
|
||||||
t.Fatalf("the token tool's per-node account = %q", got)
|
t.Fatalf("the token tool's per-node account = %q", got)
|
||||||
}
|
}
|
||||||
|
// The accounts that read every repository (readall.ts, meshAccounts in tools-account.ts) name these two users: the
|
||||||
|
// list is held to the Go constants here, so the two cannot drift apart (the operator, 2026-10-26).
|
||||||
|
ts, err := os.ReadFile("../../tools-account.ts")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for name, want := range map[string]string{"BUILDER_ACCOUNT": BuilderUser, "AGENT_ACCOUNT": AgentUser} {
|
||||||
|
m := regexp.MustCompile(`export const ` + name + ` = "([^"]+)";`).FindSubmatch(ts)
|
||||||
|
if m == nil || string(m[1]) != want {
|
||||||
|
t.Errorf("tools-account.ts %s = %q, want %q (internal/forgeaccount)", name, m, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRemoveKeysTitledTakesOnlyTheMeshKeysOfThatTitle(t *testing.T) {
|
func TestRemoveKeysTitledTakesOnlyTheMeshKeysOfThatTitle(t *testing.T) {
|
||||||
|
|||||||
@@ -40,6 +40,7 @@
|
|||||||
],
|
],
|
||||||
"emits": [
|
"emits": [
|
||||||
"repo.created",
|
"repo.created",
|
||||||
|
"repo.writable",
|
||||||
"issue.opened",
|
"issue.opened",
|
||||||
"pull.merged",
|
"pull.merged",
|
||||||
"pull.updated",
|
"pull.updated",
|
||||||
@@ -567,6 +568,7 @@
|
|||||||
"mesh-build-go.keys",
|
"mesh-build-go.keys",
|
||||||
"mesh-build-node.keys",
|
"mesh-build-node.keys",
|
||||||
"mesh-build-image.keys",
|
"mesh-build-image.keys",
|
||||||
"mesh-delivery-checker.owed"
|
"mesh-delivery-checker.owed",
|
||||||
|
"mesh-delivery-indexer.modules"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,263 @@
|
|||||||
|
// The mesh's own forge accounts read every repository on the forge (the operator, 2026-10-26: "the mesh's forge account
|
||||||
|
// should get read access to all repositories"; and "why wouldn't the forge account get access to all repo's?").
|
||||||
|
//
|
||||||
|
// **Every account the mesh's parts use, from one list** (MESH_ACCOUNTS, tools-account.ts): the tools' `mesh-tools`, the
|
||||||
|
// build modules' `nox-builder` (their fetch over ssh, ADR 0466 §3) and the machines' `nox-agent` (ssh-client's key,
|
||||||
|
// ADR 0410 §1). The first pass granted the tools' account alone, so the build modules' fetch of jschoubben/car-hunter
|
||||||
|
// still failed on ace and g14 ("nox-builder ... is not authorized to read"). The two Go names are the ones of
|
||||||
|
// internal/forgeaccount, whose test holds the list to them. An account the forge does not hold yet (its consumer never
|
||||||
|
// asked) is said and left out, never a refusal per repository.
|
||||||
|
//
|
||||||
|
// Since issue 469 the tools act as the plain account `mesh-tools`, which writes to the organisation novox's
|
||||||
|
// repositories through its team (tools-account.ts). Every other repository it saw only when public: a private
|
||||||
|
// repository of a user (jschoubben/car-hunter, jschoubben/snake) answered its tools 404 and was missing from every
|
||||||
|
// list, while that user's public ones were there. So the module makes its account a collaborator that reads only on
|
||||||
|
// each repository it cannot read, the least the forge offers: one repository, read, no organisation membership.
|
||||||
|
//
|
||||||
|
// - **at the start**, one pass over every repository of every user and organisation;
|
||||||
|
// - **on the forge's own event** (the system webhook, webhook.ts): `forge.repository.created` and `forge.fork` at
|
||||||
|
// once, and the first event of any repository this process has not yet seen readable, since the forge may not
|
||||||
|
// send a repository's creation for every owner. Nothing polls for it.
|
||||||
|
//
|
||||||
|
// **Not caught: a repository made private later.** The forge sends no event for a change of visibility, and a public
|
||||||
|
// repository is already seen readable by the start's pass, so its events ask nothing. Such a repository is granted at
|
||||||
|
// the module's next start, or now by `gitea_grant_read`; `gitea_unreadable_repos` shows it meanwhile.
|
||||||
|
//
|
||||||
|
// **The grant needs the site administrator's login.** Adding a collaborator asks the repository's administration,
|
||||||
|
// which on another user's repository only the site administrator holds; the module holds that login for setting the
|
||||||
|
// forge up (MESH_GITEA_ADMIN_PASSWORD_FILE). Here it is used through `ReadAllForge` alone: list the repositories, read
|
||||||
|
// the account's permission on one, add the account as a collaborator that reads. Nothing else, and never for the tools'
|
||||||
|
// own calls, which keep their token.
|
||||||
|
//
|
||||||
|
// What the forge refuses (a repository that went meanwhile, an archived one if the forge will not take a collaborator
|
||||||
|
// on it) is said per repository and not retried; the tool `gitea_unreadable_repos` shows what is left, and
|
||||||
|
// `gitea_grant_read` grants again now.
|
||||||
|
|
||||||
|
import type { ForgeRepository } from "./client.js";
|
||||||
|
|
||||||
|
/** What the module asks of the forge for this, with the site administrator's login (GiteaAdmin, client.ts). */
|
||||||
|
export interface ReadAllForge {
|
||||||
|
everyRepository(): Promise<ForgeRepository[]>;
|
||||||
|
repository(owner: string, repo: string): Promise<ForgeRepository | null>;
|
||||||
|
/** Whether the forge holds the account at all. */
|
||||||
|
accountExists(user: string): Promise<boolean>;
|
||||||
|
repositoryPermission(owner: string, repo: string, user: string): Promise<string>;
|
||||||
|
addReadCollaborator(owner: string, repo: string, user: string): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A repository an account cannot read, and why. */
|
||||||
|
export interface Unreadable {
|
||||||
|
account: string;
|
||||||
|
repository: string;
|
||||||
|
permission: string;
|
||||||
|
private: boolean;
|
||||||
|
archived: boolean;
|
||||||
|
why: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Granted {
|
||||||
|
/** Repositories looked at (for each account). */
|
||||||
|
checked: number;
|
||||||
|
/** The accounts looked at: those the forge holds. */
|
||||||
|
accounts: string[];
|
||||||
|
/** Accounts of the list the forge does not hold yet: nothing is granted to them until it does. */
|
||||||
|
absent: string[];
|
||||||
|
/** Made readable now, by a read collaborator: each repository and the account. */
|
||||||
|
granted: { account: string; repository: string }[];
|
||||||
|
/** The forge refused the grant: each repository, the account and the forge's own words. */
|
||||||
|
refused: { account: string; repository: string; why: string }[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const READS = new Set(["read", "write", "admin", "owner"]);
|
||||||
|
|
||||||
|
const said = (err: unknown): string => (err instanceof Error ? err.message : String(err));
|
||||||
|
|
||||||
|
/** Why the account cannot read a repository, from what the forge says of it. */
|
||||||
|
export function whyUnreadable(repo: ForgeRepository, account: string): string {
|
||||||
|
const parts: string[] = [];
|
||||||
|
if (repo.private) parts.push(`a private repository of ${repo.owner}, on which ${account} is no collaborator and in no team that covers it`);
|
||||||
|
else if (repo.owner_visibility !== "public") {
|
||||||
|
parts.push(`a public repository of ${repo.owner}, whose own visibility is ${repo.owner_visibility}: the forge shows it to members only`);
|
||||||
|
} else parts.push(`the forge answers ${account} no read on it`);
|
||||||
|
if (repo.archived) parts.push("archived");
|
||||||
|
return parts.join("; ");
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ReadAll {
|
||||||
|
/** Per account, the repositories this process has seen it read, by full name: an event of a repository every account
|
||||||
|
* was seen reading asks nothing more. */
|
||||||
|
readonly readable = new Map<string, Set<string>>();
|
||||||
|
readonly accounts: readonly string[];
|
||||||
|
|
||||||
|
constructor(private readonly forge: ReadAllForge, accounts: readonly string[] | string,
|
||||||
|
private readonly log: (line: string) => void = (l) => console.log(`[gitea] ${l}`)) {
|
||||||
|
this.accounts = [...new Set(typeof accounts === "string" ? [accounts] : accounts)];
|
||||||
|
if (!this.accounts.length) throw new Error("no forge account to give read: the list is empty");
|
||||||
|
for (const a of this.accounts) this.readable.set(a, new Set());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The accounts of the list (or of those named) the forge holds, and those it does not hold yet. */
|
||||||
|
private async present(only?: string[]): Promise<{ accounts: string[]; absent: string[] }> {
|
||||||
|
const wanted = only && only.length ? only : [...this.accounts];
|
||||||
|
for (const a of wanted) if (!this.accounts.includes(a)) throw new Error(`"${a}" is not one of the mesh's accounts: ${this.accounts.join(", ")}`);
|
||||||
|
const accounts: string[] = [];
|
||||||
|
const absent: string[] = [];
|
||||||
|
for (const a of wanted) ((await this.forge.accountExists(a)) ? accounts : absent).push(a);
|
||||||
|
return { accounts, absent };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Every repository each account cannot read, and why, as the forge says it now. */
|
||||||
|
async unreadable(only?: string[]): Promise<{ checked: number; accounts: { account: string; unreadable: number }[]; absent: string[]; unreadable: Unreadable[] }> {
|
||||||
|
const { accounts, absent } = await this.present(only);
|
||||||
|
const repos = await this.forge.everyRepository();
|
||||||
|
const out: Unreadable[] = [];
|
||||||
|
for (const account of accounts) {
|
||||||
|
for (const repo of repos) {
|
||||||
|
try {
|
||||||
|
const u = await this.judge(repo, account);
|
||||||
|
if (u) out.push(u);
|
||||||
|
} catch (err) {
|
||||||
|
// A permission the forge would not answer is said for that repository; the rest are still looked at.
|
||||||
|
out.push({ account, repository: repo.full_name, permission: "unknown", private: repo.private, archived: repo.archived,
|
||||||
|
why: `the forge did not answer ${account}'s permission on it — ${said(err)}` });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { checked: repos.length, accounts: accounts.map((account) => ({ account, unreadable: out.filter((u) => u.account === account).length })),
|
||||||
|
absent, unreadable: out };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Grant read on every repository an account cannot read (or on those named, `owner/name`), for every account of the
|
||||||
|
* list the forge holds (or for those named). */
|
||||||
|
async grant(only?: string[], accountsOnly?: string[]): Promise<Granted> {
|
||||||
|
const { accounts, absent } = await this.present(accountsOnly);
|
||||||
|
let repos: ForgeRepository[];
|
||||||
|
if (only && only.length) {
|
||||||
|
repos = [];
|
||||||
|
for (const full of only) {
|
||||||
|
const [owner, name, ...rest] = full.split("/");
|
||||||
|
if (!owner || !name || rest.length) throw new Error(`"${full}" is not a repository: say owner/name`);
|
||||||
|
const repo = await this.forge.repository(owner, name);
|
||||||
|
if (!repo) throw new Error(`the forge holds no repository ${full}`);
|
||||||
|
repos.push(repo);
|
||||||
|
}
|
||||||
|
} else repos = await this.forge.everyRepository();
|
||||||
|
const result: Granted = { checked: repos.length, accounts, absent, granted: [], refused: [] };
|
||||||
|
for (const account of accounts) {
|
||||||
|
for (const repo of repos) {
|
||||||
|
const repository = repo.full_name;
|
||||||
|
let u: Unreadable | null;
|
||||||
|
try {
|
||||||
|
u = await this.judge(repo, account);
|
||||||
|
} catch (err) {
|
||||||
|
// One repository whose permission cannot be read is a refusal of its own, never the end of the pass.
|
||||||
|
result.refused.push({ account, repository, why: `its permission could not be read — ${said(err)}` });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!u) continue;
|
||||||
|
try {
|
||||||
|
await this.forge.addReadCollaborator(repo.owner, repo.name, account);
|
||||||
|
} catch (err) {
|
||||||
|
result.refused.push({ account, repository, why: said(err) });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// Read back: the forge's own word that the account reads it now, never the grant's answer alone.
|
||||||
|
let after: string;
|
||||||
|
try {
|
||||||
|
after = await this.forge.repositoryPermission(repo.owner, repo.name, account);
|
||||||
|
} catch (err) {
|
||||||
|
result.refused.push({ account, repository, why: `granted, but its permission could not be read back — ${said(err)}` });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (READS.has(after)) {
|
||||||
|
this.readable.get(account)!.add(repository);
|
||||||
|
result.granted.push({ account, repository });
|
||||||
|
} else {
|
||||||
|
result.refused.push({ account, repository, why: `the grant was answered, but the forge still gives ${account} "${after}"` });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The start's one pass: grant what is missing and say it. Answers the result; a failure is the caller's to say. */
|
||||||
|
async atStart(): Promise<Granted> {
|
||||||
|
const r = await this.grant();
|
||||||
|
const per = r.accounts.map((a) => {
|
||||||
|
const g = r.granted.filter((x) => x.account === a).map((x) => x.repository);
|
||||||
|
return `${a} ${g.length} granted read now${g.length ? ` (${g.join(", ")})` : ""}`;
|
||||||
|
});
|
||||||
|
this.log(`the mesh's accounts read every repository: ${r.checked} looked at; ${per.join("; ")}` +
|
||||||
|
`${r.absent.length ? `; not on the forge yet, so not granted: ${r.absent.join(", ")}` : ""}` +
|
||||||
|
`${r.refused.length ? `; ${r.refused.length} refused: ` + r.refused.map((x) => `${x.account} on ${x.repository} — ${x.why}`).join("; ") : ""}`);
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A forge event: a repository made (or forked), or the first event of one some account was not seen reading, is
|
||||||
|
* granted read now to every account that cannot read it. */
|
||||||
|
async onEvent(event: { name: string; body: unknown }): Promise<void> {
|
||||||
|
const b = (event.body ?? {}) as { owner?: string; repo?: string };
|
||||||
|
if (!b.owner || !b.repo) return;
|
||||||
|
const full = `${b.owner}/${b.repo}`;
|
||||||
|
if (event.name === "forge.repository.deleted") {
|
||||||
|
for (const seen of this.readable.values()) seen.delete(full);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const made = event.name === "forge.repository.created" || event.name === "forge.fork";
|
||||||
|
// Seen readable once by every account, its events ask nothing: so a repository made private after that is not
|
||||||
|
// caught here (the forge says no change of visibility), only at the next start or by gitea_grant_read.
|
||||||
|
if (!made && [...this.readable.values()].every((seen) => seen.has(full))) return;
|
||||||
|
const r = await this.grant([full]);
|
||||||
|
for (const x of r.granted) this.log(`${x.account} was given read on ${full}, on the forge's ${event.name}`);
|
||||||
|
for (const x of r.refused) this.log(`${x.account} could not be given read on ${x.repository} — ${x.why}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async judge(repo: ForgeRepository, account: string): Promise<Unreadable | null> {
|
||||||
|
const permission = await this.forge.repositoryPermission(repo.owner, repo.name, account);
|
||||||
|
const seen = this.readable.get(account)!;
|
||||||
|
if (READS.has(permission)) {
|
||||||
|
seen.add(repo.full_name);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
seen.delete(repo.full_name);
|
||||||
|
return { account, repository: repo.full_name, permission, private: repo.private, archived: repo.archived, why: whyUnreadable(repo, account) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type Tool = { name: string; description: string; input: Record<string, unknown>; run: (args: Readonly<Record<string, unknown>>) => Promise<unknown> };
|
||||||
|
|
||||||
|
const strings = (v: unknown): string[] | undefined => (Array.isArray(v) ? v.map(String) : undefined);
|
||||||
|
|
||||||
|
/** The two tools: what each account cannot read and why, and the grant now. */
|
||||||
|
export function readAllTools(readAll: () => ReadAll): Tool[] {
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
name: "gitea_unreadable_repos",
|
||||||
|
description: "Every repository on the forge one of the mesh's own forge accounts cannot read, and why, per account: " +
|
||||||
|
"the tools' (mesh-tools), the build modules' fetch over ssh (nox-builder) and the machines' agents' (nox-agent). " +
|
||||||
|
"For each, the permission the forge gives it there, whether the repository is private or archived, and in words " +
|
||||||
|
"the reason (a private repository of a user with the account no collaborator, an owner not public); a count per " +
|
||||||
|
"account, and the accounts the forge does not hold yet. Empty when every account reads every repository. Read " +
|
||||||
|
"with the site administrator's login, which alone sees every repository.",
|
||||||
|
input: {
|
||||||
|
accounts: { type: "array", items: { type: "string" }, description: "the accounts to look at; empty or absent, every one of the mesh's" },
|
||||||
|
},
|
||||||
|
run: async (args) => readAll().unreadable(strings(args.accounts)),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "gitea_grant_read",
|
||||||
|
description: "Give each of the mesh's own forge accounts (mesh-tools, nox-builder, nox-agent) read on every " +
|
||||||
|
"repository it cannot read now, or on the repositories named, or for the accounts named: each becomes a " +
|
||||||
|
"collaborator that reads only, the least the forge offers. The module does this itself at its start and on the " +
|
||||||
|
"forge's event of a repository made; this is the same act by hand, such as when a build's fetch says its account " +
|
||||||
|
"is not authorized to read a repository. Answers how many were looked at, which were granted to which account " +
|
||||||
|
"(each read back from the forge), which the forge refused, in its own words, and the accounts it does not hold " +
|
||||||
|
"yet. Needs the site administrator's login, used for this grant alone.",
|
||||||
|
input: {
|
||||||
|
repositories: { type: "array", items: { type: "string" }, description: "owner/name of each repository; empty or absent, every repository" },
|
||||||
|
accounts: { type: "array", items: { type: "string" }, description: "the accounts to grant; empty or absent, every one of the mesh's" },
|
||||||
|
},
|
||||||
|
run: async (args) => readAll().grant(strings(args.repositories), strings(args.accounts)),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { test } from "node:test";
|
||||||
|
import { createServer } from "node:http";
|
||||||
|
|
||||||
|
// **The forge's 405 "Please try again later" on a merge is asked again briefly** (novox/hq issue 1275, 2026-10-26 13:40):
|
||||||
|
// mesh-delivery#73's merge was answered so a few seconds after its base moved, and counted as refused at once. The forge
|
||||||
|
// says it while it works out whether the pull request merges; the merge is asked again after a short wait, and every other
|
||||||
|
// refusal is answered at once. A pull request's own mergeable is carried, so a group can be read whole before it merges.
|
||||||
|
|
||||||
|
/** A forge whose merge of novox/hq#7 answers each of `answers` in turn (the last for ever), and whose pull request says
|
||||||
|
* `mergeable`. */
|
||||||
|
async function serve(answers: { status: number; body: string }[], mergeable?: boolean) {
|
||||||
|
const { GiteaClient } = await import("../client.ts");
|
||||||
|
let merges = 0;
|
||||||
|
const server = createServer((req, res) => {
|
||||||
|
const url = new URL(req.url ?? "", "http://x");
|
||||||
|
if (url.pathname === "/api/v1/repos/novox/hq/pulls/7/merge" && req.method === "POST") {
|
||||||
|
const a = answers[Math.min(merges++, answers.length - 1)];
|
||||||
|
res.statusCode = a.status;
|
||||||
|
return res.end(a.body);
|
||||||
|
}
|
||||||
|
if (url.pathname === "/api/v1/repos/novox/hq/pulls/7") {
|
||||||
|
return res.end(JSON.stringify({ number: 7, state: "open", merged: false, base: { ref: "main" },
|
||||||
|
head: { ref: "b", sha: "a".repeat(40) }, html_url: "h/7", ...(mergeable === undefined ? {} : { mergeable }) }));
|
||||||
|
}
|
||||||
|
res.statusCode = 404;
|
||||||
|
res.end("{}");
|
||||||
|
});
|
||||||
|
await new Promise<void>((r) => server.listen(0, r));
|
||||||
|
const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t");
|
||||||
|
GiteaClient.mergeRetryAfter = [1, 1, 1];
|
||||||
|
return { client, server, merges: () => merges };
|
||||||
|
}
|
||||||
|
|
||||||
|
const later = { status: 405, body: '{"message":"Please try again later"}' };
|
||||||
|
const ok = { status: 200, body: "" };
|
||||||
|
|
||||||
|
test("a merge answered 'try again later' while the forge works out its mergeability, then merged", async (t) => {
|
||||||
|
const cases: { name: string; answers: { status: number; body: string }[]; merged: boolean; asked: number }[] = [
|
||||||
|
{ name: "merged on the third ask", answers: [later, later, ok], merged: true, asked: 3 },
|
||||||
|
{ name: "still 'later' after the last wait: refused, with the forge's words", answers: [later], merged: false, asked: 4 },
|
||||||
|
{ name: "a refusal of another kind is answered at once", answers: [{ status: 409, body: '{"message":"head moved"}' }], merged: false, asked: 1 },
|
||||||
|
];
|
||||||
|
for (const c of cases) {
|
||||||
|
await t.test(c.name, async () => {
|
||||||
|
const { client, server, merges } = await serve(c.answers);
|
||||||
|
try {
|
||||||
|
const done = client.mergePullRequest("novox", "hq", 7, "merge", false, "a".repeat(40));
|
||||||
|
if (c.merged) await done;
|
||||||
|
else await assert.rejects(done, (err: Error) => /40[59]/.test(err.message));
|
||||||
|
assert.equal(merges(), c.asked);
|
||||||
|
} finally {
|
||||||
|
server.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a pull request carries the forge's mergeable, and none when the forge did not say", async () => {
|
||||||
|
for (const mergeable of [true, false, undefined]) {
|
||||||
|
const { client, server } = await serve([ok], mergeable);
|
||||||
|
try {
|
||||||
|
assert.equal((await client.getPullRequest("novox", "hq", 7)).mergeable, mergeable);
|
||||||
|
} finally {
|
||||||
|
server.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -16,7 +16,7 @@ import { join } from "node:path";
|
|||||||
|
|
||||||
import { MintedToken } from "../dist/token.js";
|
import { MintedToken } from "../dist/token.js";
|
||||||
import { GiteaClient } from "../dist/client.js";
|
import { GiteaClient } from "../dist/client.js";
|
||||||
import { TOOLS_ACCOUNT, TOOLS_TEAM_UNITS } from "../dist/tools-account.js";
|
import { TOOLS_ACCOUNT, TOOLS_TEAM_UNITS, BUILDER_ACCOUNT, AGENT_ACCOUNT, meshAccounts } from "../dist/tools-account.js";
|
||||||
import { repoCheckFacts } from "../dist/pulls.js";
|
import { repoCheckFacts } from "../dist/pulls.js";
|
||||||
|
|
||||||
const ADMIN = "mesh-admin";
|
const ADMIN = "mesh-admin";
|
||||||
@@ -26,6 +26,18 @@ interface Account { admin: boolean; password: string; active: boolean }
|
|||||||
interface Token { id: number; name: string; sha1: string; user: string; scopes: string[] }
|
interface Token { id: number; name: string; sha1: string; user: string; scopes: string[] }
|
||||||
interface Team { id: number; name: string; units: any; members: Set<string> }
|
interface Team { id: number; name: string; units: any; members: Set<string> }
|
||||||
|
|
||||||
|
interface ForgeRepo { owner: string; name: string; full: string; private: boolean; archived?: boolean; ownerVisibility: string; collaborators: Map<string, string> }
|
||||||
|
const forgeRepos: ForgeRepo[] = [];
|
||||||
|
const repoFaults = { refuse: new Set<string>(), permission: new Set<string>() };
|
||||||
|
function addRepo(full: string, isPrivate: boolean, ownerVisibility = "public"): ForgeRepo {
|
||||||
|
const [owner, name] = full.split("/");
|
||||||
|
const r = { owner, name, full, private: isPrivate, ownerVisibility, collaborators: new Map<string, string>() };
|
||||||
|
forgeRepos.push(r);
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
const repoJSON = (r: ForgeRepo) => ({ full_name: r.full, name: r.name, owner: { login: r.owner, visibility: r.ownerVisibility },
|
||||||
|
private: r.private, archived: r.archived === true });
|
||||||
|
|
||||||
function fakeForge() {
|
function fakeForge() {
|
||||||
const users = new Map<string, Account>([[ADMIN, { admin: true, password: PASSWORD, active: true }]]);
|
const users = new Map<string, Account>([[ADMIN, { admin: true, password: PASSWORD, active: true }]]);
|
||||||
const tokens: Token[] = [];
|
const tokens: Token[] = [];
|
||||||
@@ -61,6 +73,13 @@ function fakeForge() {
|
|||||||
return null;
|
return null;
|
||||||
};
|
};
|
||||||
const isAdmin = (u: string | null) => u !== null && users.get(u)?.admin === true;
|
const isAdmin = (u: string | null) => u !== null && users.get(u)?.admin === true;
|
||||||
|
const permissionOf = (u: string | null, r: ForgeRepo): string => {
|
||||||
|
if (isAdmin(u)) return "admin";
|
||||||
|
if (u === r.owner) return "owner";
|
||||||
|
if (r.owner === "novox" && teams.some((t) => t.members.has(u ?? "") && t.units.units_map?.["repo.code"] === "write")) return "write";
|
||||||
|
if (u && r.collaborators.has(u)) return r.collaborators.get(u)!;
|
||||||
|
return !r.private && r.ownerVisibility === "public" ? "read" : "none";
|
||||||
|
};
|
||||||
|
|
||||||
const server = createServer(async (req, res) => {
|
const server = createServer(async (req, res) => {
|
||||||
const url = new URL(req.url ?? "/", "http://fake");
|
const url = new URL(req.url ?? "/", "http://fake");
|
||||||
@@ -176,7 +195,35 @@ function fakeForge() {
|
|||||||
if (!writes) return json(res, 403, { message: "write on code required" });
|
if (!writes) return json(res, 403, { message: "write on code required" });
|
||||||
return json(res, 201, { context: (await read(req)).context, creator: { login: me } });
|
return json(res, 201, { context: (await read(req)).context, creator: { login: me } });
|
||||||
}
|
}
|
||||||
if (req.method === "GET" && p === "/repos/search") return json(res, 200, { data: [] });
|
// Every repository of every owner, and who reads it (readall.ts): the site administrator reads all; the tools' account
|
||||||
|
// a repository of novox through its team, a public one of a public owner, and one it is a collaborator of.
|
||||||
|
if (req.method === "GET" && p === "/repos/search") {
|
||||||
|
const page = Number(url.searchParams.get("page") ?? "1");
|
||||||
|
const seen = forgeRepos.filter((r) => permissionOf(me, r) !== "none");
|
||||||
|
return json(res, 200, { data: page === 1 ? seen.map(repoJSON) : [] });
|
||||||
|
}
|
||||||
|
m = p.match(/^\/repos\/([^/]+)\/([^/]+)(?:\/collaborators\/([^/]+)(\/permission)?)?$/);
|
||||||
|
if (m) {
|
||||||
|
const repo = forgeRepos.find((r) => r.owner === decodeURIComponent(m![1]) && r.name === decodeURIComponent(m![2]));
|
||||||
|
if (!repo || permissionOf(me, repo) === "none") return json(res, 404, { message: "repository does not exist" });
|
||||||
|
if (req.method === "GET" && !m[3]) {
|
||||||
|
const mine = permissionOf(me, repo);
|
||||||
|
return json(res, 200, { ...repoJSON(repo), permissions: { pull: true, push: ["write", "admin", "owner"].includes(mine), admin: ["admin", "owner"].includes(mine) } });
|
||||||
|
}
|
||||||
|
const user = decodeURIComponent(m[3] ?? "");
|
||||||
|
if (req.method === "GET" && m[4]) {
|
||||||
|
if (repoFaults.permission.has(repo.full)) return json(res, 500, { message: "the forge is not well" });
|
||||||
|
if (!isAdmin(me) && me !== user) return json(res, 403, { message: "only the user or an administrator" });
|
||||||
|
return json(res, 200, { permission: permissionOf(user, repo) });
|
||||||
|
}
|
||||||
|
if (req.method === "PUT" && !m[4]) {
|
||||||
|
// gitea's reqAdmin: the repository's administration, which on a repository of another user the site administrator alone holds.
|
||||||
|
if (!isAdmin(me) && me !== repo.owner) return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" });
|
||||||
|
if (repoFaults.refuse.has(repo.full)) return json(res, 422, { message: "the forge would not have it" });
|
||||||
|
repo.collaborators.set(user, (await read(req))?.permission ?? "write");
|
||||||
|
return json(res, 204, null);
|
||||||
|
}
|
||||||
|
}
|
||||||
return json(res, 404, { message: "no such route in the fake" });
|
return json(res, 404, { message: "no such route in the fake" });
|
||||||
});
|
});
|
||||||
return new Promise<{ url: string; users: typeof users; tokens: typeof tokens; teams: typeof teams; calls: string[]; mainRules: typeof mainRules;
|
return new Promise<{ url: string; users: typeof users; tokens: typeof tokens; teams: typeof teams; calls: string[]; mainRules: typeof mainRules;
|
||||||
@@ -402,3 +449,197 @@ test("a mint with keep-main-rules false (the default) writes no branch protectio
|
|||||||
const writes = forge.calls.slice(from).filter((c: string) => /branch_protections/.test(c) && !/^GET /.test(c) && !/^(GET|HEAD) /.test(c));
|
const writes = forge.calls.slice(from).filter((c: string) => /branch_protections/.test(c) && !/^GET /.test(c) && !/^(GET|HEAD) /.test(c));
|
||||||
assert.deepEqual(writes, []);
|
assert.deepEqual(writes, []);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// The mesh's accounts read every repository (readall.ts; the operator, 2026-10-26): jschoubben/car-hunter and
|
||||||
|
// jschoubben/snake, private repositories of a user, answered the tools 404 while that user's public ones were seen; and
|
||||||
|
// once the tools' account read them, the build modules' fetch as nox-builder still could not.
|
||||||
|
|
||||||
|
test("the start's pass gives each of the mesh's accounts read, as a collaborator that reads only, on each repository it could not read, and on no other; an account the forge does not hold is said, not refused", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
await fresh(env).listRepos(); // the account and its team exist, as after any mint
|
||||||
|
forge.users.set(BUILDER_ACCOUNT, { admin: false, password: "unused", active: true }); // made by the forge-account provider
|
||||||
|
forge.users.delete(AGENT_ACCOUNT); // no machine's key asked for it yet
|
||||||
|
forgeRepos.splice(0);
|
||||||
|
addRepo("novox/hq", true);
|
||||||
|
addRepo("jschoubben/txt-game", false);
|
||||||
|
addRepo("jschoubben/car-hunter", true);
|
||||||
|
addRepo("jschoubben/snake", true);
|
||||||
|
addRepo("shy/open", false, "limited");
|
||||||
|
addRepo("elsewhere/secret", true);
|
||||||
|
const { GiteaAdmin } = await import("../dist/client.js");
|
||||||
|
const { ReadAll, readAllTools } = await import("../dist/readall.js");
|
||||||
|
const lines: string[] = [];
|
||||||
|
const readAll = new ReadAll(GiteaAdmin.fromEnv(env), meshAccounts(), (l: string) => lines.push(l));
|
||||||
|
assert.deepEqual(readAll.accounts, [TOOLS_ACCOUNT, BUILDER_ACCOUNT, AGENT_ACCOUNT]);
|
||||||
|
const [unreadableTool, grantTool] = readAllTools(() => readAll);
|
||||||
|
|
||||||
|
// What the tool says before, per account: the four the tools' account cannot read, each with why; the builders'
|
||||||
|
// account, in no write team here, also not novox/hq; nox-agent not on the forge.
|
||||||
|
const before = await unreadableTool.run({}) as any;
|
||||||
|
assert.equal(before.checked, 6);
|
||||||
|
const of = (account: string) => before.unreadable.filter((u: any) => u.account === account).map((u: any) => u.repository).sort();
|
||||||
|
assert.deepEqual(of(TOOLS_ACCOUNT), ["elsewhere/secret", "jschoubben/car-hunter", "jschoubben/snake", "shy/open"]);
|
||||||
|
assert.deepEqual(of(BUILDER_ACCOUNT), ["elsewhere/secret", "jschoubben/car-hunter", "jschoubben/snake", "novox/hq", "shy/open"]);
|
||||||
|
assert.deepEqual(before.accounts, [{ account: TOOLS_ACCOUNT, unreadable: 4 }, { account: BUILDER_ACCOUNT, unreadable: 5 }]);
|
||||||
|
assert.deepEqual(before.absent, [AGENT_ACCOUNT]);
|
||||||
|
assert.match(before.unreadable.find((u: any) => u.account === BUILDER_ACCOUNT && u.repository === "jschoubben/car-hunter").why,
|
||||||
|
/a private repository of jschoubben, on which nox-builder is no collaborator/);
|
||||||
|
const builderOnly = await unreadableTool.run({ accounts: [BUILDER_ACCOUNT] }) as any;
|
||||||
|
assert.deepEqual(builderOnly.accounts, [{ account: BUILDER_ACCOUNT, unreadable: 5 }]);
|
||||||
|
await assert.rejects(unreadableTool.run({ accounts: ["someone"] }), /not one of the mesh's accounts/);
|
||||||
|
assert.match(before.unreadable.find((u: any) => u.repository === "jschoubben/car-hunter").why,
|
||||||
|
/a private repository of jschoubben, on which mesh-tools is no collaborator/);
|
||||||
|
assert.match(before.unreadable.find((u: any) => u.repository === "shy/open").why, /visibility is limited/);
|
||||||
|
assert.deepEqual((await fresh(env).listAllRepos()).map((r) => r.full_name).sort(), ["jschoubben/txt-game", "novox/hq"]);
|
||||||
|
|
||||||
|
// The account's own token cannot grant itself: the forge asks the repository's administration (the reason for the admin login).
|
||||||
|
const own = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim();
|
||||||
|
const self = await fetch(`${forge.url}/api/v1/repos/jschoubben/car-hunter/collaborators/${TOOLS_ACCOUNT}`,
|
||||||
|
{ method: "PUT", headers: { Authorization: `token ${own}`, "Content-Type": "application/json" }, body: JSON.stringify({ permission: "read" }) });
|
||||||
|
assert.equal(self.status, 404, "a repository the account cannot read is not even there for it");
|
||||||
|
|
||||||
|
const from = forge.calls.length;
|
||||||
|
const r = await readAll.atStart();
|
||||||
|
const grantedTo = (account: string) => r.granted.filter((x: any) => x.account === account).map((x: any) => x.repository).sort();
|
||||||
|
assert.deepEqual(grantedTo(TOOLS_ACCOUNT), ["elsewhere/secret", "jschoubben/car-hunter", "jschoubben/snake", "shy/open"]);
|
||||||
|
assert.deepEqual(grantedTo(BUILDER_ACCOUNT), ["elsewhere/secret", "jschoubben/car-hunter", "jschoubben/snake", "novox/hq", "shy/open"]);
|
||||||
|
assert.deepEqual(r.refused, []);
|
||||||
|
assert.deepEqual(r.absent, [AGENT_ACCOUNT]);
|
||||||
|
const puts = forge.calls.slice(from).filter((c) => c.startsWith("PUT "));
|
||||||
|
assert.equal(puts.length, 9, "the repositories each read already were left alone, and the absent account was asked nothing");
|
||||||
|
assert.ok(!puts.some((c) => c.includes(AGENT_ACCOUNT)));
|
||||||
|
assert.ok(puts.every((c) => c.endsWith(` as ${ADMIN}`)), "only the administrator's login grants");
|
||||||
|
for (const repo of forgeRepos.filter((x) => x.collaborators.size)) {
|
||||||
|
for (const [account, permission] of repo.collaborators) assert.equal(permission, "read", `${account} on ${repo.full} reads only`);
|
||||||
|
}
|
||||||
|
assert.equal(forgeRepos.find((x) => x.full === "jschoubben/car-hunter")!.collaborators.get(BUILDER_ACCOUNT), "read");
|
||||||
|
assert.match(lines.join("\n"), /6 looked at; mesh-tools 4 granted read now .*; nox-builder 5 granted read now .*; not on the forge yet, so not granted: nox-agent/);
|
||||||
|
|
||||||
|
// The account made later is granted by hand (or at the next pass), its rows like any other.
|
||||||
|
forge.users.set(AGENT_ACCOUNT, { admin: false, password: "unused", active: true });
|
||||||
|
const agent = await grantTool.run({ accounts: [AGENT_ACCOUNT], repositories: ["jschoubben/car-hunter"] }) as any;
|
||||||
|
assert.deepEqual(agent.granted, [{ account: AGENT_ACCOUNT, repository: "jschoubben/car-hunter" }]);
|
||||||
|
assert.deepEqual(agent.absent, []);
|
||||||
|
forge.users.delete(AGENT_ACCOUNT);
|
||||||
|
|
||||||
|
// After: the account's own token sees every repository, and the tool names none.
|
||||||
|
assert.equal((await fresh(env).listAllRepos()).length, 6);
|
||||||
|
assert.deepEqual(((await unreadableTool.run({})) as any).unreadable, []);
|
||||||
|
// Granting again asks nothing of the forge but reads.
|
||||||
|
const again = forge.calls.length;
|
||||||
|
assert.deepEqual(((await grantTool.run({})) as any).granted, []);
|
||||||
|
assert.ok(forge.calls.slice(again).every((c) => c.startsWith("GET ")));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("on the forge's events: a repository made is granted read at once, a known readable one asks nothing, a refusal is said, not thrown", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
await fresh(env).listRepos();
|
||||||
|
forgeRepos.splice(0);
|
||||||
|
addRepo("jschoubben/txt-game", false);
|
||||||
|
const { GiteaAdmin } = await import("../dist/client.js");
|
||||||
|
const { ReadAll, readAllTools } = await import("../dist/readall.js");
|
||||||
|
const lines: string[] = [];
|
||||||
|
forge.users.set(BUILDER_ACCOUNT, { admin: false, password: "unused", active: true });
|
||||||
|
const readAll = new ReadAll(GiteaAdmin.fromEnv(env), [TOOLS_ACCOUNT, BUILDER_ACCOUNT], (l: string) => lines.push(l));
|
||||||
|
await readAll.atStart();
|
||||||
|
|
||||||
|
const cases: { name: string; event: string; repo: string; make?: boolean; refuse?: boolean; unanswered?: boolean; asks: boolean; reads: string }[] = [
|
||||||
|
{ name: "a repository made", event: "forge.repository.created", repo: "jschoubben/new-private", make: true, asks: true, reads: "read" },
|
||||||
|
{ name: "a fork made", event: "forge.fork", repo: "someone/fork", make: true, asks: true, reads: "read" },
|
||||||
|
{ name: "a push to a repository seen readable", event: "forge.push", repo: "jschoubben/txt-game", asks: false, reads: "read" },
|
||||||
|
{ name: "a push to a private repository never seen", event: "forge.push", repo: "jschoubben/quiet", make: true, asks: true, reads: "read" },
|
||||||
|
{ name: "a repository the forge refuses the grant on", event: "forge.repository.created", repo: "jschoubben/stuck", make: true, refuse: true, asks: true, reads: "none" },
|
||||||
|
{ name: "a repository whose permission the forge will not answer", event: "forge.repository.created", repo: "jschoubben/mute", make: true, unanswered: true, asks: true, reads: "none" },
|
||||||
|
];
|
||||||
|
for (const c of cases) {
|
||||||
|
if (c.make) addRepo(c.repo, true);
|
||||||
|
if (c.refuse) repoFaults.refuse.add(c.repo);
|
||||||
|
if (c.unanswered) repoFaults.permission.add(c.repo);
|
||||||
|
const from = forge.calls.length;
|
||||||
|
const [owner, repo] = c.repo.split("/");
|
||||||
|
await readAll.onEvent({ name: c.event, body: { owner, repo } });
|
||||||
|
assert.equal(forge.calls.length > from, c.asks, `${c.name}: asks the forge`);
|
||||||
|
for (const account of [TOOLS_ACCOUNT, BUILDER_ACCOUNT]) {
|
||||||
|
assert.equal(forgeRepos.find((x) => x.full === c.repo)!.collaborators.get(account) ?? (c.make ? "none" : "read"), c.reads, `${c.name}: ${account}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.match(lines.join("\n"), /mesh-tools could not be given read on jschoubben\/stuck — .*422/);
|
||||||
|
assert.match(lines.join("\n"), /nox-builder was given read on someone\/fork, on the forge's forge.fork/);
|
||||||
|
const named = await (readAllTools(() => readAll)[1].run({ repositories: ["jschoubben/stuck"] })) as any;
|
||||||
|
assert.deepEqual(named.refused.map((x: any) => `${x.account} ${x.repository}`), [`${TOOLS_ACCOUNT} jschoubben/stuck`, `${BUILDER_ACCOUNT} jschoubben/stuck`]);
|
||||||
|
await assert.rejects(readAllTools(() => readAll)[1].run({ repositories: ["no-slash"] }), /not a repository/);
|
||||||
|
assert.match(lines.join("\n"), /could not be given read on jschoubben\/mute — its permission could not be read — .*500/);
|
||||||
|
// A whole pass goes past the repository it cannot judge: every other one is still granted, and the listing names it.
|
||||||
|
addRepo("jschoubben/after-mute", true);
|
||||||
|
const pass = await readAll.grant();
|
||||||
|
assert.ok(pass.granted.some((x: any) => x.repository === "jschoubben/after-mute" && x.account === BUILDER_ACCOUNT), "the pass did not stop at the repository it could not judge");
|
||||||
|
assert.ok(pass.refused.some((x: any) => x.repository === "jschoubben/mute" && /permission could not be read/.test(x.why)));
|
||||||
|
const listed = await readAllTools(() => readAll)[0].run({}) as any;
|
||||||
|
assert.equal(listed.unreadable.find((u: any) => u.repository === "jschoubben/mute").permission, "unknown");
|
||||||
|
repoFaults.refuse.clear();
|
||||||
|
repoFaults.permission.clear();
|
||||||
|
});
|
||||||
|
|
||||||
|
// The merging account writes every repository that builds a registered module, and no other (writes.ts; the operator,
|
||||||
|
// 2026-10-26; issue 1945): car-hunter #2 was approved and its commit status refused 403, mesh-tools reading only there.
|
||||||
|
|
||||||
|
test("the merging account is given write on each repository the indexer's modules name, read back and said; the operator's other repositories stay read-only; the tools say both", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
const client = fresh(env);
|
||||||
|
await client.listRepos(); // the account and its team exist, as after any mint
|
||||||
|
forgeRepos.splice(0);
|
||||||
|
addRepo("novox/hq", true);
|
||||||
|
for (const full of ["jschoubben/car-hunter", "jschoubben/snake", "jschoubben/dotfiles"]) addRepo(full, true).collaborators.set(TOOLS_ACCOUNT, "read");
|
||||||
|
const { GiteaAdmin } = await import("../dist/client.js");
|
||||||
|
const { DeliveredWrites, writeTools, WRITABLE_EVENT } = await import("../dist/writes.js");
|
||||||
|
const lines: string[] = [];
|
||||||
|
const said: { type: string; body: any }[] = [];
|
||||||
|
const writes = new DeliveredWrites(GiteaAdmin.fromEnv(env), TOOLS_ACCOUNT, "novox", (l: string) => lines.push(l),
|
||||||
|
async (type: string, body: unknown) => { said.push({ type, body }); });
|
||||||
|
const [unwritableTool, grantTool] = writeTools(() => writes);
|
||||||
|
await assert.rejects(grantTool.run({}), /not known yet/, "nothing is granted before the indexer's modules were read");
|
||||||
|
assert.equal((await client.getRepo("jschoubben", "car-hunter")).writes, false, "the statuses' answer says the account cannot write");
|
||||||
|
|
||||||
|
// 1945/AC1: the watch hands the indexer's modules; a user's repository a module builds from is made writable, once.
|
||||||
|
await writes.take({ key: "car-hunter", op: "put", value: { v: 1, repository: "jschoubben/car-hunter", commit: "c" }, current: true });
|
||||||
|
await writes.take({ key: "gitea", op: "put", value: { v: 1, repository: "novox/hq", commit: "c" }, current: true });
|
||||||
|
await writes.take({ key: "broken", op: "put", value: { v: 1, repository: "no-slash" }, current: true });
|
||||||
|
const from = forge.calls.length;
|
||||||
|
await writes.take({ key: "car-hunter-2", op: "put", value: { v: 1, repository: "jschoubben/car-hunter" } });
|
||||||
|
assert.equal(forge.calls.length, from, "a repository already written asks nothing more");
|
||||||
|
const repo = (full: string) => forgeRepos.find((x) => x.full === full)!;
|
||||||
|
assert.equal(repo("jschoubben/car-hunter").collaborators.get(TOOLS_ACCOUNT), "write");
|
||||||
|
assert.equal(repo("novox/hq").collaborators.size, 0, "the organisation's repository is written through its team, no collaborator");
|
||||||
|
assert.equal((await client.getRepo("jschoubben", "car-hunter")).writes, true);
|
||||||
|
assert.deepEqual(said, [{ type: WRITABLE_EVENT, body: { owner: "jschoubben", repo: "car-hunter", account: TOOLS_ACCOUNT } }]);
|
||||||
|
assert.match(lines.join("\n"), /mesh-tools was given write on jschoubben\/car-hunter, which builds the registered module car-hunter/);
|
||||||
|
assert.match(lines.join("\n"), /broken names no repository/);
|
||||||
|
|
||||||
|
// 1945/AC2: a delivered repository the forge refuses is said, in its words, by the listing and the grant; the operator's
|
||||||
|
// repository no module builds from is left read-only, named with why, and a grant naming it asks the forge nothing.
|
||||||
|
repoFaults.refuse.add("jschoubben/snake");
|
||||||
|
await writes.take({ key: "snake", op: "put", value: { v: 1, repository: "jschoubben/snake" } });
|
||||||
|
assert.match(lines.join("\n"), /could not be given write on jschoubben\/snake, which builds snake — .*422/);
|
||||||
|
const listed = await unwritableTool.run({}) as any;
|
||||||
|
assert.deepEqual(listed.unwritable.map((u: any) => [u.repository, u.permission]), [["jschoubben/snake", "read"]]);
|
||||||
|
assert.match(listed.unwritable[0].why, /builds snake.*gitea_grant_write/);
|
||||||
|
const all = await grantTool.run({}) as any;
|
||||||
|
assert.deepEqual(all.delivered, ["jschoubben/car-hunter", "jschoubben/snake", "novox/hq"]);
|
||||||
|
assert.deepEqual(all.already, ["jschoubben/car-hunter", "novox/hq"]);
|
||||||
|
assert.deepEqual(all.granted, []);
|
||||||
|
assert.match(all.refused.find((x: any) => x.repository === "jschoubben/snake").why, /422.*would not have it/);
|
||||||
|
assert.deepEqual(all.left_read_only.map((x: any) => x.repository), ["jschoubben/dotfiles"]);
|
||||||
|
assert.match(all.left_read_only[0].why, /builds no registered module/);
|
||||||
|
const puts = forge.calls.length;
|
||||||
|
const named = await grantTool.run({ repositories: ["jschoubben/dotfiles"] }) as any;
|
||||||
|
assert.deepEqual([named.granted, named.left_read_only.map((x: any) => x.repository)], [[], ["jschoubben/dotfiles"]]);
|
||||||
|
assert.ok(!forge.calls.slice(puts).some((c) => c.startsWith("PUT ")), "no write is asked on a repository no module builds from");
|
||||||
|
assert.equal(repo("jschoubben/dotfiles").collaborators.get(TOOLS_ACCOUNT), "read");
|
||||||
|
repoFaults.refuse.clear();
|
||||||
|
const after = await grantTool.run({ repositories: ["jschoubben/snake"] }) as any;
|
||||||
|
assert.deepEqual(after.granted, ["jschoubben/snake"]);
|
||||||
|
assert.deepEqual(((await unwritableTool.run({})) as any).unwritable, []);
|
||||||
|
// A module gone from the indexer leaves its repository out of the delivered list.
|
||||||
|
await writes.take({ key: "snake", op: "delete" });
|
||||||
|
assert.deepEqual(writes.delivered(), ["jschoubben/car-hunter", "novox/hq"]);
|
||||||
|
});
|
||||||
|
|||||||
@@ -34,6 +34,9 @@ test("both statuses say the head and the trunk commit the verdict judged, first,
|
|||||||
const [old] = statusesFor({ ...c, trunk: undefined, gate: { verdict: "pass", summary: "every machine composes" } });
|
const [old] = statusesFor({ ...c, trunk: undefined, gate: { verdict: "pass", summary: "every machine composes" } });
|
||||||
assert.equal(old.description, "pass: every machine composes");
|
assert.equal(old.description, "pass: every machine composes");
|
||||||
assert.equal(trunkNamed(old.description), undefined);
|
assert.equal(trunkNamed(old.description), undefined);
|
||||||
|
// ADR 0548 §8: a carried verdict's line names the head on the new trunk first, so the merge tool merges it on that trunk.
|
||||||
|
assert.equal(trunkNamed(`pass ${HEAD.slice(0, 12)} on ${MOVED.slice(0, 12)} | carried from ${TRUNK.slice(0, 12)}, mesh-catalog 0123456789ab | unit 0 run, 4 kept`),
|
||||||
|
MOVED.slice(0, 12));
|
||||||
});
|
});
|
||||||
|
|
||||||
test("a trunk's move puts an open head's merge check out of date, naming both commits, unless a check is on its way", () => {
|
test("a trunk's move puts an open head's merge check out of date, naming both commits, unless a check is on its way", () => {
|
||||||
|
|||||||
@@ -22,6 +22,15 @@
|
|||||||
import { keepMainRules } from "./mainrules.js";
|
import { keepMainRules } from "./mainrules.js";
|
||||||
|
|
||||||
export const TOOLS_ACCOUNT = "mesh-tools";
|
export const TOOLS_ACCOUNT = "mesh-tools";
|
||||||
|
/** The build modules' account, whose keys fetch over ssh (internal/forgeaccount BuilderUser, ADR 0466 §3). */
|
||||||
|
export const BUILDER_ACCOUNT = "nox-builder";
|
||||||
|
/** The machines' agents' account, whose keys ssh-client reports (internal/forgeaccount AgentUser, ADR 0410 §1). */
|
||||||
|
export const AGENT_ACCOUNT = "nox-agent";
|
||||||
|
/** Every forge account the mesh's own parts use: each reads every repository on the forge (readall.ts; the operator,
|
||||||
|
* 2026-10-26). internal/forgeaccount's test holds the two Go names to its own constants. */
|
||||||
|
export function meshAccounts(tools: string = TOOLS_ACCOUNT): string[] {
|
||||||
|
return [tools, BUILDER_ACCOUNT, AGENT_ACCOUNT];
|
||||||
|
}
|
||||||
/** The organisation whose repositories are the mesh's, and the team through which the tools' account writes there. */
|
/** The organisation whose repositories are the mesh's, and the team through which the tools' account writes there. */
|
||||||
export const TOOLS_ORG = "novox";
|
export const TOOLS_ORG = "novox";
|
||||||
export const TOOLS_TEAM = "mesh-tools";
|
export const TOOLS_TEAM = "mesh-tools";
|
||||||
|
|||||||
@@ -492,7 +492,7 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "gitea_commit_statuses",
|
name: "gitea_commit_statuses",
|
||||||
description: "Read a commit's statuses — or a pull request's head's — as the forge keeps them: the newest of each context (mesh/check, mesh/delivery, …) with its state, description, link, when it was set and by whom; and whether the branch it merges into would let it merge, every status that branch's protection requires being a success. Reads only.",
|
description: "Read a commit's statuses — or a pull request's head's — as the forge keeps them: the newest of each context (mesh/check, mesh/delivery, …) with its state, description, link, when it was set and by whom; and whether the branch it merges into would let it merge, every status that branch's protection requires being a success; for a pull request, also the forge's own mergeable (false while it is in conflict with its base or the forge is still working that out); and merge.writes, whether the mesh's merging account may write the repository at all (false: its statuses and its merge are refused until gitea_grant_write). Reads only.",
|
||||||
input: {
|
input: {
|
||||||
owner: { type: "string", description: "the repository owner" },
|
owner: { type: "string", description: "the repository owner" },
|
||||||
repo: { type: "string", description: "the repository name" },
|
repo: { type: "string", description: "the repository name" },
|
||||||
@@ -508,11 +508,12 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
|||||||
if (!owner || !repo) throw new Error("name the repository: owner and repo");
|
if (!owner || !repo) throw new Error("name the repository: owner and repo");
|
||||||
if (!Number.isInteger(number) || number < 0) throw new Error(`${args.number} is not a pull request's number`);
|
if (!Number.isInteger(number) || number < 0) throw new Error(`${args.number} is not a pull request's number`);
|
||||||
let pull: { number: number; title: string; state: string; merged: boolean; base?: string; head?: string;
|
let pull: { number: number; title: string; state: string; merged: boolean; base?: string; head?: string;
|
||||||
head_sha?: string; merge_commit_sha?: string; merged_at?: string; html_url: string } | undefined;
|
head_sha?: string; merge_commit_sha?: string; merged_at?: string; html_url: string; mergeable?: boolean } | undefined;
|
||||||
if (number > 0) {
|
if (number > 0) {
|
||||||
const p = await gitea.getPullRequest(owner, repo, number);
|
const p = await gitea.getPullRequest(owner, repo, number);
|
||||||
pull = { number: p.number, title: p.title, state: p.state, merged: p.merged, base: p.base, head: p.head,
|
pull = { number: p.number, title: p.title, state: p.state, merged: p.merged, base: p.base, head: p.head,
|
||||||
head_sha: p.head_sha, merge_commit_sha: p.merge_commit_sha, merged_at: p.merged_at, html_url: p.html_url };
|
head_sha: p.head_sha, merge_commit_sha: p.merge_commit_sha, merged_at: p.merged_at, html_url: p.html_url,
|
||||||
|
...(p.mergeable === undefined ? {} : { mergeable: p.mergeable }) };
|
||||||
if (!ref) ref = p.head_sha ?? "";
|
if (!ref) ref = p.head_sha ?? "";
|
||||||
if (!base) base = p.base ?? "";
|
if (!base) base = p.base ?? "";
|
||||||
if (!ref) throw new Error(`${owner}/${repo}#${number} names no head commit`);
|
if (!ref) throw new Error(`${owner}/${repo}#${number} names no head commit`);
|
||||||
@@ -527,8 +528,18 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
unreadable = err instanceof Error ? err.message : String(err);
|
unreadable = err instanceof Error ? err.message : String(err);
|
||||||
}
|
}
|
||||||
|
// Whether this account may write the repository at all (issue 1945): a commit status and a merge need it, and a
|
||||||
|
// repository of a user it only reads refuses both 403. Not said when the forge would not answer.
|
||||||
|
let writes: boolean | undefined;
|
||||||
|
try {
|
||||||
|
writes = (await gitea.getRepo(owner, repo)).writes;
|
||||||
|
} catch {
|
||||||
|
writes = undefined;
|
||||||
|
}
|
||||||
|
const merge = mergeableUnder(base, rule, statuses, unreadable);
|
||||||
return { commit: combined.sha || ref, pull, combined: combined.state, statuses,
|
return { commit: combined.sha || ref, pull, combined: combined.state, statuses,
|
||||||
merge: mergeableUnder(base, rule, statuses, unreadable) };
|
merge: writes === undefined ? merge : { ...merge, writes,
|
||||||
|
...(writes ? {} : { cannot_write: `the forge account cannot write ${owner}/${repo}: gitea_grant_write` }) } };
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,246 @@
|
|||||||
|
// The account that sets commit statuses and merges writes every repository that builds a registered module, and no other
|
||||||
|
// (the operator, 2026-10-26: "yes the mesh should get write access", on the repositories the mesh delivers; novox/hq issue
|
||||||
|
// 1945). car-hunter #2 (a repository of a user, not of the organisation novox) was approved, and mesh-delivery's first
|
||||||
|
// act, its commit status, was refused 403: the tools' account was only a collaborator that reads there (readall.ts).
|
||||||
|
//
|
||||||
|
// **Which account.** The tools' account, `mesh-tools`: the gitea module's own tools set every commit status and merge
|
||||||
|
// (gitea_commit_status, gitea_merge_pull_request) with its token. The build modules' `nox-builder` and the machines'
|
||||||
|
// `nox-agent` only fetch, and keep read (readall.ts).
|
||||||
|
//
|
||||||
|
// **Which repositories: the indexer's word, read over the bus.** mesh-delivery-indexer keeps every registered module in its
|
||||||
|
// state `modules` (mesh-sdk go/delivery ModulesBucket), each value naming the repository it builds from. This module reads
|
||||||
|
// that state (its manifest's `reads`) with a watch: the watch hands the latest value of every key when it starts, and then
|
||||||
|
// every change, so a repository is granted at the module's start and when a module of it is first registered. Nothing
|
||||||
|
// polls. A repository of the organisation needs nothing: the organisation's team writes it (tools-account.ts). Every other
|
||||||
|
// repository of a user (the operator's dotfiles, say) is left as it is: read only.
|
||||||
|
//
|
||||||
|
// **Every grant is read back** from the forge (its permission for the account), and said on the bus as
|
||||||
|
// `repo.writable` {owner, repo, account}: mesh-delivery-gate, which does not ask the operator's Approve of a pull request
|
||||||
|
// whose merge the forge would refuse, reads that repository again on it.
|
||||||
|
//
|
||||||
|
// Not taken away: a repository whose last module goes from the indexer keeps its collaborator until a hand removes it;
|
||||||
|
// gitea_unwritable_repos and gitea_grant_write say the delivered list as it is now.
|
||||||
|
|
||||||
|
import type { ForgeRepository } from "./client.js";
|
||||||
|
|
||||||
|
/** The indexer's state this module reads: one key per registered module, its value naming the repository. */
|
||||||
|
export const INDEXER_MODULES = "mesh-delivery-indexer.modules";
|
||||||
|
/** The event said once an account writes a repository it could not write before. */
|
||||||
|
export const WRITABLE_EVENT = "repo.writable";
|
||||||
|
|
||||||
|
/** What the module asks of the forge for this, with the site administrator's login (GiteaAdmin, client.ts). */
|
||||||
|
export interface WriteForge {
|
||||||
|
everyRepository(): Promise<ForgeRepository[]>;
|
||||||
|
repository(owner: string, repo: string): Promise<ForgeRepository | null>;
|
||||||
|
repositoryPermission(owner: string, repo: string, user: string): Promise<string>;
|
||||||
|
addWriteCollaborator(owner: string, repo: string, user: string): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One change of the indexer's modules state, as the SDK's state watch hands it. */
|
||||||
|
export interface ModulesChange {
|
||||||
|
key: string;
|
||||||
|
op: "put" | "delete";
|
||||||
|
value?: unknown;
|
||||||
|
revision?: number;
|
||||||
|
current?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Written {
|
||||||
|
account: string;
|
||||||
|
/** The repositories that build a registered module, as the indexer says now. */
|
||||||
|
delivered: string[];
|
||||||
|
/** Made writable now, each read back from the forge. */
|
||||||
|
granted: string[];
|
||||||
|
/** Written already (the organisation's team, or a collaborator that writes). */
|
||||||
|
already: string[];
|
||||||
|
/** The forge refused, in its own words; or the repository is the organisation's, whose team should write it. */
|
||||||
|
refused: { repository: string; why: string }[];
|
||||||
|
/** Repositories the account cannot write that are left so, and why: they build no registered module. */
|
||||||
|
left_read_only: { repository: string; permission: string; why: string }[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const WRITES = new Set(["write", "admin", "owner"]);
|
||||||
|
const said = (err: unknown): string => (err instanceof Error ? err.message : String(err));
|
||||||
|
|
||||||
|
function split(full: string): [string, string] {
|
||||||
|
const [owner, name, ...rest] = String(full ?? "").split("/");
|
||||||
|
if (!owner || !name || rest.length) throw new Error(`"${full}" is not a repository: say owner/name`);
|
||||||
|
return [owner, name];
|
||||||
|
}
|
||||||
|
|
||||||
|
export class DeliveredWrites {
|
||||||
|
/** Module to the repository it builds from, as the indexer's state says. */
|
||||||
|
readonly modules = new Map<string, string>();
|
||||||
|
/** Whether the watch has handed the indexer's state at all: until then, which repositories are delivered is not known. */
|
||||||
|
heard = false;
|
||||||
|
/** Repositories seen written by the account in this process: a module registered again asks nothing more. */
|
||||||
|
readonly writable = new Set<string>();
|
||||||
|
|
||||||
|
constructor(private readonly forge: WriteForge, readonly account: string, readonly org: string,
|
||||||
|
private readonly log: (line: string) => void = (l) => console.log(`[gitea] ${l}`),
|
||||||
|
private readonly emit: (type: string, body: unknown) => Promise<void> = async () => {}) {}
|
||||||
|
|
||||||
|
/** The repositories that build a registered module, sorted. */
|
||||||
|
delivered(): string[] {
|
||||||
|
return [...new Set(this.modules.values())].sort();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One change of the indexer's modules: a repository newly delivered is made writable now. Never throws. */
|
||||||
|
async take(change: ModulesChange): Promise<void> {
|
||||||
|
this.heard = true;
|
||||||
|
if (change.op === "delete") {
|
||||||
|
this.modules.delete(change.key);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const repository = String((change.value as { repository?: unknown } | undefined)?.repository ?? "").trim();
|
||||||
|
try {
|
||||||
|
split(repository);
|
||||||
|
} catch {
|
||||||
|
this.log(`${INDEXER_MODULES} ${change.key} names no repository (${JSON.stringify(repository)}): no write is granted for it`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.modules.set(change.key, repository);
|
||||||
|
if (this.writable.has(repository)) return;
|
||||||
|
try {
|
||||||
|
const r = await this.grant([repository]);
|
||||||
|
for (const g of r.granted) this.log(`${this.account} was given write on ${g}, which builds the registered module ${change.key}`);
|
||||||
|
for (const x of r.refused) this.log(`${this.account} could not be given write on ${x.repository}, which builds ${change.key} — ${x.why}`);
|
||||||
|
} catch (err) {
|
||||||
|
this.log(`whether ${this.account} writes ${repository} (module ${change.key}) could not be settled — ${said(err)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Grant write on every delivered repository the account cannot write (or on those named, each of which must be
|
||||||
|
* delivered); with none named, also say which repositories it cannot write are left so, and why. */
|
||||||
|
async grant(only?: string[]): Promise<Written> {
|
||||||
|
if (!this.heard) {
|
||||||
|
throw new Error(`which repositories build a registered module is not known yet: ${INDEXER_MODULES} has not been read ` +
|
||||||
|
"(the watch is not granted or not answered); nothing is granted");
|
||||||
|
}
|
||||||
|
const delivered = this.delivered();
|
||||||
|
const result: Written = { account: this.account, delivered, granted: [], already: [], refused: [], left_read_only: [] };
|
||||||
|
const named = only && only.length ? only.map((x) => String(x).trim()) : null;
|
||||||
|
const targets: string[] = [];
|
||||||
|
for (const full of named ?? delivered) {
|
||||||
|
split(full);
|
||||||
|
if (delivered.includes(full)) targets.push(full);
|
||||||
|
else result.left_read_only.push({ repository: full, permission: "unchanged",
|
||||||
|
why: "builds no registered module (the indexer names none from it), so the account is given no write there" });
|
||||||
|
}
|
||||||
|
for (const full of targets) {
|
||||||
|
const [owner, name] = split(full);
|
||||||
|
let permission: string;
|
||||||
|
try {
|
||||||
|
if (!(await this.forge.repository(owner, name))) {
|
||||||
|
result.refused.push({ repository: full, why: "the forge holds no repository by that name" });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
permission = await this.forge.repositoryPermission(owner, name, this.account);
|
||||||
|
} catch (err) {
|
||||||
|
result.refused.push({ repository: full, why: `its permission could not be read — ${said(err)}` });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (WRITES.has(permission)) {
|
||||||
|
this.writable.add(full);
|
||||||
|
result.already.push(full);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (owner === this.org) {
|
||||||
|
// The organisation's repositories are written through its team (tools-account.ts), never a collaborator.
|
||||||
|
result.refused.push({ repository: full, why: `a repository of ${this.org}, which ${this.account} writes through the ` +
|
||||||
|
`team ${this.org}/${this.account}; the forge gives it "${permission}", so the team is not as it should be (it is ` +
|
||||||
|
"set again at the next mint of the tools' token)" });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await this.forge.addWriteCollaborator(owner, name, this.account);
|
||||||
|
} catch (err) {
|
||||||
|
result.refused.push({ repository: full, why: said(err) });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let after: string;
|
||||||
|
try {
|
||||||
|
after = await this.forge.repositoryPermission(owner, name, this.account);
|
||||||
|
} catch (err) {
|
||||||
|
result.refused.push({ repository: full, why: `granted, but its permission could not be read back — ${said(err)}` });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!WRITES.has(after)) {
|
||||||
|
result.refused.push({ repository: full, why: `the grant was answered, but the forge still gives ${this.account} "${after}"` });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
this.writable.add(full);
|
||||||
|
result.granted.push(full);
|
||||||
|
await this.emit(WRITABLE_EVENT, { owner, repo: name, account: this.account }).catch((err) =>
|
||||||
|
this.log(`${this.account} writes ${full} now, and that could not be said on the bus — ${said(err)}`));
|
||||||
|
}
|
||||||
|
if (!named) {
|
||||||
|
for (const repo of await this.forge.everyRepository()) {
|
||||||
|
if (repo.owner === this.org || delivered.includes(repo.full_name)) continue;
|
||||||
|
let permission: string;
|
||||||
|
try {
|
||||||
|
permission = await this.forge.repositoryPermission(repo.owner, repo.name, this.account);
|
||||||
|
} catch (err) {
|
||||||
|
permission = `unknown — ${said(err)}`;
|
||||||
|
}
|
||||||
|
if (WRITES.has(permission)) continue;
|
||||||
|
result.left_read_only.push({ repository: repo.full_name, permission,
|
||||||
|
why: "builds no registered module (the indexer names none from it), so the account is given no write there" });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Every delivered repository the account cannot write, as the forge says now, and why. */
|
||||||
|
async unwritable(): Promise<{ account: string; delivered: number; unwritable: { repository: string; permission: string; why: string }[] }> {
|
||||||
|
if (!this.heard) throw new Error(`which repositories build a registered module is not known yet: ${INDEXER_MODULES} has not been read`);
|
||||||
|
const delivered = this.delivered();
|
||||||
|
const out: { repository: string; permission: string; why: string }[] = [];
|
||||||
|
for (const full of delivered) {
|
||||||
|
const [owner, name] = split(full);
|
||||||
|
let permission: string;
|
||||||
|
try {
|
||||||
|
permission = await this.forge.repositoryPermission(owner, name, this.account);
|
||||||
|
} catch (err) {
|
||||||
|
out.push({ repository: full, permission: "unknown", why: `the forge did not answer ${this.account}'s permission on it — ${said(err)}` });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (WRITES.has(permission)) continue;
|
||||||
|
const modules = [...this.modules].filter(([, r]) => r === full).map(([m]) => m).sort();
|
||||||
|
out.push({ repository: full, permission, why: `builds ${modules.join(", ")}, and the forge gives ${this.account} ` +
|
||||||
|
`"${permission}" on it: its commit status and its merge are refused until gitea_grant_write grants it` });
|
||||||
|
}
|
||||||
|
return { account: this.account, delivered: delivered.length, unwritable: out };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type Tool = { name: string; description: string; input: Record<string, unknown>; run: (args: Readonly<Record<string, unknown>>) => Promise<unknown> };
|
||||||
|
|
||||||
|
/** The two tools: which delivered repositories the merging account cannot write, and the grant now. */
|
||||||
|
export function writeTools(writes: () => DeliveredWrites): Tool[] {
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
name: "gitea_unwritable_repos",
|
||||||
|
description: "Every repository that builds a registered module (the indexer's modules, read over the bus) on which " +
|
||||||
|
"the mesh's merging forge account (mesh-tools, which sets commit statuses and merges) cannot write, with the " +
|
||||||
|
"permission the forge gives it there and why: such a pull request can be checked but never merged, and the " +
|
||||||
|
"delivery gate does not ask its Approve. Empty when it writes every one. Read with the site administrator's login.",
|
||||||
|
input: {},
|
||||||
|
run: async () => writes().unwritable(),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "gitea_grant_write",
|
||||||
|
description: "Give the mesh's merging forge account (mesh-tools) write, as a collaborator, on every repository that " +
|
||||||
|
"builds a registered module and that it cannot write now, or on the repositories named (each must build one). " +
|
||||||
|
"The module does this itself at its start and when a module of a new repository is registered; this is the same " +
|
||||||
|
"act by hand, such as when a merge or a commit status was refused 403. A repository of the organisation novox " +
|
||||||
|
"is written through its team and needs nothing. Answers what was granted (each read back from the forge), what " +
|
||||||
|
"was written already, what the forge refused in its own words, and which other repositories it left read-only " +
|
||||||
|
"and why (they build no registered module). Needs the site administrator's login, used for this grant alone.",
|
||||||
|
input: {
|
||||||
|
repositories: { type: "array", items: { type: "string" }, description: "owner/name of each repository; empty or absent, every delivered one" },
|
||||||
|
},
|
||||||
|
run: async (args) => writes().grant(Array.isArray(args.repositories) ? args.repositories.map(String) : undefined),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
@@ -43,6 +43,7 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps {
|
|||||||
return delivery.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()}, nil
|
return delivery.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()}, nil
|
||||||
},
|
},
|
||||||
Kept: m.KeptLookup,
|
Kept: m.KeptLookup,
|
||||||
|
Has: m.Has,
|
||||||
Release: func(ws string) error { return m.spaces().Release(ws) },
|
Release: func(ws string) error { return m.spaces().Release(ws) },
|
||||||
Logf: m.Say,
|
Logf: m.Say,
|
||||||
}
|
}
|
||||||
@@ -96,7 +97,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return &buildstep.Puller{Source: source, Language: delivery.AskGo, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load,
|
return &buildstep.Puller{Source: source, Language: delivery.AskGo, Node: m.Node, Put: m.Steps, PausePut: m.Pauses, Load: load,
|
||||||
Capacity: capacity, WatchConfig: watch, Logf: m.Say,
|
Capacity: capacity, WatchConfig: watch, Logf: m.Say, Has: m.Has,
|
||||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||||
if err := m.Check(ctx); err != nil {
|
if err := m.Check(ctx); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -115,3 +116,11 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
|
|||||||
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
|
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
|
||||||
return buildstep.SeatTools(Seat, "Go", p)
|
return buildstep.SeatTools(Seat, "Go", p)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes.
|
||||||
|
const StepEndTool = "build_go_step_end"
|
||||||
|
|
||||||
|
// stepScan is the module's steps state as buildstep reads it.
|
||||||
|
func (m *Module) stepScan() buildstep.Scan {
|
||||||
|
return buildstep.ScanOf(stdio.State(delivery.StepsState))
|
||||||
|
}
|
||||||
|
|||||||
@@ -183,12 +183,12 @@ func (m *Module) buildEnv(dir string) []string {
|
|||||||
} else if modfile, ok := m.pinModfile(dir); ok {
|
} else if modfile, ok := m.pinModfile(dir); ok {
|
||||||
mod += " -modfile=" + modfile // a pin on a pull request's head the fetch placed beside the workspace (pins.go)
|
mod += " -modfile=" + modfile // a pin on a pull request's head the fetch placed beside the workspace (pins.go)
|
||||||
}
|
}
|
||||||
return []string{
|
return append([]string{
|
||||||
"CGO_ENABLED=0",
|
"CGO_ENABLED=0",
|
||||||
"GOFLAGS=" + recipeFlags + " " + mod,
|
"GOFLAGS=" + recipeFlags + " " + mod,
|
||||||
"GOTOOLCHAIN=local",
|
"GOTOOLCHAIN=local",
|
||||||
"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
|
"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
|
||||||
"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
|
"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
|
||||||
"GOPROXY=off",
|
"GOPROXY=off",
|
||||||
}
|
}, m.threadEnv()...)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The runners of a digest or test step: this installed program, or the candidate's compiled from the workspace.
|
// The runners of a digest or test step: this installed program, or the candidate's compiled from the workspace.
|
||||||
@@ -108,7 +110,7 @@ func (m *Module) mergeCommit(ctx context.Context, workspace string) (string, err
|
|||||||
r := m.Run(ctx, nil, "git", "-C", filepath.Join(m.Work, workspace, "tree"), "rev-parse", "HEAD")
|
r := m.Run(ctx, nil, "git", "-C", filepath.Join(m.Work, workspace, "tree"), "rev-parse", "HEAD")
|
||||||
commit := strings.TrimSpace(r.Stdout)
|
commit := strings.TrimSpace(r.Stdout)
|
||||||
if r.Status != 0 || r.Err != "" || commit == "" {
|
if r.Status != 0 || r.Err != "" || commit == "" {
|
||||||
return "", fmt.Errorf("the workspace's commit: git rev-parse: %s", firstLine(r.Err+"\n"+r.Stderr))
|
return "", fmt.Errorf("the workspace's commit: git rev-parse: %s", forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||||
}
|
}
|
||||||
return commit, nil
|
return commit, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildcpu"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The processors a step may take (novox/hq ADR 0553, issue 1806). The node-engine holds this module's runtime, with
|
||||||
|
// every other build module's of the node, in one slice at idle priority under the node's one share of its processors:
|
||||||
|
// that quota bounds the total. Threads only size each step: the share the composer gives (buildstep.NodeLimits) over
|
||||||
|
// this module's own capacity, so a step does not wait in the kernel's throttle. Without the composer's share there is
|
||||||
|
// no slice to size a step to, and no thread limit is set.
|
||||||
|
|
||||||
|
// cpuLimits is what one step is held to now.
|
||||||
|
func (m *Module) cpuLimits() buildcpu.Limits {
|
||||||
|
slots := 1
|
||||||
|
switch {
|
||||||
|
case m.Capacity != nil:
|
||||||
|
slots = m.Capacity()
|
||||||
|
case m.Settings != "":
|
||||||
|
slots = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, nil)()
|
||||||
|
}
|
||||||
|
return buildstep.NodeLimits(slots)
|
||||||
|
}
|
||||||
|
|
||||||
|
// threadEnv is the environment (nothing without a share) every go command of a step is given for its threads: GOMAXPROCS sets its -p, the compiler's
|
||||||
|
// concurrency, a test binary's processors and -parallel.
|
||||||
|
func (m *Module) threadEnv() []string {
|
||||||
|
return buildcpu.Env(m.cpuLimits().Threads)
|
||||||
|
}
|
||||||
|
|
||||||
|
// cpuStatus is what the status tool says of it: the limits a step is given, and what the kernel holds this runtime
|
||||||
|
// to (its unit and its slice), read from its cgroup rather than assumed.
|
||||||
|
func (m *Module) cpuStatus() map[string]any {
|
||||||
|
return map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()}
|
||||||
|
}
|
||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
@@ -19,6 +20,8 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind.
|
// DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind.
|
||||||
@@ -39,6 +42,33 @@ const besideVariable = "MESH_CHECK_BESIDE"
|
|||||||
// test log) with each verdict.
|
// test log) with each verdict.
|
||||||
const ReadsOutsideMarker = "reads-outside"
|
const ReadsOutsideMarker = "reads-outside"
|
||||||
|
|
||||||
|
// NeedsMarker is the name of a file in a package's directory that names, one per line, what its integration tests need
|
||||||
|
// of the node they run on (issue 890; mesh-sdk delivery.AskNeeds): `node-build-node` for tests that read the Node build
|
||||||
|
// module's installed toolchain. The digest reports it (DigestResult.Needs); a check ask leaves such a package untested on
|
||||||
|
// a node that lacks it, and the checker asks it again of a node that has it. A marker beside the tests, as
|
||||||
|
// ReadsOutsideMarker is, and not a manifest field: it is a property of the package's tests, read only where they are
|
||||||
|
// run, and the manifest describes the module on the machines. The unit level is never given anything of the node (ADR
|
||||||
|
// 0533 §1), so a marker binds the integration level alone. Blank lines and lines starting with # are not read.
|
||||||
|
const NeedsMarker = "check-needs"
|
||||||
|
|
||||||
|
// readNeeds is what the marker in dir names; nil when there is none.
|
||||||
|
func readNeeds(dir string) ([]string, error) {
|
||||||
|
raw, err := os.ReadFile(filepath.Join(dir, NeedsMarker))
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var needs []string
|
||||||
|
for _, l := range strings.Split(string(raw), "\n") {
|
||||||
|
if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") {
|
||||||
|
needs = append(needs, l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return needs, nil
|
||||||
|
}
|
||||||
|
|
||||||
// integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation
|
// integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation
|
||||||
// for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run.
|
// for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run.
|
||||||
var integrationTags = []string{"integration", "isolation"}
|
var integrationTags = []string{"integration", "isolation"}
|
||||||
@@ -101,6 +131,8 @@ type DigestResult struct {
|
|||||||
// ReadsOutside names the packages that carry the marker: each has a digest that no kept verdict can have (a fresh
|
// ReadsOutside names the packages that carry the marker: each has a digest that no kept verdict can have (a fresh
|
||||||
// random one), so the check runs it every time.
|
// random one), so the check runs it every time.
|
||||||
ReadsOutside []string `json:"reads_outside,omitempty"`
|
ReadsOutside []string `json:"reads_outside,omitempty"`
|
||||||
|
// Needs are the packages whose tests at this level need something of the node, as their NeedsMarker names it.
|
||||||
|
Needs []delivery.PackageNeeds `json:"needs,omitempty"`
|
||||||
// JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
|
// JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
|
||||||
// runner workspace.
|
// runner workspace.
|
||||||
JudgedBy string `json:"judged_by,omitempty"`
|
JudgedBy string `json:"judged_by,omitempty"`
|
||||||
@@ -206,12 +238,13 @@ func (m *Module) digestHere(ctx context.Context, workspace, level string) (Diges
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return DigestResult{}, err
|
return DigestResult{}, err
|
||||||
}
|
}
|
||||||
if err := digestModule(&res, pkgs, tree, header, besideHeader); err != nil {
|
if err := digestModule(&res, pkgs, tree, header, besideHeader, level == LevelIntegration); err != nil {
|
||||||
return DigestResult{}, err
|
return DigestResult{}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sort.Strings(res.WithoutTests)
|
sort.Strings(res.WithoutTests)
|
||||||
sort.Strings(res.ReadsOutside)
|
sort.Strings(res.ReadsOutside)
|
||||||
|
sort.Slice(res.Needs, func(i, j int) bool { return res.Needs[i].Package < res.Needs[j].Package })
|
||||||
for _, p := range res.ReadsOutside {
|
for _, p := range res.ReadsOutside {
|
||||||
if m.Say != nil {
|
if m.Say != nil {
|
||||||
m.Say("the %s level of %s always runs: its directory carries %s, so its tests read files outside it that no digest names", level, p, ReadsOutsideMarker)
|
m.Say("the %s level of %s always runs: its directory carries %s, so its tests read files outside it that no digest names", level, p, ReadsOutsideMarker)
|
||||||
@@ -256,7 +289,7 @@ func parsePackages(out string) (map[string]*goPackage, error) {
|
|||||||
|
|
||||||
// digestModule adds to res the digest of each root package of one go.mod directory. A root is a package of the main
|
// digestModule adds to res the digest of each root package of one go.mod directory. A root is a package of the main
|
||||||
// module that is listed, not only a dependency, and not a variant of another.
|
// module that is listed, not only a dependency, and not a variant of another.
|
||||||
func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header, beside []string) error {
|
func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header, beside []string, needs bool) error {
|
||||||
roots := []string{}
|
roots := []string{}
|
||||||
for path, p := range pkgs {
|
for path, p := range pkgs {
|
||||||
if p.ForTest == "" && !p.DepOnly && !p.Standard && !strings.HasSuffix(path, ".test") && p.Module != nil && p.Module.Main {
|
if p.ForTest == "" && !p.DepOnly && !p.Standard && !strings.HasSuffix(path, ".test") && p.Module != nil && p.Module.Main {
|
||||||
@@ -274,6 +307,15 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he
|
|||||||
if testMain == nil {
|
if testMain == nil {
|
||||||
return fmt.Errorf("go list gave no test binary for %s", path)
|
return fmt.Errorf("go list gave no test binary for %s", path)
|
||||||
}
|
}
|
||||||
|
if needs {
|
||||||
|
n, err := readNeeds(root.Dir)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s: its %s: %w", path, NeedsMarker, err)
|
||||||
|
}
|
||||||
|
if len(n) > 0 {
|
||||||
|
res.Needs = append(res.Needs, delivery.PackageNeeds{Package: path, Needs: n})
|
||||||
|
}
|
||||||
|
}
|
||||||
if _, err := os.Stat(filepath.Join(root.Dir, ReadsOutsideMarker)); err == nil {
|
if _, err := os.Stat(filepath.Join(root.Dir, ReadsOutsideMarker)); err == nil {
|
||||||
// The digest of a package that reads outside itself is one nothing can have kept: the key is a fresh random
|
// The digest of a package that reads outside itself is one nothing can have kept: the key is a fresh random
|
||||||
// hex, so the ask finds no kept verdict and runs it, and what the checker writes under it is never read again.
|
// hex, so the ask finds no kept verdict and runs it, and what the checker writes under it is never read again.
|
||||||
|
|||||||
@@ -213,6 +213,16 @@ func TestAPackageWithTheReadsOutsideMarkerIsNeverKept(t *testing.T) {
|
|||||||
if first.Packages[a] != plain.Packages[a] || first.Packages[a] != second.Packages[a] {
|
if first.Packages[a] != plain.Packages[a] || first.Packages[a] != second.Packages[a] {
|
||||||
t.Errorf("a has no marker and its digest moved: %s, %s, before %s", first.Packages[a], second.Packages[a], plain.Packages[a])
|
t.Errorf("a has no marker and its digest moved: %s, %s, before %s", first.Packages[a], second.Packages[a], plain.Packages[a])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue 890/AC3: a package whose directory carries the needs marker is named with its needs at the integration level,
|
||||||
|
// comments and blank lines not read, and at the unit level not at all.
|
||||||
|
edit(t, tree, "a/"+NeedsMarker, "# its integration tests copy the installed Node toolchain\n\nnode-build-node\n")
|
||||||
|
if n := digests(t, m, LevelIntegration).Needs; len(n) != 1 || n[0].Package != a || strings.Join(n[0].Needs, ",") != "node-build-node" {
|
||||||
|
t.Errorf("the integration level's needs = %+v, want %s needing node-build-node", n, a)
|
||||||
|
}
|
||||||
|
if n := digests(t, m, LevelUnit).Needs; len(n) != 0 {
|
||||||
|
t.Errorf("the unit level names needs: %+v", n)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Issue 1935: a catalogue module's packages read ../../module.json and the module's files: a change to them changes the
|
// Issue 1935: a catalogue module's packages read ../../module.json and the module's files: a change to them changes the
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||||
)
|
)
|
||||||
|
|
||||||
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||||
@@ -255,7 +256,7 @@ func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error {
|
|||||||
func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
|
func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
|
||||||
r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "go.mod", "*/go.mod")
|
r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "go.mod", "*/go.mod")
|
||||||
if r.Status != 0 || r.Err != "" {
|
if r.Status != 0 || r.Err != "" {
|
||||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
return nil, fmt.Errorf("git ls-files: %s", forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||||
}
|
}
|
||||||
dirs := []string{}
|
dirs := []string{}
|
||||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||||
@@ -292,7 +293,7 @@ func (m *Module) goEnv() []string {
|
|||||||
if m.GoModCache != "" {
|
if m.GoModCache != "" {
|
||||||
modcache = m.GoModCache
|
modcache = m.GoModCache
|
||||||
}
|
}
|
||||||
return []string{
|
return append([]string{
|
||||||
"GOTOOLCHAIN=local",
|
"GOTOOLCHAIN=local",
|
||||||
"GOFLAGS=-mod=mod",
|
"GOFLAGS=-mod=mod",
|
||||||
"GOMODCACHE=" + modcache,
|
"GOMODCACHE=" + modcache,
|
||||||
@@ -300,7 +301,7 @@ func (m *Module) goEnv() []string {
|
|||||||
"GOPRIVATE=git.novox.be",
|
"GOPRIVATE=git.novox.be",
|
||||||
"GIT_TERMINAL_PROMPT=0",
|
"GIT_TERMINAL_PROMPT=0",
|
||||||
"GIT_SSH_COMMAND=ssh -o BatchMode=yes",
|
"GIT_SSH_COMMAND=ssh -o BatchMode=yes",
|
||||||
}
|
}, m.threadEnv()...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// goEnvFor is goEnv for a command run in dir: a directory that vendors (vendor/modules.txt) is read from its vendor
|
// goEnvFor is goEnv for a command run in dir: a directory that vendors (vendor/modules.txt) is read from its vendor
|
||||||
@@ -331,10 +332,11 @@ func (m *Module) clock() time.Time {
|
|||||||
func (m *Module) now() time.Time { return m.clock() }
|
func (m *Module) now() time.Time { return m.clock() }
|
||||||
|
|
||||||
// causeLine is the first line of a go command's output that says something other than Go's own progress: `go:
|
// causeLine is the first line of a go command's output that says something other than Go's own progress: `go:
|
||||||
// downloading ...` comes first when Go fetches a module, and the cause of a failure on a later line.
|
// downloading ...` comes first when Go fetches a module, and `# <package>` heads a compile error of go build or go vet
|
||||||
|
// (issue 1703: the reason said only the package), and the cause of a failure on a later line.
|
||||||
func causeLine(s string) string {
|
func causeLine(s string) string {
|
||||||
for _, l := range strings.Split(s, "\n") {
|
for _, l := range strings.Split(s, "\n") {
|
||||||
if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "go: downloading ") {
|
if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "go: downloading ") && !strings.HasPrefix(l, "# ") {
|
||||||
return l
|
return l
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -302,3 +302,16 @@ func TestWorkAndCacheAreTheDirectoriesTheManifestPlacesOutsideTheHome(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// issue 1703: a go command's reason is its cause, not Go's progress nor go build's `# <package>` heading.
|
||||||
|
func TestCauseLineSkipsProgressAndThePackageHeading(t *testing.T) {
|
||||||
|
for in, want := range map[string]string{
|
||||||
|
"go: downloading x v1\n# git.example/m/cmd\n./main.go:3:2: undefined: y\n": "./main.go:3:2: undefined: y",
|
||||||
|
"\ngo: example.com/x@v1: reading: 404\n": "go: example.com/x@v1: reading: 404",
|
||||||
|
"go: downloading x v1\n": "go: downloading x v1",
|
||||||
|
} {
|
||||||
|
if got := causeLine(in); got != want {
|
||||||
|
t.Errorf("causeLine(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,10 +5,13 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildcpu"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||||
"git.novox.be/novox/mesh-sdk/go/health"
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
@@ -28,6 +31,7 @@ func toolchainValues(env []string) []string {
|
|||||||
// 1572/AC1 (ADR 0477 §1): every environment a step gives its go commands says GOTOOLCHAIN=local, once, and names no
|
// 1572/AC1 (ADR 0477 §1): every environment a step gives its go commands says GOTOOLCHAIN=local, once, and names no
|
||||||
// release.
|
// release.
|
||||||
func TestEveryStepEnvironmentSaysGotoolchainLocal(t *testing.T) {
|
func TestEveryStepEnvironmentSaysGotoolchainLocal(t *testing.T) {
|
||||||
|
t.Setenv("MESH_BUILD_CPU_SHARE", "50")
|
||||||
m := &Module{Cache: t.TempDir()}
|
m := &Module{Cache: t.TempDir()}
|
||||||
vend := t.TempDir()
|
vend := t.TempDir()
|
||||||
os.MkdirAll(filepath.Join(vend, "vendor"), 0o755)
|
os.MkdirAll(filepath.Join(vend, "vendor"), 0o755)
|
||||||
@@ -41,6 +45,24 @@ func TestEveryStepEnvironmentSaysGotoolchainLocal(t *testing.T) {
|
|||||||
if got := toolchainValues(env); len(got) != 1 || got[0] != "GOTOOLCHAIN=local" {
|
if got := toolchainValues(env); len(got) != 1 || got[0] != "GOTOOLCHAIN=local" {
|
||||||
t.Errorf("%s: GOTOOLCHAIN settings %v, want exactly GOTOOLCHAIN=local", name, got)
|
t.Errorf("%s: GOTOOLCHAIN settings %v, want exactly GOTOOLCHAIN=local", name, got)
|
||||||
}
|
}
|
||||||
|
// Issue 1806: and its threads, its part of the node's processor cap (probeEnv runs no build).
|
||||||
|
if procs := strings.Count(strings.Join(env, "\n"), "GOMAXPROCS="); name != "probeEnv" && procs != 1 {
|
||||||
|
t.Errorf("%s: GOMAXPROCS said %d times, want once: %v", name, procs, env)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Without the composer's share no thread limit; with it, the share over its own three slots (ADR 0553 §4).
|
||||||
|
t.Setenv("MESH_BUILD_CPU_SHARE", "")
|
||||||
|
m = &Module{Cache: t.TempDir(), Node: "shanks", Capacity: func() int { return 3 }}
|
||||||
|
if l := m.cpuLimits(); l.Threads != 0 || strings.Contains(strings.Join(m.goEnv(), " "), "GOMAXPROCS=") {
|
||||||
|
t.Errorf("without the share: %+v, goEnv %v", l, m.goEnv())
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_BUILD_CPU_SHARE", "50")
|
||||||
|
want := buildcpu.Threads(50, runtime.NumCPU(), 3)
|
||||||
|
if l := m.cpuLimits(); l.Slots != 3 || !strings.Contains(strings.Join(m.goEnv(), " "), "GOMAXPROCS="+strconv.Itoa(want)) {
|
||||||
|
t.Errorf("limits %+v, goEnv %v; want %d threads", l, m.goEnv(), want)
|
||||||
|
}
|
||||||
|
if st := m.cpuStatus(); st["limits"].(buildcpu.Limits) != m.cpuLimits() {
|
||||||
|
t.Errorf("the status says %v", st)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ func (m *Module) isolation(workspace string, write bool) buildstep.Isolation {
|
|||||||
Workspace: filepath.Join(m.Work, workspace),
|
Workspace: filepath.Join(m.Work, workspace),
|
||||||
Caches: m.caches(),
|
Caches: m.caches(),
|
||||||
WriteCaches: write,
|
WriteCaches: write,
|
||||||
Env: []string{"PATH=" + path, "LC_ALL=C"},
|
Env: append([]string{"PATH=" + path, "LC_ALL=C"}, m.threadEnv()...),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,11 +11,13 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// levelArg is the first argument that makes this binary a step's shut-off process: the integration level's, and, when
|
// levelArg is the first argument that makes this binary a step's shut-off process: the integration level's, and, when
|
||||||
@@ -140,6 +142,49 @@ func (m *Module) nodeToolchains() []string {
|
|||||||
return []string{dir}
|
return []string{dir}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// nodeToolchainMark is the file the Node build module writes in a toolchain directory once it is installed whole
|
||||||
|
// (mesh-build-node toolchain.go installedMark).
|
||||||
|
const nodeToolchainMark = ".installed"
|
||||||
|
|
||||||
|
// Has says whether this node has a need a check ask or a package's tests name (issue 890, ADR 0552; mesh-sdk
|
||||||
|
// delivery.AskNeeds), read from the node itself at every ask: node-build-node when the Node build module's toolchains
|
||||||
|
// directory, given to the integration level as nodeToolchains gives it, holds a toolchain installed whole (its mark and
|
||||||
|
// its package-lock.json readable by this account), and node, npm and bwrap are installed: what mesh-build-node's
|
||||||
|
// integration tests (realMachine) look for before they run. Any other need it lacks.
|
||||||
|
func (m *Module) Has(need string) bool {
|
||||||
|
if m.HasNeed != nil {
|
||||||
|
return m.HasNeed(need)
|
||||||
|
}
|
||||||
|
switch need {
|
||||||
|
case delivery.NeedNodeBuildNode:
|
||||||
|
dirs := m.nodeToolchains()
|
||||||
|
if len(dirs) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, p := range []string{"node", "npm", "bwrap"} {
|
||||||
|
if _, err := exec.LookPath(p); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(dirs[0])
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
dir := filepath.Join(dirs[0], e.Name())
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, nodeToolchainMark)); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if f, err := os.Open(filepath.Join(dir, "package-lock.json")); err == nil {
|
||||||
|
f.Close()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func lastLine(s string) string {
|
func lastLine(s string) string {
|
||||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||||
return strings.TrimSpace(lines[len(lines)-1])
|
return strings.TrimSpace(lines[len(lines)-1])
|
||||||
|
|||||||
@@ -41,6 +41,8 @@ type PackageResult struct {
|
|||||||
Tests int `json:"tests"`
|
Tests int `json:"tests"`
|
||||||
// NoTests is a package with no test files (at this level: a file with the other level's build tag is none).
|
// NoTests is a package with no test files (at this level: a file with the other level's build tag is none).
|
||||||
NoTests bool `json:"no_tests,omitempty"`
|
NoTests bool `json:"no_tests,omitempty"`
|
||||||
|
// Skipped is how many of the package's tests skipped: a pass that skipped proved less than it says, so the checker keeps no verdict for it.
|
||||||
|
Skipped int `json:"skipped,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestTime is one test and the time it took.
|
// TestTime is one test and the time it took.
|
||||||
@@ -227,6 +229,7 @@ func (r *reader) event(ev event) {
|
|||||||
r.res.Tests++
|
r.res.Tests++
|
||||||
if ev.Action == "skip" {
|
if ev.Action == "skip" {
|
||||||
r.res.Skipped++
|
r.res.Skipped++
|
||||||
|
p.Skipped++
|
||||||
}
|
}
|
||||||
r.times = append(r.times, TestTime{Package: ev.Package, Test: ev.Test, Elapsed: ev.Elapsed})
|
r.times = append(r.times, TestTime{Package: ev.Package, Test: ev.Test, Elapsed: ev.Elapsed})
|
||||||
if ev.Action == "fail" {
|
if ev.Action == "fail" {
|
||||||
|
|||||||
@@ -8,9 +8,11 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"os"
|
"os"
|
||||||
"os/user"
|
"os/user"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -70,7 +72,13 @@ func main() {
|
|||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
steps := m.StepTools()
|
steps := m.StepTools()
|
||||||
|
booted := time.Now()
|
||||||
puller := m.Puller(steps, nil)
|
puller := m.Puller(steps, nil)
|
||||||
|
m.Held = puller
|
||||||
|
if m.Node != "" {
|
||||||
|
// A step a previous process left running is ended (issue 1890); this process's own steps start after booted.
|
||||||
|
go buildstep.EndStaleAtStart(ctx, m.stepScan(), m.Node, booted, logf)
|
||||||
|
}
|
||||||
if m.Node != "" {
|
if m.Node != "" {
|
||||||
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
|
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
|
||||||
go func() { _ = puller.Pull(ctx) }()
|
go func() { _ = puller.Pull(ctx) }()
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||||
)
|
)
|
||||||
|
|
||||||
// mirrorLocks hold one mutex per mirror of this process; the lock file beside the mirror holds off another process.
|
// mirrorLocks hold one mutex per mirror of this process; the lock file beside the mirror holds off another process.
|
||||||
@@ -67,7 +68,7 @@ func (m *Module) dissociate(ctx context.Context, dir string) error {
|
|||||||
return nil // no mirror was used
|
return nil // no mirror was used
|
||||||
}
|
}
|
||||||
if r := m.Run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-C", dir, "repack", "-a", "-d", "-q"); r.Status != 0 || r.Err != "" {
|
if r := m.Run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-C", dir, "repack", "-a", "-d", "-q"); r.Status != 0 || r.Err != "" {
|
||||||
return fmt.Errorf("git repack in %s: %s", dir, firstLine(r.Err+" "+r.Stderr))
|
return fmt.Errorf("git repack in %s: %s", dir, forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||||
}
|
}
|
||||||
return os.Remove(alternates)
|
return os.Remove(alternates)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
@@ -121,20 +122,35 @@ func TestTwoUpdatesOfOneMirrorNeverRunAtOnce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A mirror that cannot be updated does not fail the fetch: the fetch goes on without it.
|
// A mirror that cannot be updated does not fail the fetch: the fetch goes on without it, and the line saying so carries
|
||||||
|
// git's own words. The forge's refusal of a repository its account cannot read came out as "git -C: error:" (ace,
|
||||||
|
// 2026-10-26: git writes a bare "error:" line before the forge's message).
|
||||||
func TestAMirrorThatCannotBeUpdatedLetsTheFetchGoOnWithoutIt(t *testing.T) {
|
func TestAMirrorThatCannotBeUpdatedLetsTheFetchGoOnWithoutIt(t *testing.T) {
|
||||||
f := &fakeStep{gomods: "go.mod\n"}
|
f := &fakeStep{gomods: "go.mod\n"}
|
||||||
f.rule = func(line string) (r runAnswer, ok bool) {
|
f.rule = func(line string) (r runAnswer, ok bool) {
|
||||||
if strings.Contains(line, "mirror") && strings.Contains(line, " fetch ") {
|
if strings.Contains(line, "mirror") && strings.Contains(line, " fetch ") {
|
||||||
return runAnswer{Status: 1, Stderr: "fatal: no space left\n"}, true
|
return runAnswer{Status: 128, Stderr: "error: \nerror: User: 22:nox-builder with Key: 30:mesh:ace.mesh-build-go is not authorized to read jschoubben/car-hunter.\nerror: \nfatal: Could not read from remote repository.\n"}, true
|
||||||
}
|
}
|
||||||
return r, false
|
return r, false
|
||||||
}
|
}
|
||||||
m, states := fetchModule(t, f)
|
m, states := fetchModule(t, f)
|
||||||
|
var mu sync.Mutex
|
||||||
|
var said []string
|
||||||
|
m.Say = func(format string, a ...any) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
said = append(said, fmt.Sprintf(format, a...))
|
||||||
|
}
|
||||||
ws := fetchOnce(t, m, states, fetchArgs).Workspace
|
ws := fetchOnce(t, m, states, fetchArgs).Workspace
|
||||||
if _, err := os.Stat(filepath.Join(m.Work, ws, "tree", ".git", "objects", "info", "alternates")); err == nil {
|
if _, err := os.Stat(filepath.Join(m.Work, ws, "tree", ".git", "objects", "info", "alternates")); err == nil {
|
||||||
t.Errorf("the workspace borrows from a mirror that is not there")
|
t.Errorf("the workspace borrows from a mirror that is not there")
|
||||||
}
|
}
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
all := strings.Join(said, "\n")
|
||||||
|
if !strings.Contains(all, "is not used: git fetch: error: User: 22:nox-builder with Key: 30:mesh:ace.mesh-build-go is not authorized to read jschoubben/car-hunter.; fatal: Could not read from remote repository.") {
|
||||||
|
t.Errorf("the mirror's line lost git's words:\n%s", all)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func sleepABit() { time.Sleep(30 * time.Millisecond) }
|
func sleepABit() { time.Sleep(30 * time.Millisecond) }
|
||||||
|
|||||||
@@ -28,6 +28,8 @@ type Module struct {
|
|||||||
// Toolchains and Steps are the Put of the states of those names.
|
// Toolchains and Steps are the Put of the states of those names.
|
||||||
Toolchains, Steps buildstep.Put
|
Toolchains, Steps buildstep.Put
|
||||||
// ReadSteps and ReadToolchainsState read those two states back for the read tools (issue 1562).
|
// ReadSteps and ReadToolchainsState read those two states back for the read tools (issue 1562).
|
||||||
|
// Held is the puller whose running asks a step-end tool will not end (issue 1890); nil in a test.
|
||||||
|
Held *buildstep.Puller
|
||||||
ReadSteps, ReadToolchainsState StateRead
|
ReadSteps, ReadToolchainsState StateRead
|
||||||
// Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back.
|
// Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back.
|
||||||
Pauses buildstep.Put
|
Pauses buildstep.Put
|
||||||
@@ -73,6 +75,8 @@ type Module struct {
|
|||||||
// NodeToolchains is where the Node build module keeps its installed toolchains, bound read-only into the
|
// NodeToolchains is where the Node build module keeps its installed toolchains, bound read-only into the
|
||||||
// integration level (ADR 0533 §1); NodeToolchains when empty. A test points it elsewhere.
|
// integration level (ADR 0533 §1); NodeToolchains when empty. A test points it elsewhere.
|
||||||
NodeToolchains string
|
NodeToolchains string
|
||||||
|
// HasNeed answers Has in place of the node's own reading; a test's.
|
||||||
|
HasNeed func(need string) bool
|
||||||
// StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own
|
// StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own
|
||||||
// program, StartBus in bus.go, when nil).
|
// program, StartBus in bus.go, when nil).
|
||||||
StartBus func(ctx context.Context, dir string) (url string, stop func(), err error)
|
StartBus func(ctx context.Context, dir string) (url string, stop func(), err error)
|
||||||
@@ -121,6 +125,7 @@ func (m *Module) Start(ctx context.Context) {
|
|||||||
m.Say("the toolchain: %v", err)
|
m.Say("the toolchain: %v", err)
|
||||||
}
|
}
|
||||||
m.CleanWorkspaces(time.Now())
|
m.CleanWorkspaces(time.Now())
|
||||||
|
buildstep.EndStaleRound(m.stepScan(), m.Node, m.Say) // asks past the step bound, on the module's own round (issue 1890)
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
@@ -184,13 +189,14 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Status is what the module reports of itself: the node, the recipe, the toolchain found and why it refuses, if it
|
// Status is what the module reports of itself: the node, the recipe, the toolchain found and why it refuses, if it
|
||||||
// does.
|
// does, and the processors its steps may take (issue 1806).
|
||||||
func (m *Module) Status(ctx context.Context) map[string]any {
|
func (m *Module) Status(ctx context.Context) map[string]any {
|
||||||
err := m.Check(ctx)
|
err := m.Check(ctx)
|
||||||
installed, _ := m.Probe(ctx)
|
installed, _ := m.Probe(ctx)
|
||||||
out := map[string]any{
|
out := map[string]any{
|
||||||
"node": m.Node, "recipe": m.Recipe.Version(), "installed": installed,
|
"node": m.Node, "recipe": m.Recipe.Version(), "installed": installed,
|
||||||
"postgres-major": m.Recipe.PostgresMajor, "nats-server": m.Recipe.NatsServer, "refusing": err != nil,
|
"postgres-major": m.Recipe.PostgresMajor, "nats-server": m.Recipe.NatsServer, "refusing": err != nil,
|
||||||
|
"cpu": m.cpuStatus(),
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
out["why"] = err.Error()
|
out["why"] = err.Error()
|
||||||
|
|||||||
@@ -115,7 +115,7 @@ func TestAGoThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {
|
|||||||
{Name: "short_probe", Run: run},
|
{Name: "short_probe", Run: run},
|
||||||
}, m.StepTools()...)
|
}, m.StepTools()...)
|
||||||
for _, tool := range m.Tools(steps) {
|
for _, tool := range m.Tools(steps) {
|
||||||
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool {
|
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == StepEndTool {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var reason string
|
var reason string
|
||||||
|
|||||||
@@ -71,6 +71,47 @@ func TestANodeWithoutNodeToolchainsRunsTheLevelWithNothingMoreBound(t *testing.T
|
|||||||
if slices.Contains(s.roBinds, missing) {
|
if slices.Contains(s.roBinds, missing) {
|
||||||
t.Errorf("a missing %s is bound: %v", missing, s.roBinds)
|
t.Errorf("a missing %s is bound: %v", missing, s.roBinds)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue 890/AC4: such a node lacks node-build-node; one with an installed toolchain and node and npm has it; nothing
|
||||||
|
// else is had.
|
||||||
|
bin := t.TempDir()
|
||||||
|
for _, p := range []string{"node", "npm", "bwrap"} {
|
||||||
|
if err := os.WriteFile(filepath.Join(bin, p), []byte("#!/bin/sh\n"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Setenv("PATH", bin)
|
||||||
|
m := &Module{NodeToolchains: missing}
|
||||||
|
if m.Has(delivery.NeedNodeBuildNode) {
|
||||||
|
t.Error("a node with no Node toolchain has node-build-node")
|
||||||
|
}
|
||||||
|
installed := t.TempDir()
|
||||||
|
m.NodeToolchains = installed
|
||||||
|
if m.Has(delivery.NeedNodeBuildNode) {
|
||||||
|
t.Error("a node with an empty toolchains directory has node-build-node")
|
||||||
|
}
|
||||||
|
half := filepath.Join(installed, "0.1.0")
|
||||||
|
if err := os.Mkdir(half, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(half, "package-lock.json"), []byte("{}"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.Has(delivery.NeedNodeBuildNode) {
|
||||||
|
t.Error("a toolchain without its installed mark counts")
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(half, nodeToolchainMark), nil, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !m.Has(delivery.NeedNodeBuildNode) || m.Has("gpu") {
|
||||||
|
t.Errorf("with a toolchain installed: node-build-node %v, gpu %v; want true, false", m.Has(delivery.NeedNodeBuildNode), m.Has("gpu"))
|
||||||
|
}
|
||||||
|
if err := os.Remove(filepath.Join(bin, "bwrap")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.Has(delivery.NeedNodeBuildNode) {
|
||||||
|
t.Error("a node with no bubblewrap installed has node-build-node")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1891 (ADR 0533 §1): the unit level is not given them.
|
// 1891 (ADR 0533 §1): the unit level is not given them.
|
||||||
|
|||||||
@@ -101,3 +101,25 @@ func TestALoggedPanicIsNotTheCause(t *testing.T) { // 1901/AC4
|
|||||||
t.Errorf("1901/AC4: failures = %+v, want the timeout's panic as the cause", res.Failed)
|
t.Errorf("1901/AC4: failures = %+v, want the timeout's panic as the cause", res.Failed)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A package counts its own skipped tests, so the checker withholds a kept verdict from that package alone (issue 890).
|
||||||
|
func TestEachPackageCountsItsOwnSkippedTests(t *testing.T) {
|
||||||
|
log := `{"Action":"run","Package":"m/a","Test":"TestA"}
|
||||||
|
{"Action":"skip","Package":"m/a","Test":"TestA","Elapsed":0}
|
||||||
|
{"Action":"run","Package":"m/b","Test":"TestB"}
|
||||||
|
{"Action":"pass","Package":"m/b","Test":"TestB","Elapsed":0}
|
||||||
|
{"Action":"pass","Package":"m/a","Elapsed":0}
|
||||||
|
{"Action":"pass","Package":"m/b","Elapsed":0}
|
||||||
|
`
|
||||||
|
res := &TestResult{}
|
||||||
|
rd := newReader(res, ".")
|
||||||
|
rd.read(log)
|
||||||
|
rd.finish()
|
||||||
|
got := map[string]int{}
|
||||||
|
for _, p := range res.Packages {
|
||||||
|
got[p.Package] = p.Skipped
|
||||||
|
}
|
||||||
|
if res.Skipped != 1 || got["m/a"] != 1 || got["m/b"] != 0 {
|
||||||
|
t.Errorf("skipped = %d, per package %v, want 1 with m/a 1 and m/b 0", res.Skipped, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||||
)
|
)
|
||||||
|
|
||||||
const forgePrefix = "git.novox.be/novox/"
|
const forgePrefix = "git.novox.be/novox/"
|
||||||
@@ -125,7 +127,7 @@ func (m *Module) placePin(ctx context.Context, root string, p pin, failure strin
|
|||||||
repository := "novox/" + p.repo
|
repository := "novox/" + p.repo
|
||||||
r := m.Run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-c", "credential.helper=", "ls-remote", m.remote(repository), "refs/pull/*/head")
|
r := m.Run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-c", "credential.helper=", "ls-remote", m.remote(repository), "refs/pull/*/head")
|
||||||
if r.Status != 0 || r.Err != "" {
|
if r.Status != 0 || r.Err != "" {
|
||||||
return "", fmt.Errorf("the pin of %s at commit %s: listing the pull requests of %s: %s", p.module, p.short, repository, firstLine(r.Err+" "+r.Stderr))
|
return "", fmt.Errorf("the pin of %s at commit %s: listing the pull requests of %s: %s", p.module, p.short, repository, forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||||
}
|
}
|
||||||
commit, ref := "", ""
|
commit, ref := "", ""
|
||||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||||
|
|||||||
@@ -128,6 +128,7 @@ type StepLine struct {
|
|||||||
Started string `json:"started"`
|
Started string `json:"started"`
|
||||||
Ended string `json:"ended,omitempty"`
|
Ended string `json:"ended,omitempty"`
|
||||||
Reason string `json:"reason,omitempty"`
|
Reason string `json:"reason,omitempty"`
|
||||||
|
What string `json:"what,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// RecentSteps are this node's steps, the latest started first, at most limit (ten when limit is not above zero). Only
|
// RecentSteps are this node's steps, the latest started first, at most limit (ten when limit is not above zero). Only
|
||||||
@@ -180,7 +181,7 @@ func (m *Module) RecentSteps(limit int) (lines []StepLine, cut bool, err error)
|
|||||||
}
|
}
|
||||||
out := make([]StepLine, 0, len(steps))
|
out := make([]StepLine, 0, len(steps))
|
||||||
for _, s := range steps {
|
for _, s := range steps {
|
||||||
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason}
|
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason, What: s.What}
|
||||||
if !s.Ended.IsZero() {
|
if !s.Ended.IsZero() {
|
||||||
l.Ended = s.Ended.UTC().Format("2006-01-02T15:04:05Z")
|
l.Ended = s.Ended.UTC().Format("2006-01-02T15:04:05Z")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -96,7 +96,7 @@ func TestStepsToolListsRecentStepsOfThisNode(t *testing.T) {
|
|||||||
t0 := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
|
t0 := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
|
||||||
putStep(t, steps, delivery.Step{ID: "a", Tool: "build_go_fetch", Node: "ace", State: delivery.StepPassed, Started: t0, Ended: t0, Result: json.RawMessage(`1`)})
|
putStep(t, steps, delivery.Step{ID: "a", Tool: "build_go_fetch", Node: "ace", State: delivery.StepPassed, Started: t0, Ended: t0, Result: json.RawMessage(`1`)})
|
||||||
putStep(t, steps, delivery.Step{ID: "b", Tool: "build_go_test", Node: "ace", State: delivery.StepRunning, Started: t0.Add(time.Minute)})
|
putStep(t, steps, delivery.Step{ID: "b", Tool: "build_go_test", Node: "ace", State: delivery.StepRunning, Started: t0.Add(time.Minute)})
|
||||||
putStep(t, steps, delivery.Step{ID: "c", Tool: "build_go_build", Node: "ace", State: delivery.StepFailed, Started: t0.Add(2 * time.Minute), Reason: "boom"})
|
putStep(t, steps, delivery.Step{ID: "c", Tool: "build_go_build", Node: "ace", State: delivery.StepFailed, Started: t0.Add(2 * time.Minute), Reason: "boom", What: "check ask, novox/x@abc"})
|
||||||
putStep(t, steps, delivery.Step{ID: "d", Tool: "build_go_build", Node: "g14", State: delivery.StepPassed, Started: t0.Add(time.Hour)})
|
putStep(t, steps, delivery.Step{ID: "d", Tool: "build_go_build", Node: "g14", State: delivery.StepPassed, Started: t0.Add(time.Hour)})
|
||||||
m := readModule(steps, fakeState{})
|
m := readModule(steps, fakeState{})
|
||||||
out, err := tool(t, m, StepsReadTool)(map[string]any{})
|
out, err := tool(t, m, StepsReadTool)(map[string]any{})
|
||||||
@@ -111,7 +111,7 @@ func TestStepsToolListsRecentStepsOfThisNode(t *testing.T) {
|
|||||||
if !slices.Equal(ids, []string{"c", "b", "a"}) {
|
if !slices.Equal(ids, []string{"c", "b", "a"}) {
|
||||||
t.Errorf("ids = %v, want c b a", ids)
|
t.Errorf("ids = %v, want c b a", ids)
|
||||||
}
|
}
|
||||||
if lines[0].Reason != "boom" || lines[0].State != delivery.StepFailed {
|
if lines[0].Reason != "boom" || lines[0].What != "check ask, novox/x@abc" || lines[0].State != delivery.StepFailed {
|
||||||
t.Errorf("first = %+v", lines[0])
|
t.Errorf("first = %+v", lines[0])
|
||||||
}
|
}
|
||||||
out, _ = tool(t, m, StepsReadTool)(map[string]any{"limit": float64(1)})
|
out, _ = tool(t, m, StepsReadTool)(map[string]any{"limit": float64(1)})
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
|
|
||||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
)
|
)
|
||||||
@@ -40,6 +41,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
|||||||
tools = append(tools, t)
|
tools = append(tools, t)
|
||||||
}
|
}
|
||||||
tools = append(tools, m.ReadTools()...)
|
tools = append(tools, m.ReadTools()...)
|
||||||
|
tools = append(tools, buildstep.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held))
|
||||||
tools = append(tools, m.releaseTool())
|
tools = append(tools, m.releaseTool())
|
||||||
return append(tools,
|
return append(tools,
|
||||||
stdio.Tool{Name: StatusTool,
|
stdio.Tool{Name: StatusTool,
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module meshbuildgo
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f
|
||||||
github.com/distribution/distribution/v3 v3.1.2
|
github.com/distribution/distribution/v3 v3.1.2
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4 h1:05sVel/UOvz4/g0dxrDOeVUL7BGakVrxPBlqRq02ZhM=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||||
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
|
|||||||
@@ -39,6 +39,7 @@
|
|||||||
"build_go_release",
|
"build_go_release",
|
||||||
"build_go_step",
|
"build_go_step",
|
||||||
"build_go_steps",
|
"build_go_steps",
|
||||||
|
"build_go_step_end",
|
||||||
"build_go_toolchains",
|
"build_go_toolchains",
|
||||||
"build_go_status",
|
"build_go_status",
|
||||||
"build_go_health"
|
"build_go_health"
|
||||||
|
|||||||
@@ -66,3 +66,11 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
|
|||||||
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
|
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
|
||||||
return buildstep.SeatTools(Seat, "image", p)
|
return buildstep.SeatTools(Seat, "image", p)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes.
|
||||||
|
const StepEndTool = "build_image_step_end"
|
||||||
|
|
||||||
|
// stepScan is the module's steps state as buildstep reads it.
|
||||||
|
func (m *Module) stepScan() buildstep.Scan {
|
||||||
|
return buildstep.ScanOf(stdio.State(delivery.StepsState))
|
||||||
|
}
|
||||||
|
|||||||
@@ -155,14 +155,18 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
local := localName(workspace, manifest.Module, art.Name)
|
local := localName(workspace, manifest.Module, art.Name)
|
||||||
argv := append(m.storeFlags(), "bud", "--isolation", "oci", "--tag", local, "--file", dockerfile)
|
// --network pasta: a RUN step's network is pasta's, in user space. buildah's default for a rootless RUN is a
|
||||||
|
// netavark bridge whose nftables rules cannot be applied for an account without a user session, so every RUN
|
||||||
|
// failed before it started ("setup network: netavark: nftables error"; issue 1703, reproduced with buildah 1.45.2).
|
||||||
|
argv := append(m.storeFlags(), "bud", "--isolation", "oci", "--network", "pasta", "--tag", local, "--file", dockerfile)
|
||||||
if art.Target != "" {
|
if art.Target != "" {
|
||||||
argv = append(argv, "--target", art.Target)
|
argv = append(argv, "--target", art.Target)
|
||||||
}
|
}
|
||||||
argv = append(argv, imageArgs(manifest)...)
|
argv = append(argv, imageArgs(manifest)...)
|
||||||
argv = append(argv, contextDir)
|
argv = append(argv, contextDir)
|
||||||
if r := m.Run(ctx, m.env(), "buildah", argv...); r.Status != 0 || r.Err != "" {
|
if r := m.Run(ctx, m.env(), "buildah", argv...); r.Status != 0 || r.Err != "" {
|
||||||
return nil, fmt.Errorf("buildah bud of %s: %s", art.Name, lastLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
m.sayFailure("buildah bud of "+art.Name+" of "+manifest.Module, r)
|
||||||
|
return nil, fmt.Errorf("buildah bud of %s: %s", art.Name, cause(r))
|
||||||
}
|
}
|
||||||
defer m.remove(local)
|
defer m.remove(local)
|
||||||
digest, err := m.localDigest(ctx, local)
|
digest, err := m.localDigest(ctx, local)
|
||||||
@@ -204,8 +208,11 @@ func localName(workspace, module, artifact string) string {
|
|||||||
func (m *Module) localDigest(ctx context.Context, name string) (string, error) {
|
func (m *Module) localDigest(ctx context.Context, name string) (string, error) {
|
||||||
r := m.Run(ctx, m.env(), "buildah", append(m.storeFlags(), "images", "--no-trunc", "--format", "{{.Digest}}", name)...)
|
r := m.Run(ctx, m.env(), "buildah", append(m.storeFlags(), "images", "--no-trunc", "--format", "{{.Digest}}", name)...)
|
||||||
digest := strings.TrimSpace(r.Stdout)
|
digest := strings.TrimSpace(r.Stdout)
|
||||||
if r.Status != 0 || r.Err != "" || !digestOK.MatchString(digest) {
|
if r.Status != 0 || r.Err != "" {
|
||||||
return "", fmt.Errorf("the digest of the image %s could not be read: %s", name, lastLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
return "", fmt.Errorf("the digest of the image %s could not be read: %s", name, cause(r))
|
||||||
|
}
|
||||||
|
if !digestOK.MatchString(digest) {
|
||||||
|
return "", fmt.Errorf("the digest of the image %s could not be read: buildah images answered %q, not a digest", name, cutFront(digest, CauseBytes))
|
||||||
}
|
}
|
||||||
return digest, nil
|
return digest, nil
|
||||||
}
|
}
|
||||||
@@ -216,17 +223,6 @@ var digestOK = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
|||||||
func (m *Module) remove(name string) {
|
func (m *Module) remove(name string) {
|
||||||
r := m.Run(m.Ctx, m.env(), "buildah", append(m.storeFlags(), "rmi", "--force", name)...)
|
r := m.Run(m.Ctx, m.env(), "buildah", append(m.storeFlags(), "rmi", "--force", name)...)
|
||||||
if (r.Status != 0 || r.Err != "") && m.Say != nil {
|
if (r.Status != 0 || r.Err != "") && m.Say != nil {
|
||||||
m.Say("the image %s could not be removed from the store: %s", name, lastLine(r.Err+"\n"+r.Stderr))
|
m.Say("the image %s could not be removed from the store: %s", name, cause(r))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// lastLine is the last line of a command's output that says something: buildah's error is its last line.
|
|
||||||
func lastLine(s string) string {
|
|
||||||
lines := strings.Split(s, "\n")
|
|
||||||
for i := len(lines) - 1; i >= 0; i-- {
|
|
||||||
if l := strings.TrimSpace(lines[i]); l != "" {
|
|
||||||
return l
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -69,6 +70,7 @@ func setup(t *testing.T, edit func(string) string) (*Module, *fake, *keptStates)
|
|||||||
// Dockerfile of the module's own directory and that directory as the context, the target, the pinned base as a build
|
// Dockerfile of the module's own directory and that directory as the context, the target, the pinned base as a build
|
||||||
// argument; no mount, no credential file, a runtime directory of its own, and it answers the image's digest.
|
// argument; no mount, no credential file, a runtime directory of its own, and it answers the image's digest.
|
||||||
func TestBuildRunsBuildahRootlessAndAnswersTheDigest(t *testing.T) {
|
func TestBuildRunsBuildahRootlessAndAnswersTheDigest(t *testing.T) {
|
||||||
|
t.Setenv("MESH_BUILD_CPU_SHARE", "50") // the composer's share (ADR 0553): buildah is given its threads
|
||||||
m, f, states := setup(t, same)
|
m, f, states := setup(t, same)
|
||||||
res, st := runBuild(t, m, states, nil)
|
res, st := runBuild(t, m, states, nil)
|
||||||
if st.State != delivery.StepPassed {
|
if st.State != delivery.StepPassed {
|
||||||
@@ -83,7 +85,7 @@ func TestBuildRunsBuildahRootlessAndAnswersTheDigest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo")
|
dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo")
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
"--root " + m.Store + "/root", "--runroot " + m.Store + "/run", "--isolation oci",
|
"--root " + m.Store + "/root", "--runroot " + m.Store + "/run", "--isolation oci", "--network pasta",
|
||||||
"--file " + dir + "/Dockerfile", "--target final",
|
"--file " + dir + "/Dockerfile", "--target final",
|
||||||
"--build-arg DEMO_BASE=docker.io/library/busybox@sha256:0000000000000000000000000000000000000000000000000000000000000001",
|
"--build-arg DEMO_BASE=docker.io/library/busybox@sha256:0000000000000000000000000000000000000000000000000000000000000001",
|
||||||
} {
|
} {
|
||||||
@@ -110,6 +112,9 @@ func TestBuildRunsBuildahRootlessAndAnswersTheDigest(t *testing.T) {
|
|||||||
if !strings.Contains(env, "XDG_RUNTIME_DIR="+m.Store+"/xdg") {
|
if !strings.Contains(env, "XDG_RUNTIME_DIR="+m.Store+"/xdg") {
|
||||||
t.Errorf("the environment of %q lacks the module's own runtime directory: %s", c.line, env)
|
t.Errorf("the environment of %q lacks the module's own runtime directory: %s", c.line, env)
|
||||||
}
|
}
|
||||||
|
if !strings.Contains(env, "GOMAXPROCS=") {
|
||||||
|
t.Errorf("the environment of %q gives buildah no thread limit under a share (ADR 0553): %s", c.line, env)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// The image is removed from the store once the digest is read, so the store does not grow with every build.
|
// The image is removed from the store once the digest is read, so the store does not grow with every build.
|
||||||
if len(f.called("rmi")) != 1 {
|
if len(f.called("rmi")) != 1 {
|
||||||
@@ -117,16 +122,50 @@ func TestBuildRunsBuildahRootlessAndAnswersTheDigest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1703c: a failing Dockerfile step fails the step, saying buildah's last line.
|
// 1703c, issue 1703: a failing Dockerfile step fails the step, its reason saying buildah's error from stderr, never the
|
||||||
func TestBuildFailsWithBuildahsLastLine(t *testing.T) {
|
// last STEP line buildah writes on stdout; nothing is pushed.
|
||||||
m, f, states := setup(t, same)
|
func TestBuildFailsWithBuildahsError(t *testing.T) {
|
||||||
f.failBud = "Error: building at STEP \"RUN false\": exit status 1"
|
steps := "STEP 1/3: FROM docker.io/library/alpine\nSTEP 2/3: RUN apk add --no-cache ca-certificates\nfetch https://dl-cdn.alpinelinux.org/x\n"
|
||||||
_, st := runBuild(t, m, states, nil)
|
long := strings.Repeat("x", 2*CauseBytes)
|
||||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, `STEP "RUN false"`) {
|
cases := map[string]struct {
|
||||||
t.Errorf("step = %+v", st)
|
ran forgekey.Ran
|
||||||
|
want []string
|
||||||
|
wantNot []string
|
||||||
|
}{
|
||||||
|
"buildah's RUN failure": {forgekey.Ran{Status: 1, Stdout: steps + "STEP 3/3: COPY . /app\n",
|
||||||
|
Stderr: "error running container: from /usr/bin/crun creating container for [/bin/sh -c apk add]: sd-bus call: Interactive authentication required.\nError: building at STEP \"RUN apk add --no-cache ca-certificates\": while running runtime: exit status 1\n"},
|
||||||
|
[]string{"exit status 1", "Interactive authentication required", `building at STEP "RUN apk add`}, []string{"STEP 3/3", "COPY"}},
|
||||||
|
"a failed RUN": {forgekey.Ran{Status: 1, Stdout: steps, Stderr: "Error: building at STEP \"RUN false\": exit status 1\n"},
|
||||||
|
[]string{`STEP "RUN false"`}, []string{"dl-cdn"}},
|
||||||
|
"a long stderr is bounded, its end kept": {forgekey.Ran{Status: 125, Stdout: steps, Stderr: "early noise\n" + long + "\nError: the end\n"},
|
||||||
|
[]string{"Error: the end"}, []string{"early noise", long}},
|
||||||
|
"nothing on stderr": {forgekey.Ran{Status: 2, Stdout: steps}, []string{"exit status 2", "nothing on stderr", "dl-cdn"}, nil},
|
||||||
|
"never ran": {forgekey.Ran{Err: "ENOENT"}, []string{"ENOENT", "wrote nothing"}, nil},
|
||||||
}
|
}
|
||||||
if len(f.called("push")) != 0 {
|
for name, c := range cases {
|
||||||
t.Error("a failed build pushed")
|
m, f, states := setup(t, same)
|
||||||
|
f.failBud = c.ran
|
||||||
|
_, st := runBuild(t, m, states, nil)
|
||||||
|
if st.State != delivery.StepFailed {
|
||||||
|
t.Errorf("%s: step = %+v, want failed", name, st)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, w := range c.want {
|
||||||
|
if !strings.Contains(st.Reason, w) {
|
||||||
|
t.Errorf("%s: the reason %q lacks %q", name, st.Reason, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, w := range c.wantNot {
|
||||||
|
if strings.Contains(st.Reason, w) {
|
||||||
|
t.Errorf("%s: the reason %q carries %q", name, st.Reason, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(st.Reason) > CauseBytes+200 {
|
||||||
|
t.Errorf("%s: the reason is %d bytes, more than its bound", name, len(st.Reason))
|
||||||
|
}
|
||||||
|
if len(f.called("push")) != 0 {
|
||||||
|
t.Errorf("%s: a failed build pushed", name)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// Why a command failed, as the command said it (issue 1703). buildah writes its progress (STEP n/m, the RUN's own
|
||||||
|
// output) on stdout and its error on stderr, so the last line of both together is a STEP line and hides the error:
|
||||||
|
// a failure's reason is stderr's last lines, never stdout's, and the whole tail of stderr is said in the journal.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CauseBytes bounds what a failure's reason carries of the command's output; TailBytes bounds what the journal gets.
|
||||||
|
const (
|
||||||
|
CauseBytes = 600
|
||||||
|
TailBytes = 4096
|
||||||
|
)
|
||||||
|
|
||||||
|
// cause says why a command failed: its exit status (or why it never ran to one) and the last lines it wrote on stderr,
|
||||||
|
// at most CauseBytes of them, joined with " | ". Only when stderr is empty does stdout's tail stand in, said as such.
|
||||||
|
func cause(r forgekey.Ran) string {
|
||||||
|
head := exitOf(r)
|
||||||
|
if said := tailLines(r.Stderr, CauseBytes, " | "); said != "" {
|
||||||
|
return head + ": " + said
|
||||||
|
}
|
||||||
|
if said := tailLines(r.Stdout, CauseBytes, " | "); said != "" {
|
||||||
|
return head + ", nothing on stderr; stdout ends: " + said
|
||||||
|
}
|
||||||
|
return head + ", and it wrote nothing"
|
||||||
|
}
|
||||||
|
|
||||||
|
// exitOf is how a command ended: why it never ran to an exit status, or the status.
|
||||||
|
func exitOf(r forgekey.Ran) string {
|
||||||
|
if r.Err != "" {
|
||||||
|
return r.Err
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("exit status %d", r.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// tailLines is the last non-empty lines of s, trimmed, joined with sep, at most max bytes: the lines are taken whole
|
||||||
|
// from the end while they fit, and when even the last one does not, its end is kept behind "…".
|
||||||
|
func tailLines(s string, max int, sep string) string {
|
||||||
|
var lines []string
|
||||||
|
for _, l := range strings.Split(s, "\n") {
|
||||||
|
if l = strings.TrimSpace(l); l != "" {
|
||||||
|
lines = append(lines, l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var keep []string
|
||||||
|
n := 0
|
||||||
|
for i := len(lines) - 1; i >= 0; i-- {
|
||||||
|
add := len(lines[i])
|
||||||
|
if len(keep) > 0 {
|
||||||
|
add += len(sep)
|
||||||
|
}
|
||||||
|
if n+add > max {
|
||||||
|
if len(keep) == 0 {
|
||||||
|
keep = append(keep, "…"+cutFront(lines[i], max-len("…")))
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
keep = append([]string{lines[i]}, keep...)
|
||||||
|
n += add
|
||||||
|
}
|
||||||
|
return strings.Join(keep, sep)
|
||||||
|
}
|
||||||
|
|
||||||
|
// cutFront is the last at most max bytes of s, starting on a whole character.
|
||||||
|
func cutFront(s string, max int) string {
|
||||||
|
if max <= 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if len(s) <= max {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
s = s[len(s)-max:]
|
||||||
|
for len(s) > 0 && !utf8.RuneStart(s[0]) {
|
||||||
|
s = s[1:]
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayFailure writes the tail of a failed command's stderr (and of its stdout, where buildah's RUN output goes) to the
|
||||||
|
// journal, so what the reason had to leave out can still be read there.
|
||||||
|
func (m *Module) sayFailure(what string, r forgekey.Ran) {
|
||||||
|
m.say("%s failed (%s); stderr ends:\n%s\nstdout ends:\n%s", what, exitOf(r),
|
||||||
|
tailLines(r.Stderr, TailBytes, "\n"), tailLines(r.Stdout, TailBytes, "\n"))
|
||||||
|
}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildcpu"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The processors a step may take (novox/hq ADR 0553, issue 1806). The node-engine holds this module's runtime, with
|
||||||
|
// every other build module's of the node, in one slice at idle priority under the node's one share of its processors:
|
||||||
|
// that quota bounds the total. Threads only size each step: the share the composer gives (buildstep.NodeLimits) over
|
||||||
|
// this module's own capacity, so a step does not wait in the kernel's throttle. Without the composer's share there is
|
||||||
|
// no slice to size a step to, and no thread limit is set.
|
||||||
|
|
||||||
|
// cpuLimits is what one step is held to now.
|
||||||
|
func (m *Module) cpuLimits() buildcpu.Limits {
|
||||||
|
slots := 1
|
||||||
|
switch {
|
||||||
|
case m.Capacity != nil:
|
||||||
|
slots = m.Capacity()
|
||||||
|
case m.Settings != "":
|
||||||
|
slots = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, nil)()
|
||||||
|
}
|
||||||
|
return buildstep.NodeLimits(slots)
|
||||||
|
}
|
||||||
|
|
||||||
|
// threadEnv is the environment (nothing without a share) buildah is given for its threads (it is a Go program). What a build's RUN lines start
|
||||||
|
// runs in the container's own environment, inside the runtime's cgroup: the slice's quota and the nice level bind it.
|
||||||
|
func (m *Module) threadEnv() []string {
|
||||||
|
return buildcpu.Env(m.cpuLimits().Threads)
|
||||||
|
}
|
||||||
|
|
||||||
|
// cpuStatus is what the status tool says of it: the limits a step is given, and what the kernel holds this runtime
|
||||||
|
// to (its unit and its slice), read from its cgroup rather than assumed.
|
||||||
|
func (m *Module) cpuStatus() map[string]any {
|
||||||
|
return map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()}
|
||||||
|
}
|
||||||
@@ -70,8 +70,8 @@ type fake struct {
|
|||||||
calls []cmdRecord
|
calls []cmdRecord
|
||||||
digest string // what `buildah images` prints
|
digest string // what `buildah images` prints
|
||||||
pushes int
|
pushes int
|
||||||
refuse int // the first n pushes answer "connection refused"
|
refuse int // the first n pushes answer "connection refused"
|
||||||
failBud string
|
failBud forgekey.Ran // what a failing buildah bud answers; a zero Status is a passing bud
|
||||||
// digestFile: "" writes the digest, "none" writes no file, "empty" writes an empty one.
|
// digestFile: "" writes the digest, "none" writes no file, "empty" writes an empty one.
|
||||||
digestFile string
|
digestFile string
|
||||||
// registry: repository:tag -> digest
|
// registry: repository:tag -> digest
|
||||||
@@ -116,8 +116,8 @@ func (f *fake) run(ctx context.Context, env []string, name string, args ...strin
|
|||||||
case name == "buildah" && strings.Contains(line, "--version"):
|
case name == "buildah" && strings.Contains(line, "--version"):
|
||||||
return forgekey.Ran{Stdout: "buildah version 1.45.1 (image-spec 1.1.0, runtime-spec 1.1.0)\n"}
|
return forgekey.Ran{Stdout: "buildah version 1.45.1 (image-spec 1.1.0, runtime-spec 1.1.0)\n"}
|
||||||
case name == "buildah" && strings.Contains(line, " bud "):
|
case name == "buildah" && strings.Contains(line, " bud "):
|
||||||
if f.failBud != "" {
|
if f.failBud.Status != 0 || f.failBud.Err != "" {
|
||||||
return forgekey.Ran{Status: 1, Stderr: "STEP 1/2: FROM x\n" + f.failBud + "\n"}
|
return f.failBud
|
||||||
}
|
}
|
||||||
case name == "buildah" && strings.Contains(line, " images "):
|
case name == "buildah" && strings.Contains(line, " images "):
|
||||||
return forgekey.Ran{Stdout: f.digest + "\n"}
|
return forgekey.Ran{Stdout: f.digest + "\n"}
|
||||||
|
|||||||
@@ -203,6 +203,8 @@ func TestARealBuildIsRootlessWithNoSocketAndNoWayOut(t *testing.T) {
|
|||||||
imageModule(t, m, "FROM "+base+"\nRUN echo b > /proc/sysrq-trigger\n", nil)
|
imageModule(t, m, "FROM "+base+"\nRUN echo b > /proc/sysrq-trigger\n", nil)
|
||||||
if _, st := buildReal(t, m, states); st.State != delivery.StepFailed {
|
if _, st := buildReal(t, m, states); st.State != delivery.StepFailed {
|
||||||
t.Errorf("a write to the host's /proc/sysrq-trigger passed: %+v", st)
|
t.Errorf("a write to the host's /proc/sysrq-trigger passed: %+v", st)
|
||||||
|
} else if strings.Contains(st.Reason, "setup network") {
|
||||||
|
t.Errorf("the RUN never started, so the write was not what failed (issue 1703): %s", st.Reason)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,9 +9,11 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"os"
|
"os"
|
||||||
"os/user"
|
"os/user"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -57,7 +59,13 @@ func main() {
|
|||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
steps := m.StepTools()
|
steps := m.StepTools()
|
||||||
|
booted := time.Now()
|
||||||
puller := m.Puller(steps, nil)
|
puller := m.Puller(steps, nil)
|
||||||
|
m.Held = puller
|
||||||
|
if m.Node != "" {
|
||||||
|
// A step a previous process left running is ended (issue 1890); this process's own steps start after booted.
|
||||||
|
go buildstep.EndStaleAtStart(ctx, m.stepScan(), m.Node, booted, logf)
|
||||||
|
}
|
||||||
if m.Node != "" {
|
if m.Node != "" {
|
||||||
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
|
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
|
||||||
go func() { _ = puller.Pull(ctx) }()
|
go func() { _ = puller.Pull(ctx) }()
|
||||||
|
|||||||
@@ -22,7 +22,9 @@ type Module struct {
|
|||||||
// Account and UID are the account the module runs as; Home is its home.
|
// Account and UID are the account the module runs as; Home is its home.
|
||||||
Account, UID, Home string
|
Account, UID, Home string
|
||||||
// Steps is the Put of the steps state; ReadSteps reads it back for the read tools.
|
// Steps is the Put of the steps state; ReadSteps reads it back for the read tools.
|
||||||
Steps buildstep.Put
|
Steps buildstep.Put
|
||||||
|
// Held is the puller whose running asks a step-end tool will not end (issue 1890); nil in a test.
|
||||||
|
Held *buildstep.Puller
|
||||||
ReadSteps StateRead
|
ReadSteps StateRead
|
||||||
// Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back.
|
// Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back.
|
||||||
Pauses buildstep.Put
|
Pauses buildstep.Put
|
||||||
@@ -104,6 +106,7 @@ func (m *Module) Start(ctx context.Context) {
|
|||||||
m.Say("this machine cannot build images: %v", err)
|
m.Say("this machine cannot build images: %v", err)
|
||||||
}
|
}
|
||||||
m.CleanWorkspaces(m.clock())
|
m.CleanWorkspaces(m.clock())
|
||||||
|
buildstep.EndStaleRound(m.stepScan(), m.Node, m.Say) // asks past the step bound, on the module's own round (issue 1890)
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
@@ -128,7 +131,8 @@ func (m *Module) Health(ctx context.Context) health.Answer {
|
|||||||
// Status is what the module reports of itself.
|
// Status is what the module reports of itself.
|
||||||
func (m *Module) Status(ctx context.Context) map[string]any {
|
func (m *Module) Status(ctx context.Context) map[string]any {
|
||||||
l, err := m.look(ctx)
|
l, err := m.look(ctx)
|
||||||
out := map[string]any{"node": m.Node, "account": m.Account, "buildah": l.Buildah, "subuids": l.UIDs, "subgids": l.GIDs, "refusing": err != nil}
|
out := map[string]any{"node": m.Node, "account": m.Account, "buildah": l.Buildah, "subuids": l.UIDs, "subgids": l.GIDs, "refusing": err != nil,
|
||||||
|
"cpu": m.cpuStatus()}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
out["why"] = err.Error()
|
out["why"] = err.Error()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -117,7 +117,7 @@ func (m *Module) storeFlags() []string {
|
|||||||
// error, and the next look refuses with it.
|
// error, and the next look refuses with it.
|
||||||
func (m *Module) env() []string {
|
func (m *Module) env() []string {
|
||||||
_ = m.ensureRuntimeDir()
|
_ = m.ensureRuntimeDir()
|
||||||
return []string{"XDG_RUNTIME_DIR=" + m.runtimeDir()}
|
return append([]string{"XDG_RUNTIME_DIR=" + m.runtimeDir()}, m.threadEnv()...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// runtimeDir is buildah's runtime directory, inside the store.
|
// runtimeDir is buildah's runtime directory, inside the store.
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RegistryWait is how long a call waits for a registry that refuses connections, longer than the nightly collection
|
// RegistryWait is how long a call waits for a registry that refuses connections, longer than the nightly collection
|
||||||
@@ -47,13 +49,15 @@ func (m *Module) push(ctx context.Context, workspace, module, local string, res
|
|||||||
dest := addr + "/" + repository + ":" + tag
|
dest := addr + "/" + repository + ":" + tag
|
||||||
digestFile := filepath.Join(m.Work, workspace, "pushed."+tag)
|
digestFile := filepath.Join(m.Work, workspace, "pushed."+tag)
|
||||||
argv := append(m.storeFlags(), "push", "--tls-verify=false", "--digestfile", digestFile, local, "docker://"+dest)
|
argv := append(m.storeFlags(), "push", "--tls-verify=false", "--digestfile", digestFile, local, "docker://"+dest)
|
||||||
|
var last forgekey.Ran
|
||||||
if err := m.waitForRegistry(ctx, addr, "buildah push "+dest, func() error {
|
if err := m.waitForRegistry(ctx, addr, "buildah push "+dest, func() error {
|
||||||
r := m.Run(ctx, m.env(), "buildah", argv...)
|
last = m.Run(ctx, m.env(), "buildah", argv...)
|
||||||
if r.Status != 0 || r.Err != "" {
|
if last.Status != 0 || last.Err != "" {
|
||||||
return errors.New(lastLine(r.Err + "\n" + r.Stderr + "\n" + r.Stdout))
|
return errors.New(cause(last))
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
|
m.sayFailure("buildah push "+dest, last)
|
||||||
return nil, fmt.Errorf("pushing %s: %w", dest, err)
|
return nil, fmt.Errorf("pushing %s: %w", dest, err)
|
||||||
}
|
}
|
||||||
var served string
|
var served string
|
||||||
|
|||||||
@@ -127,6 +127,7 @@ type StepLine struct {
|
|||||||
Started string `json:"started"`
|
Started string `json:"started"`
|
||||||
Ended string `json:"ended,omitempty"`
|
Ended string `json:"ended,omitempty"`
|
||||||
Reason string `json:"reason,omitempty"`
|
Reason string `json:"reason,omitempty"`
|
||||||
|
What string `json:"what,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// RecentSteps are this node's steps, the latest started first, at most limit (ten when limit is not above zero). Only
|
// RecentSteps are this node's steps, the latest started first, at most limit (ten when limit is not above zero). Only
|
||||||
@@ -179,7 +180,7 @@ func (m *Module) RecentSteps(limit int) (lines []StepLine, cut bool, err error)
|
|||||||
}
|
}
|
||||||
out := make([]StepLine, 0, len(steps))
|
out := make([]StepLine, 0, len(steps))
|
||||||
for _, s := range steps {
|
for _, s := range steps {
|
||||||
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason}
|
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason, What: s.What}
|
||||||
if !s.Ended.IsZero() {
|
if !s.Ended.IsZero() {
|
||||||
l.Ended = s.Ended.UTC().Format("2006-01-02T15:04:05Z")
|
l.Ended = s.Ended.UTC().Format("2006-01-02T15:04:05Z")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
|
|
||||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
)
|
)
|
||||||
@@ -44,6 +45,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
|||||||
tools = append(tools, t)
|
tools = append(tools, t)
|
||||||
}
|
}
|
||||||
tools = append(tools, m.ReadTools()...)
|
tools = append(tools, m.ReadTools()...)
|
||||||
|
tools = append(tools, buildstep.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held))
|
||||||
tools = append(tools, m.releaseTool())
|
tools = append(tools, m.releaseTool())
|
||||||
return append(tools,
|
return append(tools,
|
||||||
stdio.Tool{Name: StatusTool,
|
stdio.Tool{Name: StatusTool,
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module meshbuildimage
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025175617-518dae5ec9c8
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025175617-518dae5ec9c8 h1:x4mvY7yiGTP5lnzxbiwyj1o24p05hAmzZP7xHg3XX2M=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025175617-518dae5ec9c8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -33,6 +33,7 @@
|
|||||||
"build_image_release",
|
"build_image_release",
|
||||||
"build_image_step",
|
"build_image_step",
|
||||||
"build_image_steps",
|
"build_image_steps",
|
||||||
|
"build_image_step_end",
|
||||||
"build_image_status",
|
"build_image_status",
|
||||||
"build_image_health"
|
"build_image_health"
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -74,3 +74,11 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
|
|||||||
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
|
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
|
||||||
return buildstep.SeatTools(Seat, "Node", p)
|
return buildstep.SeatTools(Seat, "Node", p)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes.
|
||||||
|
const StepEndTool = "build_node_step_end"
|
||||||
|
|
||||||
|
// stepScan is the module's steps state as buildstep reads it.
|
||||||
|
func (m *Module) stepScan() buildstep.Scan {
|
||||||
|
return buildstep.ScanOf(stdio.State(delivery.StepsState))
|
||||||
|
}
|
||||||
|
|||||||
@@ -267,3 +267,38 @@ func TestTheLaunchersAreTheSDKsRule(t *testing.T) {
|
|||||||
t.Errorf("launchers %v, mesh-sdk's rule %v", got, want)
|
t.Errorf("launchers %v, mesh-sdk's rule %v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// issue 1703: a failed command's reason keeps stderr's end and stdout's start, each bounded, so a long stdout (npm's
|
||||||
|
// progress) never pushes npm's error out, and tsc's first error on stdout stays in.
|
||||||
|
func TestRanFailedKeepsTheError(t *testing.T) {
|
||||||
|
noise := strings.Repeat("npm progress line\n", 200)
|
||||||
|
cases := map[string]struct {
|
||||||
|
ran forgekey.Ran
|
||||||
|
want []string
|
||||||
|
wantNot []string
|
||||||
|
}{
|
||||||
|
"passed": {forgekey.Ran{Stdout: "ok"}, nil, nil},
|
||||||
|
"npm, long stdout": {forgekey.Ran{Status: 1, Stdout: noise, Stderr: "npm ERR! early\n" + noise + "npm ERR! code E404\n"}, []string{"status 1", "npm ERR! code E404"}, []string{"npm ERR! early"}},
|
||||||
|
"tsc on stdout": {forgekey.Ran{Status: 2, Stdout: "src/a.ts(1,7): error TS2322: wrong\n" + noise}, []string{"error TS2322"}, nil},
|
||||||
|
"never ran": {forgekey.Ran{Err: "ENOENT"}, []string{"ENOENT"}, nil},
|
||||||
|
}
|
||||||
|
for name, c := range cases {
|
||||||
|
got := ranFailed(c.ran)
|
||||||
|
if c.want == nil && got != "" {
|
||||||
|
t.Errorf("%s: %q, want none", name, got)
|
||||||
|
}
|
||||||
|
for _, w := range c.want {
|
||||||
|
if !strings.Contains(got, w) {
|
||||||
|
t.Errorf("%s: %q lacks %q", name, got, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, w := range c.wantNot {
|
||||||
|
if strings.Contains(got, w) {
|
||||||
|
t.Errorf("%s: %q carries %q", name, got, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(got) > failedBytes+100 {
|
||||||
|
t.Errorf("%s: %d bytes, past its bound", name, len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
# Its integration tests copy the Node toolchain the Node build module installed on this node and run node and npm
|
||||||
|
# (realMachine, build_integration_test.go): they run only where the node holds node-build-node (issue 890).
|
||||||
|
node-build-node
|
||||||
@@ -56,11 +56,15 @@ type TestResult struct {
|
|||||||
Failed []Failure `json:"failed,omitempty"`
|
Failed []Failure `json:"failed,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Failure is a check or script that failed; Output is the first lines it wrote.
|
// Failure is a check or script that failed, or one failing test of a test script.
|
||||||
type Failure struct {
|
type Failure struct {
|
||||||
Package string `json:"package"`
|
Package string `json:"package"`
|
||||||
Test string `json:"test,omitempty"`
|
Test string `json:"test,omitempty"`
|
||||||
Output string `json:"output,omitempty"`
|
// File is the failing test's location (file:line:column) when node --test named it, Cause the first line of its
|
||||||
|
// error, and Output its own output, Cause first, up to 4096 bytes (issue 1901), as mesh-build-go's.
|
||||||
|
File string `json:"file,omitempty"`
|
||||||
|
Cause string `json:"cause,omitempty"`
|
||||||
|
Output string `json:"output,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Module) testTool() StepTool {
|
func (m *Module) testTool() StepTool {
|
||||||
@@ -128,11 +132,19 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
|||||||
var check, run time.Duration
|
var check, run time.Duration
|
||||||
var failed []Failure
|
var failed []Failure
|
||||||
// a command that could not start is not a verdict: the step fails. One that ran and said no is.
|
// a command that could not start is not a verdict: the step fails. One that ran and said no is.
|
||||||
ran := func(name string, r forgekey.Ran) error {
|
ran := func(name string, r forgekey.Ran, tests bool) error {
|
||||||
if r.Err != "" {
|
if r.Err != "" {
|
||||||
return fmt.Errorf("%s could not be run: status %d %s: %s", name, r.Status, r.Err, firstLine(r.Stderr))
|
return fmt.Errorf("%s could not be run: status %d %s: %s", name, r.Status, r.Err, firstLine(r.Stderr))
|
||||||
}
|
}
|
||||||
if r.Status != 0 {
|
if r.Status != 0 && tests {
|
||||||
|
// the test script ran under node's TAP reporter: each failing test is a failure of its own, never a passing line
|
||||||
|
for _, f := range tapFailures(where, r.Stdout+"\n"+r.Stderr) {
|
||||||
|
if f.Test == "" {
|
||||||
|
f.Test = name
|
||||||
|
}
|
||||||
|
failed = append(failed, f)
|
||||||
|
}
|
||||||
|
} else if r.Status != 0 {
|
||||||
failed = append(failed, Failure{Package: where, Test: name, Output: lines(r.Stdout + "\n" + r.Stderr)})
|
failed = append(failed, Failure{Package: where, Test: name, Output: lines(r.Stdout + "\n" + r.Stderr)})
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -148,17 +160,17 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
|||||||
start := m.now()
|
start := m.now()
|
||||||
r := m.runIn(ctx, root, false, dir, nil, "node", line...)
|
r := m.runIn(ctx, root, false, dir, nil, "node", line...)
|
||||||
check = m.now().Sub(start)
|
check = m.now().Sub(start)
|
||||||
if err := ran(name, r); err != nil {
|
if err := ran(name, r, false); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
res.Ran = append(res.Ran, name)
|
res.Ran = append(res.Ran, name)
|
||||||
}
|
}
|
||||||
if hasScript && len(failed) == 0 {
|
if hasScript && len(failed) == 0 {
|
||||||
start := m.now()
|
start := m.now()
|
||||||
env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm")}
|
env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "NODE_OPTIONS=" + TestReporterOption, "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm")}
|
||||||
r := m.runIn(ctx, root, false, dir, env, "npm", scriptArgs(script)...)
|
r := m.runIn(ctx, root, false, dir, env, "npm", scriptArgs(script)...)
|
||||||
run = m.now().Sub(start)
|
run = m.now().Sub(start)
|
||||||
if err := ran(script, r); err != nil {
|
if err := ran(script, r, true); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
res.Ran = append(res.Ran, script)
|
res.Ran = append(res.Ran, script)
|
||||||
@@ -166,7 +178,7 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
|||||||
res.Failed = failed
|
res.Failed = failed
|
||||||
var why []string
|
var why []string
|
||||||
for _, f := range failed {
|
for _, f := range failed {
|
||||||
why = append(why, fmt.Sprintf("%s failed in %s: %s", f.Test, f.Package, firstLine(f.Output)))
|
why = append(why, failureWhy(f))
|
||||||
}
|
}
|
||||||
if level == "unit" {
|
if level == "unit" {
|
||||||
res.Elapsed, res.Bound = (check + run).Seconds(), UnitBound.Seconds()
|
res.Elapsed, res.Bound = (check + run).Seconds(), UnitBound.Seconds()
|
||||||
@@ -280,3 +292,16 @@ func typeScriptFiles(dir string) ([]string, error) {
|
|||||||
})
|
})
|
||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// failureWhy is the words of one failure: the check or test that failed, where, and its cause (the first line of its output
|
||||||
|
// when it has none).
|
||||||
|
func failureWhy(f Failure) string {
|
||||||
|
if f.Cause == "" {
|
||||||
|
return fmt.Sprintf("%s failed in %s: %s", f.Test, f.Package, firstLine(f.Output))
|
||||||
|
}
|
||||||
|
file := ""
|
||||||
|
if f.File != "" {
|
||||||
|
file = " (" + f.File + ")"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("test %q failed in %s%s: %s", f.Test, f.Package, file, firstLine(f.Cause))
|
||||||
|
}
|
||||||
|
|||||||
@@ -138,7 +138,7 @@ func TestAFailingScriptIsAVerdictAndACommandThatCannotStartIsAFailedStep(t *test
|
|||||||
f.answer["npm"] = forgekey.Ran{Status: 1, Stdout: "not ok 1 - adds\n"}
|
f.answer["npm"] = forgekey.Ran{Status: 1, Stdout: "not ok 1 - adds\n"}
|
||||||
out, err := m.test(context.Background(), testArgs("unit"))
|
out, err := m.test(context.Background(), testArgs("unit"))
|
||||||
r, _ := out.(*TestResult)
|
r, _ := out.(*TestResult)
|
||||||
if err != nil || r == nil || r.Passed || len(r.Failed) != 1 || r.Failed[0].Test != "test" || !strings.Contains(r.Failed[0].Output, "not ok 1") {
|
if err != nil || r == nil || r.Passed || len(r.Failed) != 1 || r.Failed[0].Test != "adds" || !strings.Contains(r.Failed[0].Output, "not ok 1") {
|
||||||
t.Errorf("a failing script = %+v, %v, want passed false and its output", r, err)
|
t.Errorf("a failing script = %+v, %v, want passed false and its output", r, err)
|
||||||
}
|
}
|
||||||
f.answer["npm"] = forgekey.Ran{Status: 127, Err: "ENOENT"}
|
f.answer["npm"] = forgekey.Ran{Status: 127, Err: "ENOENT"}
|
||||||
@@ -199,3 +199,38 @@ func TestTheTestScriptFindsTscAndEsbuildOnItsPath(t *testing.T) {
|
|||||||
t.Errorf("npm ran with PATH %q, want %s first", path, bin)
|
t.Errorf("npm ran with PATH %q, want %s first", path, bin)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1901: a failing node --test run is reported test by test, from node's TAP reporter, in the output captured in
|
||||||
|
// testdata/node-test-tap: each failing test with its name, file and cause, never a passing line of an earlier file.
|
||||||
|
func TestFailingNodeTestsAreReportedEachWithTheirCause(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
file, test, location, cause, inOutput string
|
||||||
|
n int
|
||||||
|
}{
|
||||||
|
{"two-files", "fails in the second file", "/work/two-files/test/b.test.mjs:3:1", "one plus one is not three", "2 !== 3", 1},
|
||||||
|
{"suite", "fails inside", "/work/suite/test/s.test.mjs:5:3", "Expected values to be strictly deep-equal:", "operator: 'deepStrictEqual'", 1},
|
||||||
|
{"syntax", "test/b.test.mjs", "/work/syntax/test/b.test.mjs:1:1", "", "SyntaxError: Unexpected end of input", 1},
|
||||||
|
} {
|
||||||
|
out, err := os.ReadFile(filepath.Join("testdata", "node-test-tap", c.file+".tap"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := tapFailures("p", string(out))
|
||||||
|
if len(got) != c.n {
|
||||||
|
t.Fatalf("%s: %d failures %+v, want %d", c.file, len(got), got, c.n)
|
||||||
|
}
|
||||||
|
f := got[0]
|
||||||
|
if c.file == "syntax" {
|
||||||
|
c.cause = "SyntaxError: Unexpected end of input" // node names the file, the error is in the lines before it
|
||||||
|
}
|
||||||
|
if f.Test != c.test || f.File != c.location || !strings.HasPrefix(f.Cause, c.cause) ||
|
||||||
|
!strings.Contains(f.Output, c.inOutput) || len(f.Output) > 4096+len(f.Cause)+64 {
|
||||||
|
t.Errorf("%s: failure %+v, want test %q at %q, cause %q, output with %q", c.file, f, c.test, c.location, c.cause, c.inOutput)
|
||||||
|
}
|
||||||
|
for _, l := range strings.Split(f.Output, "\n") {
|
||||||
|
if strings.HasPrefix(strings.TrimSpace(l), "ok ") || strings.Contains(l, "passes in the first file") {
|
||||||
|
t.Errorf("%s: output keeps the passing line %q", c.file, l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildcpu"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The processors a step may take (novox/hq ADR 0553, issue 1806). The node-engine holds this module's runtime, with
|
||||||
|
// every other build module's of the node, in one slice at idle priority under the node's one share of its processors:
|
||||||
|
// that quota bounds the total. Threads only size each step: the share the composer gives (buildstep.NodeLimits) over
|
||||||
|
// this module's own capacity, so a step does not wait in the kernel's throttle. Without the composer's share there is
|
||||||
|
// no slice to size a step to, and no thread limit is set.
|
||||||
|
|
||||||
|
// cpuLimits is what one step is held to now.
|
||||||
|
func (m *Module) cpuLimits() buildcpu.Limits {
|
||||||
|
slots := 1
|
||||||
|
switch {
|
||||||
|
case m.Capacity != nil:
|
||||||
|
slots = m.Capacity()
|
||||||
|
case m.Settings != "":
|
||||||
|
slots = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, nil)()
|
||||||
|
}
|
||||||
|
return buildstep.NodeLimits(slots)
|
||||||
|
}
|
||||||
|
|
||||||
|
// threadEnv is the environment (nothing without a share) every command of a step is given for its threads: GOMAXPROCS for esbuild and every Go
|
||||||
|
// program, UV_THREADPOOL_SIZE for Node's worker pool.
|
||||||
|
func (m *Module) threadEnv() []string {
|
||||||
|
return buildcpu.Env(m.cpuLimits().Threads)
|
||||||
|
}
|
||||||
|
|
||||||
|
// cpuStatus is what the status tool says of it: the limits a step is given, and what the kernel holds this runtime
|
||||||
|
// to (its unit and its slice), read from its cgroup rather than assumed.
|
||||||
|
func (m *Module) cpuStatus() map[string]any {
|
||||||
|
return map[string]any{"limits": m.cpuLimits(), "in-force": buildcpu.ReadInForce()}
|
||||||
|
}
|
||||||
@@ -212,7 +212,7 @@ func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, grou
|
|||||||
func (m *Module) typeScriptModules(ctx context.Context, tree string) (dirs []string, packages map[string]bool, err error) {
|
func (m *Module) typeScriptModules(ctx context.Context, tree string) (dirs []string, packages map[string]bool, err error) {
|
||||||
r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json")
|
r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json")
|
||||||
if r.Status != 0 || r.Err != "" {
|
if r.Status != 0 || r.Err != "" {
|
||||||
return nil, nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
return nil, nil, fmt.Errorf("git ls-files: %s", forgekey.Said(r.Err+"\n"+r.Stderr))
|
||||||
}
|
}
|
||||||
packages = map[string]bool{}
|
packages = map[string]bool{}
|
||||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -225,6 +226,7 @@ func mustReadDir(t *testing.T, dir string) []string {
|
|||||||
// are given PATH, HOME and the locale and their own words; and the real clean runner, run on `env` here, prints only
|
// are given PATH, HOME and the locale and their own words; and the real clean runner, run on `env` here, prints only
|
||||||
// those.
|
// those.
|
||||||
func TestNoCommandOfTheStepSeesATokenOfTheProcess(t *testing.T) {
|
func TestNoCommandOfTheStepSeesATokenOfTheProcess(t *testing.T) {
|
||||||
|
t.Setenv("MESH_BUILD_CPU_SHARE", "50")
|
||||||
t.Setenv("MESH_BUS_CREDENTIAL", "hunter2")
|
t.Setenv("MESH_BUS_CREDENTIAL", "hunter2")
|
||||||
t.Setenv("NPM_TOKEN", "hunter3")
|
t.Setenv("NPM_TOKEN", "hunter3")
|
||||||
files := typeScriptModule(`{"dependencies":{"lodash":"^4"}}`)
|
files := typeScriptModule(`{"dependencies":{"lodash":"^4"}}`)
|
||||||
@@ -245,6 +247,11 @@ func TestNoCommandOfTheStepSeesATokenOfTheProcess(t *testing.T) {
|
|||||||
if len(c.env) < 3 || !strings.HasPrefix(c.env[0], "PATH=") || !strings.HasPrefix(c.env[1], "HOME=") || c.env[2] != "LC_ALL=C" {
|
if len(c.env) < 3 || !strings.HasPrefix(c.env[0], "PATH=") || !strings.HasPrefix(c.env[1], "HOME=") || c.env[2] != "LC_ALL=C" {
|
||||||
t.Errorf("%s was given %v, want PATH, HOME and the locale first", c.name, c.env)
|
t.Errorf("%s was given %v, want PATH, HOME and the locale first", c.name, c.env)
|
||||||
}
|
}
|
||||||
|
// And its threads, its part of the node's share the composer gave (ADR 0553).
|
||||||
|
if !slices.ContainsFunc(c.env, func(e string) bool { return strings.HasPrefix(e, "GOMAXPROCS=") }) ||
|
||||||
|
!slices.ContainsFunc(c.env, func(e string) bool { return strings.HasPrefix(e, "UV_THREADPOOL_SIZE=") }) {
|
||||||
|
t.Errorf("%s was given %v, without its threads", c.name, c.env)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// The runner main wires: what it hands a command is exactly what the step gave.
|
// The runner main wires: what it hands a command is exactly what the step gave.
|
||||||
ran := forgekey.ExecRunnerClean(10*time.Second)(context.Background(), append(m.baseEnv(), "ONLY=this"), "env")
|
ran := forgekey.ExecRunnerClean(10*time.Second)(context.Background(), append(m.baseEnv(), "ONLY=this"), "env")
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ func (m *Module) isolation(root string, write bool) buildstep.Isolation {
|
|||||||
ReadOnly: []string{m.toolchainDir()},
|
ReadOnly: []string{m.toolchainDir()},
|
||||||
Caches: []string{m.npmCache()},
|
Caches: []string{m.npmCache()},
|
||||||
WriteCaches: write,
|
WriteCaches: write,
|
||||||
Env: []string{"PATH=" + pathOrDefault(), "LC_ALL=C"},
|
Env: append([]string{"PATH=" + pathOrDefault(), "LC_ALL=C"}, m.threadEnv()...),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,10 +9,12 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"os"
|
"os"
|
||||||
"os/user"
|
"os/user"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -69,7 +71,13 @@ func main() {
|
|||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
steps := m.StepTools()
|
steps := m.StepTools()
|
||||||
|
booted := time.Now()
|
||||||
puller := m.Puller(steps, nil)
|
puller := m.Puller(steps, nil)
|
||||||
|
m.Held = puller
|
||||||
|
if m.Node != "" {
|
||||||
|
// A step a previous process left running is ended (issue 1890); this process's own steps start after booted.
|
||||||
|
go buildstep.EndStaleAtStart(ctx, m.stepScan(), m.Node, booted, logf)
|
||||||
|
}
|
||||||
if m.Node != "" {
|
if m.Node != "" {
|
||||||
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
|
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
|
||||||
go func() { _ = puller.Pull(ctx) }()
|
go func() { _ = puller.Pull(ctx) }()
|
||||||
|
|||||||
@@ -42,6 +42,8 @@ type Module struct {
|
|||||||
// Toolchains and Steps are the Put of the states of those names.
|
// Toolchains and Steps are the Put of the states of those names.
|
||||||
Toolchains, Steps buildstep.Put
|
Toolchains, Steps buildstep.Put
|
||||||
// ReadSteps and ReadToolchainsState read those two states back for the read tools (issue 1562).
|
// ReadSteps and ReadToolchainsState read those two states back for the read tools (issue 1562).
|
||||||
|
// Held is the puller whose running asks a step-end tool will not end (issue 1890); nil in a test.
|
||||||
|
Held *buildstep.Puller
|
||||||
ReadSteps, ReadToolchainsState StateRead
|
ReadSteps, ReadToolchainsState StateRead
|
||||||
// Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back.
|
// Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back.
|
||||||
Pauses buildstep.Put
|
Pauses buildstep.Put
|
||||||
@@ -137,6 +139,7 @@ func (m *Module) Start(ctx context.Context) {
|
|||||||
m.Say("the toolchain: %v", err)
|
m.Say("the toolchain: %v", err)
|
||||||
}
|
}
|
||||||
m.CleanWorkspaces(m.now())
|
m.CleanWorkspaces(m.now())
|
||||||
|
buildstep.EndStaleRound(m.stepScan(), m.Node, m.Say) // asks past the step bound, on the module's own round (issue 1890)
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
@@ -202,7 +205,8 @@ func (m *Module) Status(ctx context.Context) map[string]any {
|
|||||||
err := m.Check(ctx)
|
err := m.Check(ctx)
|
||||||
installed, _ := m.Probe(ctx)
|
installed, _ := m.Probe(ctx)
|
||||||
r := m.recipe()
|
r := m.recipe()
|
||||||
out := map[string]any{"node": m.Node, "recipe": r.Version(), "installed": installed, "refusing": err != nil}
|
out := map[string]any{"node": m.Node, "recipe": r.Version(), "installed": installed, "refusing": err != nil,
|
||||||
|
"cpu": m.cpuStatus()}
|
||||||
if m.Toolchain != nil {
|
if m.Toolchain != nil {
|
||||||
out["toolchain-dir"] = m.Toolchain.Paths.Dir(r.Version())
|
out["toolchain-dir"] = m.Toolchain.Paths.Dir(r.Version())
|
||||||
out["toolchain-installed"] = m.Toolchain.Present(r.Version())
|
out["toolchain-installed"] = m.Toolchain.Present(r.Version())
|
||||||
@@ -227,7 +231,7 @@ const defaultPath = "/usr/local/bin:/usr/bin:/bin"
|
|||||||
// files and git configuration) and a fixed locale. A command is given these and its own words, never this process's
|
// files and git configuration) and a fixed locale. A command is given these and its own words, never this process's
|
||||||
// environment, which holds the module's bus credentials (ADR 0466 §4).
|
// environment, which holds the module's bus credentials (ADR 0466 §4).
|
||||||
func (m *Module) baseEnv() []string {
|
func (m *Module) baseEnv() []string {
|
||||||
return []string{"PATH=" + pathOrDefault(), "HOME=" + m.Toolchain.homeOrEmpty(), "LC_ALL=C"}
|
return append([]string{"PATH=" + pathOrDefault(), "HOME=" + m.Toolchain.homeOrEmpty(), "LC_ALL=C"}, m.threadEnv()...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func pathOrDefault() string {
|
func pathOrDefault() string {
|
||||||
|
|||||||
@@ -117,7 +117,7 @@ func TestANodeThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T)
|
|||||||
{Name: "short_probe", Run: run},
|
{Name: "short_probe", Run: run},
|
||||||
}, (&Module{}).StepTools()...)
|
}, (&Module{}).StepTools()...)
|
||||||
for _, tool := range m.Tools(steps) {
|
for _, tool := range m.Tools(steps) {
|
||||||
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool {
|
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == StepEndTool {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var reason string
|
var reason string
|
||||||
|
|||||||
@@ -128,6 +128,7 @@ type StepLine struct {
|
|||||||
Started string `json:"started"`
|
Started string `json:"started"`
|
||||||
Ended string `json:"ended,omitempty"`
|
Ended string `json:"ended,omitempty"`
|
||||||
Reason string `json:"reason,omitempty"`
|
Reason string `json:"reason,omitempty"`
|
||||||
|
What string `json:"what,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// RecentSteps are this node's steps, the latest started first, at most limit (ten when limit is not above zero). Only
|
// RecentSteps are this node's steps, the latest started first, at most limit (ten when limit is not above zero). Only
|
||||||
@@ -180,7 +181,7 @@ func (m *Module) RecentSteps(limit int) (lines []StepLine, cut bool, err error)
|
|||||||
}
|
}
|
||||||
out := make([]StepLine, 0, len(steps))
|
out := make([]StepLine, 0, len(steps))
|
||||||
for _, s := range steps {
|
for _, s := range steps {
|
||||||
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason}
|
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason, What: s.What}
|
||||||
if !s.Ended.IsZero() {
|
if !s.Ended.IsZero() {
|
||||||
l.Ended = s.Ended.UTC().Format("2006-01-02T15:04:05Z")
|
l.Ended = s.Ended.UTC().Format("2006-01-02T15:04:05Z")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -96,7 +96,7 @@ func TestStepsToolListsRecentStepsOfThisNode(t *testing.T) {
|
|||||||
t0 := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
|
t0 := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
|
||||||
putStep(t, steps, delivery.Step{ID: "a", Tool: "build_node_fetch", Node: "ace", State: delivery.StepPassed, Started: t0, Ended: t0, Result: json.RawMessage(`1`)})
|
putStep(t, steps, delivery.Step{ID: "a", Tool: "build_node_fetch", Node: "ace", State: delivery.StepPassed, Started: t0, Ended: t0, Result: json.RawMessage(`1`)})
|
||||||
putStep(t, steps, delivery.Step{ID: "b", Tool: "build_node_test", Node: "ace", State: delivery.StepRunning, Started: t0.Add(time.Minute)})
|
putStep(t, steps, delivery.Step{ID: "b", Tool: "build_node_test", Node: "ace", State: delivery.StepRunning, Started: t0.Add(time.Minute)})
|
||||||
putStep(t, steps, delivery.Step{ID: "c", Tool: "build_node_build", Node: "ace", State: delivery.StepFailed, Started: t0.Add(2 * time.Minute), Reason: "boom"})
|
putStep(t, steps, delivery.Step{ID: "c", Tool: "build_node_build", Node: "ace", State: delivery.StepFailed, Started: t0.Add(2 * time.Minute), Reason: "boom", What: "check ask, novox/x@abc"})
|
||||||
putStep(t, steps, delivery.Step{ID: "d", Tool: "build_node_build", Node: "g14", State: delivery.StepPassed, Started: t0.Add(time.Hour)})
|
putStep(t, steps, delivery.Step{ID: "d", Tool: "build_node_build", Node: "g14", State: delivery.StepPassed, Started: t0.Add(time.Hour)})
|
||||||
m := readModule(steps, fakeState{})
|
m := readModule(steps, fakeState{})
|
||||||
out, err := tool(t, m, StepsReadTool)(map[string]any{})
|
out, err := tool(t, m, StepsReadTool)(map[string]any{})
|
||||||
@@ -111,7 +111,7 @@ func TestStepsToolListsRecentStepsOfThisNode(t *testing.T) {
|
|||||||
if !slices.Equal(ids, []string{"c", "b", "a"}) {
|
if !slices.Equal(ids, []string{"c", "b", "a"}) {
|
||||||
t.Errorf("ids = %v, want c b a", ids)
|
t.Errorf("ids = %v, want c b a", ids)
|
||||||
}
|
}
|
||||||
if lines[0].Reason != "boom" || lines[0].State != delivery.StepFailed {
|
if lines[0].Reason != "boom" || lines[0].What != "check ask, novox/x@abc" || lines[0].State != delivery.StepFailed {
|
||||||
t.Errorf("first = %+v", lines[0])
|
t.Errorf("first = %+v", lines[0])
|
||||||
}
|
}
|
||||||
out, _ = tool(t, m, StepsReadTool)(map[string]any{"limit": float64(1)})
|
out, _ = tool(t, m, StepsReadTool)(map[string]any{"limit": float64(1)})
|
||||||
|
|||||||
@@ -0,0 +1,165 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ansi = regexp.MustCompile("\x1b\\[[0-9;]*m")
|
||||||
|
|
||||||
|
// TestReporterOption is what the test script runs under: node's TAP reporter, which writes each failing test as a
|
||||||
|
// "not ok" line with a YAML block (its error, its location, its stack), whatever the package's own script prints
|
||||||
|
// otherwise (issue 1901: the default output of a run over several files kept the first file's passing lines).
|
||||||
|
const TestReporterOption = "--test-reporter=tap"
|
||||||
|
|
||||||
|
const outputBytes = 4096
|
||||||
|
|
||||||
|
// tapFailures are the failing tests of a `node --test` run read from its TAP output, each with its name, its file
|
||||||
|
// (the test's location), its Cause (the first line of its error) and its own block up to outputBytes, Cause first.
|
||||||
|
// A suite that failed only because a test inside it did (failureType subtestsFailed) is not reported twice. When the
|
||||||
|
// output holds no failing test (node itself failed: a syntax error, a file that could not load, a crash) the failure
|
||||||
|
// is the last lines of the output without its passing lines, and its Cause the first line that names an error.
|
||||||
|
func tapFailures(pkg, out string) []Failure {
|
||||||
|
var failed []Failure
|
||||||
|
all := strings.Split(ansi.ReplaceAllString(strings.ReplaceAll(out, "\r\n", "\n"), ""), "\n")
|
||||||
|
prev := 0 // the line after the last test block: what the run wrote since is the next test's own
|
||||||
|
for i := 0; i < len(all); i++ {
|
||||||
|
name, ok := notOk(all[i])
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
indent := len(all[i]) - len(strings.TrimLeft(all[i], " "))
|
||||||
|
block := []string{strings.TrimSpace(all[i])}
|
||||||
|
fields := map[string]string{}
|
||||||
|
var key string
|
||||||
|
inYAML := false
|
||||||
|
j := i + 1
|
||||||
|
for ; j < len(all); j++ {
|
||||||
|
t := strings.TrimSpace(all[j])
|
||||||
|
if !inYAML {
|
||||||
|
if t == "---" {
|
||||||
|
inYAML = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if t == "..." {
|
||||||
|
j++
|
||||||
|
break
|
||||||
|
}
|
||||||
|
block = append(block, strings.TrimRight(all[j], " \t"))
|
||||||
|
if k, v, found := yamlKey(all[j], indent); found {
|
||||||
|
key = k
|
||||||
|
if v != "|-" && v != "|" {
|
||||||
|
fields[k] = strings.Trim(v, "'")
|
||||||
|
}
|
||||||
|
} else if key != "" && fields[key] == "" && t != "" {
|
||||||
|
fields[key] = t // the first line of a block scalar
|
||||||
|
}
|
||||||
|
}
|
||||||
|
from := prev
|
||||||
|
i, prev = j-1, j
|
||||||
|
if fields["failureType"] == "subtestsFailed" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cause := fields["error"]
|
||||||
|
if cause == "" {
|
||||||
|
cause = fields["failureType"]
|
||||||
|
}
|
||||||
|
if cause == "test failed" || cause == "" {
|
||||||
|
// a file that did not load (a syntax error, a crash): node says only "test failed" and writes the error
|
||||||
|
// as comment lines before it
|
||||||
|
var own []string
|
||||||
|
for _, l := range all[from:i] {
|
||||||
|
if t := strings.TrimSpace(l); strings.HasPrefix(t, "# ") && !strings.HasPrefix(t, "# Subtest") {
|
||||||
|
own = append(own, strings.TrimPrefix(t, "# "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(own) > 0 {
|
||||||
|
cause = own[0]
|
||||||
|
for _, l := range own {
|
||||||
|
if strings.Contains(l, "Error") {
|
||||||
|
cause = l
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
block = append(own, block...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
failed = append(failed, Failure{Package: pkg, Test: name, File: fields["location"], Cause: cause, Output: clipBlock(cause, block)})
|
||||||
|
}
|
||||||
|
if len(failed) > 0 {
|
||||||
|
return failed
|
||||||
|
}
|
||||||
|
var rest []string
|
||||||
|
for _, l := range all {
|
||||||
|
t := strings.TrimSpace(l)
|
||||||
|
if t == "" || strings.HasPrefix(t, "ok ") || strings.HasPrefix(t, "✔") || strings.HasPrefix(t, "# Subtest") || strings.HasPrefix(t, "TAP version") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rest = append(rest, t)
|
||||||
|
}
|
||||||
|
cause := ""
|
||||||
|
for _, l := range rest {
|
||||||
|
if strings.Contains(l, "Error") || strings.HasPrefix(l, "not ok") {
|
||||||
|
cause = l
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return []Failure{{Package: pkg, Cause: cause, Output: clipBlock(cause, rest)}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// notOk is the test name of a "not ok N - name" line, indented or not.
|
||||||
|
func notOk(l string) (string, bool) {
|
||||||
|
t := strings.TrimSpace(l)
|
||||||
|
rest, ok := strings.CutPrefix(t, "not ok ")
|
||||||
|
if !ok {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
if _, after, found := strings.Cut(rest, " - "); found {
|
||||||
|
rest = after
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(rest), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// yamlKey is "key: value" of a field of a TAP block, whose lines sit two spaces deeper than their "not ok" line; a
|
||||||
|
// deeper line is part of a block scalar (an error's lines, a stack).
|
||||||
|
func yamlKey(l string, indent int) (key, val string, ok bool) {
|
||||||
|
t := strings.TrimLeft(l, " ")
|
||||||
|
if len(l)-len(t) != indent+2 {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
k, v, found := strings.Cut(t, ": ")
|
||||||
|
if !found {
|
||||||
|
if k, found = strings.CutSuffix(t, ":"); !found {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if k == "" || strings.ContainsAny(k, " ()/") {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
return k, strings.TrimSpace(v), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// clipBlock is a failing test's block, Cause first, kept to its last lines within outputBytes.
|
||||||
|
func clipBlock(cause string, lines []string) string {
|
||||||
|
size, from := 0, len(lines)
|
||||||
|
for from > 0 {
|
||||||
|
n := len(lines[from-1]) + 1
|
||||||
|
if size+n > outputBytes && from < len(lines) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
size += n
|
||||||
|
from--
|
||||||
|
}
|
||||||
|
var kept []string
|
||||||
|
if cause = strings.TrimSpace(cause); cause != "" && (from > 0 || len(lines) == 0 || strings.TrimSpace(lines[from]) != cause) {
|
||||||
|
kept = append(kept, cause)
|
||||||
|
}
|
||||||
|
if from > 0 {
|
||||||
|
kept = append(kept, fmt.Sprintf("... (%d earlier lines)", from))
|
||||||
|
}
|
||||||
|
kept = append(kept, lines[from:]...)
|
||||||
|
return strings.Join(kept, "\n")
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
# node --test TAP output, captured (issue 1901)
|
||||||
|
|
||||||
|
Real output of `NODE_OPTIONS=--test-reporter=tap node --test 'test/*.test.mjs'` (node v23.9.0, the Node the build
|
||||||
|
module runs with) in each project folder beside it, stdout and stderr together; every run exits 1.
|
||||||
|
|
||||||
|
- `two-files`: a.test.mjs passes, b.test.mjs has one failing and one passing test (`two-files.tap`).
|
||||||
|
- `suite`: a describe whose second test fails, and prints a line first (`suite.tap`); the suite itself is reported
|
||||||
|
`not ok` too, with failureType subtestsFailed.
|
||||||
|
- `syntax`: a.test.mjs passes, b.test.mjs has a syntax error, so node reports the file `not ok` (`syntax.tap`).
|
||||||
|
|
||||||
|
Captured by running the command in each folder. The only change to the captures: the absolute path of the capturing
|
||||||
|
checkout is replaced by `/work/`. To capture again, run the command in the folder and redirect `2>&1` to the file.
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
TAP version 13
|
||||||
|
# a line the test printed
|
||||||
|
# Subtest: a suite
|
||||||
|
# Subtest: passes inside
|
||||||
|
ok 1 - passes inside
|
||||||
|
---
|
||||||
|
duration_ms: 0.786439
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
# Subtest: fails inside
|
||||||
|
not ok 2 - fails inside
|
||||||
|
---
|
||||||
|
duration_ms: 4.551994
|
||||||
|
type: 'test'
|
||||||
|
location: '/work/suite/test/s.test.mjs:5:3'
|
||||||
|
failureType: 'testCodeFailure'
|
||||||
|
error: |-
|
||||||
|
Expected values to be strictly deep-equal:
|
||||||
|
[32m+ actual[39m [31m- expected[39m
|
||||||
|
|
||||||
|
[39m {
|
||||||
|
[32m+[39m a: 1
|
||||||
|
[31m-[39m a: 2
|
||||||
|
[39m }
|
||||||
|
|
||||||
|
code: 'ERR_ASSERTION'
|
||||||
|
name: 'AssertionError'
|
||||||
|
expected:
|
||||||
|
a: 2
|
||||||
|
actual:
|
||||||
|
a: 1
|
||||||
|
operator: 'deepStrictEqual'
|
||||||
|
stack: |-
|
||||||
|
TestContext.<anonymous> (file:///work/suite/test/s.test.mjs:5:77)
|
||||||
|
Test.runInAsyncScope (node:async_hooks:214:14)
|
||||||
|
Test.run (node:internal/test_runner/test:1047:25)
|
||||||
|
Suite.processPendingSubtests (node:internal/test_runner/test:744:18)
|
||||||
|
Test.postRun (node:internal/test_runner/test:1173:19)
|
||||||
|
Test.run (node:internal/test_runner/test:1101:12)
|
||||||
|
async Promise.all (index 0)
|
||||||
|
async Suite.run (node:internal/test_runner/test:1442:7)
|
||||||
|
async startSubtestAfterBootstrap (node:internal/test_runner/harness:297:3)
|
||||||
|
...
|
||||||
|
1..2
|
||||||
|
not ok 1 - a suite
|
||||||
|
---
|
||||||
|
duration_ms: 6.475177
|
||||||
|
type: 'suite'
|
||||||
|
location: '/work/suite/test/s.test.mjs:3:1'
|
||||||
|
failureType: 'subtestsFailed'
|
||||||
|
error: '1 subtest failed'
|
||||||
|
code: 'ERR_TEST_FAILURE'
|
||||||
|
...
|
||||||
|
1..1
|
||||||
|
# tests 2
|
||||||
|
# suites 1
|
||||||
|
# pass 1
|
||||||
|
# fail 1
|
||||||
|
# cancelled 0
|
||||||
|
# skipped 0
|
||||||
|
# todo 0
|
||||||
|
# duration_ms 69.244721
|
||||||
+1
@@ -0,0 +1 @@
|
|||||||
|
{"name":"p","type":"module"}
|
||||||
+6
@@ -0,0 +1,6 @@
|
|||||||
|
import { describe, it } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
describe('a suite', () => {
|
||||||
|
it('passes inside', () => {});
|
||||||
|
it('fails inside', () => { console.log('a line the test printed'); assert.deepEqual({ a: 1 }, { a: 2 }); });
|
||||||
|
});
|
||||||
+36
@@ -0,0 +1,36 @@
|
|||||||
|
TAP version 13
|
||||||
|
# Subtest: passes
|
||||||
|
ok 1 - passes
|
||||||
|
---
|
||||||
|
duration_ms: 0.929406
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
# file:///work/syntax/test/b.test.mjs:3
|
||||||
|
# SyntaxError: Unexpected end of input
|
||||||
|
# [90m at compileSourceTextModule (node:internal/modules/esm/utils:344:16)[39m
|
||||||
|
# [90m at ModuleLoader.moduleStrategy (node:internal/modules/esm/translators:106:18)[39m
|
||||||
|
# [90m at \#translate (node:internal/modules/esm/loader:490:12)[39m
|
||||||
|
# [90m at ModuleLoader.loadAndTranslate (node:internal/modules/esm/loader:537:27)[39m
|
||||||
|
# [90m at async ModuleJob._link (node:internal/modules/esm/module_job:116:19)[39m
|
||||||
|
# Node.js v23.9.0
|
||||||
|
# Subtest: test/b.test.mjs
|
||||||
|
not ok 2 - test/b.test.mjs
|
||||||
|
---
|
||||||
|
duration_ms: 37.927068
|
||||||
|
type: 'test'
|
||||||
|
location: '/work/syntax/test/b.test.mjs:1:1'
|
||||||
|
failureType: 'testCodeFailure'
|
||||||
|
exitCode: 1
|
||||||
|
signal: ~
|
||||||
|
error: 'test failed'
|
||||||
|
code: 'ERR_TEST_FAILURE'
|
||||||
|
...
|
||||||
|
1..2
|
||||||
|
# tests 2
|
||||||
|
# suites 0
|
||||||
|
# pass 1
|
||||||
|
# fail 1
|
||||||
|
# cancelled 0
|
||||||
|
# skipped 0
|
||||||
|
# todo 0
|
||||||
|
# duration_ms 68.264065
|
||||||
+1
@@ -0,0 +1 @@
|
|||||||
|
{"name":"p","type":"module"}
|
||||||
+2
@@ -0,0 +1,2 @@
|
|||||||
|
import test from 'node:test';
|
||||||
|
test('passes', () => {});
|
||||||
+2
@@ -0,0 +1,2 @@
|
|||||||
|
import test from "node:test";
|
||||||
|
test("never loads", () => {;
|
||||||
+46
@@ -0,0 +1,46 @@
|
|||||||
|
TAP version 13
|
||||||
|
# Subtest: passes in the first file
|
||||||
|
ok 1 - passes in the first file
|
||||||
|
---
|
||||||
|
duration_ms: 0.596042
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
# Subtest: fails in the second file
|
||||||
|
not ok 2 - fails in the second file
|
||||||
|
---
|
||||||
|
duration_ms: 1.962872
|
||||||
|
type: 'test'
|
||||||
|
location: '/work/two-files/test/b.test.mjs:3:1'
|
||||||
|
failureType: 'testCodeFailure'
|
||||||
|
error: |-
|
||||||
|
one plus one is not three
|
||||||
|
|
||||||
|
2 !== 3
|
||||||
|
|
||||||
|
code: 'ERR_ASSERTION'
|
||||||
|
name: 'AssertionError'
|
||||||
|
expected: 3
|
||||||
|
actual: 2
|
||||||
|
operator: 'strictEqual'
|
||||||
|
stack: |-
|
||||||
|
TestContext.<anonymous> (file:///work/two-files/test/b.test.mjs:3:49)
|
||||||
|
Test.runInAsyncScope (node:async_hooks:214:14)
|
||||||
|
Test.run (node:internal/test_runner/test:1047:25)
|
||||||
|
Test.start (node:internal/test_runner/test:944:17)
|
||||||
|
startSubtestAfterBootstrap (node:internal/test_runner/harness:297:17)
|
||||||
|
...
|
||||||
|
# Subtest: also passes
|
||||||
|
ok 3 - also passes
|
||||||
|
---
|
||||||
|
duration_ms: 0.148378
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
1..3
|
||||||
|
# tests 3
|
||||||
|
# suites 0
|
||||||
|
# pass 2
|
||||||
|
# fail 1
|
||||||
|
# cancelled 0
|
||||||
|
# skipped 0
|
||||||
|
# todo 0
|
||||||
|
# duration_ms 69.115323
|
||||||
+1
@@ -0,0 +1 @@
|
|||||||
|
{"name":"p","type":"module"}
|
||||||
Vendored
+2
@@ -0,0 +1,2 @@
|
|||||||
|
import test from 'node:test';
|
||||||
|
test('passes in the first file', () => {});
|
||||||
Vendored
+4
@@ -0,0 +1,4 @@
|
|||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
test('fails in the second file', () => { assert.equal(1 + 1, 3, 'one plus one is not three'); });
|
||||||
|
test('also passes', () => {});
|
||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
|
|
||||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
)
|
)
|
||||||
@@ -40,6 +41,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
|||||||
tools = append(tools, t)
|
tools = append(tools, t)
|
||||||
}
|
}
|
||||||
tools = append(tools, m.ReadTools()...)
|
tools = append(tools, m.ReadTools()...)
|
||||||
|
tools = append(tools, buildstep.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held))
|
||||||
tools = append(tools, m.releaseTool())
|
tools = append(tools, m.releaseTool())
|
||||||
return append(tools,
|
return append(tools,
|
||||||
stdio.Tool{Name: StatusTool,
|
stdio.Tool{Name: StatusTool,
|
||||||
|
|||||||
@@ -180,14 +180,28 @@ func (m *Module) runIn(ctx context.Context, root string, write bool, dir string,
|
|||||||
return run(ctx, extra, "env", append([]string{"-C", dir, name}, args...)...)
|
return run(ctx, extra, "env", append([]string{"-C", dir, name}, args...)...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ranFailed says why a command did not pass, or "" when it did.
|
// ranFailed says why a command did not pass, or "" when it did (issue 1703): its exit status, the end of what it wrote
|
||||||
|
// on stderr (npm's error is its last lines) and the start of what it wrote on stdout (tsc writes its errors there,
|
||||||
|
// first first), each bounded, so a long stdout never pushes stderr's error out of the reason.
|
||||||
func ranFailed(r forgekey.Ran) string {
|
func ranFailed(r forgekey.Ran) string {
|
||||||
if r.Status == 0 && r.Err == "" {
|
if r.Status == 0 && r.Err == "" {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
out := strings.TrimSpace(r.Stdout + "\n" + r.Stderr)
|
why := fmt.Sprintf("status %d %s", r.Status, r.Err)
|
||||||
if len(out) > 1500 {
|
if e := strings.TrimSpace(r.Stderr); e != "" {
|
||||||
out = out[:1500] + " ..."
|
if len(e) > failedBytes/2 {
|
||||||
|
e = "... " + e[len(e)-failedBytes/2:]
|
||||||
|
}
|
||||||
|
why += ": stderr: " + e
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("status %d %s: %s", r.Status, r.Err, out)
|
if o := strings.TrimSpace(r.Stdout); o != "" {
|
||||||
|
if len(o) > failedBytes/2 {
|
||||||
|
o = o[:failedBytes/2] + " ..."
|
||||||
|
}
|
||||||
|
why += ": stdout: " + o
|
||||||
|
}
|
||||||
|
return why
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// failedBytes bounds what ranFailed carries of a command's output, stderr and stdout half each.
|
||||||
|
const failedBytes = 1500
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module meshbuildnode
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f
|
||||||
github.com/distribution/distribution/v3 v3.1.2
|
github.com/distribution/distribution/v3 v3.1.2
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4 h1:05sVel/UOvz4/g0dxrDOeVUL7BGakVrxPBlqRq02ZhM=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f h1:GYLCvz2x+JNUOTbXzyh2QwzX0sTzvRlXR7lGocWjk3A=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026142629-ff81bdf8cc8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||||
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
|
|||||||
@@ -34,6 +34,7 @@
|
|||||||
"build_node_release",
|
"build_node_release",
|
||||||
"build_node_step",
|
"build_node_step",
|
||||||
"build_node_steps",
|
"build_node_steps",
|
||||||
|
"build_node_step_end",
|
||||||
"build_node_toolchains",
|
"build_node_toolchains",
|
||||||
"build_node_status",
|
"build_node_status",
|
||||||
"build_node_health"
|
"build_node_health"
|
||||||
|
|||||||
@@ -36,6 +36,14 @@ names every refusal and rejection at once, and the pull request is told at once;
|
|||||||
`needs: new-head` and names its author: the branch, and the session a `Pushed-by: <who@node>` line in the pull
|
`needs: new-head` and names its author: the branch, and the session a `Pushed-by: <who@node>` line in the pull
|
||||||
request's description names.
|
request's description names.
|
||||||
|
|
||||||
|
## A merge the forge account cannot carry out is not asked (issue 1945)
|
||||||
|
|
||||||
|
Before the operator's Approve is asked, the gate reads through `gitea_commit_statuses` whether the forge's merging
|
||||||
|
account writes each repository the ask covers (`merge.writes`). Where it does not, nothing is asked: the delivery waits
|
||||||
|
on "the forge account cannot write <repo>: gitea_grant_write" (`show`), raised at once as a condition. The read holds
|
||||||
|
until the gitea module's `gitea.repo.writable` (its grant, read back) or ten minutes, then the ask is made again; an
|
||||||
|
approved merge of that repository the forge refused is asked again on the same event (`writes.go`).
|
||||||
|
|
||||||
## The seat's hand-over
|
## The seat's hand-over
|
||||||
|
|
||||||
The gate took the seat `mesh-delivery` over from the module `mesh-delivery` (ADR 0449 §4, ADR 0450 §6, ADR 0454) and carried its
|
The gate took the seat `mesh-delivery` over from the module `mesh-delivery` (ADR 0449 §4, ADR 0450 §6, ADR 0454) and carried its
|
||||||
|
|||||||
@@ -0,0 +1,246 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
sdk "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// One check per delivery (novox/hq ADR 0548), the gate's part: nothing merges into a repository while a check of other heads
|
||||||
|
// judges it (§2); an approved merge holds a turn on its repositories (§3); a held merge past its bound is a stalled line, read
|
||||||
|
// and handed by two tools (§4); an approved delivery whose trunk moved merges on its carried verdict with no second ask (§8).
|
||||||
|
|
||||||
|
const otherHead = "c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3"
|
||||||
|
|
||||||
|
// approvedLone is a lone delivery into oneRepo whose check passed, approved by the operator (the world approves at once).
|
||||||
|
func approvedLone(w *world) string {
|
||||||
|
w.t.Helper()
|
||||||
|
p := pr(oneRepo, 1, oneHead, "feat/one")
|
||||||
|
requiresCheck(w, p)
|
||||||
|
w.h.PullUpdated(p)
|
||||||
|
putCheckerVerdict(w, "pass", headOf(oneRepo, 1, oneHead))
|
||||||
|
w.h.CommitStatus(checkStatus(oneRepo, oneHead, "success", oneTrunk))
|
||||||
|
return IDOf(oneRepo, oneHead)
|
||||||
|
}
|
||||||
|
|
||||||
|
// memTurns is the bucket turns in memory; it logs each write with the merges the fake bridge had made by then.
|
||||||
|
type memTurns struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
m map[string]sdk.Turn
|
||||||
|
log []string
|
||||||
|
merges func() int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMemTurns(w *world) *memTurns {
|
||||||
|
return &memTurns{m: map[string]sdk.Turn{}, merges: func() int { return len(w.forge.merges) }}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *memTurns) All() (map[string]sdk.Turn, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
out := map[string]sdk.Turn{}
|
||||||
|
for k, v := range f.m {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *memTurns) Put(t sdk.Turn) error {
|
||||||
|
if _, err := sdk.EncodeTurn(t); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.m[t.Repository] = t
|
||||||
|
f.log = append(f.log, fmt.Sprintf("put %s after %d merges", t.Repository, f.merges()))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *memTurns) Delete(repository string) error {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
delete(f.m, repository)
|
||||||
|
f.log = append(f.log, fmt.Sprintf("delete %s after %d merges", repository, f.merges()))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *memTurns) said() string { f.mu.Lock(); defer f.mu.Unlock(); return strings.Join(f.log, "; ") }
|
||||||
|
|
||||||
|
func TestOneCheckPerDeliveryAtTheGate(t *testing.T) {
|
||||||
|
for name, run := range map[string]func(t *testing.T){
|
||||||
|
// §2: a merge into a repository a running verdict names is not written to merges, and is written on that verdict's pass.
|
||||||
|
"an approved merge waits for a running check on its repository and merges on its end": func(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
putCheckerVerdict(w, "running", headOf(oneRepo, 9, otherHead))
|
||||||
|
id := approvedLone(w)
|
||||||
|
w.settleAll()
|
||||||
|
if w.op.count() != 1 || len(w.forge.merges) != 0 {
|
||||||
|
t.Fatalf("asked %d times, merges %v: want the Approve asked and the merge held", w.op.count(), w.forge.merges)
|
||||||
|
}
|
||||||
|
d := w.delivery(id)
|
||||||
|
if d.Ready == nil || !strings.Contains(d.Ready.Waits, "waits for the check of g") || !strings.Contains(d.Ready.Waits, oneRepo) ||
|
||||||
|
d.Ready.HeldSince == nil {
|
||||||
|
t.Fatalf("the merge does not say what it waits for: %+v", d.Ready)
|
||||||
|
}
|
||||||
|
// §9: deliveries says what it waits for.
|
||||||
|
for _, l := range w.h.Deliveries("", "", "", false) {
|
||||||
|
if l.ID == id && !strings.Contains(l.Waits, "approved; waits for the check of g") {
|
||||||
|
t.Errorf("deliveries says it waits for %q", l.Waits)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// §4: past 30 minutes the held merge is a stalled line naming the check it waits for and since when.
|
||||||
|
for _, l := range w.h.Stalled() {
|
||||||
|
if l.ID == id {
|
||||||
|
t.Fatalf("a stalled line within the bound: %+v", l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.later(turnBound + time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
var line *StalledLine
|
||||||
|
for _, l := range w.h.Stalled() {
|
||||||
|
if l.ID == id {
|
||||||
|
line = &l
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if line == nil || line.Bound != "30m0s" || !strings.Contains(line.Says, "waits for the check of g") || !strings.Contains(line.Says, "held since") ||
|
||||||
|
!strings.Contains(line.Says, "checker_status") || !strings.Contains(line.H2, "delivery_hand_now") {
|
||||||
|
t.Fatalf("the stalled line of the held merge = %+v", line)
|
||||||
|
}
|
||||||
|
putCheckerVerdict(w, "pass", headOf(oneRepo, 9, otherHead))
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.forge.merges) != 1 || w.op.count() != 1 {
|
||||||
|
t.Fatalf("after the check's end: merges %v, asked %d times; want merged with no second ask", w.forge.merges, w.op.count())
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"a running check on another repository holds nothing": func(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
putCheckerVerdict(w, "running", headOf(oneRepo2, 9, otherHead))
|
||||||
|
approvedLone(w)
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.forge.merges) != 1 {
|
||||||
|
t.Fatalf("merges %v: a check of another repository held the merge (%s)", w.forge.merges, w.logs())
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"the delivery's own running verdict does not hold it": func(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
w.op.auto = false
|
||||||
|
id := approvedLone(w)
|
||||||
|
w.settleAll()
|
||||||
|
putCheckerVerdict(w, "running", headOf(oneRepo, 1, oneHead)) // its own heads judged again (a carry)
|
||||||
|
for _, k := range w.op.open() {
|
||||||
|
w.op.answer(k, OptionApprove)
|
||||||
|
}
|
||||||
|
w.settleAll()
|
||||||
|
if d := w.delivery(id); len(w.forge.merges) != 1 || d.Ready == nil || d.Ready.HeldSince != nil {
|
||||||
|
t.Fatalf("merges %v, run %+v: its own check held it", w.forge.merges, d.Ready)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// §3: the turn is written before the verdicts are read for the merge, and deleted after the last merge.
|
||||||
|
"a lone merge's turn is written before it is handed and deleted once it merged": func(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
turns := newMemTurns(w)
|
||||||
|
w.h.Turns = turns
|
||||||
|
approvedLone(w)
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.forge.merges) != 1 || turns.said() != "put "+oneRepo+" after 0 merges; delete "+oneRepo+" after 1 merges" {
|
||||||
|
t.Fatalf("merges %v; the turns were %q", w.forge.merges, turns.said())
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"a group's turns stand on every member's repository until its last member merged": func(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
turns := newMemTurns(w)
|
||||||
|
w.h.Turns = turns
|
||||||
|
a, b := pr(oneRepo, 1, oneHead, "feat/two"), pr(oneRepo2, 2, oneHead2, "feat/two")
|
||||||
|
requiresCheck(w, a, b)
|
||||||
|
w.h.PullUpdated(a)
|
||||||
|
w.h.PullUpdated(b)
|
||||||
|
w.h.wantForge()
|
||||||
|
putCheckerVerdict(w, "pass", headOf(oneRepo, 1, oneHead), headOf(oneRepo2, 2, oneHead2))
|
||||||
|
w.h.CommitStatus(checkStatus(oneRepo, oneHead, "success", oneTrunk))
|
||||||
|
w.h.CommitStatus(checkStatus(oneRepo2, oneHead2, "success", oneTrunk))
|
||||||
|
for range 3 {
|
||||||
|
w.settleAll()
|
||||||
|
}
|
||||||
|
want := "put " + oneRepo2 + " after 0 merges; put " + oneRepo + " after 0 merges; delete " + oneRepo2 + " after 2 merges; delete " + oneRepo + " after 2 merges"
|
||||||
|
if len(w.forge.merges) != 2 || turns.said() != want {
|
||||||
|
t.Fatalf("merges %v; the turns were %q, want %q", w.forge.merges, turns.said(), want)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// §8: an approved delivery whose trunk moved goes back to proposed, waits for its carried verdict rather than being
|
||||||
|
// handed on the old one, and merges on it with no second ask.
|
||||||
|
"an approved merge whose trunk moved merges on its carried verdict with no second ask": func(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
putCheckerVerdict(w, "running", headOf(oneRepo, 9, otherHead)) // holds the approved merge (§2)
|
||||||
|
id := approvedLone(w)
|
||||||
|
w.settleAll()
|
||||||
|
if w.op.count() != 1 || len(w.forge.merges) != 0 {
|
||||||
|
t.Fatalf("asked %d times, merges %v; want approved and held", w.op.count(), w.forge.merges)
|
||||||
|
}
|
||||||
|
w.h.PullMerged(merged(oneRepo, 5, otherHead, oneTrunk2)) // a merge at the forge moves the trunk
|
||||||
|
putCheckerVerdict(w, "pass", headOf(oneRepo, 9, otherHead))
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.forge.merges) != 0 {
|
||||||
|
t.Fatalf("merges %v: handed on a verdict judged on the old trunk", w.forge.merges)
|
||||||
|
}
|
||||||
|
d := w.delivery(id)
|
||||||
|
if d.Ready == nil || !strings.Contains(d.Ready.Waits, "being carried") {
|
||||||
|
t.Fatalf("after the trunk moved it waits for %q, want its verdict carried", d.Ready.Waits)
|
||||||
|
}
|
||||||
|
// The checker's carried line: the head on the new trunk, carried from the old one.
|
||||||
|
carried := checkStatus(oneRepo, oneHead, "success", oneTrunk2)
|
||||||
|
carried.Description = "pass " + oneHead[:12] + " on " + oneTrunk2[:12] + " | carried from " + oneTrunk[:12] + " | unit 0 run, 4 kept"
|
||||||
|
if head, trunk := checkLine(carried.Description); head != oneHead[:12] || trunk != oneTrunk2[:12] {
|
||||||
|
t.Fatalf("the carried line reads as %s on %s", head, trunk)
|
||||||
|
}
|
||||||
|
if head, trunk := checkLine("failed: a test"); head != "" || trunk != "" {
|
||||||
|
t.Fatalf("a line that names no commits gave %q %q", head, trunk)
|
||||||
|
}
|
||||||
|
putCheckerVerdict(w, "pass", headOf(oneRepo, 1, oneHead))
|
||||||
|
w.h.CommitStatus(carried)
|
||||||
|
w.settleAll()
|
||||||
|
if w.op.count() != 1 {
|
||||||
|
t.Errorf("the Approve was asked %d times, want once: it stands through the carry", w.op.count())
|
||||||
|
}
|
||||||
|
if d := w.delivery(id); d.Check == nil || d.Check.Trunk != oneTrunk2[:12] || len(w.forge.merges) != 1 {
|
||||||
|
t.Errorf("after the carry: verdict %+v, merges %v; want merged on the carried verdict", d.Check, w.forge.merges)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// §4: delivery_turns reads every turn with the check it waits for; delivery_hand_now hands a held merge at once on the
|
||||||
|
// operator's word, and refuses one no check holds.
|
||||||
|
"the turns are read, and a held merge is handed now on the operator's word": func(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
w.h.Turns = newMemTurns(w)
|
||||||
|
putCheckerVerdict(w, "running", headOf(oneRepo, 9, otherHead))
|
||||||
|
id := approvedLone(w)
|
||||||
|
w.settleAll()
|
||||||
|
read := w.h.TurnsRead()
|
||||||
|
lines := read["turns"].([]TurnLine)
|
||||||
|
if len(lines) != 1 || lines[0].Repository != oneRepo || len(lines[0].Turn.Waiting) != 1 || lines[0].Turn.Waiting[0].WaitsFor == "" ||
|
||||||
|
!strings.Contains(lines[0].Waits["feat/one"].(string), "the check g.") || read["written"] != true {
|
||||||
|
t.Fatalf("delivery_turns = %+v", read)
|
||||||
|
}
|
||||||
|
if _, err := w.h.HandNow(id, ""); err == nil || !strings.Contains(err.Error(), "why") {
|
||||||
|
t.Errorf("a hand without the operator's why: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := w.h.HandNow(IDOf(oneRepo2, oneHead2), "x"); err == nil {
|
||||||
|
t.Error("an unknown delivery was handed")
|
||||||
|
}
|
||||||
|
if _, err := w.h.HandNow(id, "the operator, on telegram: merge it now"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.forge.merges) != 1 || w.delivery(id).Ready.HandNow == "" {
|
||||||
|
t.Fatalf("merges %v after the hand", w.forge.merges)
|
||||||
|
}
|
||||||
|
if _, err := w.h.HandNow(id, "again"); err == nil {
|
||||||
|
t.Error("a merged delivery was handed again")
|
||||||
|
}
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(name, run)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -339,6 +339,9 @@ func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) {
|
|||||||
return "the operator cannot be asked from here", nil
|
return "the operator cannot be asked from here", nil
|
||||||
}
|
}
|
||||||
members := h.askMembers(d)
|
members := h.askMembers(d)
|
||||||
|
if waits := h.writeWaits(d, members, now); waits != "" {
|
||||||
|
return waits, nil // asked only once the forge account can carry the merge out (writes.go, issue 1945)
|
||||||
|
}
|
||||||
group := ""
|
group := ""
|
||||||
if len(members) > 1 {
|
if len(members) > 1 {
|
||||||
group = d.Group
|
group = d.Group
|
||||||
@@ -390,12 +393,12 @@ func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) {
|
|||||||
if gid == "" {
|
if gid == "" {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
says, err := h.notASuccess(m)
|
says, forge, err := h.notASuccess(m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
readErr = err
|
readErr = err
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if says != "" {
|
if says != "" && !forge { // the forge's merge answer is weighed at the merge, where it is asked again (group_merge.go)
|
||||||
holdMember, holdSays = m.id, says
|
holdMember, holdSays = m.id, says
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -423,6 +426,16 @@ func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) {
|
|||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Nothing is asked that the forge account could not merge (writes.go, issue 1945): said at the next look.
|
||||||
|
if says := h.cannotWrite(reads); says != "" {
|
||||||
|
stale()
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
undo()
|
||||||
|
h.Logf("[mesh-delivery] %s passed its checks and the operator is not asked: %s", d.ID, says)
|
||||||
|
h.lookAt(h.Now(), "an ask not made: the forge account cannot write a repository it covers")
|
||||||
|
return
|
||||||
|
}
|
||||||
if readErr != nil || holdSays != "" {
|
if readErr != nil || holdSays != "" {
|
||||||
stale()
|
stale()
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
|
|||||||
@@ -224,6 +224,9 @@ func (h *Holder) nextStall(now time.Time) time.Time {
|
|||||||
consider(r.RefusedSince.Add(h.answeredRefusalBound(d.Repository)))
|
consider(r.RefusedSince.Add(h.answeredRefusalBound(d.Repository)))
|
||||||
consider(r.RefusedSince.Add(outOfDateFor))
|
consider(r.RefusedSince.Add(outOfDateFor))
|
||||||
}
|
}
|
||||||
|
if r.HeldSince != nil {
|
||||||
|
consider(r.HeldSince.Add(turnBound))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return next
|
return next
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -101,7 +101,9 @@ type fakeForge struct {
|
|||||||
viewCalls int
|
viewCalls int
|
||||||
// What Statuses reads: the pull requests by owner/repo#number, the base's required statuses, whether the
|
// What Statuses reads: the pull requests by owner/repo#number, the base's required statuses, whether the
|
||||||
// protection cannot be read, and what it was asked.
|
// protection cannot be read, and what it was asked.
|
||||||
pulls map[string]ForgePull
|
pulls map[string]ForgePull
|
||||||
|
// unwritable are the repositories (owner/repo) the forge's merging account cannot write (issue 1945).
|
||||||
|
unwritable map[string]bool
|
||||||
required []string
|
required []string
|
||||||
protectionDown bool
|
protectionDown bool
|
||||||
asked []string
|
asked []string
|
||||||
@@ -220,6 +222,10 @@ func (f *fakeForge) Statuses(owner, repo string, number int, commit, base string
|
|||||||
base = "main"
|
base = "main"
|
||||||
}
|
}
|
||||||
out.Merge = ForgeMerge{Branch: base, Protected: f.required != nil, Required: f.required}
|
out.Merge = ForgeMerge{Branch: base, Protected: f.required != nil, Required: f.required}
|
||||||
|
if f.unwritable[owner+"/"+repo] {
|
||||||
|
no := false
|
||||||
|
out.Merge.Writes, out.Merge.CannotWrite = &no, "the forge account cannot write "+owner+"/"+repo+": gitea_grant_write"
|
||||||
|
}
|
||||||
if f.protectionDown {
|
if f.protectionDown {
|
||||||
out.Merge.Says = "the protection of " + base + " could not be read"
|
out.Merge.Says = "the protection of " + base + " could not be read"
|
||||||
return out, nil
|
return out, nil
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -28,26 +29,46 @@ func readOf(d *Delivery, head string) memberRead {
|
|||||||
return memberRead{id: d.ID, owner: d.Owner(), repo: d.Repo(), base: d.Base, head: head, number: d.Number}
|
return memberRead{id: d.ID, owner: d.Owner(), repo: d.Repo(), base: d.Base, head: head, number: d.Number}
|
||||||
}
|
}
|
||||||
|
|
||||||
// notAsSuccess reads a member's required statuses at its head and says, in the forge's words, which are not a success;
|
// forgeMergeGrace is how long the forge may say a member does not merge now before the group is held for it (issue 1275):
|
||||||
// empty when all are. An error is a
|
// the forge says so for a few seconds after a base moved, while it works out whether the pull request still merges; within
|
||||||
|
// the grace the merge is asked again every forgeMergeAgain. Past it, the member is in conflict with its trunk: held, said
|
||||||
|
// urgent, and asked again on the next event (a new head, a status).
|
||||||
|
const (
|
||||||
|
forgeMergeGrace = 2 * time.Minute
|
||||||
|
forgeMergeAgain = 5 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// notASuccess reads a member's required statuses at its head and says, in the forge's words, which are not a success,
|
||||||
|
// and whether the forge says the pull request merges into its base now (its head still the one approved, and not in
|
||||||
|
// conflict); empty when all hold. forge says the reason is the forge's merge answer rather than a status. An error is a
|
||||||
// read that could not be made: the member then counts as not mergeable.
|
// read that could not be made: the member then counts as not mergeable.
|
||||||
func (h *Holder) notASuccess(m memberRead) (says string, err error) {
|
func (h *Holder) notASuccess(m memberRead) (says string, forge bool, err error) {
|
||||||
fc, err := h.Forge.Statuses(m.owner, m.repo, m.number, m.head, m.base)
|
fc, err := h.Forge.Statuses(m.owner, m.repo, m.number, m.head, m.base)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", false, err
|
||||||
}
|
}
|
||||||
if fc.Merge.Mergeable == nil {
|
if fc.Merge.Mergeable == nil {
|
||||||
return "", fmt.Errorf("the forge could not say whether %s may merge: %s", m.id, fc.Merge.Says)
|
return "", false, fmt.Errorf("the forge could not say whether %s may merge: %s", m.id, fc.Merge.Says)
|
||||||
}
|
}
|
||||||
if *fc.Merge.Mergeable {
|
if !*fc.Merge.Mergeable {
|
||||||
return "", nil
|
var parts []string
|
||||||
|
for _, b := range fc.Merge.Blocking {
|
||||||
|
parts = append(parts, fmt.Sprintf("%s (%s)", b.Context, b.State))
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s requires %s; not a success: %s", fc.Merge.Branch, strings.Join(fc.Merge.Required, ", "),
|
||||||
|
strings.Join(parts, ", ")), false, nil
|
||||||
}
|
}
|
||||||
var parts []string
|
if p := fc.Pull; p != nil && m.number > 0 {
|
||||||
for _, b := range fc.Merge.Blocking {
|
if p.HeadSHA != "" && !sameCommit(p.HeadSHA, m.head) {
|
||||||
parts = append(parts, fmt.Sprintf("%s (%s)", b.Context, b.State))
|
return fmt.Sprintf("its head is now %s, not the %s approved", shortOf(p.HeadSHA, 12), shortOf(m.head, 12)), false, nil
|
||||||
|
}
|
||||||
|
if p.Mergeable != nil && !*p.Mergeable {
|
||||||
|
base := firstOf(p.Base, m.base)
|
||||||
|
return fmt.Sprintf("the forge says it does not merge into %s now (in conflict with it, a work in progress, or still "+
|
||||||
|
"being worked out after %s moved)", base, base), true, nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("%s requires %s; not a success: %s", fc.Merge.Branch, strings.Join(fc.Merge.Required, ", "),
|
return "", false, nil
|
||||||
strings.Join(parts, ", ")), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// unmergedLive are a group's members not yet merged and not over, in the group's order. Under the lock.
|
// unmergedLive are a group's members not yet merged and not over, in the group's order. Under the lock.
|
||||||
@@ -61,10 +82,13 @@ func (h *Holder) unmergedLive(g *Group) []*Delivery {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// groupMergeable reads every member's required statuses at the head the Approve covers, and says whether all are a success.
|
// groupMergeable reads every member's required statuses, its head and the forge's merge answer at the head the Approve
|
||||||
// When one is not, or cannot be read, nothing is handed to the bridge: the group is held, said urgent (ADR 0441 §2, §4), and
|
// covers, and says whether all hold: no member is merged unless every member merges now (issue 1275, 2026-10-26: two of
|
||||||
// the target's turn is given back so the next event that decides whether it may merge asks it again. A read that could not
|
// three merged and the third refused). When one does not, or cannot be read, nothing is handed to the bridge: the group is
|
||||||
// be made is a failed ask, tried again after askAgainAfter. Outside the lock.
|
// held, said urgent (ADR 0441 §2, §4), and the target's turn is given back so the next event that decides whether it may
|
||||||
|
// merge asks it again. A read that could not be made is a failed ask, tried again after askAgainAfter; the forge saying a
|
||||||
|
// member does not merge now is asked again every forgeMergeAgain for forgeMergeGrace before the group is held. Outside
|
||||||
|
// the lock.
|
||||||
func (h *Holder) groupMergeable(gid, target string, reads []memberRead) bool {
|
func (h *Holder) groupMergeable(gid, target string, reads []memberRead) bool {
|
||||||
release := func() {
|
release := func() {
|
||||||
if d := h.deliveries[target]; d != nil && d.Ready != nil {
|
if d := h.deliveries[target]; d != nil && d.Ready != nil {
|
||||||
@@ -73,7 +97,7 @@ func (h *Holder) groupMergeable(gid, target string, reads []memberRead) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, m := range reads {
|
for _, m := range reads {
|
||||||
says, err := h.notASuccess(m)
|
says, forge, err := h.notASuccess(m)
|
||||||
if err == nil && says == "" {
|
if err == nil && says == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -85,6 +109,25 @@ func (h *Holder) groupMergeable(gid, target string, reads []memberRead) bool {
|
|||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
if forge {
|
||||||
|
if h.unmergeableSince == nil {
|
||||||
|
h.unmergeableSince = map[string]time.Time{}
|
||||||
|
}
|
||||||
|
key, now := gid+" "+m.id, h.Now()
|
||||||
|
first, seen := h.unmergeableSince[key]
|
||||||
|
if !seen {
|
||||||
|
first = now
|
||||||
|
h.unmergeableSince[key] = now
|
||||||
|
h.Logf("[mesh-delivery] group %s: nothing is merged yet: %s at %s: %s; asked again every %s for %s", gid, m.id,
|
||||||
|
shortOf(m.head, 12), says, forgeMergeAgain, forgeMergeGrace)
|
||||||
|
}
|
||||||
|
if now.Sub(first) < forgeMergeGrace {
|
||||||
|
release()
|
||||||
|
h.lookAt(now.Add(forgeMergeAgain), "a group's member the forge was still weighing")
|
||||||
|
h.mu.Unlock()
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
if g := h.groups[gid]; g != nil {
|
if g := h.groups[gid]; g != nil {
|
||||||
if g.Hold == nil || g.Hold.Kind != "blocked" || g.Hold.Says != says || g.Hold.Member != m.id {
|
if g.Hold == nil || g.Hold.Kind != "blocked" || g.Hold.Says != says || g.Hold.Member != m.id {
|
||||||
h.Logf("[mesh-delivery] group %s is approved and NOT merged: %s at %s: %s", gid, m.id, shortOf(m.head, 12), says)
|
h.Logf("[mesh-delivery] group %s is approved and NOT merged: %s at %s: %s", gid, m.id, shortOf(m.head, 12), says)
|
||||||
@@ -99,13 +142,18 @@ func (h *Holder) groupMergeable(gid, target string, reads []memberRead) bool {
|
|||||||
g.Hold = &GroupHold{Kind: "blocked", Member: m.id, Says: says, Since: since}
|
g.Hold = &GroupHold{Kind: "blocked", Member: m.id, Says: says, Since: since}
|
||||||
h.keepGroup(g)
|
h.keepGroup(g)
|
||||||
}
|
}
|
||||||
// No timer: the status turning success is an event on the forge (CommitStatus, mergeAgainNow), and it asks the
|
// No timer: the status turning success, or a new head, is an event on the forge (CommitStatus, mergeAgainNow), and
|
||||||
// merge again at once (issue 1275: a pass that stops part way resumes on an event).
|
// it asks the merge again at once (issue 1275: a pass that stops part way resumes on an event).
|
||||||
release()
|
release()
|
||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
|
for key := range h.unmergeableSince {
|
||||||
|
if strings.HasPrefix(key, gid+" ") {
|
||||||
|
delete(h.unmergeableSince, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
if g := h.groups[gid]; g != nil && g.Hold != nil {
|
if g := h.groups[gid]; g != nil && g.Hold != nil {
|
||||||
g.Hold = nil
|
g.Hold = nil
|
||||||
h.keepGroup(g)
|
h.keepGroup(g)
|
||||||
@@ -114,6 +162,98 @@ func (h *Holder) groupMergeable(gid, target string, reads []memberRead) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// passWaits is why an approved merge is not handed yet because another approved merge's pass is under way into a
|
||||||
|
// repository it goes into; empty when none is (issue 1275, 2026-10-26 13:40: group feat/890-placement merged
|
||||||
|
// mesh-delivery#72 between the members of fix/734-lost-ask, whose mesh-delivery#73 then conflicted). A pass is under way
|
||||||
|
// from its first member handed until every member is merged; one half merged stays under way while it is refused or held,
|
||||||
|
// until it is whole, declined, or its remaining members are gone or asked anew (a new head is a new Approve). The first
|
||||||
|
// merge handed has the repositories; the other is handed once that pass ends, on the event that ends it. This holds
|
||||||
|
// whether or not mesh/check gates the merge: it waits for a merge, never for a check (ADR 0551). Under the lock.
|
||||||
|
func (h *Holder) passWaits(d *Delivery) string {
|
||||||
|
c := h.covering(d)
|
||||||
|
if c == nil || c.Approved == nil || c.AskID == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
mine := h.unmergedRepos(c)
|
||||||
|
for _, p := range h.passes() {
|
||||||
|
if p.run.AskID == c.AskID || !h.passUnderWay(p.run) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for repo := range h.unmergedRepos(p.run) {
|
||||||
|
if mine[repo] {
|
||||||
|
return fmt.Sprintf("approved; waits for the merge of %s into %s to end, so no other merge lands there between "+
|
||||||
|
"its members (issue 1275)", p.name, repo)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// pass is an approved merge not merged whole: the run holding its Approve and the name of its group (or pull request).
|
||||||
|
type pass struct {
|
||||||
|
run *ReadyRun
|
||||||
|
name string
|
||||||
|
}
|
||||||
|
|
||||||
|
// passes are the approved merges not merged whole and not declined, each ask once. Unlike approvedMerges it keeps one whose
|
||||||
|
// members merged or were refused: a half-merged pass still holds its repositories (issue 1275). Under the lock.
|
||||||
|
func (h *Holder) passes() []pass {
|
||||||
|
ids := make([]string, 0, len(h.deliveries))
|
||||||
|
for id := range h.deliveries {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
sort.Strings(ids)
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []pass
|
||||||
|
for _, id := range ids {
|
||||||
|
d := h.deliveries[id]
|
||||||
|
r := d.Ready
|
||||||
|
if r == nil || r.Approved == nil || r.Declined != nil || r.AskID == "" || len(r.Heads) == 0 || seen[r.AskID] ||
|
||||||
|
d.Base != MergeBranch || h.allMerged(r.Heads) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[r.AskID] = true
|
||||||
|
out = append(out, pass{run: r, name: askGroup(d)})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// unmergedRepos are the repositories (lowercase) of the deliveries an Approve covers that are not merged and may still be
|
||||||
|
// merged on it: not over (closed, held), at the head approved, and covered by this Approve, not a newer one. Under the lock.
|
||||||
|
func (h *Holder) unmergedRepos(r *ReadyRun) map[string]bool {
|
||||||
|
out := map[string]bool{}
|
||||||
|
for id, sha := range r.Heads {
|
||||||
|
m := h.deliveries[id]
|
||||||
|
if m == nil || m.MergedAs != "" || (m.Ready != nil && m.Ready.Merged != nil) || m.State.Final() || !sameCommit(m.Commit, sha) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c := h.covering(m); c == nil || c.AskID != r.AskID {
|
||||||
|
continue // asked anew (its new head, or a new Approve): that ask is the pass now
|
||||||
|
}
|
||||||
|
out[strings.ToLower(m.Repository)] = true
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// passUnderWay is whether an approved merge's pass has started and not ended: a member is handed to the bridge, or some
|
||||||
|
// are merged and others may still merge on this Approve, refused or held or not (issue 1275, review: a held half-merged
|
||||||
|
// group that let another group start, then asked again, waited on that group while it waited on it). Under the lock.
|
||||||
|
func (h *Holder) passUnderWay(r *ReadyRun) bool {
|
||||||
|
if len(h.unmergedRepos(r)) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for id := range r.Heads {
|
||||||
|
m := h.deliveries[id]
|
||||||
|
if m == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m.MergedAs != "" || (m.Ready != nil && m.Ready.Merged != nil) || (m.Ready != nil && m.Ready.merging) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// groupLines are the groups that are held or half merged, as urgent stalled lines the gate says as conditions (conditions.go):
|
// groupLines are the groups that are held or half merged, as urgent stalled lines the gate says as conditions (conditions.go):
|
||||||
// one for each, on its first member not merged, until every member is merged or a person decides (issue 1275). A member
|
// one for each, on its first member not merged, until every member is merged or a person decides (issue 1275). A member
|
||||||
// the merge tool refused is said by readyLines, with its group; this adds a group held for a required status, and a
|
// the merge tool refused is said by readyLines, with its group; this adds a group held for a required status, and a
|
||||||
@@ -149,9 +289,10 @@ func (h *Holder) groupLines(now time.Time) []StalledLine {
|
|||||||
}
|
}
|
||||||
out = append(out, StalledLine{ID: g.Hold.Member, Number: numberOf(h.deliveries[g.Hold.Member]), State: first.State,
|
out = append(out, StalledLine{ID: g.Hold.Member, Number: numberOf(h.deliveries[g.Hold.Member]), State: first.State,
|
||||||
For: now.Sub(g.Hold.Since).Round(time.Second).String(), Bound: "0s", Severity: "urgent",
|
For: now.Sub(g.Hold.Since).Round(time.Second).String(), Bound: "0s", Severity: "urgent",
|
||||||
H2: "none: the required status turns success, which asks the merge again — or the operator's decision",
|
H2: "none: the required status turns success or a new head merges, which asks the merge again — or the operator's decision",
|
||||||
Says: fmt.Sprintf("group %s %s (merged %d of %d): %s waits on a required status "+
|
Says: fmt.Sprintf("group %s %s (merged %d of %d): %s does not merge now "+
|
||||||
"at its head: %s. Needed: that status turns success (its check asked again), or the operator decides.",
|
"at its head: %s. Needed: that status turns success (its check asked again), a new head that merges "+
|
||||||
|
"(rebased on its trunk), or the operator decides.",
|
||||||
g.ID, stands, merged, len(h.membersOf(g)), g.Hold.Member, g.Hold.Says)})
|
g.ID, stands, merged, len(h.membersOf(g)), g.Hold.Member, g.Hold.Says)})
|
||||||
case merged > 0 && !refused && !running:
|
case merged > 0 && !refused && !running:
|
||||||
out = append(out, StalledLine{ID: first.ID, Number: first.Number, State: first.State,
|
out = append(out, StalledLine{ID: first.ID, Number: first.Number, State: first.State,
|
||||||
@@ -177,3 +318,69 @@ func holdMember(g *Group) string {
|
|||||||
}
|
}
|
||||||
return g.Hold.Member
|
return g.Hold.Member
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MergeAgain is the tool delivery_merge_again: on the operator's word, the merge of an approved delivery the forge refused,
|
||||||
|
// or of a group held part way, is asked again at once at the heads the Approve named (issue 1275): a refusal the forge
|
||||||
|
// meant as "later", a conflict cleared on the trunk, or a refusal repeated until the gate stopped asking. A delivery whose
|
||||||
|
// head moved is asked again by its new head, never by this. Refused, naming why, for one that is not an approved merge
|
||||||
|
// waiting after a refusal or a hold.
|
||||||
|
func (h *Holder) MergeAgain(id, why string) (string, error) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
d := h.deliveries[id]
|
||||||
|
if d == nil {
|
||||||
|
return "", fmt.Errorf("no delivery %q", id)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(why) == "" {
|
||||||
|
return "", fmt.Errorf("say why: the merge asked again is the operator's word, kept in the log")
|
||||||
|
}
|
||||||
|
c := h.covering(d)
|
||||||
|
if c == nil || c.Approved == nil || c.Declined != nil || c.Heads[d.ID] != d.Commit {
|
||||||
|
return "", fmt.Errorf("%s is not approved at its head %s: nothing to ask again (a new head is asked on its own)", id, shortOf(d.Commit, 12))
|
||||||
|
}
|
||||||
|
if d.MergedAs != "" || (d.Ready != nil && d.Ready.Merged != nil) {
|
||||||
|
return "", fmt.Errorf("%s is merged already", id)
|
||||||
|
}
|
||||||
|
g := h.groups[d.Group]
|
||||||
|
if d.Group == "" {
|
||||||
|
g = nil
|
||||||
|
}
|
||||||
|
held := g != nil && g.Hold != nil && g.Hold.Kind == "blocked"
|
||||||
|
refused := false
|
||||||
|
for hid := range c.Heads {
|
||||||
|
if m := h.deliveries[hid]; m != nil && m.Ready != nil && m.Ready.Merged == nil && m.Ready.RefusedSince != nil {
|
||||||
|
refused = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !held && !refused {
|
||||||
|
return "", fmt.Errorf("%s was neither refused nor held: %s", id, firstOf(d.readyWords(), "it is merged at its turn"))
|
||||||
|
}
|
||||||
|
if waits := h.passWaits(d); waits != "" {
|
||||||
|
return "", fmt.Errorf("%s is not asked again yet: it %s; it is asked at that pass's end", id, strings.TrimPrefix(waits, "approved; "))
|
||||||
|
}
|
||||||
|
for hid := range c.Heads {
|
||||||
|
if m := h.deliveries[hid]; m != nil && m.Ready != nil && m.Ready.Merged == nil {
|
||||||
|
m.Ready.Repeats, m.Ready.LastTry, m.Ready.merging = 0, time.Time{}, false
|
||||||
|
h.keep(m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if held {
|
||||||
|
g.Hold = nil
|
||||||
|
h.keepGroup(g)
|
||||||
|
}
|
||||||
|
for key := range h.unmergeableSince {
|
||||||
|
if g != nil && strings.HasPrefix(key, g.ID+" ") {
|
||||||
|
delete(h.unmergeableSince, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
h.Logf("[mesh-delivery] %s: its merge is asked again on the operator's word, at the heads approved: %s", id, why)
|
||||||
|
return fmt.Sprintf("%s's merge is asked again at the next look, every member read first; nothing merges unless every member merges now", id), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readyWords is what an approved delivery's run says it waits for, if anything.
|
||||||
|
func (d *Delivery) readyWords() string {
|
||||||
|
if d.Ready == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return d.Ready.Waits
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
sdk "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The holder of the mesh-delivery seat (novox/hq ADR 0239, to-be 47): the one writer of deliveries and
|
// The holder of the mesh-delivery seat (novox/hq ADR 0239, to-be 47): the one writer of deliveries and
|
||||||
@@ -75,8 +76,19 @@ type Holder struct {
|
|||||||
// unchecked are the open pull requests' heads seen without a merge check (unchecked.go).
|
// unchecked are the open pull requests' heads seen without a merge check (unchecked.go).
|
||||||
unchecked map[string]unchecked
|
unchecked map[string]unchecked
|
||||||
// checker are the checker's verdicts by the hash of the heads they judged, as its verdicts bucket holds them
|
// checker are the checker's verdicts by the hash of the heads they judged, as its verdicts bucket holds them
|
||||||
// (onestatus.go, issue 1302).
|
// (onestatus.go, issue 1302); runningSince when this owner first saw each running (ADR 0548 §2, turns.go).
|
||||||
checker map[string]checkerVerdict
|
checker map[string]checkerVerdict
|
||||||
|
runningSince map[string]time.Time
|
||||||
|
// unmergeableSince is when the forge first said a group's member does not merge now, by group and member (issue 1275,
|
||||||
|
// group_merge.go): within forgeMergeGrace that is the forge still working out mergeability, asked again shortly.
|
||||||
|
unmergeableSince map[string]time.Time
|
||||||
|
// writeReads is whether the forge's merging account writes each repository, as last read (writes.go, issue 1945).
|
||||||
|
writeReads map[string]*writeRead
|
||||||
|
// Turns is the gate's bucket turns (turns.go, ADR 0548 §3); nil keeps none. turnsHeld are the turns as written there,
|
||||||
|
// by repository, read once at the first look (turnsLoaded).
|
||||||
|
Turns TurnStore
|
||||||
|
turnsHeld map[string]sdk.Turn
|
||||||
|
turnsLoaded bool
|
||||||
// tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407).
|
// tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407).
|
||||||
tried owedTries
|
tried owedTries
|
||||||
// forgeWanted is whether the forge's open pull requests are to be read at the next React: a check may have
|
// forgeWanted is whether the forge's open pull requests are to be read at the next React: a check may have
|
||||||
@@ -126,6 +138,7 @@ func (h *Holder) init() {
|
|||||||
h.lostRead = map[string]time.Time{}
|
h.lostRead = map[string]time.Time{}
|
||||||
h.unchecked = map[string]unchecked{}
|
h.unchecked = map[string]unchecked{}
|
||||||
h.checker = map[string]checkerVerdict{}
|
h.checker = map[string]checkerVerdict{}
|
||||||
|
h.runningSince, h.turnsHeld = map[string]time.Time{}, map[string]sdk.Turn{}
|
||||||
h.started = h.Now()
|
h.started = h.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,347 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
sdk "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Issue 1275, met a third time (2026-10-26 13:40, group fix/734-lost-ask): the operator approved mesh-sdk#92,
|
||||||
|
// mesh-controller#472 and mesh-delivery#73; the first two merged, the forge answered #73's merge 405 "Please try again
|
||||||
|
// later", and in the seconds between, group feat/890-placement merged mesh-delivery#72, so #73 then conflicted. A group is
|
||||||
|
// merged only when every member merges now, and no other approved merge lands in a repository between its members.
|
||||||
|
|
||||||
|
const (
|
||||||
|
wSDK, wCtl, wDel = "novox/mesh-sdk", "novox/mesh-controller", "novox/mesh-delivery"
|
||||||
|
wH92, wH472, wH73 = "3203358c4cb5fa5bed223f7ce85577bbb70a346e", "ccc7ba334c82554d809b5b38d8eb0eb5f36be7be", "bf574172154a9f6ed7d4e790643e0dcab2a10a91"
|
||||||
|
wH72, wH73b = "72727272727272727272727272727272727272aa", "b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0"
|
||||||
|
forgeLater = "Gitea API /repos/novox/mesh-delivery/pulls/73/merge: 405 {\"message\":\"Please try again later\"}"
|
||||||
|
conflictWords = "does not merge into main now"
|
||||||
|
headMovedWords = "its head is now"
|
||||||
|
)
|
||||||
|
|
||||||
|
var lostAsk = []member{
|
||||||
|
{wSDK, 92, wH92, "go/buildstep/ask.go"},
|
||||||
|
{wCtl, 472, wH472, "internal/grants.go"},
|
||||||
|
{wDel, 73, wH73, "modules/mesh-delivery-checker/internal/check/lostask.go,modules/mesh-delivery-checker/go.mod"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgePull says what the forge answers of a pull request: its head and whether it merges now.
|
||||||
|
func forgePull(w *world, repository string, number int, head string, mergeable bool) {
|
||||||
|
w.forge.mu.Lock()
|
||||||
|
defer w.forge.mu.Unlock()
|
||||||
|
m := mergeable
|
||||||
|
w.forge.pulls[fmt.Sprintf("%s#%d", repository, number)] = ForgePull{Number: number, State: "open", Base: "main", HeadSHA: head, Mergeable: &m}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnApprovedGroupMergesNoMemberUnlessEveryMemberMergesNow(t *testing.T) {
|
||||||
|
approve := func(t *testing.T) *world {
|
||||||
|
w := approvedGroupOf(t, lostAsk...)
|
||||||
|
allGreenFor(w, lostAsk...)
|
||||||
|
for _, m := range lostAsk {
|
||||||
|
forgePull(w, m.repo, m.number, m.head, true)
|
||||||
|
}
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
// before is the forge as the Approve lands; after, once the grace has passed (nil: as before).
|
||||||
|
before, after func(w *world)
|
||||||
|
// merged is how many members merge in the end; line what the urgent line says, when one stands.
|
||||||
|
merged int
|
||||||
|
line string
|
||||||
|
}{
|
||||||
|
{name: "a member in conflict with its trunk: none merged, the group held and said urgent", merged: 0, line: conflictWords,
|
||||||
|
before: func(w *world) {
|
||||||
|
forgePull(w, wDel, 73, wH73, false)
|
||||||
|
w.forge.refuseRepo = map[string]error{wDel: errors.New(forgeLater)} // what the merge tool would answer
|
||||||
|
}},
|
||||||
|
{name: "the forge still weighing a member: none merged, then every member once it says yes", merged: 3,
|
||||||
|
before: func(w *world) { forgePull(w, wDel, 73, wH73, false) },
|
||||||
|
after: func(w *world) { forgePull(w, wDel, 73, wH73, true) }},
|
||||||
|
{name: "a member's head moved after the Approve: none merged, said urgent", merged: 0, line: headMovedWords,
|
||||||
|
before: func(w *world) { forgePull(w, wDel, 73, wH73b, true) }},
|
||||||
|
{name: "every member merges now: all merged in one pass", merged: 3, before: func(*world) {}},
|
||||||
|
}
|
||||||
|
// The recovery of a group that ended half merged: its remaining member's new head, or the operator's word.
|
||||||
|
t.Run("a half-merged group is made whole by its remaining member's new head", halfMergedMadeWholeByANewHead)
|
||||||
|
t.Run("a member refused part way is asked again on the operator's word", refusedPartWayAskedAgainOnTheOperatorsWord)
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
w := approve(t)
|
||||||
|
c.before(w)
|
||||||
|
w.op.answer(w.op.open()[0], OptionApprove)
|
||||||
|
w.settleAll()
|
||||||
|
if c.after == nil && c.merged == 3 {
|
||||||
|
if len(w.forge.merges) != 3 {
|
||||||
|
t.Fatalf("merged %v, wanted all three", w.forge.merges)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(w.forge.merges) != 0 {
|
||||||
|
t.Fatalf("a member was merged while another does not merge now: %v", w.forge.merges)
|
||||||
|
}
|
||||||
|
if c.after != nil {
|
||||||
|
c.after(w)
|
||||||
|
}
|
||||||
|
w.later(forgeMergeGrace + time30s)
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.forge.merges) != c.merged {
|
||||||
|
t.Fatalf("merged %v, wanted %d", w.forge.merges, c.merged)
|
||||||
|
}
|
||||||
|
lines := urgentLines(w)
|
||||||
|
if c.line == "" {
|
||||||
|
if len(lines) != 0 {
|
||||||
|
t.Fatalf("a group merged whole still says %+v", lines)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(lines) != 1 || !strings.Contains(lines[0].Says, "feat/g") ||
|
||||||
|
!strings.Contains(lines[0].Says, IDOf(wDel, wH73)) || !strings.Contains(lines[0].Says, c.line) ||
|
||||||
|
!strings.Contains(lines[0].Says, "nothing is merged") {
|
||||||
|
t.Fatalf("the held group is not one urgent line naming the member and why: %+v", lines)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handing are the pull requests the gate has handed the forge bridge and the bridge has not answered, in every merges key.
|
||||||
|
func handing(w *world) []sdk.MergePull {
|
||||||
|
w.merges.mu.Lock()
|
||||||
|
defer w.merges.mu.Unlock()
|
||||||
|
var out []sdk.MergePull
|
||||||
|
for _, v := range w.merges.values {
|
||||||
|
for _, p := range v.Pulls {
|
||||||
|
if p.State == sdk.MergeMerging {
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Repository < out[j].Repository })
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func handedWords(ps []sdk.MergePull) string {
|
||||||
|
var out []string
|
||||||
|
for _, p := range ps {
|
||||||
|
out = append(out, fmt.Sprintf("%s#%d", p.Repository, p.Number))
|
||||||
|
}
|
||||||
|
return strings.Join(out, " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two approved merges into one repository: the one whose pass started has the repository until every member merged, and
|
||||||
|
// the other is handed only then (2026-10-26: mesh-delivery#72 of feat/890-placement merged between fix/734-lost-ask's
|
||||||
|
// members). This holds while mesh/check gates no merge too (ADR 0551): it waits for a merge, never for a check.
|
||||||
|
func TestTwoApprovedMergesIntoOneRepositoryDoNotInterleave(t *testing.T) {
|
||||||
|
for _, gates := range []bool{true, false} {
|
||||||
|
t.Run(fmt.Sprintf("check-gates-merge %v", gates), func(t *testing.T) {
|
||||||
|
w := approvedGroupOf(t, lostAsk...)
|
||||||
|
if !gates {
|
||||||
|
ungated(w)
|
||||||
|
}
|
||||||
|
w.merges.manual = true
|
||||||
|
allGreenFor(w, lostAsk...)
|
||||||
|
w.op.answer(w.op.open()[0], OptionApprove)
|
||||||
|
w.settleAll()
|
||||||
|
if got := handedWords(handing(w)); got != "novox/mesh-sdk#92" {
|
||||||
|
t.Fatalf("handed %q, wanted the group's first member", got)
|
||||||
|
}
|
||||||
|
// feat/890-placement: a lone pull request into mesh-delivery, approved while the group's pass is under way.
|
||||||
|
w.h.PullUpdated(pr(wDel, 72, wH72, "feat/890-placement", "modules/mesh-delivery-checker/go.mod"))
|
||||||
|
w.h.Checked(judged(wDel, 72, wH72, "pass", gTrunk0))
|
||||||
|
setChecks(w, wDel, wH72, "success", "success")
|
||||||
|
w.settleAll()
|
||||||
|
if open := w.op.open(); len(open) != 1 {
|
||||||
|
t.Fatalf("the lone pull request is asked once: %v", open)
|
||||||
|
} else {
|
||||||
|
w.op.answer(open[0], OptionApprove)
|
||||||
|
}
|
||||||
|
w.settleAll()
|
||||||
|
var order []string
|
||||||
|
steps := []struct {
|
||||||
|
repo string
|
||||||
|
number int
|
||||||
|
head string
|
||||||
|
}{{wSDK, 92, wH92}, {wCtl, 472, wH472}, {wDel, 73, wH73}}
|
||||||
|
if !gates {
|
||||||
|
// While the check gates the merge, #72's verdict went out of date with #73's merge and it waits for its
|
||||||
|
// check against the new trunk; while it gates none, #72 is handed once the group's pass ended.
|
||||||
|
steps = append(steps, struct {
|
||||||
|
repo string
|
||||||
|
number int
|
||||||
|
head string
|
||||||
|
}{wDel, 72, wH72})
|
||||||
|
}
|
||||||
|
for _, next := range steps {
|
||||||
|
handed := handing(w)
|
||||||
|
if len(handed) != 1 || handed[0].Repository != next.repo || handed[0].Number != next.number {
|
||||||
|
t.Fatalf("after %v: handed %q, wanted only %s#%d", order, handedWords(handed), next.repo, next.number)
|
||||||
|
}
|
||||||
|
merge := strings.Repeat(string(rune('a'+next.number%20)), 40)
|
||||||
|
bridgeMerged(w, next.repo, next.number, next.head, merge)
|
||||||
|
w.h.PullMerged(merged(next.repo, next.number, next.head, merge))
|
||||||
|
w.settleAll()
|
||||||
|
order = append(order, fmt.Sprintf("%s#%d", next.repo, next.number))
|
||||||
|
}
|
||||||
|
if left := handing(w); len(left) != 0 {
|
||||||
|
t.Fatalf("still handed: %q", handedWords(left))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A group that ended half merged (its last member refused after the others merged) is made whole by its remaining member's
|
||||||
|
// new head: rebased on its trunk, checked, composed with the merged members, asked once and merged, and the urgent line
|
||||||
|
// is cleared (issue 1275, the recovery of 2026-10-26 13:40).
|
||||||
|
func halfMergedMadeWholeByANewHead(t *testing.T) {
|
||||||
|
w := approvedGroupOf(t, lostAsk...)
|
||||||
|
allGreenFor(w, lostAsk...)
|
||||||
|
w.forge.refuseRepo = map[string]error{wDel: errors.New(forgeLater)}
|
||||||
|
w.op.answer(w.op.open()[0], OptionApprove)
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.forge.merges) != 3 || len(urgentLines(w)) != 1 {
|
||||||
|
t.Fatalf("the case: two merged and the third refused, said urgent: %v %+v", w.forge.merges, urgentLines(w))
|
||||||
|
}
|
||||||
|
w.forge.refuseRepo = nil
|
||||||
|
// #73 rebased on its trunk: a new head, checked, composed with the members merged.
|
||||||
|
w.h.PullUpdated(pr(wDel, 73, wH73b, "feat/g", "modules/mesh-delivery-checker/go.mod"))
|
||||||
|
w.h.Checked(judged(wDel, 73, wH73b, "pass", gT1))
|
||||||
|
setChecks(w, wDel, wH73b, "success", "success")
|
||||||
|
w.settleAll()
|
||||||
|
open := w.op.open()
|
||||||
|
if len(open) != 1 {
|
||||||
|
t.Fatalf("the remaining member's new head is asked once: %v", open)
|
||||||
|
}
|
||||||
|
w.op.answer(open[0], OptionApprove)
|
||||||
|
w.settleAll()
|
||||||
|
if last := w.forge.merges[len(w.forge.merges)-1]; len(w.forge.merges) != 4 || !strings.Contains(last, "mesh-delivery#73@"+wH73b) {
|
||||||
|
t.Fatalf("the new head is not merged alone: %v", w.forge.merges)
|
||||||
|
}
|
||||||
|
if lines := urgentLines(w); len(lines) != 0 {
|
||||||
|
t.Fatalf("the group is whole and still said: %+v", lines)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// delivery_merge_again (issue 1275, the tools rule): a member refused after its siblings merged, with no new head and no
|
||||||
|
// trunk move to ask it again, is asked again on the operator's word, at the head approved; a delivery neither refused nor
|
||||||
|
// held is refused, naming why.
|
||||||
|
func refusedPartWayAskedAgainOnTheOperatorsWord(t *testing.T) {
|
||||||
|
w := approvedGroupOf(t, lostAsk...)
|
||||||
|
allGreenFor(w, lostAsk...)
|
||||||
|
if _, err := w.h.MergeAgain(IDOf(wDel, wH73), "too early"); err == nil {
|
||||||
|
t.Fatal("a delivery not approved is asked again")
|
||||||
|
}
|
||||||
|
w.forge.refuseRepo = map[string]error{wDel: errors.New(forgeLater)}
|
||||||
|
w.op.answer(w.op.open()[0], OptionApprove)
|
||||||
|
w.settleAll()
|
||||||
|
w.forge.refuseRepo = nil
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.forge.merges) != 3 {
|
||||||
|
t.Fatalf("the case: the third member refused and not asked again by itself: %v", w.forge.merges)
|
||||||
|
}
|
||||||
|
if _, err := w.h.MergeAgain(IDOf(wDel, wH73), ""); err == nil {
|
||||||
|
t.Fatal("asked again with no why")
|
||||||
|
}
|
||||||
|
if _, err := w.h.MergeAgain(IDOf(wDel, wH73), "the forge said try again later"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.forge.merges) != 4 || len(urgentLines(w)) != 0 {
|
||||||
|
t.Fatalf("not merged on the operator's word: %v %+v", w.forge.merges, urgentLines(w))
|
||||||
|
}
|
||||||
|
if _, err := w.h.MergeAgain(IDOf(wDel, wH73), "again"); err == nil || !strings.Contains(err.Error(), "merged already") {
|
||||||
|
t.Fatalf("a merged delivery is asked again: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// approveSecondGroup makes a second group of these members on branch, checked, composed together and approved; the world
|
||||||
|
// already holds group feat/g.
|
||||||
|
func approveSecondGroup(t *testing.T, w *world, branch string, ms ...member) {
|
||||||
|
t.Helper()
|
||||||
|
for _, m := range ms {
|
||||||
|
w.h.PullUpdated(pr(m.repo, m.number, m.head, branch, strings.Split(m.paths, ",")...))
|
||||||
|
w.h.Checked(judged(m.repo, m.number, m.head, "pass", gTrunk0))
|
||||||
|
}
|
||||||
|
allGreenFor(w, ms...)
|
||||||
|
w.settleAll()
|
||||||
|
g := w.h.groups[branch]
|
||||||
|
if g == nil || g.Check == nil {
|
||||||
|
t.Fatalf("no composed check was asked for %s: %+v", branch, g)
|
||||||
|
}
|
||||||
|
c := CheckedEvent{Group: branch, ID: g.Check.Asked, Verdict: "pass", Summary: "composed"}
|
||||||
|
for _, m := range ms {
|
||||||
|
owner, repo, _ := strings.Cut(m.repo, "/")
|
||||||
|
c.Members = append(c.Members, struct {
|
||||||
|
Owner string `json:"owner"`
|
||||||
|
Repo string `json:"repo"`
|
||||||
|
Commit string `json:"commit"`
|
||||||
|
Trunk string `json:"trunk,omitempty"`
|
||||||
|
}{Owner: owner, Repo: repo, Commit: m.head, Trunk: gTrunk0[:12]})
|
||||||
|
}
|
||||||
|
w.h.Checked(c)
|
||||||
|
w.settleAll()
|
||||||
|
for _, key := range w.op.open() {
|
||||||
|
if strings.Contains(key, strings.ReplaceAll(branch, "/", "_")) || strings.Contains(key, branch) {
|
||||||
|
w.op.answer(key, OptionApprove)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A group held half merged keeps its repositories until it is whole or declined: another group sharing one waits, and the
|
||||||
|
// operator's word that asks the held group again never leaves the two waiting on each other (issue 1275, review round 1:
|
||||||
|
// a2 waited for group B, b2 waited for group A, for ever). And the operator's word is refused while another pass it would
|
||||||
|
// wait for is under way, naming it.
|
||||||
|
func TestAHeldHalfMergedGroupKeepsItsRepositoriesAndNoTwoPassesWaitOnEachOther(t *testing.T) {
|
||||||
|
groupA := []member{lostAsk[0], lostAsk[2]} // mesh-sdk#92, then mesh-delivery#73
|
||||||
|
groupB := []member{lostAsk[1], {wDel, 72, wH72, "modules/mesh-delivery-checker/go.mod"}}
|
||||||
|
t.Run("half merged and held: the other group waits, and both merge once the held one is asked again", func(t *testing.T) {
|
||||||
|
w := approvedGroupOf(t, groupA...)
|
||||||
|
allGreenFor(w, groupA...)
|
||||||
|
w.forge.refuseRepo = map[string]error{wDel: errors.New(forgeLater)}
|
||||||
|
w.op.answer(w.op.open()[0], OptionApprove)
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.forge.merges) != 2 {
|
||||||
|
t.Fatalf("the case: mesh-sdk#92 merged, mesh-delivery#73 refused: %v", w.forge.merges)
|
||||||
|
}
|
||||||
|
approveSecondGroup(t, w, "feat/890-placement", groupB...)
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.forge.merges) != 2 {
|
||||||
|
t.Fatalf("group feat/890-placement merged while feat/g is half merged into a repository both go into: %v", w.forge.merges)
|
||||||
|
}
|
||||||
|
w.forge.refuseRepo = nil
|
||||||
|
if _, err := w.h.MergeAgain(IDOf(wDel, wH73), "the forge said try again later"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.settleAll()
|
||||||
|
want := []string{"mesh-sdk#92", "mesh-delivery#73", "mesh-delivery#73", "mesh-controller#472", "mesh-delivery#72"}
|
||||||
|
if len(w.forge.merges) != len(want) {
|
||||||
|
t.Fatalf("merged %v, wanted %v: the held group first, then the other", w.forge.merges, want)
|
||||||
|
}
|
||||||
|
for i, m := range want {
|
||||||
|
if !strings.Contains(w.forge.merges[i], m+"@") {
|
||||||
|
t.Fatalf("merged %v, wanted %v in that order", w.forge.merges, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
t.Run("the operator's word is refused while another pass it would wait for is under way", func(t *testing.T) {
|
||||||
|
w := approvedGroupOf(t, groupA...)
|
||||||
|
allGreenFor(w, groupA...)
|
||||||
|
w.forge.refuseRepo = map[string]error{wSDK: errors.New(forgeLater)} // refused first: nothing of feat/g merged
|
||||||
|
w.op.answer(w.op.open()[0], OptionApprove)
|
||||||
|
w.settleAll()
|
||||||
|
w.merges.manual = true
|
||||||
|
approveSecondGroup(t, w, "feat/890-placement", groupB...)
|
||||||
|
w.settleAll()
|
||||||
|
if got := handedWords(handing(w)); got != "novox/mesh-controller#472" {
|
||||||
|
t.Fatalf("the other group's pass did not start: handed %q", got)
|
||||||
|
}
|
||||||
|
_, err := w.h.MergeAgain(IDOf(wSDK, wH92), "asked again")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "feat/890-placement") {
|
||||||
|
t.Fatalf("asked again while a pass it waits for is under way: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user