Compare commits
90
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8d3f81c2c0 | ||
|
|
e9e8c7c901 | ||
|
|
db35f2b0cb | ||
|
|
f90d3bfcc0 | ||
|
|
37c47f8f95 | ||
|
|
a45e467ddd | ||
|
|
d3f9a1855d | ||
|
|
c97cea21cf | ||
|
|
2f05e70734 | ||
|
|
5ab9d6c296 | ||
|
|
e61c5ee5bf | ||
|
|
b5f812270e | ||
|
|
0786408109 | ||
|
|
a18eeeef5a | ||
|
|
5b6dfc70f4 | ||
|
|
59c28971cb | ||
|
|
035c950ff2 | ||
|
|
5a79776625 | ||
|
|
b9cdca78b0 | ||
|
|
fde561075d | ||
|
|
3cce15fa81 | ||
|
|
889fa09ea4 | ||
|
|
3a25b6e1f8 | ||
|
|
22992da15d | ||
|
|
b6247f837d | ||
|
|
9c5b7e14e1 | ||
|
|
c0873728e2 | ||
|
|
86ba54b06e | ||
|
|
af91a7e660 | ||
|
|
d45767bb2f | ||
|
|
d108487396 | ||
|
|
027ea030fa | ||
|
|
6eb9eb53de | ||
|
|
eed6be0f4a | ||
|
|
51900367a0 | ||
|
|
77596f37a3 | ||
|
|
56f842c3af | ||
|
|
075b61901c | ||
|
|
0270027bf9 | ||
|
|
835bf22b7d | ||
|
|
e85b7da63b | ||
|
|
c112df0c1c | ||
|
|
8e791896f1 | ||
|
|
b07a101caf | ||
|
|
72effa1fb1 | ||
|
|
76a0a726f0 | ||
|
|
df821c8a4a | ||
|
|
f40269caec | ||
|
|
2149d8f3e7 | ||
|
|
e65c8c8961 | ||
|
|
ea8e0def6e | ||
|
|
f8ac498d64 | ||
|
|
276422a85b | ||
|
|
28ef6e9e27 | ||
|
|
f339d0aab7 | ||
|
|
43fccb843c | ||
|
|
f550f1c6b2 | ||
|
|
71f47c3291 | ||
|
|
dc703d1511 | ||
|
|
d70135bf08 | ||
|
|
ff0f43ea2f | ||
|
|
2b2517e4a5 | ||
|
|
b7a4a1a6b0 | ||
|
|
c1a9ad2abc | ||
|
|
c93f77ac19 | ||
|
|
75f3f52589 | ||
|
|
900a2da3e3 | ||
|
|
b7f3664839 | ||
|
|
9511074f40 | ||
|
|
9651a8b2da | ||
|
|
32c66a98c1 | ||
|
|
62437f8964 | ||
|
|
ac4d156cb7 | ||
|
|
95e69276c3 | ||
|
|
56b285d3e8 | ||
|
|
5cc401683c | ||
|
|
79ce72a7b7 | ||
|
|
f40fa6e8e0 | ||
|
|
b015576a13 | ||
|
|
e4fb4cc793 | ||
|
|
1de9a903c8 | ||
|
|
0bb6584437 | ||
|
|
c0bc0d4826 | ||
|
|
efac1b4856 | ||
|
|
bc77afa6bb | ||
|
|
e4373768af | ||
|
|
2cb2e27cdf | ||
|
|
0f4e631cdd | ||
|
|
802c72e4ed | ||
|
|
eb717cc514 |
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/locks
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,105 @@
|
||||
// Command locks refuses a TypeScript module that names packages beyond the SDK in its package.json and commits no
|
||||
// package-lock.json (novox/hq ADR 0469 §5, issue 1868): the Node build module installs a module's own packages only
|
||||
// from its committed lock, so such a module is refused by the builder and could not be delivered.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding. The packages that count are those the build module counts (mesh-build-node fetch.go
|
||||
// ownDependencies): the `dependencies` beyond the SDK, typescript and the @types packages.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Finding is one module refused: it names Packages beyond the SDK and has no lock file.
|
||||
type Finding struct {
|
||||
Module string
|
||||
Packages []string
|
||||
}
|
||||
|
||||
// Check reads every module under root/modules that builds a TypeScript artifact.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Finding
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Language string `json:"language"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
typescript := false
|
||||
for _, a := range m.Build.Artifacts {
|
||||
typescript = typescript || a.Language == "typescript"
|
||||
}
|
||||
if !typescript {
|
||||
continue
|
||||
}
|
||||
dir := filepath.Dir(path)
|
||||
pkgRaw, err := os.ReadFile(filepath.Join(dir, "package.json"))
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pkg struct {
|
||||
Dependencies map[string]json.RawMessage `json:"dependencies"`
|
||||
}
|
||||
if err := json.Unmarshal(pkgRaw, &pkg); err != nil {
|
||||
return nil, fmt.Errorf("%s: package.json: %w", m.Module, err)
|
||||
}
|
||||
var names []string
|
||||
for n := range pkg.Dependencies {
|
||||
if n != "@novox/mesh-sdk" && n != "typescript" && !strings.HasPrefix(n, "@types/") {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "package-lock.json")); err != nil {
|
||||
sort.Strings(names)
|
||||
out = append(out, Finding{Module: m.Module, Packages: names})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Printf("%s names packages beyond the SDK (%s) and has no package-lock.json: commit one made by the build module's toolchain (novox/hq ADR 0469 §5)\n", f.Module, strings.Join(f.Packages, ", "))
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("locks: every TypeScript module that names packages beyond the SDK commits its lock")
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// module writes modules/<name> with a TypeScript artifact, the package.json given ("" for none) and, when lock, a lock.
|
||||
func module(t *testing.T, root, name, language, pkg string, lock bool) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", name)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write := func(file, body string) {
|
||||
if err := os.WriteFile(filepath.Join(dir, file), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write("module.json", `{"module":"`+name+`","build":{"artifacts":[{"name":"tools","kind":"bundle","language":"`+language+`","from":"index.ts"}]}}`)
|
||||
if pkg != "" {
|
||||
write("package.json", pkg)
|
||||
}
|
||||
if lock {
|
||||
write("package-lock.json", "{}")
|
||||
}
|
||||
}
|
||||
|
||||
const sdkOnly = `{"dependencies":{"@novox/mesh-sdk":"^0.1.1"},"devDependencies":{"typescript":"^5","@types/node":"^22"}}`
|
||||
const withPg = `{"dependencies":{"@novox/mesh-sdk":"^0.1.1","pg":"^8","@types/pg":"^8"}}`
|
||||
|
||||
func TestAModuleNamingOnlyTheSDKNeedsNoLock(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "plain", "typescript", sdkOnly, false)
|
||||
module(t, root, "none", "typescript", "", false)
|
||||
got, err := Check(root)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("%v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleNamingMorePackagesWithoutALockIsRefusedNamingThem(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "db", "typescript", withPg, false)
|
||||
module(t, root, "db-locked", "typescript", withPg, true)
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []Finding{{Module: "db", Packages: []string{"pg"}}}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleThatIsNotTypeScriptIsNotJudged(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "go", "go", withPg, false)
|
||||
if got, err := Check(root); err != nil || len(got) != 0 {
|
||||
t.Fatalf("%v %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,8 @@ set -eu
|
||||
(cd checks/imagebuild && go test -count=1 . && go run . ../..)
|
||||
# The image's place (novox/hq ADR 0487 §1): every image builds from a Dockerfile in its own module's directory.
|
||||
(cd checks/imageplace && go test -count=1 . && go run . ../..)
|
||||
# The Node lock files (novox/hq ADR 0469 §5, issue 1868): a TypeScript module that names packages beyond the SDK commits its lock.
|
||||
(cd checks/locks && go test -count=1 . && go run . ../..)
|
||||
# The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
|
||||
(cd checks/agenthome && go test -count=1 . && go run . ../..)
|
||||
# The mail-store seat (novox/hq ADR 0378 §3): only mesh-mail may claim it.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
@@ -170,20 +171,33 @@ func TestTheLauncherWaitsQuietlyWhileAnotherClipmenudHoldsTheLock(t *testing.T)
|
||||
store := dir + "/cm/clipmenu.6.op"
|
||||
os.MkdirAll(store, 0o700)
|
||||
lock := store + "/session_lock"
|
||||
// the older clipmenud: it holds the lock the way clipmenud does, until it is killed
|
||||
holder := exec.Command("flock", "-x", "-n", lock, "sleep", "60")
|
||||
// the older clipmenud: it holds the lock the way clipmenud does, until it is killed. It says "held" once it has the
|
||||
// lock, so the test goes on only when the lock is held for certain: a holder that is slow to start (the checker
|
||||
// runs thousands of tests at once) must not let the launcher find the lock free (issue 1814).
|
||||
holder := exec.Command("flock", "-x", "-n", lock, "sh", "-c", "echo held; exec sleep 60")
|
||||
holder.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
held, err := holder.StdoutPipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := holder.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// flock's child shares the lock, so the whole group goes
|
||||
release := func() { syscall.Kill(-holder.Process.Pid, syscall.SIGKILL); holder.Wait() }
|
||||
defer release()
|
||||
for i := 0; i < 50; i++ {
|
||||
if probe := exec.Command("flock", "-n", lock, "true"); probe.Run() != nil {
|
||||
break
|
||||
said := make(chan string, 1)
|
||||
go func() {
|
||||
line, _ := bufio.NewReader(held).ReadString('\n')
|
||||
said <- strings.TrimSpace(line)
|
||||
}()
|
||||
select {
|
||||
case line := <-said:
|
||||
if line != "held" {
|
||||
t.Fatalf("the older clipmenud did not take the lock: %q", line)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the older clipmenud did not take the lock within 30 s")
|
||||
}
|
||||
var out syncBuffer
|
||||
cmd := exec.Command("sh", file)
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
// **mesh/check is written from the forge bridge's own event, never from a tool** (review finding K1 of R4e; novox/hq issue
|
||||
// 1301; ADR 0447 §1, ADR 0503 §3, ADR 0513 §4: main requires mesh/check alone).
|
||||
//
|
||||
// A tool is told no caller, and the console every agent session speaks through (node-tools) invokes every tool, so a tool
|
||||
// that wrote mesh/check could be called by any session to pass a merge past the checker: the fault issue 293 closed for
|
||||
// the two statuses before it. So the checker's forge bridge emits `check.status` instead. The bus grants the subject
|
||||
// mesh.mod.mesh-delivery-forge-bridge.event.check.status to the bridge's own account alone (its manifest's `emits`; a
|
||||
// module publishes under its own name only), and the bridge runs on that account, never carried by a machine's tool
|
||||
// runner. This module consumes the event, as it consumes mesh-controller.checked, and writes the status.
|
||||
//
|
||||
// The event's body: owner, repo, sha (the head, whole), state, description, target_url, key (the checker's owed key) and
|
||||
// revision (that key's revision in the checker's bucket). Its source is checked against the bridge too: the bus already
|
||||
// said it, and a body read from elsewhere says it again. Delivery is at-least-once and a failed write is delivered again
|
||||
// later, so a head's revision older than one already written is dropped: a stale pending never hides a verdict.
|
||||
|
||||
import type { CommitStatus } from "./client.js";
|
||||
import { checkName } from "./guard.js";
|
||||
import { CHECK_STATUS } from "./pulls.js";
|
||||
|
||||
/** The module that alone writes mesh/check, and the event it writes it with. */
|
||||
export const CHECK_WRITER = "mesh-delivery-forge-bridge";
|
||||
export const CHECK_EVENT = `${CHECK_WRITER}.check.status`;
|
||||
|
||||
const states = new Set(["pending", "success", "error", "failure", "warning"]);
|
||||
|
||||
/** What writing the check needs of the forge. */
|
||||
export interface CheckForge {
|
||||
setCommitStatus(owner: string, repo: string, sha: string, status: CommitStatus): Promise<void>;
|
||||
}
|
||||
|
||||
/** An event as the SDK's `on` hands it. */
|
||||
export interface CheckEvent {
|
||||
source: string;
|
||||
id?: string;
|
||||
body: unknown;
|
||||
}
|
||||
|
||||
/** The status a check.status asks for, checked; or why it is not one. */
|
||||
export function checkStatusOf(body: unknown): { owner: string; repo: string; sha: string; revision: number; status: CommitStatus } {
|
||||
if (!body || typeof body !== "object") throw new Error("the event carries no body");
|
||||
const b = body as Record<string, unknown>;
|
||||
const owner = checkName("owner", b.owner), repo = checkName("name", b.repo);
|
||||
const sha = String(b.sha ?? "");
|
||||
if (!/^([0-9a-f]{40}|[0-9a-f]{64})$/i.test(sha)) throw new Error(`${JSON.stringify(b.sha)} is not a head's whole sha`);
|
||||
if (b.context !== undefined && b.context !== null && b.context !== CHECK_STATUS) {
|
||||
throw new Error(`${JSON.stringify(b.context)}: check.status writes ${CHECK_STATUS} alone`);
|
||||
}
|
||||
const state = String(b.state ?? "");
|
||||
if (!states.has(state)) throw new Error(`${JSON.stringify(b.state)} is not a status the forge keeps`);
|
||||
const revision = b.revision;
|
||||
if (typeof revision !== "number" || !Number.isSafeInteger(revision) || revision < 0) {
|
||||
throw new Error(`${JSON.stringify(revision)} is not a revision of the checker's owed key`);
|
||||
}
|
||||
let description = String(b.description ?? "").replace(/\s+/g, " ").trim();
|
||||
if (description.length > 140) description = description.slice(0, 139) + "…";
|
||||
const target = typeof b.target_url === "string" && /^https?:\/\//.test(b.target_url) ? b.target_url : undefined;
|
||||
const status: CommitStatus = { state: state as CommitStatus["state"], context: CHECK_STATUS, description,
|
||||
...(target ? { target_url: target } : {}) };
|
||||
return { owner, repo, sha, revision, status };
|
||||
}
|
||||
|
||||
/** The one writer of mesh/check: each check.status the bridge emits, in revision order per head. */
|
||||
export class CheckStatuses {
|
||||
/** The newest revision written on each head, owner/repo@sha; the oldest heads go past what any forge holds open. */
|
||||
private written = new Map<string, number>();
|
||||
/** The turn each head's writes wait on. */
|
||||
private turns = new Map<string, Promise<void>>();
|
||||
|
||||
private forge: CheckForge;
|
||||
private say: (line: string) => void;
|
||||
|
||||
constructor(forge: CheckForge, say: (line: string) => void = (l) => console.error(l)) {
|
||||
this.forge = forge;
|
||||
this.say = say;
|
||||
}
|
||||
|
||||
/** Write the event's status: "written", or "dropped" for one that is not the bridge's, malformed or stale. A forge
|
||||
* that refuses throws, so the bus delivers the event again. */
|
||||
async apply(event: CheckEvent): Promise<"written" | "dropped"> {
|
||||
if (event.source !== CHECK_WRITER) {
|
||||
this.say(`[gitea] a check.status from ${JSON.stringify(event.source)}, not the forge bridge, is dropped: mesh/check is the bridge's alone`);
|
||||
return "dropped";
|
||||
}
|
||||
let asked;
|
||||
try {
|
||||
asked = checkStatusOf(event.body);
|
||||
} catch (err) {
|
||||
this.say(`[gitea] the forge bridge's check.status ${event.id ?? ""} is dropped: ${err instanceof Error ? err.message : err}`);
|
||||
return "dropped";
|
||||
}
|
||||
const head = `${asked.owner}/${asked.repo}@${asked.sha}`.toLowerCase();
|
||||
// One head's writes take turns, so two deliveries at once are weighed one after the other.
|
||||
const before = this.turns.get(head) ?? Promise.resolve();
|
||||
const turn = before.then(async (): Promise<"written" | "dropped"> => {
|
||||
const newest = this.written.get(head);
|
||||
if (newest !== undefined && asked.revision < newest) {
|
||||
this.say(`[gitea] ${asked.owner}/${asked.repo}@${asked.sha.slice(0, 12)}: mesh/check at revision ${asked.revision} is older than ${newest}, already written, and is dropped`);
|
||||
return "dropped";
|
||||
}
|
||||
await this.forge.setCommitStatus(asked.owner, asked.repo, asked.sha, asked.status);
|
||||
this.written.delete(head);
|
||||
this.written.set(head, asked.revision);
|
||||
if (this.written.size > 2000) this.written.delete(this.written.keys().next().value as string);
|
||||
return "written";
|
||||
});
|
||||
const settled = turn.then(() => {}, () => {});
|
||||
this.turns.set(head, settled);
|
||||
void settled.then(() => { if (this.turns.get(head) === settled) this.turns.delete(head); });
|
||||
return turn;
|
||||
}
|
||||
}
|
||||
+17
-2
@@ -533,7 +533,7 @@ export class GiteaClient {
|
||||
* **A listing the forge cut is no listing** (it says `truncated` on a page with nothing more to read, or the pages
|
||||
* run past a bound): the call fails naming the repository and the commit, never answering the directories it did
|
||||
* see, since a directory left out would be taken for one without a module. Reads only. */
|
||||
async commitTrees(owner: string, repo: string, sha: string): Promise<{ root: string; trees: Record<string, string> }> {
|
||||
async commitTrees(owner: string, repo: string, sha: string, paths?: Record<string, string[]>): Promise<{ root: string; trees: Record<string, string>; paths?: Record<string, Record<string, string>> }> {
|
||||
const base = `${repoPath(owner, repo)}/git/trees/${commitSegment(sha)}?recursive=true`;
|
||||
const cut = (why: string) => new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} was cut (${why}); no trees are answered from a part`);
|
||||
const entries: any[] = [];
|
||||
@@ -560,7 +560,22 @@ export class GiteaClient {
|
||||
if (!tree) throw new Error(`the forge's tree listing of ${owner}/${repo} at ${sha} holds ${e.path} but not the tree of ${dir}`);
|
||||
trees[dir] = tree;
|
||||
}
|
||||
return { root, trees };
|
||||
if (paths === undefined) return { root, trees };
|
||||
// **Named paths** (novox/hq ADR 0499 §3, issue 1758): each asked path's git object id at the commit, a file's or a
|
||||
// directory's, from the same listing, by directory and by path relative to it. A path the listing lacks fails
|
||||
// naming both, and nothing is answered in part.
|
||||
const objectOf = new Map<string, string>();
|
||||
for (const e of entries) if (typeof e?.path === "string" && (e.type === "tree" || e.type === "blob")) objectOf.set(e.path, String(e.sha));
|
||||
const named: Record<string, Record<string, string>> = {};
|
||||
for (const [dir, list] of Object.entries(paths)) {
|
||||
named[dir] = {};
|
||||
for (const p of list) {
|
||||
const id = objectOf.get(dir === "" ? p : `${dir}/${p}`);
|
||||
if (!id) throw new Error(`the path ${p} of ${dir === "" ? "the repository root" : dir} is not in ${owner}/${repo} at ${sha}`);
|
||||
named[dir][p] = id;
|
||||
}
|
||||
}
|
||||
return { root, trees, paths: named };
|
||||
}
|
||||
|
||||
/** Whether a repository holds a file at a commit: true, false for a 404, thrown when the forge cannot say. */
|
||||
|
||||
@@ -101,10 +101,10 @@ export function deliveryStatus(context: string, state: string, description: stri
|
||||
if (!/^mesh\/[a-z-]+$/.test(context)) throw new Error(`${context} is not a status of the mesh's own`);
|
||||
if (!states.has(state)) throw new Error(`${state} is not a status the forge keeps`);
|
||||
// mesh/check is the checker's verdict and the one status main requires (ADR 0447 §1): no hand sets it in any
|
||||
// state, as issue 293 closed for the two before it. The forge bridge writes it through gitea_check_status, a tool
|
||||
// only its account may call (issue 1301).
|
||||
// state, as issue 293 closed for the two before it. No tool writes it, since a tool is told no caller: the forge
|
||||
// bridge emits check.status on a subject only its own account may publish, and check.ts writes it (R4e finding K1).
|
||||
if (context === CHECK_STATUS) {
|
||||
throw new Error(`${CHECK_STATUS} is the checker's status: the forge bridge writes it through gitea_check_status, never this tool`);
|
||||
throw new Error(`${CHECK_STATUS} is the checker's status: it is written from the forge bridge's check.status event alone, never by a tool`);
|
||||
}
|
||||
// The merge check's statuses are its verdict's, set when the controller says it, and a branch may require
|
||||
// them: never set to a verdict by hand, so nothing passes — or fails — a merge past the check (novox/hq
|
||||
@@ -136,7 +136,7 @@ export interface StatusWriters {
|
||||
export function commitStatusTool(writers: StatusWriters): ToolDefinition {
|
||||
return {
|
||||
name: "gitea_commit_status",
|
||||
description: "Set one of the mesh's statuses on a commit (mesh/delivery, mesh/delivery-group): pending, success, error, failure or warning, a short description, and the page it links to. mesh/check, the status main requires, is refused: the checker's forge bridge writes it through gitea_check_status. hq's mesh/repo-check (and mesh/merge-gate, kept for a revert) take pending only — a recheck putting them back until the fresh verdict sets them — on the head's whole sha, in their turn after the check's earlier writes.",
|
||||
description: "Set one of the mesh's statuses on a commit (mesh/delivery, mesh/delivery-group): pending, success, error, failure or warning, a short description, and the page it links to. mesh/check, the status main requires, is refused: no tool writes it, only the checker's forge bridge's check.status event. hq's mesh/repo-check (and mesh/merge-gate, kept for a revert) take pending only — a recheck putting them back until the fresh verdict sets them — on the head's whole sha, in their turn after the check's earlier writes.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
@@ -163,48 +163,3 @@ export function commitStatusTool(writers: StatusWriters): ToolDefinition {
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** What the check's status tool writes with: the forge. */
|
||||
export interface CheckStatusWriter {
|
||||
setCommitStatus(owner: string, repo: string, sha: string, status: CommitStatus): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* The `gitea_check_status` tool (novox/hq issue 1301, ADR 0447 §1): the one way `mesh/check` is written. Its arguments
|
||||
* are gitea_commit_status's, so the forge bridge sends an owed status's arguments unchanged to it: owner, repo, sha
|
||||
* (the head's whole sha), state, description, target_url, and context, which may be left out and is mesh/check when
|
||||
* given. **Who may write it is the bus's grant**: the runtime tells a tool no caller, so only a module that lists
|
||||
* `gitea.gitea_check_status` in its manifest's `invokes` may call it, and the forge bridge alone does. Served beside
|
||||
* gitea_commit_status, from the watcher's bundle.
|
||||
*/
|
||||
export function checkStatusTool(writer: CheckStatusWriter): ToolDefinition {
|
||||
return {
|
||||
name: "gitea_check_status",
|
||||
description: "Set mesh/check, the checker's verdict and the one status main requires, on a pull request's head: pending, success, error, failure or warning, a short description, and the page it links to. The checker's forge bridge alone may call it (its manifest's invokes); no other status is written here, and no other tool writes this one.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
sha: { type: "string", description: "the head's whole sha" },
|
||||
context: { type: "string", description: "mesh/check, or left out" },
|
||||
state: { type: "string", description: "pending | success | error | failure | warning" },
|
||||
description: { type: "string", description: "one line" },
|
||||
target_url: { type: "string", description: "the page it links to (the check's report)" },
|
||||
},
|
||||
run: async (args) => {
|
||||
const owner = checkName("owner", args.owner), repo = checkName("name", args.repo), sha = String(args.sha ?? "");
|
||||
if (args.context !== undefined && args.context !== null && String(args.context) !== CHECK_STATUS) {
|
||||
throw new Error(`${JSON.stringify(args.context)}: this tool writes ${CHECK_STATUS} alone. Nothing was sent`);
|
||||
}
|
||||
if (!/^([0-9a-f]{40}|[0-9a-f]{64})$/i.test(sha)) {
|
||||
throw new Error(`${JSON.stringify(args.sha)} is not a head's whole sha: ${CHECK_STATUS} names the head whole. Nothing was sent`);
|
||||
}
|
||||
// deliveryStatus checks the state and cuts the description; it refuses mesh/check, so it is asked for a
|
||||
// context of the mesh's own and the status is named mesh/check after.
|
||||
const checked = deliveryStatus("mesh/delivery", String(args.state), String(args.description ?? ""),
|
||||
args.target_url ? String(args.target_url) : undefined);
|
||||
const status = { ...checked, context: CHECK_STATUS };
|
||||
await writer.setCommitStatus(owner, repo, sha, status);
|
||||
return { set: status };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
+20
-5
@@ -13,8 +13,11 @@
|
||||
// commit's statuses: `mesh/merge-gate`, the modules of the mesh's graph the change touches, and
|
||||
// `mesh/repo-check`, the repository's own merge-check.sh (novox/hq ADR 0237 as amended); with a comment
|
||||
// saying why when the gate is not a pass or the repository's own check failed. Of the two, only those the base
|
||||
// still requires are written (issue 1301): main requires `mesh/check` — the checker's, written by its forge bridge
|
||||
// through gitea_check_status — everywhere but hq, whose main requires `mesh/repo-check` alone (ADR 0503 §5).
|
||||
// still requires are written (issue 1301): main requires `mesh/check` — the checker's — everywhere but hq, whose main
|
||||
// requires `mesh/repo-check` alone (ADR 0503 §5).
|
||||
//
|
||||
// Consumes mesh-delivery-forge-bridge.check.status — the checker's forge bridge's mesh/check on a head — and writes it
|
||||
// (check.ts). No tool writes mesh/check: the bus lets the bridge's own account alone publish that event (R4e finding K1).
|
||||
//
|
||||
// Consumes node-build-agent.started — a build machine took an ask — and says, on the head a pending check is of,
|
||||
// which machine builds it since when (novox/hq issue 446). This module is the one writer of mesh/merge-gate.
|
||||
@@ -40,7 +43,9 @@
|
||||
|
||||
import { emit, on } from "@novox/mesh-sdk/events";
|
||||
import { registerModuleTools } from "@novox/mesh-sdk/tools";
|
||||
import { checkStatusTool, commitStatusTool } from "./delivery.js";
|
||||
import { statesAnnouncement } from "./kept.js";
|
||||
import { commitStatusTool } from "./delivery.js";
|
||||
import { CHECK_EVENT, CheckStatuses } from "./check.js";
|
||||
import { announce, keepWatermark, readWatermark } from "./announce.js";
|
||||
import { mergeTool } from "./merge.js";
|
||||
import { GiteaAdmin, GiteaClient, movedSince } from "./client.js";
|
||||
@@ -474,6 +479,11 @@ const gates = new GateWriter({
|
||||
headChecks: (owner, repo, sha) => (gitea as GiteaClient).headChecks(owner, repo, sha),
|
||||
});
|
||||
|
||||
// The one writer of mesh/check, from the forge bridge's check.status (check.ts, R4e finding K1).
|
||||
const checks = new CheckStatuses({
|
||||
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
||||
});
|
||||
|
||||
// **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own
|
||||
// process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to
|
||||
// reach, nothing is served, as the tools' bundle serves nothing then.
|
||||
@@ -482,12 +492,14 @@ const gates = new GateWriter({
|
||||
const pace = new Pace();
|
||||
if (!gitea) pace.unheard("the forge is not watched: no way to a token", false);
|
||||
|
||||
// The states this module keeps and which others may read, said on the bus before it serves (novox/hq ADR 0495 clause 2).
|
||||
// Said with `emit`, not the SDK's `keeps`, which the toolchain's SDK does not have yet: switch to `keeps` once that SDK is 0.1.15 or later (issue 1852).
|
||||
await emit("states", statesAnnouncement(process.env.MESH_MODULE || "gitea", process.env.MESH_NODE));
|
||||
|
||||
// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729).
|
||||
registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({
|
||||
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
||||
gates,
|
||||
}), checkStatusTool({
|
||||
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
||||
}), mergeTool(gitea, { gates })] : [])]);
|
||||
|
||||
// **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when
|
||||
@@ -615,5 +627,8 @@ if (gitea) {
|
||||
return setVerdict(client, event);
|
||||
})],
|
||||
["node-build-agent.started", () => on("node-build-agent.started", buildStarted)],
|
||||
// The checker's mesh/check, from the one account the bus lets publish it (check.ts): a forge that refuses throws,
|
||||
// and the event is delivered again.
|
||||
[CHECK_EVENT, () => on(CHECK_EVENT, async (event) => { await checks.apply(event); })],
|
||||
], () => console.log("[gitea] watching for new repositories and merged pull requests"));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// The states this module keeps on the bus and whether other modules may read them (novox/hq ADR 0495 clause 2; issue
|
||||
// 1790): the module says it itself, from its code, as it starts (index.ts calls keeps with this list), and a reader is
|
||||
// granted what it said (clause 3). A state is readable when another module's manifest reads it; none reads gitea's.
|
||||
// test/kept.test.ts holds the list to the manifest's `state`.
|
||||
export const KEPT: readonly { readonly name: string; readonly readable?: boolean }[] = [
|
||||
{ name: "announced" },
|
||||
{ name: "bound" },
|
||||
];
|
||||
|
||||
/** The announcement of these states, in the layout mesh-sdk's go/announce reads (novox/hq ADR 0495 clause 2): version 1,
|
||||
* the module, the machine when known, and the states sorted by name, `readable` only where true. It is said as the event
|
||||
* `states`, which is the subject mesh.mod.gitea.event.states. gitea says it with the SDK's `emit` because the
|
||||
* toolchain's SDK, which judges R4e's group, predates the SDK's `keeps` (0.1.15); switch to `keeps` once the toolchain's
|
||||
* SDK is 0.1.15 or later (issue 1852). */
|
||||
export function statesAnnouncement(module: string, node: string | undefined) {
|
||||
const states = [...KEPT]
|
||||
.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
|
||||
.map((k) => (k.readable ? { name: k.name, readable: true } : { name: k.name }));
|
||||
return { v: 1, module, ...(node ? { node } : {}), states };
|
||||
}
|
||||
@@ -87,7 +87,8 @@
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-controller.checked",
|
||||
"node-build-agent.started"
|
||||
"node-build-agent.started",
|
||||
"mesh-delivery-forge-bridge.check.status"
|
||||
],
|
||||
"listens": [
|
||||
{
|
||||
|
||||
@@ -36,3 +36,60 @@ test("importing the bundle resolves while the broker keeps refusing the binds",
|
||||
clearTimeout(timer);
|
||||
assert.match(out, /imported/, `the import did not resolve within 3 s while a bind was refused; stderr: ${err.slice(0, 600)}`);
|
||||
});
|
||||
|
||||
// **The bundle binds the forge bridge's check.status and serves no tool that writes mesh/check** (review finding K1 of R4e;
|
||||
// issue 1301 S4; ADR 0503 §3). The bundle is imported whole, as the runtime imports it; the consumer it binds is handed the
|
||||
// bridge's event and one from another source, and the forge stand-in sees mesh/check written for the bridge's alone.
|
||||
test("the bundle writes mesh/check from the bridge's check.status alone, and serves no gitea_check_status", async () => {
|
||||
const index = pathToFileURL(join(here, "..", "dist", "index.js")).href;
|
||||
const sha = "0123456789abcdef0123456789abcdef01234567";
|
||||
const script = `
|
||||
const http = await import("node:http");
|
||||
const statuses = [];
|
||||
const forge = http.createServer((req, res) => {
|
||||
let b = ""; req.on("data", (d) => (b += d)); req.on("end", () => {
|
||||
if (req.method === "POST" && /\\/statuses\\//.test(req.url)) statuses.push(req.url + " " + b);
|
||||
res.setHeader("content-type", "application/json"); res.end(req.method === "GET" ? "[]" : "{}");
|
||||
});
|
||||
});
|
||||
await new Promise((r) => forge.listen(0, "127.0.0.1", r));
|
||||
process.env.MESH_GITEA_URL = "http://127.0.0.1:" + forge.address().port;
|
||||
const bound = {};
|
||||
const { useBroker } = await import("@novox/mesh-sdk/messaging");
|
||||
useBroker(() => ({
|
||||
subscribe: async (pattern, handler) => { bound[pattern] = handler; return () => {}; },
|
||||
publish: async () => {}, request: async () => { throw new Error("no responders"); },
|
||||
handle: async () => () => {}, close: async () => {},
|
||||
}));
|
||||
await import(${JSON.stringify(index)});
|
||||
const { collectTools } = await import("@novox/mesh-sdk/tools");
|
||||
const tools = collectTools().flatMap((r) => r.tools.map((t) => t.name));
|
||||
for (let i = 0; i < 100 && !bound["mesh-delivery-forge-bridge.check.status"]; i++) await new Promise((r) => setTimeout(r, 20));
|
||||
const h = bound["mesh-delivery-forge-bridge.check.status"];
|
||||
const env = (source, state) => ({ key: "check.status", node: "n1",
|
||||
headers: { "x-event-id": "e", "x-source": source, "x-node": "n1", "x-time": "", "content-type": "application/json" },
|
||||
body: { owner: "novox", repo: "mesh-sdk", sha: ${JSON.stringify(sha)}, state, description: "by " + source, revision: 1 } });
|
||||
if (h) { await h(env("node-tools", "success")); await h(env("mesh-delivery-forge-bridge", "pending")); }
|
||||
console.log(JSON.stringify({ tools, bound: Object.keys(bound), statuses }));
|
||||
process.exit(0);`;
|
||||
const env = { ...process.env, MESH_GITEA_TOKEN: "x" };
|
||||
for (const k of ["MESH_GITEA_STATE_DIR", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD", "MESH_SERVED_MODULE", "MESH_GITEA_URL"]) delete env[k];
|
||||
const child = spawn(process.execPath, ["--input-type=module", "-e", script], { cwd: join(here, ".."), env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let out = "";
|
||||
let err = "";
|
||||
child.stdout.on("data", (d) => (out += d));
|
||||
child.stderr.on("data", (d) => (err += d));
|
||||
const ended = new Promise<void>((r) => child.on("exit", () => r()));
|
||||
const timer = setTimeout(() => child.kill("SIGKILL"), 8000);
|
||||
await ended;
|
||||
clearTimeout(timer);
|
||||
const line = out.split("\n").find((l) => l.startsWith("{"));
|
||||
assert.ok(line, `the bundle said nothing; stdout: ${out.slice(0, 400)} stderr: ${err.slice(0, 600)}`);
|
||||
const got = JSON.parse(line!) as { tools: string[]; bound: string[]; statuses: string[] };
|
||||
assert.ok(got.tools.includes("gitea_commit_status"), `the bundle's tools: ${got.tools}`);
|
||||
assert.ok(!got.tools.includes("gitea_check_status"), "no tool writes mesh/check");
|
||||
assert.ok(got.bound.includes("mesh-delivery-forge-bridge.check.status"), `bound: ${got.bound}`);
|
||||
assert.equal(got.statuses.length, 1, `statuses written: ${got.statuses}`);
|
||||
assert.match(got.statuses[0], new RegExp(`/repos/novox/mesh-sdk/statuses/${sha} .*"context":"mesh/check"`));
|
||||
assert.match(got.statuses[0], /"state":"pending".*by mesh-delivery-forge-bridge/);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
// **mesh/check is written from the forge bridge's own event, never from a tool** (review finding K1 of R4e; issue 1301 S4;
|
||||
// ADR 0503 §3, ADR 0513 §4). A tool cannot know its caller, and node-tools invokes every tool ("*"), so any session could
|
||||
// have set mesh/check through gitea_check_status. The bus grants the subject mesh.mod.mesh-delivery-forge-bridge.event.
|
||||
// check.status to the bridge's own account alone; this module consumes it and writes the status.
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
const sha = "0123456789abcdef0123456789abcdef01234567";
|
||||
const body = (over: Record<string, unknown> = {}) => ({ owner: "novox", repo: "mesh-sdk", sha, state: "success",
|
||||
description: "pass 0123456789ab on aaaaaaaaaaaa", target_url: "https://view.example/checks/g.1",
|
||||
key: `novox/mesh-sdk.12.${sha}.status=mesh~check`, revision: 7, ...over });
|
||||
const event = (b: unknown, source = "mesh-delivery-forge-bridge") => ({ type: "check.status", id: "x", source, node: "n1", at: "", body: b });
|
||||
|
||||
function forge() {
|
||||
const sent: string[] = [];
|
||||
let fail = false;
|
||||
return {
|
||||
sent,
|
||||
failNext: () => { fail = true; },
|
||||
setCommitStatus: async (o: string, r: string, s: string, st: any) => {
|
||||
if (fail) { fail = false; throw new Error("Gitea API /repos/x/statuses: 502 bad gateway"); }
|
||||
sent.push(`${o}/${r}@${s.slice(0, 8)} ${st.context} ${st.state} ${st.description}${st.target_url ? " " + st.target_url : ""}`);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test("the bridge's check.status event is the subject this module consumes, and the bridge is its one source", async () => {
|
||||
const { CHECK_EVENT, CHECK_WRITER } = await import("../check.ts");
|
||||
assert.equal(CHECK_EVENT, "mesh-delivery-forge-bridge.check.status");
|
||||
assert.equal(CHECK_WRITER, "mesh-delivery-forge-bridge");
|
||||
const manifest = JSON.parse(readFileSync(new URL("../module.json", import.meta.url), "utf8"));
|
||||
assert.ok(manifest.consumes.includes(CHECK_EVENT), "gitea's manifest consumes the bridge's check.status");
|
||||
});
|
||||
|
||||
test("a check.status from the bridge writes mesh/check on the head, its state, its line cut and its link", async () => {
|
||||
const { CheckStatuses } = await import("../check.ts");
|
||||
const f = forge();
|
||||
const checks = new CheckStatuses(f, () => {});
|
||||
assert.equal(await checks.apply(event(body())), "written");
|
||||
assert.equal(await checks.apply(event(body({ sha: sha.replace(/0/g, "f"), state: "pending", description: "y".repeat(300), target_url: "javascript:x", revision: 8 }))), "written");
|
||||
assert.equal(f.sent[0], "novox/mesh-sdk@01234567 mesh/check success pass 0123456789ab on aaaaaaaaaaaa https://view.example/checks/g.1");
|
||||
assert.ok(f.sent[1].startsWith("novox/mesh-sdk@f1234567 mesh/check pending yyyy"), f.sent[1]);
|
||||
assert.ok(f.sent[1].length < 200, "the line is cut as every status's is");
|
||||
assert.ok(!f.sent[1].includes("javascript"), "a link that is no web page is not written");
|
||||
});
|
||||
|
||||
test("an event whose source is not the bridge writes nothing", async () => {
|
||||
const { CheckStatuses } = await import("../check.ts");
|
||||
const f = forge();
|
||||
const said: string[] = [];
|
||||
const checks = new CheckStatuses(f, (l) => said.push(l));
|
||||
for (const source of ["node-tools", "gitea", "unknown", ""]) assert.equal(await checks.apply(event(body(), source)), "dropped", source);
|
||||
assert.deepEqual(f.sent, []);
|
||||
assert.match(said[0], /not the forge bridge/);
|
||||
});
|
||||
|
||||
test("a malformed event writes nothing and is not asked for again", async () => {
|
||||
const { CheckStatuses } = await import("../check.ts");
|
||||
const f = forge();
|
||||
const checks = new CheckStatuses(f, () => {});
|
||||
for (const b of [body({ sha: sha.slice(0, 12) }), body({ sha: "../../admin" }), body({ state: "green" }), body({ owner: "../x" }),
|
||||
body({ repo: ".." }), body({ revision: undefined }), body({ revision: -1 }), body({ revision: "7" }), null, "text", body({ context: "mesh/delivery" })]) {
|
||||
assert.equal(await checks.apply(event(b)), "dropped", JSON.stringify(b));
|
||||
}
|
||||
assert.deepEqual(f.sent, []);
|
||||
// The context may be named, and is mesh/check then.
|
||||
assert.equal(await checks.apply(event(body({ context: "mesh/check" }))), "written");
|
||||
});
|
||||
|
||||
test("a revision older than one written on the head is dropped; the same one is written again (a redelivery)", async () => {
|
||||
const { CheckStatuses } = await import("../check.ts");
|
||||
const f = forge();
|
||||
const checks = new CheckStatuses(f, () => {});
|
||||
await checks.apply(event(body({ state: "success", revision: 10 })));
|
||||
assert.equal(await checks.apply(event(body({ state: "pending", revision: 9 }))), "dropped", "a stale pending never hides the verdict");
|
||||
assert.equal(await checks.apply(event(body({ state: "success", revision: 10 }))), "written");
|
||||
// Another head is its own.
|
||||
assert.equal(await checks.apply(event(body({ sha: sha.replace(/1/g, "2"), state: "pending", revision: 3 }))), "written");
|
||||
assert.equal(await checks.apply(event(body({ state: "pending", revision: 11 }))), "written");
|
||||
assert.deepEqual(f.sent.map((s) => s.split(" ").slice(0, 3).join(" ")), ["novox/mesh-sdk@01234567 mesh/check success",
|
||||
"novox/mesh-sdk@01234567 mesh/check success", "novox/mesh-sdk@02234567 mesh/check pending", "novox/mesh-sdk@01234567 mesh/check pending"]);
|
||||
});
|
||||
|
||||
test("a forge that refuses throws, so the bus delivers the event again, and the revision is not taken as written", async () => {
|
||||
const { CheckStatuses } = await import("../check.ts");
|
||||
const f = forge();
|
||||
const checks = new CheckStatuses(f, () => {});
|
||||
f.failNext();
|
||||
await assert.rejects(checks.apply(event(body({ revision: 5 }))), /502/);
|
||||
assert.equal(await checks.apply(event(body({ state: "pending", revision: 4 }))), "written", "nothing of 5 was kept");
|
||||
assert.equal(await checks.apply(event(body({ revision: 5 }))), "written");
|
||||
});
|
||||
|
||||
test("no tool writes mesh/check: gitea_check_status is gone and gitea_commit_status refuses it", async () => {
|
||||
const delivery = await import("../delivery.ts");
|
||||
assert.equal((delivery as Record<string, unknown>).checkStatusTool, undefined, "delivery.ts serves no gitea_check_status");
|
||||
assert.throws(() => delivery.deliveryStatus("mesh/check", "success", "x"), /check\.status/);
|
||||
const index = readFileSync(new URL("../index.ts", import.meta.url), "utf8");
|
||||
assert.ok(!/gitea_check_status|checkStatusTool/.test(index), "index.ts serves no gitea_check_status");
|
||||
});
|
||||
@@ -171,3 +171,53 @@ test("the root tree is git's hash of the root's own entries, never the listing's
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("named paths: each asked path's git object id at the commit, a file's or a directory's, by directory (1758 S1, ADR 0499 §3)", async () => {
|
||||
const { client, server } = await serve([{ sha: COMMIT, url: "u", tree: LISTING, truncated: false, page: 1, total_count: LISTING.length }]);
|
||||
try {
|
||||
const got = await client.commitTrees("novox", "hq", COMMIT, { "modules/a": ["main.go", "module.json"], "modules/b": ["c"], "": ["README.md", "modules/d"] });
|
||||
assert.deepEqual(got.paths, {
|
||||
"modules/a": { "main.go": h("6"), "module.json": h("5") },
|
||||
"modules/b": { c: h("8") },
|
||||
"": { "README.md": h("1"), "modules/d": h("a") },
|
||||
});
|
||||
assert.deepEqual(got.trees, { "": ROOT, "modules/a": h("4"), "modules/b/c": h("8") });
|
||||
assert.equal((await client.commitTrees("novox", "hq", COMMIT)).paths, undefined, "no paths asked, none answered");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("a named path the listing lacks fails naming the directory and the path, and answers nothing", async () => {
|
||||
const { client, server } = await serve([{ sha: COMMIT, url: "u", tree: LISTING, truncated: false, page: 1, total_count: LISTING.length }]);
|
||||
try {
|
||||
await assert.rejects(client.commitTrees("novox", "hq", COMMIT, { "modules/a": ["main.go", "gone.go"] }), (err: Error) => {
|
||||
assert.match(err.message, /modules\/a/);
|
||||
assert.match(err.message, /gone\.go/);
|
||||
return true;
|
||||
});
|
||||
for (const bad of [[""], ["../x"], ["/abs"], ["a//b"]]) {
|
||||
await assert.rejects(client.commitTrees("novox", "hq", COMMIT, { "modules/a": bad }), /./, JSON.stringify(bad));
|
||||
}
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("gitea_commit_trees takes paths as {directory: [paths]} and refuses another shape before the forge is asked", async () => {
|
||||
const { getGiteaTools } = await import("../tools/index.ts");
|
||||
const { client, server, asked } = await serve([{ sha: COMMIT, url: "u", tree: LISTING, truncated: false, page: 1, total_count: LISTING.length }]);
|
||||
try {
|
||||
const tool = getGiteaTools(client).find((t) => t.name === "gitea_commit_trees")!;
|
||||
assert.ok("paths" in tool.input, "the tool names the argument");
|
||||
const got: any = await tool.run({ owner: "novox", repo: "hq", sha: COMMIT, paths: { "modules/a": ["main.go"] } } as any);
|
||||
assert.deepEqual(got.paths, { "modules/a": { "main.go": h("6") } });
|
||||
const before = asked.length;
|
||||
for (const paths of ["modules/a", ["main.go"], { "modules/a": "main.go" }, { "modules/a": [1] }]) {
|
||||
await assert.rejects(tool.run({ owner: "novox", repo: "hq", sha: COMMIT, paths } as any), /paths/, JSON.stringify(paths));
|
||||
}
|
||||
assert.equal(asked.length, before, "the forge was asked for a refused call");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -62,37 +62,20 @@ test("one view per pull request, found by its marker; only the mesh's statuses",
|
||||
assert.equal(deliveryStatus("mesh/delivery", "success", "x", "javascript:alert(1)").target_url, undefined);
|
||||
});
|
||||
|
||||
// Issue 1301-S4 (ADR 0447 §1; the lead's choice A): main requires mesh/check, so no hand may set it. The status tool
|
||||
// every caller may use refuses it in every state; it is written only by gitea_check_status, which the forge bridge
|
||||
// alone lists in its manifest's `invokes`, so only its account may call it.
|
||||
test("mesh/check is refused by the status tool and written by the check's own tool alone", async () => {
|
||||
const { commitStatusTool, checkStatusTool, deliveryStatus, CHECK_STATUS } = await import("../delivery.ts");
|
||||
// Issue 1301-S4 (ADR 0447 §1; R4e finding K1): main requires mesh/check, so no hand may set it. The status tool every
|
||||
// caller may use refuses it in every state; no tool writes it at all: it is written from the forge bridge's check.status
|
||||
// event (check.ts, test/check-event.test.ts).
|
||||
test("mesh/check is refused by the status tool in every state", async () => {
|
||||
const { commitStatusTool, deliveryStatus, CHECK_STATUS } = await import("../delivery.ts");
|
||||
assert.equal(CHECK_STATUS, "mesh/check");
|
||||
const sha = "0123456789abcdef0123456789abcdef01234567";
|
||||
const sent: string[] = [];
|
||||
const setCommitStatus = async (o: string, r: string, s: string, st: any) =>
|
||||
void sent.push(`${o}/${r}@${s.slice(0, 8)} ${st.context} ${st.state} ${st.description}${st.target_url ? " " + st.target_url : ""}`);
|
||||
const setCommitStatus = async (o: string, r: string, s: string, st: any) => void sent.push(`${o}/${r}@${s} ${st.context}`);
|
||||
const gates = { rechecked: async () => { throw new Error("not the gate writer's"); } };
|
||||
for (const state of ["pending", "success", "failure", "error", "warning"]) {
|
||||
assert.throws(() => deliveryStatus("mesh/check", state, "x"), /mesh\/check is the checker's status/);
|
||||
await assert.rejects(commitStatusTool({ setCommitStatus, gates } as any).run({ owner: "novox", repo: "mesh-sdk", sha,
|
||||
context: "mesh/check", state, description: "by hand" }), /gitea_check_status/);
|
||||
context: "mesh/check", state, description: "by hand" }), /check\.status/);
|
||||
}
|
||||
assert.deepEqual(sent, [], "nothing was sent for a hand-set mesh/check");
|
||||
|
||||
const tool = checkStatusTool({ setCommitStatus });
|
||||
assert.equal(tool.name, "gitea_check_status");
|
||||
for (const k of ["owner", "repo", "sha", "state", "description", "target_url", "context"]) assert.ok(k in tool.input, k);
|
||||
// The bridge's arguments as it sends them for gitea_commit_status: the context may be given, and is mesh/check.
|
||||
await tool.run({ owner: "novox", repo: "mesh-sdk", sha, context: "mesh/check", state: "success", description: "pass",
|
||||
target_url: "https://forge.invalid/novox/mesh-sdk/pulls/1" });
|
||||
await tool.run({ owner: "novox", repo: "mesh-sdk", sha, state: "pending", description: "y".repeat(300) });
|
||||
assert.equal(sent[0], "novox/mesh-sdk@01234567 mesh/check success pass https://forge.invalid/novox/mesh-sdk/pulls/1");
|
||||
assert.ok(sent[1].startsWith("novox/mesh-sdk@01234567 mesh/check pending yyyy"), sent[1]);
|
||||
assert.ok(sent[1].length < 200, "the description is cut as every status's is");
|
||||
await assert.rejects(tool.run({ owner: "novox", repo: "mesh-sdk", sha, context: "mesh/delivery", state: "success" }), /mesh\/check alone/);
|
||||
await assert.rejects(tool.run({ owner: "novox", repo: "mesh-sdk", sha: sha.slice(0, 12), state: "success" }), /whole sha/);
|
||||
await assert.rejects(tool.run({ owner: "novox", repo: "mesh-sdk", sha, state: "green" }), /not a status/);
|
||||
await assert.rejects(tool.run({ owner: "../x", repo: "mesh-sdk", sha, state: "success" }));
|
||||
assert.equal(sent.length, 2, "nothing more was sent");
|
||||
});
|
||||
|
||||
@@ -15,7 +15,7 @@ import { GiteaClient } from "../dist/client.js";
|
||||
import { ConfiguredToken } from "../dist/token.js";
|
||||
import { getGiteaTools } from "../dist/tools/index.js";
|
||||
import { mergeTool } from "../dist/merge.js";
|
||||
import { checkStatusTool, commitStatusTool } from "../dist/delivery.js";
|
||||
import { commitStatusTool } from "../dist/delivery.js";
|
||||
import { GateWriter } from "../dist/gate.js";
|
||||
import { judgeApiCall, judgeProtectionChange, ruleCovers, routedPath } from "../dist/guard.js";
|
||||
|
||||
@@ -168,7 +168,7 @@ const gates = { rechecked: (o: string, r: string, sha: string, s: any) => {
|
||||
} };
|
||||
const tools = [...getGiteaTools(client), commitStatusTool({
|
||||
setCommitStatus: (o, r, sha, s) => client.setCommitStatus(o, r, sha, s), gates: writer,
|
||||
}), checkStatusTool({ setCommitStatus: (o, r, sha, s) => client.setCommitStatus(o, r, sha, s) }), mergeTool(client, { gates, stateDir, emit: async (type, body) => { state.emitted.push({ type, body }); },
|
||||
}), mergeTool(client, { gates, stateDir, emit: async (type, body) => { state.emitted.push({ type, body }); },
|
||||
now: () => new Date(2026, 9, 12, 15, 1) })];
|
||||
const tool = (name: string) => tools.find((t) => t.name === name)!;
|
||||
const writes = () => state.sent.map((s) => `${s.method} ${s.path}`);
|
||||
@@ -568,9 +568,6 @@ test("no tool's caller text becomes another path or another user: `..` and `?sud
|
||||
["gitea_get_file", { ...repo, path: "README.md?sudo=jschoubben" }],
|
||||
["gitea_commit_status", { ...repo, sha: "abc1234?sudo=jschoubben", context: "mesh/delivery", state: "success" }],
|
||||
["gitea_commit_status", { ...repo, sha: "../../../admin/users", context: "mesh/delivery", state: "success" }],
|
||||
["gitea_check_status", { ...repo, sha: "abc1234?sudo=jschoubben", state: "success" }],
|
||||
["gitea_check_status", { ...repo, sha: "../../../admin/users", state: "success" }],
|
||||
["gitea_check_status", { owner: "novox", repo: "..", sha: "0123456789abcdef0123456789abcdef01234567", state: "success" }],
|
||||
["gitea_commit_status", { ...repo, sha: "0123456789abcdef0123456789abcdef01234567", context: "mesh/check", state: "success" }],
|
||||
["gitea_commit_statuses", { ...repo, sha: ".." }],
|
||||
["gitea_commit_statuses", { ...repo, sha: "main?sudo=jschoubben" }],
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
// 1790: the states the module announces are the states its manifest declares, no more, no fewer; none is readable
|
||||
// because no other module's manifest reads one of them.
|
||||
test("the states the module announces are the states its manifest declares", async () => {
|
||||
const { KEPT } = await import("../kept.ts");
|
||||
const manifest = JSON.parse(readFileSync(new URL("../module.json", import.meta.url), "utf8"));
|
||||
const declared = (manifest.state as (string | { name: string })[]).map((s) => (typeof s === "string" ? s : s.name)).sort();
|
||||
assert.deepEqual(KEPT.map((k) => k.name).sort(), declared);
|
||||
assert.ok(KEPT.every((k) => !k.readable), "no other module's manifest reads a state of gitea");
|
||||
});
|
||||
|
||||
test("the announcement has the layout mesh-sdk's go/announce reads: version 1, the module, the node, the states sorted by name", async () => {
|
||||
const { statesAnnouncement } = await import("../kept.ts");
|
||||
assert.deepEqual(statesAnnouncement("gitea", "ace"), { v: 1, module: "gitea", node: "ace", states: [{ name: "announced" }, { name: "bound" }] });
|
||||
assert.deepEqual(statesAnnouncement("gitea", undefined), { v: 1, module: "gitea", states: [{ name: "announced" }, { name: "bound" }] });
|
||||
});
|
||||
|
||||
test("index.ts says the announcement as the event `states` before the tools are served", () => {
|
||||
const source = readFileSync(new URL("../index.ts", import.meta.url), "utf8");
|
||||
assert.match(source, /await emit\("states", statesAnnouncement\(/);
|
||||
assert.ok(source.indexOf('await emit("states"') < source.indexOf('registerModuleTools("gitea"'), "announced before the tools are served");
|
||||
});
|
||||
@@ -60,6 +60,22 @@ function namesChecked(tools: ToolDefinition[]): ToolDefinition[] {
|
||||
}));
|
||||
}
|
||||
|
||||
/** The `paths` argument of gitea_commit_trees: {directory: [path relative to it]}, each path plain (no empty, `.` or `..`
|
||||
* segment, not absolute); anything else is refused before the forge is asked. Absent is absent. */
|
||||
function namedPaths(raw: unknown): Record<string, string[]> | undefined {
|
||||
if (raw === undefined || raw === null) return undefined;
|
||||
if (typeof raw !== "object" || Array.isArray(raw)) throw new Error("paths is {directory: [path, ...]}");
|
||||
const out: Record<string, string[]> = {};
|
||||
for (const [dir, list] of Object.entries(raw as Record<string, unknown>)) {
|
||||
if (!Array.isArray(list) || list.some((p) => typeof p !== "string")) throw new Error(`paths of ${dir || "the root"} is a list of paths`);
|
||||
for (const p of list as string[]) {
|
||||
if (p === "" || p.split("/").some((seg) => seg === "" || seg === "." || seg === "..")) throw new Error(`paths: ${JSON.stringify(p)} of ${dir || "the root"} is not a plain relative path`);
|
||||
}
|
||||
out[dir] = list as string[];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function getGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] {
|
||||
return namesChecked(allGiteaTools(gitea, admin));
|
||||
}
|
||||
@@ -525,18 +541,19 @@ function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefini
|
||||
},
|
||||
{
|
||||
name: "gitea_commit_trees",
|
||||
description: "A commit's trees from the forge's one recursive listing: the root tree, and the tree of every directory that holds a module.json (the root as an empty name), each by its git hash. What the forge bridge puts in a merge's message (novox/hq ADR 0437 §5). The commit is a full 40-hex sha. A listing the forge cut is refused, never answered in part. Reads only.",
|
||||
description: "A commit's trees from the forge's one recursive listing: the root tree, and the tree of every directory that holds a module.json (the root as an empty name), each by its git hash. What the forge bridge puts in a merge's message (novox/hq ADR 0437 §5). With paths, {directory: [path relative to it]}, the answer also carries paths, {directory: {path: git object id}} of each named file or directory at the commit (novox/hq ADR 0499 §3); a path the commit lacks fails naming the directory and the path. The commit is a full 40-hex sha. A listing the forge cut is refused, never answered in part. Reads only.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
sha: { type: "string", description: "the commit, its full 40-hex sha" },
|
||||
paths: { type: "object", description: "optional, {directory: [path relative to it]}: the git object id of each named file or directory is answered under paths; the root directory is the empty name" },
|
||||
},
|
||||
run: async (args) => {
|
||||
const owner = String(args.owner ?? "").trim(), repo = String(args.repo ?? "").trim();
|
||||
const sha = String(args.sha ?? "").trim();
|
||||
if (!owner || !repo) throw new Error("name the repository: owner and repo");
|
||||
if (!/^[0-9a-f]{40}$/i.test(sha)) throw new Error("sha is a commit's full 40-hex sha");
|
||||
return gitea.commitTrees(owner, repo, sha);
|
||||
return gitea.commitTrees(owner, repo, sha, namedPaths(args.paths));
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -204,41 +203,37 @@ func TestTheLoginManagersEntryRunsTheSessionsStart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Every module of the catalogue that contributes window-manager lines (novox/hq ADR 0212) is placed in
|
||||
// the main file as the controller places them — in module order, each module's under a line naming it
|
||||
// — and loaded as i3 would load them on a machine with all of them assigned: no two bind one key, and
|
||||
// every line parses.
|
||||
// The window-manager lines the catalogue's modules contribute to the main file (novox/hq ADR 0212) are kept in
|
||||
// testdata/window-manager-lines.json, one entry per contributing module in module order, as the controller places
|
||||
// them. The test reads that file, never the other modules' manifests: a module may not read a manifest (ADR 0495
|
||||
// §1), and a test that reads its siblings' fails wherever they are not beside it (issue 1814). The main file and
|
||||
// those lines are placed together — each module's under a line naming it — and loaded as i3 would load them on a
|
||||
// machine with all of them assigned: no two bind one key, and every line parses.
|
||||
func TestTheMainFileAndEveryModulesContributionLoadTogether(t *testing.T) {
|
||||
i3, err := exec.LookPath("i3")
|
||||
if err != nil {
|
||||
t.Skip("no i3 here to check with")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
manifests, _ := filepath.Glob(filepath.Join("..", "..", "..", "*", "module.json"))
|
||||
sort.Strings(manifests)
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", "window-manager-lines.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var contributions []struct {
|
||||
Module string `json:"module"`
|
||||
Content []string `json:"content"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &contributions); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var placed strings.Builder
|
||||
var found []string
|
||||
for _, p := range manifests {
|
||||
raw, _ := os.ReadFile(p)
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Contributions []struct {
|
||||
Seat, Kind, Content string
|
||||
} `json:"contributions"`
|
||||
}
|
||||
json.Unmarshal(raw, &m)
|
||||
named := false
|
||||
for _, c := range m.Contributions {
|
||||
if c.Seat != "node-display-session" || c.Kind != "config" {
|
||||
continue
|
||||
}
|
||||
if !named {
|
||||
placed.WriteString("# " + m.Module + "\n")
|
||||
named = true
|
||||
found = append(found, m.Module)
|
||||
}
|
||||
placed.WriteString(c.Content)
|
||||
if !strings.HasSuffix(c.Content, "\n") {
|
||||
for _, c := range contributions {
|
||||
placed.WriteString("# " + c.Module + "\n")
|
||||
found = append(found, c.Module)
|
||||
for _, content := range c.Content {
|
||||
placed.WriteString(content)
|
||||
if !strings.HasSuffix(content, "\n") {
|
||||
placed.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
[
|
||||
{
|
||||
"module": "asus-zephyrus-g14",
|
||||
"content": [
|
||||
"# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the\n# mesh: replaced at every push. Once the controller places contributions to node-display-session\n# (ADR 0210), these become the module's contribution instead of a file in i3's directory.\n#\n# The keys the firmware turns into ordinary key presses. The vendor keys that reach no X client are\n# triggerhappy's (/etc/triggerhappy/triggers.d/asus-g14.conf): M4 and Fn+F4/F5 for media, Fn+F7/F8 for\n# the panel, Fn+F10 for the touchpad. The keyboard backlight (Fn+F2/F3) is the firmware's and asusd's.\n\n# Fn+F6, the screenshot key: the firmware sends Super+Shift+S. Released before it runs, because the\n# screenshot grabs the pointer to select a region, which fails while the key is still held.\nbindsym --release $mod+Shift+s exec --no-startup-id $XDG_CONFIG_HOME/i3/scripts/screenshot.sh\n\n# Fn+F9, the display key: the firmware sends Super+P. Odd workspaces to the panel, even ones to the\n# external output.\nbindsym $mod+p exec --no-startup-id /usr/local/lib/example-laptop/bin/zephyrus-display order\n\n# The keyboard backlight's level, shown when it changes.\nexec --no-startup-id /usr/local/lib/example-laptop/bin/zephyrus-kbd-notify\n# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the\n# mesh: replaced at every push. Once the controller places contributions to node-display-session\n# (ADR 0210), these become the module's contribution instead of a file in i3's directory.\n#\n# The model's panel and touchpad.\n\n# The internal panel is the primary output, where the bars' tray goes. Which monitors are on and where\n# is the display server's (autorandr, run at every session start).\nexec --no-startup-id /usr/local/lib/example-laptop/bin/zephyrus-display primary\n\n# The touchpad's settings again, by hand. X applies them itself whenever the device appears.\nbindsym $mod+Shift+x exec --no-startup-id /usr/local/lib/example-laptop/bin/zephyrus-touchpad reset\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "clipmenu",
|
||||
"content": [
|
||||
"# The clipboard's history key (module clipmenu, novox/hq ADR 0208). Owned by the mesh: replaced at\n# every push. clipmenu shows the history through `dmenu`, the node's dmenu-compatible command, which\n# the holder of node-launcher answers (rofi on the workstations); the chosen entry is put back on the\n# clipboard.\nbindsym $mod+period exec --no-startup-id clipmenu -p Clipboard\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "dunst",
|
||||
"content": [
|
||||
"# The notifications' key (module dunst, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# Super+N opens the menu (rofi) of the actions and links of every notification on screen, so one is\n# chosen from the keyboard as a tap chooses it on the notification.\nbindsym $mod+n exec --no-startup-id dunstctl context\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "feh",
|
||||
"content": [
|
||||
"# The wallpaper's key (module feh, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# It puts the declared wallpaper back, after a monitor change or a wallpaper set for the session.\nbindsym $mod+Shift+b exec --no-startup-id ~/.fehbg\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "i3status-rust",
|
||||
"content": [
|
||||
"# The bars (module i3status-rust, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# i3 reads this file through its configuration's `include ~/.config/i3/config.d/*.conf`. The bottom\n# bar shows the machine; the top bar the focused window and the tray, on the primary output. The\n# face is the monospace one every desktop module names.\nbar {\n font pango:JetBrainsMono Nerd Font 11\n position bottom\n status_command i3status-rs ~/.config/i3status-rust/bottom-bar.toml\n tray_output none\n colors {\n separator #ffffff\n background #000000\n statusline #ffffff\n # The text on an accent-coloured button is dark: light text on the accent was barely\n # legible.\n focused_workspace #de5200 #de5200 #000000\n active_workspace #de5200 #de5200 #000000\n inactive_workspace #000000 #000000 #ffffff\n urgent_workspace #2f343a #900000 #ffffff\n }\n}\n\nbar {\n font pango:JetBrainsMono Nerd Font 11\n position top\n status_command i3status-rs ~/.config/i3status-rust/top-bar.toml\n workspace_buttons no\n tray_output primary\n colors {\n separator #ffffff\n background #000000\n statusline #ffffff\n }\n}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "polychromatic",
|
||||
"content": [
|
||||
"# The Razer peripherals' tray (module polychromatic, novox/hq ADR 0208, ADR 0212). Owned by the mesh:\n# replaced at every push. This line is the tray's one start, at the session's start (exec, not\n# exec_always, so a reload starts nothing). The package's own login helper starts it too while the\n# application's setting \"Start the tray applet when I log on\" is ticked; polychromatic_check names\n# that as a second start.\nexec --no-startup-id polychromatic-tray-applet\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "rofi",
|
||||
"content": [
|
||||
"# The launcher's key bindings (module rofi, novox/hq ADR 0208). Owned by the mesh: replaced at every\n# push. i3 reads this file through its configuration's `include ~/.config/i3/config.d/*.conf`, after\n# the variables it sets, so $mod is i3's.\nbindsym $mod+d exec --no-startup-id ~/.local/bin/rofi-launch drun\nbindsym $mod+t exec --no-startup-id ~/.local/bin/rofi-launch run\nbindsym $mod+Shift+t exec --no-startup-id ~/.local/bin/rofi-launch sudo\nbindsym $mod+Shift+w exec --no-startup-id ~/.local/bin/rofi-launch window\nbindsym $mod+Escape exec --no-startup-id ~/.local/bin/rofi-powermenu\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "slack",
|
||||
"content": [
|
||||
"# Slack (module slack, novox/hq ADR 0208, ADR 0212). Owned by the mesh: replaced at every push.\n# Its one start: at login, to the tray (-s), with the arguments of the package's own desktop entry.\n# No XDG autostart entry starts it as well; slack_check names one if it appears.\nexec --no-startup-id /usr/bin/slack --gtk-version=3 -s\n# The chat window. Slack owns four X windows, and the only one i3 manages has the class \"slack\" in\n# lower case; the three of class \"Slack\" (the packaged entry's StartupWMClass) are its unmanaged\n# helpers and its tray icon, so a rule on class=\"Slack\" matches nothing. window_role keeps a call or\n# screen-share window out of the rules. $ws3 is i3's, in scope here.\nfor_window [class=\"(?i)^slack$\" window_role=\"browser-window\"] move to workspace $ws3\nfor_window [class=\"(?i)^slack$\" window_role=\"browser-window\"] floating disable\nfor_window [class=\"(?i)^slack$\" window_role=\"browser-window\"] border pixel 2\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,8 +1,9 @@
|
||||
# kernel
|
||||
|
||||
The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). Today it holds one:
|
||||
**transparent huge pages**, set per machine. The rest of the row (the kernel package, microcode, boot
|
||||
entries) is not built yet.
|
||||
The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). It holds two things:
|
||||
**transparent huge pages**, set per machine, and **the fallback kernel** (`linux-lts` with its headers) and
|
||||
its place on the boot menu (novox/hq ADR 0508 §9 and §10). The default kernel's package, and microcode, are
|
||||
not built yet.
|
||||
|
||||
## Why (novox/hq issue 746)
|
||||
|
||||
@@ -35,16 +36,34 @@ under the new mode. A failed rotation is said in the unit's journal and does not
|
||||
rotation is made while the machine is still starting (`systemctl is-system-running` says `initializing`
|
||||
or `starting`): nothing is in huge pages yet, and the persistent journal opens after this unit.
|
||||
|
||||
## The fallback kernel (novox/hq ADR 0508, issue 1841)
|
||||
|
||||
Every machine keeps a second, older kernel to start from, so a kernel upgrade that does not boot is not the
|
||||
end of the machine. The module declares the packages `linux-lts` and `linux-lts-headers` (so every DKMS module is
|
||||
built for the fallback kernel too) after its boot-entries service, so the entry is in place before the kernel is.
|
||||
|
||||
| | what | notes |
|
||||
|---|---|---|
|
||||
| file | `${dir:state}/boot-entries` | the script. It reads from the machine which loader booted it: the EFI variable `LoaderEntrySelected` present means systemd-boot, a `/boot/grub/grub.cfg` without it means GRUB, neither fails the unit loudly |
|
||||
| | systemd-boot | writes `/boot/loader/entries/mesh-linux-lts.conf` from `/proc/cmdline` without `initrd=` and `BOOT_IMAGE=`, with the microcode images present in `/boot` and the fallback initramfs as initrd lines; `loader.conf` and the default entry are never touched |
|
||||
| | GRUB | runs `grub-mkconfig -o /boot/grub/grub.cfg` only when the set of `/boot/vmlinuz-*` differs from the kernels the menu names, with `GRUB_TOP_LEVEL` naming the kernel the machine booted (`BOOT_IMAGE`; else the non-fallback kernel), since GRUB lists kernels in reverse version order and would otherwise make the fallback kernel the default; `/etc/default/grub` still overrides it when it sets one |
|
||||
| | room | while `/boot/vmlinuz-linux-lts` is not there yet, less than 96 MiB free on `/boot` fails the unit and says so (the fallback kernel writes about 72 MB, issue 1836) |
|
||||
| | a kernel-copying hook | a hook in `/etc/pacman.d/hooks` that the mesh did not write, triggered by kernel images and copying to one fixed file name in `/boot`, would put the fallback kernel in the default kernel's place: it is moved into `${dir:state}/taken-over/` and named in the unit's journal. Each kernel is then placed under its own package's name by the distribution's hooks |
|
||||
| file | `/etc/systemd/system/mesh-kernel-boot-entries.service` | oneshot, sandboxed: the file system read-only but for `/boot`, `/etc/pacman.d/hooks` and the module's state; no home, no network |
|
||||
| file | `/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook` | runs the script after a kernel image is installed, upgraded or removed |
|
||||
| packages | `linux-lts`, `linux-lts-headers` | declared after the service and the hook |
|
||||
|
||||
## Tools
|
||||
|
||||
| tool | | answers |
|
||||
|---|---|---|
|
||||
| `kernel_boot` | r | the loader (`systemd-boot`, `grub` or `none`), the menu's entries, the kernels in `/boot`, the entry booted (systemd-boot) or the kernel image booted (GRUB), whether the fallback kernel is on the menu with its kernel in `/boot`, and the room free on `/boot` in KiB |
|
||||
| `kernel_huge_pages` | r | the mode, defrag and shared-memory mode; each per-size control and what `inherit` resolves to; huge pages by kind from `/proc/meminfo` (MiB); the units holding the most file pages in huge folios (`memory.stat` `file_thp`) with their dirty and writeback bytes, khugepaged's collapses and the bytes their cgroup wrote (`io.stat`, the most any one device saw: a mirror and its members each count the same write) |
|
||||
|
||||
## Tests
|
||||
|
||||
`go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the
|
||||
manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); and the script as the mesh
|
||||
manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); the boot-entries script run with `sh` over stand-in roots (a systemd-boot entries directory, a GRUB menu, a hook that copies kernels to one fixed name): the entry from the booted command line, a menu generated again only when the kernels changed, no loader, no room, the taken-over hook; `kernel_boot` over the same roots; and the huge-pages script as the mesh
|
||||
writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once,
|
||||
rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a
|
||||
reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
// kernel_boot: which loader booted the machine, its entries, the kernels in /boot, the entry it booted and
|
||||
// whether the fallback kernel is on the menu (novox/hq ADR 0508 §9, issue 1841). Reads only.
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"unicode/utf16"
|
||||
)
|
||||
|
||||
const loaderSelected = "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f"
|
||||
|
||||
// BootState is what kernel_boot answers.
|
||||
type BootState struct {
|
||||
Loader string `json:"loader"`
|
||||
Entries []string `json:"entries"`
|
||||
Kernels []string `json:"kernels"`
|
||||
Booted string `json:"booted"`
|
||||
FallbackPresent bool `json:"fallback_present"`
|
||||
BootFreeKiB int64 `json:"boot_free_kib"`
|
||||
}
|
||||
|
||||
var grubKernel = regexp.MustCompile(`(?m)^\s*linux\s+(?:/boot)?/(vmlinuz-\S+)`)
|
||||
var grubTitle = regexp.MustCompile(`(?m)^\s*menuentry\s+'([^']*)'`)
|
||||
var bootImage = regexp.MustCompile(`BOOT_IMAGE=(?:/boot)?/?(\S+)`)
|
||||
|
||||
// Boot reads the machine, the loader the way the boot-entries script does: the loader's EFI variable means
|
||||
// systemd-boot, a grub.cfg without it GRUB.
|
||||
func (m *Machine) Boot() (*BootState, error) {
|
||||
b := &BootState{Loader: "none", Entries: []string{}, Kernels: []string{}}
|
||||
files, _ := filepath.Glob(m.path("boot/vmlinuz-*"))
|
||||
for _, f := range files {
|
||||
b.Kernels = append(b.Kernels, filepath.Base(f))
|
||||
}
|
||||
sort.Strings(b.Kernels)
|
||||
hasKernel := func(name string) bool {
|
||||
for _, k := range b.Kernels {
|
||||
if k == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
var s syscall.Statfs_t
|
||||
if err := syscall.Statfs(m.path("boot"), &s); err == nil {
|
||||
b.BootFreeKiB = int64(s.Bavail) * int64(s.Bsize) / 1024
|
||||
}
|
||||
if raw, err := os.ReadFile(m.path(loaderSelected)); err == nil {
|
||||
b.Loader = "systemd-boot"
|
||||
if len(raw) > 4 {
|
||||
var u []uint16
|
||||
for i := 4; i+1 < len(raw); i += 2 {
|
||||
u = append(u, uint16(raw[i])|uint16(raw[i+1])<<8)
|
||||
}
|
||||
b.Booted = strings.TrimRight(string(utf16.Decode(u)), "\x00")
|
||||
}
|
||||
files, _ := filepath.Glob(m.path("boot/loader/entries/*.conf"))
|
||||
for _, f := range files {
|
||||
b.Entries = append(b.Entries, filepath.Base(f))
|
||||
}
|
||||
sort.Strings(b.Entries)
|
||||
if raw, err := os.ReadFile(m.path("boot/loader/entries/mesh-linux-lts.conf")); err == nil {
|
||||
b.FallbackPresent = strings.Contains(string(raw), "/vmlinuz-linux-lts") && hasKernel("vmlinuz-linux-lts")
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
if raw, err := os.ReadFile(m.path("boot/grub/grub.cfg")); err == nil {
|
||||
b.Loader = "grub"
|
||||
for _, t := range grubTitle.FindAllStringSubmatch(string(raw), -1) {
|
||||
b.Entries = append(b.Entries, t[1])
|
||||
}
|
||||
for _, k := range grubKernel.FindAllStringSubmatch(string(raw), -1) {
|
||||
if k[1] == "vmlinuz-linux-lts" && hasKernel("vmlinuz-linux-lts") {
|
||||
b.FallbackPresent = true
|
||||
}
|
||||
}
|
||||
if cmd, err := m.read("proc/cmdline"); err == nil {
|
||||
if i := bootImage.FindStringSubmatch(cmd); i != nil {
|
||||
b.Booted = i[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
// kernel_boot over stand-in roots: a systemd-boot machine with and without the fallback entry, a GRUB machine
|
||||
// with and without the fallback kernel in its menu, and a machine with neither (novox/hq ADR 0508 §9, issue 1841).
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const selectedVar = "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f"
|
||||
|
||||
func put(t *testing.T, root, p, content string) {
|
||||
t.Helper()
|
||||
full := filepath.Join(root, p)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write(t, full, content, 0o644)
|
||||
}
|
||||
|
||||
func bootOf(t *testing.T, root string) *BootState {
|
||||
t.Helper()
|
||||
got, err := (&Machine{Root: root}).Boot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func TestKernelBootReadsASystemdBootMachineWithItsFallbackEntry(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
put(t, root, selectedVar, "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00")
|
||||
put(t, root, "boot/loader/entries/arch.conf", defaultEntry)
|
||||
put(t, root, "boot/loader/entries/mesh-linux-lts.conf", "title Arch Linux (linux-lts, fallback)\nlinux /vmlinuz-linux-lts\n")
|
||||
put(t, root, "boot/vmlinuz-linux", "k")
|
||||
put(t, root, "boot/vmlinuz-linux-lts", "k")
|
||||
got := bootOf(t, root)
|
||||
if got.Loader != "systemd-boot" || got.Booted != "arch.conf" || !got.FallbackPresent {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if len(got.Entries) != 2 || got.Entries[0] != "arch.conf" || got.Entries[1] != "mesh-linux-lts.conf" {
|
||||
t.Fatalf("entries: %v", got.Entries)
|
||||
}
|
||||
if len(got.Kernels) != 2 || got.Kernels[0] != "vmlinuz-linux" || got.Kernels[1] != "vmlinuz-linux-lts" {
|
||||
t.Fatalf("kernels: %v", got.Kernels)
|
||||
}
|
||||
if got.BootFreeKiB <= 0 {
|
||||
t.Fatalf("free: %d", got.BootFreeKiB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKernelBootSaysASystemdBootMachineHasNoFallbackYet(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
put(t, root, selectedVar, "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00")
|
||||
put(t, root, "boot/loader/entries/arch.conf", defaultEntry)
|
||||
put(t, root, "boot/vmlinuz-linux", "k")
|
||||
// The entry file alone is not the fallback: the kernel it names must be in /boot.
|
||||
put(t, root, "boot/loader/entries/mesh-linux-lts.conf", "linux /vmlinuz-linux-lts\n")
|
||||
if got := bootOf(t, root); got.FallbackPresent {
|
||||
t.Fatalf("a fallback entry without its kernel: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKernelBootReadsAGrubMachine(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
put(t, root, "boot/grub/grub.cfg", grubMenu)
|
||||
put(t, root, "boot/vmlinuz-linux", "k")
|
||||
put(t, root, "proc/cmdline", bootedCmdline)
|
||||
got := bootOf(t, root)
|
||||
if got.Loader != "grub" || got.FallbackPresent || got.Booted != "vmlinuz-linux" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if len(got.Entries) != 2 || got.Entries[0] != "Arch Linux" {
|
||||
t.Fatalf("entries: %v", got.Entries)
|
||||
}
|
||||
put(t, root, "boot/vmlinuz-linux-lts", "k")
|
||||
put(t, root, "boot/grub/grub.cfg", grubMenu+"menuentry 'Arch Linux LTS' {\n\tlinux\t/boot/vmlinuz-linux-lts root=x rw\n}\n")
|
||||
if got := bootOf(t, root); !got.FallbackPresent {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKernelBootSaysWhenThereIsNoLoader(t *testing.T) {
|
||||
got := bootOf(t, t.TempDir())
|
||||
if got.Loader != "none" || got.FallbackPresent || len(got.Entries) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package main
|
||||
|
||||
// The boot-entries script as the mesh writes it, run with sh over a stand-in root (novox/hq ADR 0508 §9 and
|
||||
// §10, issue 1841). The fixtures are the shapes read from machines: a systemd-boot entries directory, a GRUB
|
||||
// menu, a hook that copies every kernel image to the default kernel's name, and the command line a machine
|
||||
// booted with.
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const bootedCmdline = "initrd=\\initramfs-linux.img BOOT_IMAGE=/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet loglevel=3 initrd=\\intel-ucode.img\n"
|
||||
|
||||
const defaultEntry = "title Arch Linux\nlinux /vmlinuz-linux\ninitrd /intel-ucode.img\ninitrd /initramfs-linux.img\noptions root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet\n"
|
||||
|
||||
const loaderConf = "default arch.conf\ntimeout 3\nconsole-mode max\n"
|
||||
|
||||
// grubMenu is the part of a generated grub.cfg that names kernels, for a machine holding one kernel.
|
||||
const grubMenu = `menuentry 'Arch Linux' --class arch {
|
||||
set root='hd0,gpt2'
|
||||
linux /boot/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet
|
||||
initrd /boot/initramfs-linux.img
|
||||
}
|
||||
menuentry 'Arch Linux, with fallback initramfs' --class arch {
|
||||
linux /boot/vmlinuz-linux root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet
|
||||
initrd /boot/initramfs-linux-fallback.img
|
||||
}
|
||||
`
|
||||
|
||||
// kernelCopyingHook copies every installed kernel image to the default kernel's name on the boot partition.
|
||||
const kernelCopyingHook = `[Trigger]
|
||||
Operation = Install
|
||||
Operation = Upgrade
|
||||
Type = Path
|
||||
Target = usr/lib/modules/*/vmlinuz
|
||||
|
||||
[Action]
|
||||
Description = Copying the kernel to the default name
|
||||
When = PostTransaction
|
||||
Exec = /usr/bin/sh -c '/usr/bin/cp /usr/lib/modules/$(uname -r)/vmlinuz /boot/vmlinuz-linux'
|
||||
`
|
||||
|
||||
// otherHook is a hook of the distribution's shape that has nothing to do with kernel images.
|
||||
const otherHook = `[Trigger]
|
||||
Operation = Upgrade
|
||||
Type = Package
|
||||
Target = pacman-mirrorlist
|
||||
|
||||
[Action]
|
||||
Description = Updating the mirror list
|
||||
When = PostTransaction
|
||||
Exec = /usr/bin/cp /etc/pacman.d/mirrorlist.pacnew /etc/pacman.d/mirrorlist
|
||||
`
|
||||
|
||||
type bootBed struct {
|
||||
script, root, state, bin, mkconfig string
|
||||
need string
|
||||
}
|
||||
|
||||
func newBootBed(t *testing.T) bootBed {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
b := bootBed{script: filepath.Join(dir, "boot-entries"), root: filepath.Join(dir, "root"), state: filepath.Join(dir, "state"),
|
||||
bin: filepath.Join(dir, "bin"), mkconfig: filepath.Join(dir, "mkconfig")}
|
||||
content := manifest(t).resource(t, "boot-entries").text(t)
|
||||
content = strings.ReplaceAll(content, "${dir:state}", b.state)
|
||||
if strings.Contains(content, "${") {
|
||||
t.Fatalf("a substitution the test does not make is left in the script")
|
||||
}
|
||||
for _, d := range []string{b.root + "/boot", b.root + "/proc", b.root + "/etc/pacman.d/hooks", b.state, b.bin} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write(t, b.script, content, 0o755)
|
||||
write(t, b.root+"/proc/cmdline", bootedCmdline, 0o644)
|
||||
// the stand-in grub-mkconfig records the kernel it was told to put first, then its arguments.
|
||||
write(t, filepath.Join(b.bin, "grub-mkconfig"), "#!/bin/sh\necho \"top=$GRUB_TOP_LEVEL $@\" >>"+b.mkconfig+"\n", 0o755)
|
||||
return b
|
||||
}
|
||||
|
||||
func (b bootBed) put(t *testing.T, p, content string) {
|
||||
t.Helper()
|
||||
full := filepath.Join(b.root, p)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write(t, full, content, 0o644)
|
||||
}
|
||||
|
||||
func (b bootBed) systemdBoot(t *testing.T) {
|
||||
b.put(t, "sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f", "\x07\x00\x00\x00a\x00r\x00c\x00h\x00.\x00c\x00o\x00n\x00f\x00\x00\x00")
|
||||
b.put(t, "boot/loader/loader.conf", loaderConf)
|
||||
b.put(t, "boot/loader/entries/arch.conf", defaultEntry)
|
||||
b.put(t, "boot/vmlinuz-linux", "k")
|
||||
}
|
||||
|
||||
func (b bootBed) run(t *testing.T) (string, error) {
|
||||
t.Helper()
|
||||
args := []string{b.script, "run", b.root, b.state}
|
||||
if b.need != "" {
|
||||
args = append(args, b.need)
|
||||
}
|
||||
cmd := exec.Command("sh", args...)
|
||||
cmd.Env = append(os.Environ(), "PATH="+b.bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
out, err := cmd.CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
func (b bootBed) read(t *testing.T, p string) string {
|
||||
t.Helper()
|
||||
raw, _ := os.ReadFile(filepath.Join(b.root, p))
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func TestASystemdBootMachineGetsTheFallbackEntryFromItsBootedCommandLine(t *testing.T) {
|
||||
b := newBootBed(t)
|
||||
b.systemdBoot(t)
|
||||
b.put(t, "boot/intel-ucode.img", "u")
|
||||
out, err := b.run(t)
|
||||
if err != nil || !strings.Contains(out, "wrote") {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
want := "title Arch Linux (linux-lts, fallback)\nlinux /vmlinuz-linux-lts\ninitrd /intel-ucode.img\ninitrd /initramfs-linux-lts.img\n" +
|
||||
"options root=UUID=2f6c1e0a-5c0e-4d52-9a57-1d8a2a7a3c11 rw quiet loglevel=3\n"
|
||||
if got := b.read(t, "boot/loader/entries/mesh-linux-lts.conf"); got != want {
|
||||
t.Fatalf("entry:\n%s\nwant:\n%s", got, want)
|
||||
}
|
||||
// The default entry and the loader's own file are as they were.
|
||||
if b.read(t, "boot/loader/entries/arch.conf") != defaultEntry || b.read(t, "boot/loader/loader.conf") != loaderConf {
|
||||
t.Fatalf("the default entry or loader.conf was touched")
|
||||
}
|
||||
if out, err := b.run(t); err != nil || !strings.Contains(out, "as it was") {
|
||||
t.Fatalf("again: %v %s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithoutMicrocodeThereIsNoMicrocodeLine(t *testing.T) {
|
||||
b := newBootBed(t)
|
||||
b.systemdBoot(t)
|
||||
if out, err := b.run(t); err != nil {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
if got := b.read(t, "boot/loader/entries/mesh-linux-lts.conf"); strings.Contains(got, "ucode") || !strings.Contains(got, "initrd /initramfs-linux-lts.img") {
|
||||
t.Fatalf("entry: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGrubMachineIsGeneratedAgainOnlyWhenItsKernelsChanged(t *testing.T) {
|
||||
b := newBootBed(t)
|
||||
b.put(t, "boot/grub/grub.cfg", grubMenu)
|
||||
b.put(t, "boot/vmlinuz-linux", "k")
|
||||
if out, err := b.run(t); err != nil || !strings.Contains(out, "left as it is") {
|
||||
t.Fatalf("same: %v %s", err, out)
|
||||
}
|
||||
if _, err := os.Stat(b.mkconfig); err == nil {
|
||||
t.Fatalf("the menu was generated again with the same kernels")
|
||||
}
|
||||
b.put(t, "boot/vmlinuz-linux-lts", "k")
|
||||
if out, err := b.run(t); err != nil || !strings.Contains(out, "generated again") {
|
||||
t.Fatalf("changed: %v %s", err, out)
|
||||
}
|
||||
raw, _ := os.ReadFile(b.mkconfig)
|
||||
if string(raw) != "top=/boot/vmlinuz-linux -o "+filepath.Join(b.root, "boot/grub/grub.cfg")+"\n" {
|
||||
t.Fatalf("grub-mkconfig was asked %q", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0508 §9: GRUB's menu lists kernels in reverse version order, so linux-lts would come first and become the
|
||||
// default entry (GRUB_DEFAULT=0); the kernel the machine booted is put first, and the fallback kernel never.
|
||||
func TestGrubKeepsTheBootedKernelFirstAndNeverTheFallback(t *testing.T) {
|
||||
for name, c := range map[string]struct{ cmdline, want string }{
|
||||
"booted from the default kernel": {"BOOT_IMAGE=/vmlinuz-linux root=UUID=x rw\n", "top=/boot/vmlinuz-linux"},
|
||||
"booted from a grub device path": {"BOOT_IMAGE=(hd0,msdos1)/vmlinuz-linux-zen root=UUID=x rw\n", "top=/boot/vmlinuz-linux-zen"},
|
||||
"booted from the fallback kernel": {"BOOT_IMAGE=/vmlinuz-linux-lts root=UUID=x rw\n", "top=/boot/vmlinuz-linux"},
|
||||
"no image on the command line": {"root=UUID=x rw\n", "top=/boot/vmlinuz-linux"},
|
||||
"an image no longer in /boot": {"BOOT_IMAGE=/vmlinuz-linux-gone root=UUID=x rw\n", "top=/boot/vmlinuz-linux"},
|
||||
} {
|
||||
b := newBootBed(t)
|
||||
b.put(t, "boot/grub/grub.cfg", grubMenu)
|
||||
b.put(t, "proc/cmdline", c.cmdline)
|
||||
for _, k := range []string{"vmlinuz-linux", "vmlinuz-linux-lts", "vmlinuz-linux-zen"} {
|
||||
b.put(t, "boot/"+k, "k")
|
||||
}
|
||||
if out, err := b.run(t); err != nil {
|
||||
t.Fatalf("%s: %v %s", name, err, out)
|
||||
}
|
||||
raw, _ := os.ReadFile(b.mkconfig)
|
||||
if !strings.HasPrefix(string(raw), c.want+" -o ") {
|
||||
t.Errorf("%s: grub-mkconfig was asked %q, want %s first", name, raw, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMachineWithNoLoaderFailsLoudly(t *testing.T) {
|
||||
b := newBootBed(t)
|
||||
out, err := b.run(t)
|
||||
if err == nil || !strings.Contains(out, "no boot loader found") {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABootPartitionWithoutRoomFailsBeforeAnythingIsWritten(t *testing.T) {
|
||||
b := newBootBed(t)
|
||||
b.systemdBoot(t)
|
||||
b.need = "999999999999"
|
||||
out, err := b.run(t)
|
||||
if err == nil || !strings.Contains(out, "free room there before the kernel is installed") {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
if b.read(t, "boot/loader/entries/mesh-linux-lts.conf") != "" {
|
||||
t.Fatalf("an entry was written")
|
||||
}
|
||||
// Once the fallback kernel is in place the room is not asked for again.
|
||||
b.put(t, "boot/vmlinuz-linux-lts", "k")
|
||||
if out, err := b.run(t); err != nil {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHookThatCopiesKernelsToOneFixedNameIsMovedIntoTheStateAndNamed(t *testing.T) {
|
||||
b := newBootBed(t)
|
||||
b.systemdBoot(t)
|
||||
b.put(t, "etc/pacman.d/hooks/60-copy-kernel.hook", kernelCopyingHook)
|
||||
b.put(t, "etc/pacman.d/hooks/70-mirrorlist.hook", otherHook)
|
||||
b.put(t, "etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook", "[Trigger]\nTarget = usr/lib/modules/*/vmlinuz\n[Action]\nExec = /bin/cp x /boot/y\n")
|
||||
out, err := b.run(t)
|
||||
if err != nil || !strings.Contains(out, "taken over: the pacman hook 60-copy-kernel.hook") {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
if b.read(t, "etc/pacman.d/hooks/60-copy-kernel.hook") != "" {
|
||||
t.Fatalf("the copying hook is still in the hook directory")
|
||||
}
|
||||
if raw, _ := os.ReadFile(filepath.Join(b.state, "taken-over", "60-copy-kernel.hook")); string(raw) != kernelCopyingHook {
|
||||
t.Fatalf("moved hook: %q", raw)
|
||||
}
|
||||
if b.read(t, "etc/pacman.d/hooks/70-mirrorlist.hook") != otherHook || b.read(t, "etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook") == "" {
|
||||
t.Fatalf("a hook that copies no kernel was moved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBootEntriesUnitAndHookAreTheModules(t *testing.T) {
|
||||
m := manifest(t)
|
||||
unit := m.resource(t, "boot-unit").text(t)
|
||||
for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/boot-entries run", "ProtectSystem=strict",
|
||||
"ReadWritePaths=/boot ${dir:state} /etc/pacman.d/hooks", "NoNewPrivileges=yes", "WantedBy=multi-user.target"} {
|
||||
if !strings.Contains(unit, line+"\n") {
|
||||
t.Errorf("the unit lacks %q", line)
|
||||
}
|
||||
}
|
||||
hook := m.resource(t, "boot-hook")
|
||||
text := hook.text(t)
|
||||
if hook["path"] != "/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook" || !strings.Contains(text, "Target = usr/lib/modules/*/vmlinuz\n") ||
|
||||
!strings.Contains(text, "When = PostTransaction\n") || !strings.Contains(text, "Exec = ${dir:state}/boot-entries run\n") {
|
||||
t.Fatalf("the hook: %v\n%s", hook["path"], text)
|
||||
}
|
||||
apply := m.resource(t, "boot-apply")
|
||||
if apply["unit"] != "mesh-kernel-boot-entries.service" || apply["state"] != "running" || apply["boot"] != "enabled" {
|
||||
t.Fatalf("apply: %v", apply)
|
||||
}
|
||||
}
|
||||
|
||||
// The fallback kernel and its headers are declared after the boot-entries service, so the entry is placed
|
||||
// before the kernel is installed and every DKMS module is built for it (ADR 0508 §9).
|
||||
func TestTheFallbackKernelAndItsHeadersAreDeclaredAfterTheBootEntriesUnit(t *testing.T) {
|
||||
m := manifest(t)
|
||||
at := map[string]int{}
|
||||
for i, r := range m.Resources {
|
||||
at[r["id"].(string)] = i
|
||||
}
|
||||
for id, pkg := range map[string]string{"linux-lts": "linux-lts", "linux-lts-headers": "linux-lts-headers"} {
|
||||
r := m.resource(t, id)
|
||||
if r["type"] != "package" || r["package"] != pkg || r["absent"] == true {
|
||||
t.Errorf("%s: %v", id, r)
|
||||
}
|
||||
if at[id] < at["boot-apply"] || at[id] < at["boot-hook"] {
|
||||
t.Errorf("%s is declared before the boot-entries unit and hook", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -46,5 +46,15 @@ func tools(m *Machine) []stdio.Tool {
|
||||
return m.HugePages(n)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "kernel_boot",
|
||||
Description: "How this machine boots and whether the fallback kernel is on its menu (novox/hq ADR 0508 §9): the loader that booted it " +
|
||||
"(systemd-boot, read from the EFI variable it sets, grub, or none), the menu's entries, the kernels in /boot, the entry " +
|
||||
"the machine booted (systemd-boot) or the kernel image it booted (grub), whether the fallback kernel linux-lts is on " +
|
||||
"the menu with its kernel in /boot, and the room free on /boot in KiB (the fallback kernel needs about 72 MB). " +
|
||||
"Replaces ls /boot, bootctl list, reading grub.cfg and df /boot. (r)",
|
||||
Input: schema(map[string]any{}),
|
||||
Run: func(args map[string]any) (any, error) { return m.Boot() },
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) {
|
||||
for _, r := range m.Resources {
|
||||
ids = append(ids, r["id"].(string))
|
||||
}
|
||||
if strings.Join(ids, " ") != "state huge-pages unit apply" {
|
||||
if strings.Join(ids, " ") != "state huge-pages unit apply boot-entries boot-unit boot-hook boot-apply linux-lts linux-lts-headers" {
|
||||
t.Fatalf("order: %v", ids)
|
||||
}
|
||||
script := m.resource(t, "huge-pages")
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# Module kernel, novox/hq ADR 0508 §9, issue 1841. After a kernel image is installed, upgraded or removed, the
|
||||
# fallback kernel's boot entry (or the GRUB menu) is placed again. Written by the mesh; change the module.
|
||||
[Trigger]
|
||||
Operation = Install
|
||||
Operation = Upgrade
|
||||
Operation = Remove
|
||||
Type = Path
|
||||
Target = usr/lib/modules/*/vmlinuz
|
||||
|
||||
[Action]
|
||||
Description = Placing the fallback kernel on the boot menu
|
||||
When = PostTransaction
|
||||
Exec = ${dir:state}/boot-entries run
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/bin/sh
|
||||
# The fallback kernel's place on this machine's boot menu (module kernel, novox/hq ADR 0508 §9 and §10,
|
||||
# issue 1841). Written by the mesh and run as root by mesh-kernel-boot-entries.service at boot and when the
|
||||
# module is applied, and by the pacman hook 90-mesh-kernel-boot-entries.hook after a kernel is installed,
|
||||
# upgraded or removed. Replaced on every push; change the module, never this file.
|
||||
#
|
||||
# boot-entries run [<root> [<this module's state> [<room needed in /boot, KiB>]]]
|
||||
#
|
||||
# The three are arguments only so the module's tests can run this against a stand-in tree.
|
||||
#
|
||||
# It reads from the machine which loader booted it, never from a setting:
|
||||
# systemd-boot the EFI variable LoaderEntrySelected is present: the entry mesh-linux-lts.conf is written
|
||||
# from the command line the machine booted with.
|
||||
# GRUB /boot/grub/grub.cfg exists and that variable does not: the menu is generated again, only
|
||||
# when the set of kernels in /boot differs from the menu's.
|
||||
# neither it fails loudly. The default entry is never touched.
|
||||
# And a pacman hook of someone else's that copies kernel images to one fixed name in /boot is moved into this
|
||||
# module's state first (ADR 0508 §10): it would put the fallback kernel in the default kernel's place.
|
||||
set -eu
|
||||
|
||||
root=/
|
||||
state='${dir:state}'
|
||||
[ $# -ge 2 ] && root=$(printf '%s' "$2" | sed 's|/$||')
|
||||
[ $# -ge 3 ] && state=$3
|
||||
boot=$root/boot
|
||||
hooks=$root/etc/pacman.d/hooks
|
||||
efivar=$root/sys/firmware/efi/efivars/LoaderEntrySelected-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f
|
||||
ours=90-mesh-kernel-boot-entries.hook
|
||||
# The fallback kernel and its initramfs are about 72 MB in /boot (issue 1836): less room fails the unit
|
||||
# before a package writes half of them.
|
||||
need_kib=98304
|
||||
[ $# -ge 4 ] && need_kib=$4
|
||||
|
||||
[ $# -ge 1 ] && [ "$1" = run ] || {
|
||||
echo "usage: boot-entries run [<root> [<state directory> [<KiB needed>]]]" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
# A hook that is not ours, triggered by kernel images, whose action copies to one fixed name in /boot.
|
||||
copies_kernels() {
|
||||
grep -Eqi '^[[:space:]]*Target[[:space:]]*=.*(vmlinuz|usr/lib/modules)' "$1" &&
|
||||
grep -Eq '^[[:space:]]*Exec[[:space:]]*=.*(cp|install|ln|rsync)[[:space:]].*/boot/[A-Za-z0-9._+-]+([^A-Za-z0-9._+/%$*-]|$)' "$1"
|
||||
}
|
||||
|
||||
if [ -d "$hooks" ]; then
|
||||
for hook in "$hooks"/*.hook; do
|
||||
[ -f "$hook" ] || continue
|
||||
[ "$(basename "$hook")" = "$ours" ] && continue
|
||||
copies_kernels "$hook" || continue
|
||||
mkdir -p "$state/taken-over"
|
||||
mv -f "$hook" "$state/taken-over/$(basename "$hook")"
|
||||
echo "taken over: the pacman hook $(basename "$hook") copied kernel images to one fixed name in /boot; moved to $state/taken-over/ so each kernel keeps its own package's name"
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -e "$efivar" ]; then
|
||||
loader=systemd-boot
|
||||
elif [ -f "$boot/grub/grub.cfg" ]; then
|
||||
loader=grub
|
||||
else
|
||||
echo "no boot loader found: the machine was not booted by systemd-boot (no LoaderEntrySelected variable) and there is no $boot/grub/grub.cfg; the fallback kernel has no menu to be placed on" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Room first, only while the fallback kernel is not in /boot yet.
|
||||
if [ ! -e "$boot/vmlinuz-linux-lts" ]; then
|
||||
free=$(df -Pk "$boot" | awk 'NR==2 {print $4}')
|
||||
[ -n "$free" ] || free=0
|
||||
if [ "$free" -lt "$need_kib" ]; then
|
||||
echo "$boot has $free KiB free and the fallback kernel needs $need_kib KiB: free room there before the kernel is installed" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
case "$loader" in
|
||||
systemd-boot)
|
||||
entries=$boot/loader/entries
|
||||
cmdline=$(sed -e 's/BOOT_IMAGE=[^ ]*//g' -e 's/initrd=[^ ]*//g' "$root/proc/cmdline" | tr -s ' \n' ' ' | sed -e 's/^ //' -e 's/ $//')
|
||||
[ -n "$cmdline" ] || {
|
||||
echo "$root/proc/cmdline holds no command line to boot the fallback kernel with" >&2
|
||||
exit 1
|
||||
}
|
||||
want=$(
|
||||
echo "title Arch Linux (linux-lts, fallback)"
|
||||
echo "linux /vmlinuz-linux-lts"
|
||||
for ucode in intel-ucode.img amd-ucode.img; do
|
||||
[ -e "$boot/$ucode" ] && echo "initrd /$ucode"
|
||||
done
|
||||
echo "initrd /initramfs-linux-lts.img"
|
||||
echo "options $cmdline"
|
||||
)
|
||||
mkdir -p "$entries"
|
||||
if [ "$(cat "$entries/mesh-linux-lts.conf" 2>/dev/null || true)" = "$want" ]; then
|
||||
echo "systemd-boot: mesh-linux-lts.conf, as it was"
|
||||
else
|
||||
printf '%s\n' "$want" >"$entries/mesh-linux-lts.conf"
|
||||
echo "systemd-boot: wrote $entries/mesh-linux-lts.conf from the command line the machine booted with; the default entry is as it was"
|
||||
fi
|
||||
;;
|
||||
grub)
|
||||
cfg=$boot/grub/grub.cfg
|
||||
menu=$(sed -n 's|^[[:space:]]*linux[[:space:]]*\(/boot\)\{0,1\}/\(vmlinuz-[^[:space:]]*\).*|\2|p' "$cfg" | sort -u)
|
||||
have=$(cd "$boot" && ls -d vmlinuz-* 2>/dev/null | sort -u || true)
|
||||
if [ "$menu" = "$have" ]; then
|
||||
echo "grub: the menu lists the kernels in /boot, left as it is"
|
||||
else
|
||||
# GRUB lists kernels in reverse version order, so the fallback kernel would come first and become the
|
||||
# default entry: the kernel the machine booted is put first (GRUB_TOP_LEVEL; /etc/default/grub still
|
||||
# overrides it when it sets one), the fallback kernel never.
|
||||
top=$(tr ' ' '\n' <"$root/proc/cmdline" | sed -n 's|^BOOT_IMAGE=\(.*\)$|\1|p' | head -n 1)
|
||||
top=$(printf '%s' "$top" | sed 's|.*/||')
|
||||
case "$top" in
|
||||
vmlinuz-linux-lts | "") top= ;;
|
||||
esac
|
||||
[ -n "$top" ] && [ -e "$boot/$top" ] || top=
|
||||
if [ -z "$top" ]; then
|
||||
if [ -e "$boot/vmlinuz-linux" ]; then
|
||||
top=vmlinuz-linux
|
||||
else
|
||||
top=$(printf '%s\n' $have | grep -v '^vmlinuz-linux-lts$' | head -n 1 || true)
|
||||
fi
|
||||
fi
|
||||
if [ -n "$top" ]; then
|
||||
GRUB_TOP_LEVEL=/boot/$top grub-mkconfig -o "$cfg"
|
||||
else
|
||||
grub-mkconfig -o "$cfg"
|
||||
fi
|
||||
first=$top
|
||||
[ -n "$first" ] || first="the menu's own order"
|
||||
echo "grub: the kernels in /boot changed ($(echo $have | tr '\n' ' ')); the menu is generated again with $first first; the default entry is as it was"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,22 @@
|
||||
[Unit]
|
||||
Description=The fallback kernel's place on this machine's boot menu
|
||||
# Module kernel, novox/hq ADR 0508 §9, issue 1841. Starting this unit reads which loader booted the machine and
|
||||
# places the fallback kernel's boot entry (systemd-boot) or generates the menu again (GRUB); it fails, and says
|
||||
# why, when there is no loader or the boot partition has no room. The kernel packages are declared after this
|
||||
# unit, so the entry is in place before they are installed.
|
||||
After=local-fs.target
|
||||
Wants=local-fs.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=${dir:state}/boot-entries run
|
||||
# It writes the boot partition, the pacman hook directory and its own state, and nothing else.
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/boot ${dir:state} /etc/pacman.d/hooks
|
||||
ProtectHome=yes
|
||||
PrivateNetwork=yes
|
||||
NoNewPrivileges=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -13,7 +13,8 @@
|
||||
}
|
||||
},
|
||||
"tools": [
|
||||
"kernel_huge_pages"
|
||||
"kernel_huge_pages",
|
||||
"kernel_boot"
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
@@ -53,6 +54,52 @@
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "boot-entries",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/boot-entries",
|
||||
"mode": "0755",
|
||||
"trusted": true,
|
||||
"template": "files/boot-entries.sh"
|
||||
},
|
||||
{
|
||||
"id": "boot-unit",
|
||||
"type": "file",
|
||||
"path": "/etc/systemd/system/mesh-kernel-boot-entries.service",
|
||||
"mode": "0644",
|
||||
"template": "files/mesh-kernel-boot-entries.service"
|
||||
},
|
||||
{
|
||||
"id": "boot-hook",
|
||||
"type": "file",
|
||||
"path": "/etc/pacman.d/hooks/90-mesh-kernel-boot-entries.hook",
|
||||
"mode": "0644",
|
||||
"template": "files/90-mesh-kernel-boot-entries.hook"
|
||||
},
|
||||
{
|
||||
"id": "boot-apply",
|
||||
"type": "service",
|
||||
"unit": "mesh-kernel-boot-entries.service",
|
||||
"state": "running",
|
||||
"boot": "enabled",
|
||||
"restart-on": [
|
||||
"boot-unit",
|
||||
"boot-entries"
|
||||
],
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "linux-lts",
|
||||
"type": "package",
|
||||
"package": "linux-lts"
|
||||
},
|
||||
{
|
||||
"id": "linux-lts-headers",
|
||||
"type": "package",
|
||||
"package": "linux-lts-headers"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
|
||||
@@ -65,14 +65,16 @@ integration-tagged test runs the real go on the fixtures.
|
||||
|
||||
## The integration level
|
||||
|
||||
`build_go_test` with `level: integration` runs `go test -tags integration` against a store and a bus started for the
|
||||
run alone (novox/hq ADR 0470). The store is a postgres of the installed package: `initdb` and `pg_ctl` into
|
||||
`build_go_test` with `level: integration` runs `go test -tags integration,isolation` (isolation: the bubblewrap proofs,
|
||||
which a build node can run) against a store and a bus started for the run alone (novox/hq ADR 0470). The store is a postgres of the installed package: `initdb` and `pg_ctl` into
|
||||
`<work>/<workspace>/run-<id>/pg`, as the module's account, listening on a unix socket there and on no TCP address, with
|
||||
durability off; the tests get `MESH_TEST_POSTGRES` (the socket directory percent-encoded in the URL, because the
|
||||
controller's tests cut the URL at its last slash). The bus is a nats-server from the Go library at the version this
|
||||
module's `go.mod` names, run by this same binary started as `mesh-build-go bus <dir>` (`bus.go`: JetStream on, a
|
||||
loopback port of its own, its address on the first line, stopped when its input closes); the tests get `MESH_TEST_NATS`.
|
||||
Both run inside the step, are stopped, and the run's directory is removed whatever the tests did.
|
||||
Both run inside the step, are stopped, and the run's directory is removed whatever the tests did. The store, the bus and
|
||||
the tests run together in one bubblewrap (ADR 0448, issue 1309): this binary again as `mesh-build-go level <spec>`, which
|
||||
prints the level's result as one line of JSON, so the tests reach the level's own store and bus and nothing of the machine.
|
||||
|
||||
Setup (compiling the tests, the store, the bus) is judged within 30 s and the tests' running time (the wall time of
|
||||
`go test`) within 60 s, apart; a level over either says `over_bound` and names the bound, with the slowest tests for the running
|
||||
@@ -87,7 +89,9 @@ is said until a source exists.
|
||||
one digest of everything those tests read (ADR 0445 §5): the files, by content and under their path in the tree, of the
|
||||
package and every package its test binary reaches (`go list -test -deps`) that lies in the module's own tree; the version
|
||||
and checksum of every module outside it; the package's own `testdata/` files; and the level, the build tags and, at the
|
||||
integration level, the postgres major and the nats-server version. The Go version is not in it: the toolchain enters a
|
||||
integration level, the postgres major and the nats-server version; and, when the fetch placed siblings beside the
|
||||
workspace (`beside`), each one's repository and commit, which the answer echoes as `beside` (the checker keeps a verdict only
|
||||
for the siblings it was judged with). The text's form is `DigestVersion` 3. The Go version is not in it: the toolchain enters a
|
||||
fingerprint once, as the version the build nodes agree on (ADR 0477 §3), and the standard library's packages are that
|
||||
version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors
|
||||
is listed and tested from its vendor directory (`-mod=vendor`).
|
||||
@@ -101,11 +105,16 @@ that vendors), `go build -ldflags "-s -w -X main.builtFor=<system>" -o .mesh-bui
|
||||
module's directory, by the installed go (`GOTOOLCHAIN=local`) from the module cache the fetch filled and no network
|
||||
(`GOPROXY=off`). The binary is named as the build agent names it. The output directory is packed with mesh-sdk's `Pack`,
|
||||
as the build agent packs; two build nodes on the same toolchain give the same digest for the same tree (ADR 0477 §6). The step ends with the artifact, the binary, the digest, the size and
|
||||
the path of the archive, which stays in the workspace's `artifacts` directory: publishing it is the builder's, in a later
|
||||
release. An artifact that is neither a Go bundle nor an archive fails the step naming it.
|
||||
the reference of the archive: once the build ended, this process pushes it to the artifact store (novox/hq ADR 0498 §1 §2,
|
||||
mesh-sdk's `buildstep.PushBlob`) under `<module>/<artifact>` by its digest, a candidate's as a trunk build's, and answers
|
||||
`artifact-store://<module>/<artifact>/blobs/<digest>`; a push that fails fails the step `not stored: <artifact>: <the
|
||||
store's words>`. The packed bytes never stay on the node. The store's address is `${dir:state}/registry`, placed by the
|
||||
manifest, which requires `artifact-store`. The go build runs shut off in bubblewrap (ADR 0448, `isolate.go`): no
|
||||
network, none of the account, the caches beneath a throwaway layer unless the step is told `publish` (a trunk build). An artifact that is neither a Go
|
||||
bundle nor an archive fails the step naming it.
|
||||
|
||||
The same tool packs an archive artifact of any module's tree, whatever its language (ADR 0478 §1, issue 1589): the
|
||||
directory named by `from`, or a file pinned by `url` and `sha256`, packed with the same `Pack` as the build agent's `pack`
|
||||
(`testdata/captured/README` says how its digest was captured). A pinned file is fetched, checked against its sha256 (a
|
||||
mismatch fails the step naming both digests) and laid out as the artifact says (`file`, or unpacked from `within`; gzip and
|
||||
bzip2, not yet xz); it runs no command and needs no Go.
|
||||
mismatch fails the step naming both digests) and laid out as the artifact says (`file`, or unpacked from `within`; gzip, bzip2
|
||||
and xz, read in Go as the build agent reads it, issue 1590); it runs no command and needs no Go.
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package main
|
||||
|
||||
// What this module keeps on the bus, announced from its code (novox/hq ADR 0495 §2 §5; issue 1790): every state it
|
||||
// keeps, each readable by others, since another module reads each today (the builder, the checker and the indexer read toolchains; the builder and the checker read steps; gitea reads keys). A state of its own that it
|
||||
// did not name here is refused at the call (stdio.Keeps).
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// Kept are the states the module keeps.
|
||||
var Kept = []announce.State{
|
||||
{Name: "toolchains", Readable: true},
|
||||
{Name: "keys", Readable: true},
|
||||
{Name: "steps", Readable: true},
|
||||
}
|
||||
|
||||
// keepStates announces Kept as this module's (stdio.Keeps): main calls it before it serves.
|
||||
func keepStates() error { return stdio.Keeps(Kept...) }
|
||||
@@ -0,0 +1,63 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announcetest"
|
||||
)
|
||||
|
||||
// 1790 (ADR 0495 §5): served as the runtime serves it, the module announces on the bus every state it keeps, as its
|
||||
// code declares them. The runtime's part, publishing what the bundle asks to publish, is played by relay.
|
||||
func TestTheModuleAnnouncesEveryStateItKeeps(t *testing.T) {
|
||||
const module = "mesh-build-go"
|
||||
announcetest.Check(t, module, Kept, func(url string) func() {
|
||||
t.Setenv("MESH_MODULE", module)
|
||||
t.Setenv("MESH_NODE", "shanks")
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := keepStates(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inR, inW := io.Pipe()
|
||||
outR, outW := io.Pipe()
|
||||
go func() { _ = stdio.ServeOn(module, nil, inR, outW) }()
|
||||
go relay(t, module, nc, outR, inW)
|
||||
return func() { _ = inW.Close(); _ = outW.Close(); nc.Close() }
|
||||
})
|
||||
}
|
||||
|
||||
// relay is the runtime's side of the channel: each mesh/publish the bundle asks is published on its module's event
|
||||
// subject, mesh.mod.<module>.event.<key>, and answered.
|
||||
func relay(t *testing.T, module string, nc *nats.Conn, from io.Reader, to io.Writer) {
|
||||
sc := bufio.NewScanner(from)
|
||||
sc.Buffer(make([]byte, 1<<20), 1<<24)
|
||||
enc := json.NewEncoder(to)
|
||||
for sc.Scan() {
|
||||
var m struct {
|
||||
ID json.RawMessage `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params struct {
|
||||
Key string `json:"key"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
} `json:"params"`
|
||||
}
|
||||
if json.Unmarshal(sc.Bytes(), &m) != nil || m.Method != "mesh/publish" {
|
||||
continue
|
||||
}
|
||||
if err := nc.Publish("mesh.mod."+module+".event."+m.Params.Key, m.Params.Body); err != nil {
|
||||
t.Errorf("publishing %s: %v", m.Params.Key, err)
|
||||
}
|
||||
_ = nc.Flush()
|
||||
_ = enc.Encode(map[string]any{"id": m.ID, "result": map[string]any{}})
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/therootcompany/xz"
|
||||
)
|
||||
|
||||
// maxUpstream is the largest pinned file fetched and the largest it may unpack to, as the build agent's and the
|
||||
@@ -42,9 +43,9 @@ func (m *Module) httpClient() *http.Client {
|
||||
|
||||
// buildArchive packs one archive artifact of the module in dir: a pinned file when it names a url, else the directory
|
||||
// `from` of the module's tree.
|
||||
func (m *Module) buildArchive(ctx context.Context, workspace, module, dir string, art catalogue.Artifact) (any, error) {
|
||||
func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir string, art catalogue.Artifact) (any, error) {
|
||||
if art.URL != "" {
|
||||
return m.pinnedArchive(ctx, workspace, module, art)
|
||||
return m.pinnedArchive(ctx, workspace, module, name, art)
|
||||
}
|
||||
from := strings.Trim(path.Clean("/"+filepath.ToSlash(art.From)), "/")
|
||||
if strings.TrimSpace(art.From) == "" {
|
||||
@@ -76,12 +77,12 @@ func (m *Module) buildArchive(ctx context.Context, workspace, module, dir string
|
||||
if !within(realDir, realRoot) {
|
||||
return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module: it resolves to %s, and the module is %s", art.Name, art.From, realRoot, realDir)
|
||||
}
|
||||
return m.packInto(workspace, module, art.Name, root)
|
||||
return m.packInto(ctx, workspace, module, name, art.Name, root)
|
||||
}
|
||||
|
||||
// pinnedArchive fetches the file an artifact is pinned to, checks it against the pin, lays it out as the artifact says
|
||||
// (kept whole as `file`, or unpacked from `within`) and packs it.
|
||||
func (m *Module) pinnedArchive(ctx context.Context, workspace, module string, art catalogue.Artifact) (any, error) {
|
||||
func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name string, art catalogue.Artifact) (any, error) {
|
||||
file, err := m.pinnedFile(ctx, workspace, art)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", art.Name, err)
|
||||
@@ -110,7 +111,7 @@ func (m *Module) pinnedArchive(ctx context.Context, workspace, module string, ar
|
||||
} else if err := unpackUpstream(file, laid, art.Within); err != nil {
|
||||
return nil, fmt.Errorf("%s: %s (%s): %w", art.Name, art.URL, art.Version, err)
|
||||
}
|
||||
return m.packInto(workspace, module, art.Name, laid)
|
||||
return m.packInto(ctx, workspace, module, name, art.Name, laid)
|
||||
}
|
||||
|
||||
// pinnedFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes are still the
|
||||
@@ -190,10 +191,13 @@ func sumOf(file string) (string, error) {
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// unpackUpstream unpacks a tar archive compressed with gzip or bzip2 into a directory: of its entries only those within
|
||||
// `within` (all, when empty), with that prefix taken off. Only files and directories, and nothing outside: the
|
||||
// node-engine unpacks nothing else. An xz archive is refused by name; the build agent reads it, this module does not
|
||||
// yet (it is a later issue, not a quiet difference in the archive).
|
||||
// xzDictMax is the largest xz dictionary read (the build agent's): xz's own presets use at most 64 MiB (-9); a stream
|
||||
// asking for more is refused rather than given the memory.
|
||||
const xzDictMax = 64 << 20
|
||||
|
||||
// unpackUpstream unpacks a tar archive compressed with gzip, bzip2 or xz into a directory: of its entries only those
|
||||
// within `within` (all, when empty), with that prefix taken off. Only files and directories, and nothing outside: the
|
||||
// node-engine unpacks nothing else.
|
||||
func unpackUpstream(file, into, within string) error {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
@@ -214,9 +218,22 @@ func unpackUpstream(file, into, within string) error {
|
||||
case bytes.HasPrefix(magic, []byte("BZh")):
|
||||
stream = bzip2.NewReader(buffered)
|
||||
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
|
||||
return errors.New("it is an xz archive, which this build module does not read yet; the build agent does")
|
||||
// As the build agent reads it (mesh-controller internal/builder/upstream_file.go; issue 1590): in Go
|
||||
// (github.com/therootcompany/xz, a port of xz-embedded), never by a tool the node may not carry. xzDictMax is
|
||||
// a real maximum: a block whose LZMA2 dictionary is larger is refused before its dictionary is allocated. The
|
||||
// reader keeps a filter decoder for every block of a stream, so the file's framing is walked first and a file
|
||||
// of more blocks than xzBlocksMax is refused before anything is decoded (xz_framing.go). What the stream
|
||||
// unpacks to is bounded by untar (maxUpstream, maxEntries).
|
||||
if err := xzWithinLimits(file); err != nil {
|
||||
return err
|
||||
}
|
||||
zipped, err := xz.NewReader(buffered, xzDictMax)
|
||||
if err != nil {
|
||||
return fmt.Errorf("it is not an xz stream this build module reads (a dictionary of %d MiB at most): %v", xzDictMax>>20, err)
|
||||
}
|
||||
stream = zipped
|
||||
default:
|
||||
return errors.New("it is not a tar archive compressed with gzip or bzip2; a single file is kept whole by naming it in `file`")
|
||||
return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept whole by naming it in `file`")
|
||||
}
|
||||
return untar(stream, into, within)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
@@ -40,16 +41,11 @@ const (
|
||||
capturedWithGo = "go1.27.1"
|
||||
)
|
||||
|
||||
// sameAsBuildAgent says the archive at res.Path holds the tar stream the build agent's pack wrote, and, under the Go the
|
||||
// sameAsBuildAgent says the archive the build pushed holds the tar stream the build agent's pack wrote, and, under the Go the
|
||||
// digests were captured with, has its digest.
|
||||
func sameAsBuildAgent(t *testing.T, res BuildResult, stream, digest string) {
|
||||
t.Helper()
|
||||
f, err := os.Open(res.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
z, err := gzip.NewReader(f)
|
||||
z, err := gzip.NewReader(bytes.NewReader(archiveOf(t, res)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -157,8 +153,8 @@ func TestAnArchiveOfADirectoryHasTheBuildAgentsDigest(t *testing.T) {
|
||||
if res.Artifact != "files" || res.Size == 0 || res.Binary != "" {
|
||||
t.Errorf("result = %+v", res)
|
||||
}
|
||||
if info, err := os.Stat(res.Path); err != nil || info.Size() != int64(res.Size) {
|
||||
t.Errorf("the archive at %q: %v", res.Path, err)
|
||||
if len(archiveOf(t, res)) != res.Size {
|
||||
t.Errorf("the archive pushed is not %d bytes", res.Size)
|
||||
}
|
||||
if len(*calls) != 0 {
|
||||
t.Errorf("an archive ran %d commands", len(*calls))
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The lead's decision L5 (the checker keeps a verdict of a core repository): the fetch writes the siblings it placed
|
||||
// beside the workspace, and the digest folds each one's repository and commit into every package's digest and echoes
|
||||
// them, so a verdict is kept only for the siblings it was judged with.
|
||||
func TestTheSiblingsCommitsAreInEveryDigestAndEchoed(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, _ := digestFixture(t)
|
||||
alone := digests(t, m, "unit")
|
||||
if len(alone.Beside) != 0 {
|
||||
t.Errorf("a workspace without siblings echoes %v", alone.Beside)
|
||||
}
|
||||
write := func(placed []Sibling) {
|
||||
raw, _ := json.Marshal(placed)
|
||||
if err := os.WriteFile(filepath.Join(m.Work, "ws1", besideRecord), raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
other := strings.Repeat("b", 40)
|
||||
write([]Sibling{{Repository: "novox/mesh-sdk", Commit: commitSHA, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}})
|
||||
with := digests(t, m, "unit")
|
||||
want := []string{"novox/mesh-catalog " + commitSHA, "novox/mesh-sdk " + commitSHA}
|
||||
if !slices.Equal(with.Beside, want) {
|
||||
t.Errorf("echoed %v, want %v", with.Beside, want)
|
||||
}
|
||||
write([]Sibling{{Repository: "novox/mesh-sdk", Commit: other, Dir: "mesh-sdk"}, {Repository: "novox/mesh-catalog", Commit: commitSHA, Dir: "mesh-catalog"}})
|
||||
moved := digests(t, m, "unit")
|
||||
for p, d := range alone.Packages {
|
||||
if with.Packages[p] == d {
|
||||
t.Errorf("%s: the siblings left the digest as it was without them", p)
|
||||
}
|
||||
if moved.Packages[p] == with.Packages[p] {
|
||||
t.Errorf("%s: a sibling's new commit left the digest as it was", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The fetch writes the siblings it placed, which the digest reads.
|
||||
func TestTheFetchRecordsTheSiblingsItPlaced(t *testing.T) {
|
||||
f := &fakeStep{gomods: "go.mod\n"}
|
||||
m, states := fetchModule(t, f)
|
||||
args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{
|
||||
map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}}}
|
||||
if _, err := fetchTool(t, m)(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res FetchResult
|
||||
json.Unmarshal(st.Result, &res)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(m.Work, res.Workspace, besideRecord))
|
||||
var placed []Sibling
|
||||
if err != nil || json.Unmarshal(raw, &placed) != nil || len(placed) != 1 || placed[0].Repository != "novox/mesh-sdk" || placed[0].Commit != commitSHA {
|
||||
t.Errorf("the record is %s (%v)", raw, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The checker's agreement: the integration level runs and digests under the tags integration and isolation, so the
|
||||
// bubblewrap proofs tagged isolation run in every check on a build node.
|
||||
func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) {
|
||||
t.Parallel()
|
||||
m, _ := digestFixture(t)
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, m.Run, &seen)
|
||||
digests(t, m, LevelIntegration)
|
||||
for _, s := range seen {
|
||||
if !strings.Contains(s.command, " -tags integration,isolation ") {
|
||||
t.Errorf("the integration digest listed %q", s.command)
|
||||
}
|
||||
}
|
||||
if DigestVersion != "3" {
|
||||
t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion)
|
||||
}
|
||||
}
|
||||
@@ -3,8 +3,8 @@ package main
|
||||
// The build step (novox/hq ADR 0405 §3 §7, ADR 0453 §7; issue 1487): a Go bundle artifact of a module of a fetched
|
||||
// workspace, built with today's recipe — the build agent's, mesh-controller's internal/builder/toolchain.go (the Go
|
||||
// entry) and builder.go (compile, binaryName, pack) — by the installed Go and no network, packed with mesh-sdk's Pack
|
||||
// so that its digest is the build agent's for the same tree. The archive is left in the workspace; publishing it is
|
||||
// the builder's, in a later release.
|
||||
// so that its digest is the build agent's for the same tree, and pushed to the artifact store by this process once the
|
||||
// build ended (ADR 0498 §1 §2, push.go).
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The Go recipe of the build agent (toolchain.go, the entry of language go): static, trimmed of its own paths and of
|
||||
@@ -33,8 +34,9 @@ type BuildResult struct {
|
||||
Binary string `json:"binary"`
|
||||
Digest string `json:"digest"`
|
||||
Size int `json:"size"`
|
||||
// Path is the archive, in the workspace's directory of the work directory.
|
||||
Path string `json:"path"`
|
||||
// Reference is where the archive is in the artifact store (novox/hq ADR 0498 §1):
|
||||
// artifact-store://<module>/<artifact>/blobs/<digest>.
|
||||
Reference string `json:"reference"`
|
||||
}
|
||||
|
||||
func (m *Module) buildTool() StepTool {
|
||||
@@ -49,6 +51,7 @@ func (m *Module) buildTool() StepTool {
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
|
||||
"module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"},
|
||||
"artifact": map[string]any{"type": "string", "description": "the artifact's name in the module's manifest"},
|
||||
"publish": map[string]any{"type": "boolean", "description": "true for a merged trunk commit alone: the build writes the shared Go caches; without it (a candidate) they are read beneath a throwaway layer"},
|
||||
},
|
||||
"required": []string{"workspace", "module", "artifact"},
|
||||
},
|
||||
@@ -76,6 +79,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
name, _ := args["artifact"].(string)
|
||||
publish, _ := args["publish"].(bool)
|
||||
if !workspaceOK.MatchString(workspace) {
|
||||
return nil, fmt.Errorf("%q is not a workspace id", workspace)
|
||||
}
|
||||
@@ -106,7 +110,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", "))
|
||||
}
|
||||
if art.Kind == "archive" {
|
||||
return m.buildArchive(ctx, workspace, module, dir, *art)
|
||||
return m.buildArchive(ctx, workspace, module, manifest.Module, dir, *art)
|
||||
}
|
||||
if art.Kind != "bundle" || art.Language != "go" {
|
||||
return nil, fmt.Errorf("the artifact %s of %s is a %s of language %q: this module builds Go bundles and archives", name, manifest.Module, art.Kind, art.Language)
|
||||
@@ -133,13 +137,19 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
linker += " -X " + recipeSystemStamp + "=" + system
|
||||
}
|
||||
cmd := []string{"-C", dir, "build", "-ldflags", linker, "-o", filepath.Join(out, binary), "./" + strings.Trim(art.From, "./")}
|
||||
if r := m.Run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" {
|
||||
// Shut off (ADR 0448): the shared caches are written by a trunk build alone (publish), a candidate's go build writes
|
||||
// beneath a throwaway layer.
|
||||
run, err := m.stepRunner(workspace, publish)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r := run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("go build of %s: %s", art.Name, causeLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
if err := m.insideModule(workspace, dir, out); err != nil {
|
||||
return nil, fmt.Errorf("the output directory of %s: %w", art.Name, err)
|
||||
}
|
||||
res, err := m.packInto(workspace, module, art.Name, out)
|
||||
res, err := m.packInto(ctx, workspace, module, manifest.Module, art.Name, out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -147,22 +157,20 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// packInto packs a directory with mesh-sdk's Pack, as the build agent packs every artifact, and leaves the archive in
|
||||
// the workspace's artifacts directory.
|
||||
func (m *Module) packInto(workspace, module, artifact, root string) (BuildResult, error) {
|
||||
// packInto packs a directory with mesh-sdk's Pack, as the build agent packs every artifact, and pushes the archive to
|
||||
// the artifact store from this process, after the build step ended (novox/hq ADR 0498 §1 §2): a candidate's as a trunk
|
||||
// build's, under <module>/<artifact> by its digest. The step answers the reference; a push that fails fails the step
|
||||
// `not stored:`. The packed bytes never stay on the node: they are held in this process until the push answered (the
|
||||
// lead's decision L9).
|
||||
func (m *Module) packInto(ctx context.Context, workspace, module, name, artifact, root string) (BuildResult, error) {
|
||||
archive, digest, err := buildstep.Pack(root)
|
||||
if err != nil {
|
||||
return BuildResult{}, fmt.Errorf("packing %s: %w", artifact, err)
|
||||
}
|
||||
artifacts := filepath.Join(m.Work, workspace, "artifacts")
|
||||
if err := os.MkdirAll(artifacts, 0o700); err != nil {
|
||||
return BuildResult{}, err
|
||||
if err := m.push(ctx, name+"/"+artifact, digest, archive); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("%s%s: %v", delivery.NotStored, artifact, err)
|
||||
}
|
||||
path := filepath.Join(artifacts, strings.ReplaceAll(module, "/", "_")+"."+artifact+".tar.gz")
|
||||
if err := os.WriteFile(path, archive, 0o600); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
return BuildResult{Artifact: artifact, Digest: digest, Size: len(archive), Path: path}, nil
|
||||
return BuildResult{Artifact: artifact, Digest: digest, Size: len(archive), Reference: buildstep.BlobReference(name, artifact, digest)}, nil
|
||||
}
|
||||
|
||||
// buildEnv is the environment of the compile: the build agent's (CGO_ENABLED=0, GOFLAGS) with the installed go
|
||||
|
||||
@@ -41,7 +41,7 @@ func buildModule(t *testing.T, manifest, content string) (*Module, *keptStates,
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(manifest), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
*calls = append(*calls, cmdRecord{env: env, line: name + " " + strings.Join(args, " ")})
|
||||
for i, a := range args {
|
||||
if a == "-o" {
|
||||
@@ -51,7 +51,8 @@ func buildModule(t *testing.T, manifest, content string) (*Module, *keptStates,
|
||||
}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
}, &[]shutOff{}) // the compile runs shut off (1309)
|
||||
withStore(t, m) // the archive is pushed to a store of the test's own (ADR 0498)
|
||||
return m, states, calls
|
||||
}
|
||||
|
||||
@@ -143,13 +144,10 @@ func TestTheArchiveHoldsOneExecutableAndTwoBuildsAreOneDigest(t *testing.T) {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
first, _ := runBuild(t, m, states, "tools")
|
||||
second, _ := runBuild(t, m, states, "tools")
|
||||
if first.Digest != second.Digest || first.Path != second.Path {
|
||||
if first.Digest != second.Digest || first.Reference != second.Reference {
|
||||
t.Errorf("two builds: %+v and %+v", first, second)
|
||||
}
|
||||
raw, err := os.ReadFile(first.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := archiveOf(t, first)
|
||||
sum := sha256.Sum256(raw)
|
||||
if first.Digest != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
t.Errorf("the digest %s is not the archive's", first.Digest)
|
||||
|
||||
@@ -20,7 +20,13 @@ import (
|
||||
)
|
||||
|
||||
// DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind.
|
||||
const DigestVersion = "2" // 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3)
|
||||
// 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3). 3: the siblings'
|
||||
// commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement).
|
||||
const DigestVersion = "3"
|
||||
|
||||
// integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation
|
||||
// for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run.
|
||||
var integrationTags = []string{"integration", "isolation"}
|
||||
|
||||
// goPackage is the part of `go list -json` a digest reads.
|
||||
type goPackage struct {
|
||||
@@ -71,9 +77,12 @@ func (p *goPackage) hasTests() bool { return len(p.TestGoFiles)+len(p.XTestGoFil
|
||||
// DigestResult is what a digest step answers: one digest per package that has tests at the level, and the packages
|
||||
// that have none.
|
||||
type DigestResult struct {
|
||||
Level string `json:"level"`
|
||||
Packages map[string]string `json:"packages"`
|
||||
WithoutTests []string `json:"without_tests,omitempty"`
|
||||
Level string `json:"level"`
|
||||
Packages map[string]string `json:"packages"`
|
||||
// Beside echoes the siblings folded into every digest, "<owner/repo> <commit>" sorted: the checker keeps a verdict
|
||||
// only when they are the siblings it gave the fetch.
|
||||
Beside []string `json:"beside,omitempty"`
|
||||
WithoutTests []string `json:"without_tests,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Module) digestTool() StepTool {
|
||||
@@ -113,20 +122,27 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
|
||||
if _, err := m.checkPostgres(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tags = []string{"integration"}
|
||||
tags = integrationTags
|
||||
header = append(header, "postgres "+strconv.Itoa(m.Recipe.PostgresMajor), "nats-server "+m.Recipe.NatsServer)
|
||||
default:
|
||||
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
|
||||
}
|
||||
header = append(header, "tags "+strings.Join(tags, ","))
|
||||
beside, err := m.besideLines(workspace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, b := range beside {
|
||||
header = append(header, "beside "+b)
|
||||
}
|
||||
|
||||
dirs, err := m.goModDirs(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := DigestResult{Level: level, Packages: map[string]string{}}
|
||||
res := DigestResult{Level: level, Packages: map[string]string{}, Beside: beside}
|
||||
for _, d := range dirs {
|
||||
pkgs, err := m.listPackages(ctx, tree, d, tags)
|
||||
pkgs, err := m.listPackages(ctx, workspace, tree, d, tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -139,14 +155,18 @@ func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
|
||||
}
|
||||
|
||||
// listPackages is `go list -test -deps -json` of the directory's packages, by import path.
|
||||
func (m *Module) listPackages(ctx context.Context, tree, dir string, tags []string) (map[string]*goPackage, error) {
|
||||
func (m *Module) listPackages(ctx context.Context, workspace, tree, dir string, tags []string) (map[string]*goPackage, error) {
|
||||
abs := filepath.Join(tree, dir)
|
||||
args := []string{"-C", abs, "list", "-test", "-deps", "-json"}
|
||||
if len(tags) > 0 {
|
||||
args = append(args, "-tags", strings.Join(tags, ","))
|
||||
}
|
||||
args = append(args, "./...")
|
||||
r := m.Run(ctx, m.goEnvFor(abs), "go", args...)
|
||||
run, err := m.stepRunner(workspace, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r := run(ctx, m.goEnvFor(abs), "go", args...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("go list in %s: %s", dir, causeLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ func digestFixture(t *testing.T) (*Module, string) {
|
||||
m := moduleWith(newStates(), "go1.27.1")
|
||||
m.Home, m.Work, m.Cache = t.TempDir(), work, t.TempDir()
|
||||
real := StepRunner(m.Home)
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch name {
|
||||
case "git":
|
||||
return forgekey.Ran{Stdout: "go.mod\n"}
|
||||
@@ -38,7 +38,7 @@ func digestFixture(t *testing.T) (*Module, string) {
|
||||
return forgekey.Ran{Stdout: "postgres (PostgreSQL) " + strconv.Itoa(m.Recipe.PostgresMajor) + ".1\n"}
|
||||
}
|
||||
return real(ctx, env, name, args...)
|
||||
}
|
||||
}, &[]shutOff{}) // go list runs shut off (1309); here the real go, unshut, reads the fixture
|
||||
return m, tree
|
||||
}
|
||||
|
||||
|
||||
@@ -67,6 +67,8 @@ func (m *Module) fetchTool() StepTool {
|
||||
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
||||
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
|
||||
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
|
||||
"dir": buildstep.FetchRequestProperties()["dir"],
|
||||
"paths": buildstep.FetchRequestProperties()["paths"],
|
||||
"beside": map[string]any{"type": "array", "description": "optional sibling repositories the tests read, each placed as a plain directory (no symlink) under the workspace's beside/<name>, which build_go_test gives the tests as MESH_CHECK_BESIDE",
|
||||
"items": map[string]any{"type": "object", "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
||||
@@ -81,19 +83,16 @@ func (m *Module) fetchTool() StepTool {
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
commit, _ := args["commit"].(string)
|
||||
ref, _ := args["ref"].(string)
|
||||
base, _ := args["base"].(string)
|
||||
if base != "" && !commitOK.MatchString(base) {
|
||||
return nil, fmt.Errorf("%q is not a trunk commit (40 hex digits)", base)
|
||||
}
|
||||
siblings, err := siblingsArg(args["beside"])
|
||||
// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
|
||||
// named paths, the workspace is narrowed to them (buildstep.Narrow).
|
||||
req, err := buildstep.ParseFetchRequest(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
|
||||
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
|
||||
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
||||
siblings, err := siblingsArg(args["beside"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
id := buildstep.NewID()
|
||||
@@ -115,6 +114,15 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
if res.Beside, err = m.placeSiblings(ctx, root, siblings); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if (base == "" || res.Merge != "") && len(req.Paths) > 0 {
|
||||
if err := buildstep.Narrow(tree, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Said beside the tree, so the steps that list the tree's go.mod files leave out those narrowed away.
|
||||
if err := os.WriteFile(filepath.Join(root, narrowedMark), nil, 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
dirs, err := m.goModDirs(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -177,6 +185,11 @@ func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
|
||||
dirs := []string{}
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" && !inTestdata(l) {
|
||||
if narrowed(tree) {
|
||||
if _, err := os.Stat(filepath.Join(tree, filepath.FromSlash(l))); err != nil {
|
||||
continue // outside a narrowed workspace's paths (buildstep.Narrow): no module of this build
|
||||
}
|
||||
}
|
||||
dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
|
||||
}
|
||||
}
|
||||
@@ -268,3 +281,12 @@ func firstLine(s string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// narrowedMark is the file beside a workspace's tree saying the fetch narrowed it to a module's named paths.
|
||||
const narrowedMark = "narrowed"
|
||||
|
||||
// narrowed says whether the fetch narrowed the workspace whose tree this is.
|
||||
func narrowed(tree string) bool {
|
||||
_, err := os.Stat(filepath.Join(filepath.Dir(tree), narrowedMark))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
@@ -143,8 +143,9 @@ func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) {
|
||||
// template in the files/ archive (novox/hq ADR 0349 §2).
|
||||
func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) {
|
||||
m := readForgeManifest(t)
|
||||
if len(m.Requires) != 2 || m.Requires[0] != "forge-account" || m.Requires[1] != "go-toolchain" {
|
||||
t.Errorf("requires = %v, want [forge-account go-toolchain]", m.Requires)
|
||||
// artifact-store: the module pushes a done build's archive itself (ADR 0498 §2).
|
||||
if len(m.Requires) != 3 || m.Requires[0] != "forge-account" || m.Requires[1] != "go-toolchain" || m.Requires[2] != "artifact-store" {
|
||||
t.Errorf("requires = %v, want [forge-account go-toolchain artifact-store]", m.Requires)
|
||||
}
|
||||
keeps := false
|
||||
for _, s := range m.State {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
@@ -33,11 +34,22 @@ func (m *Module) integrationBounds() Bounds {
|
||||
return b
|
||||
}
|
||||
|
||||
func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string) (res *TestResult, err error) {
|
||||
func (m *Module) integration(ctx context.Context, workspace, tree string, packages []string, pattern string) (res *TestResult, err error) {
|
||||
major, err := m.checkPostgres(ctx)
|
||||
if err != nil {
|
||||
return nil, err // a *buildstep.Refusal: this level only
|
||||
}
|
||||
if !m.Inside {
|
||||
// The store, the bus and the tests run together in one shut-off process (level.go), so the tests reach the
|
||||
// level's own store and bus and nothing else.
|
||||
return m.integrationShutOff(ctx, workspace, packages, pattern)
|
||||
}
|
||||
return m.integrationHere(ctx, workspace, tree, packages, pattern, major)
|
||||
}
|
||||
|
||||
// integrationHere is the level itself, in the shut-off process: the store and the bus started for the run, the tests
|
||||
// compiled and run against them, both stopped and the run's directory removed however it ends.
|
||||
func (m *Module) integrationHere(ctx context.Context, workspace, tree string, packages []string, pattern string, major int) (res *TestResult, err error) {
|
||||
run := filepath.Join(m.Work, workspace, "run-"+buildstep.NewID())
|
||||
if err := os.MkdirAll(run, 0o700); err != nil {
|
||||
return nil, err
|
||||
@@ -64,12 +76,13 @@ func (m *Module) integration(ctx context.Context, workspace, tree string, packag
|
||||
// Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run
|
||||
// below finds them built. Whether they compile is for the run to say; its failure is not read here.
|
||||
for _, d := range dirs {
|
||||
m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", "integration", "-run", "^$", "-count=1", "./...")
|
||||
m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), env...), "go", "-C", filepath.Join(tree, d), "test", "-tags", strings.Join(integrationTags, ","), "-run", "^$", "-count=1", "./...")
|
||||
}
|
||||
setup := m.clock().Sub(began)
|
||||
|
||||
return m.runLevel(ctx, tree, levelOpts{
|
||||
level: LevelIntegration, tags: []string{"integration"}, packages: packages, bounds: m.integrationBounds(),
|
||||
workspace: workspace, run: pattern,
|
||||
level: LevelIntegration, tags: integrationTags, packages: packages, bounds: m.integrationBounds(),
|
||||
env: env,
|
||||
setup: setup, wall: true,
|
||||
store: "postgres " + strconv.Itoa(major), bus: "nats-server " + m.Recipe.NatsServer,
|
||||
|
||||
@@ -107,6 +107,9 @@ func integModule(t *testing.T, f *integFake) (*Module, *keptStates) {
|
||||
m := moduleWith(states, "go1.27.1")
|
||||
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
|
||||
m.Run, m.Now, m.StartBus = f.run, f.now, f.startBus
|
||||
// These tests judge the level as it runs in its shut-off process (level.go, 1309-S5): the store, the bus and the
|
||||
// tests side by side. That the step starts that one process shut off is isolate_test.go's.
|
||||
m.Inside = true
|
||||
if err := os.MkdirAll(filepath.Join(m.Work, "ws1", "tree"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -177,7 +180,7 @@ func TestTheStoreListensOnASocketUnderTheRunAndNowhereElse(t *testing.T) {
|
||||
test = c
|
||||
}
|
||||
}
|
||||
if !strings.Contains(test.line, "-tags integration") {
|
||||
if !strings.Contains(test.line, "-tags integration,isolation") { // the bubblewrap proofs run too (the checker's agreement)
|
||||
t.Errorf("the run does not ask for the tag: %s", test.line)
|
||||
}
|
||||
env := map[string]string{}
|
||||
@@ -277,7 +280,7 @@ func TestAFailedOrPanickedRunStopsTheServersAndRemovesTheRunDirectory(t *testing
|
||||
m, _ = integModule(t, g)
|
||||
func() {
|
||||
defer func() { recover() }()
|
||||
m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil)
|
||||
m.integration(context.Background(), "ws1", filepath.Join(m.Work, "ws1", "tree"), nil, "")
|
||||
}()
|
||||
checkCleaned(g, m)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
// The run steps shut off (novox/hq ADR 0448; issue 1309, lead decision L2 of R4e): every command that runs a change's
|
||||
// code (go test, go build, go list) runs through mesh-sdk's buildstep.Isolated, in bubblewrap: no network, none of the
|
||||
// account (its home, its ssh key, its bus login, the runtime directory), the workspace read-write, the shared Go caches
|
||||
// read-only under a throwaway writable layer, or read-write for a trunk build alone (publish). The fetch step (git, go
|
||||
// mod download) holds the key and the network and runs none of the change's code, so it runs as before.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// BubblewrapLeast is the oldest bubblewrap a run step is shut off with: --overlay-src and --tmp-overlay, which keep the
|
||||
// shared caches unchanged by a candidate, came in 0.10.
|
||||
const BubblewrapLeast = "0.10"
|
||||
|
||||
// caches are the shared warm caches of the module: Go's module cache and its build cache.
|
||||
func (m *Module) caches() []string {
|
||||
return []string{filepath.Join(m.Cache, "gomod"), filepath.Join(m.Cache, "gobuild")}
|
||||
}
|
||||
|
||||
// isolation is how a run step of the workspace is shut off: the workspace (its tree, its siblings, its run directories,
|
||||
// its packed artifacts) read-write, the caches beneath a throwaway layer unless write, and an environment of a PATH and
|
||||
// a locale alone (the step's HOME is its own, inside its /tmp).
|
||||
func (m *Module) isolation(workspace string, write bool) buildstep.Isolation {
|
||||
path := os.Getenv("PATH")
|
||||
if path == "" {
|
||||
path = defaultPath
|
||||
}
|
||||
return buildstep.Isolation{
|
||||
Workspace: filepath.Join(m.Work, workspace),
|
||||
Caches: m.caches(),
|
||||
WriteCaches: write,
|
||||
Env: []string{"PATH=" + path, "LC_ALL=C"},
|
||||
}
|
||||
}
|
||||
|
||||
// stepRunner is the runner of a run step of the workspace: buildstep.Isolated over m.Run, so every command it runs is
|
||||
// shut off; write is true for a trunk build alone (the shared caches are written by trunk builds only). Inside is the
|
||||
// integration level's own process, already running shut off (level.go): it runs its commands as they are.
|
||||
func (m *Module) stepRunner(workspace string, write bool) (forgekey.Runner, error) {
|
||||
if m.Inside {
|
||||
return m.Run, nil
|
||||
}
|
||||
for _, c := range m.caches() {
|
||||
// bubblewrap binds only a directory that is there; the fetch makes them, a step on a fresh node may come first.
|
||||
if err := os.MkdirAll(c, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("the shared cache %s: %w", c, err)
|
||||
}
|
||||
}
|
||||
return buildstep.Isolated(m.Run, m.isolation(workspace, write)), nil
|
||||
}
|
||||
|
||||
// ProbeBubblewrap reads the installed bubblewrap's version (`bwrap --version`: "bubblewrap 0.13.0").
|
||||
func ProbeBubblewrap(ctx context.Context) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, ProbeTimeout)
|
||||
defer cancel()
|
||||
r := forgekey.ExecRunnerClean(0)(ctx, []string{"PATH=" + defaultPath}, buildstep.Bubblewrap, "--version")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("`%s --version` failed: %s", buildstep.Bubblewrap, firstLine(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
return parseBubblewrapVersion(r.Stdout)
|
||||
}
|
||||
|
||||
var bubblewrapVersion = regexp.MustCompile(`^bubblewrap (\d+)\.(\d+)(\.\d+)?\s*$`)
|
||||
|
||||
func parseBubblewrapVersion(out string) (string, error) {
|
||||
f := bubblewrapVersion.FindStringSubmatch(strings.TrimSpace(out))
|
||||
if f == nil {
|
||||
return "", fmt.Errorf("`%s --version` answered %q", buildstep.Bubblewrap, strings.TrimSpace(out))
|
||||
}
|
||||
return f[1] + "." + f[2] + f[3], nil
|
||||
}
|
||||
|
||||
// bubblewrapProblem says why the installed bubblewrap cannot shut a run step off, or "" when it can: it is missing, or
|
||||
// older than BubblewrapLeast.
|
||||
func bubblewrapProblem(version string, err error) string {
|
||||
if err != nil {
|
||||
return "bubblewrap cannot be run, and every build and test step runs in it (ADR 0448): " + err.Error()
|
||||
}
|
||||
if !atLeast(version, BubblewrapLeast) {
|
||||
return fmt.Sprintf("bubblewrap %s installed, and a build or test step needs %s or later (ADR 0448)", version, BubblewrapLeast)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// atLeast says whether the dotted version v is least or later.
|
||||
func atLeast(v, least string) bool {
|
||||
a, b := strings.Split(v, "."), strings.Split(least, ".")
|
||||
for i := 0; i < len(b); i++ {
|
||||
var x, y int
|
||||
if i < len(a) {
|
||||
x, _ = strconv.Atoi(a[i])
|
||||
}
|
||||
y, _ = strconv.Atoi(b[i])
|
||||
if x != y {
|
||||
return x > y
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// checkBubblewrap refuses a step when bubblewrap cannot shut it off; nil Bubblewrap probes nothing (a module that only
|
||||
// guards).
|
||||
func (m *Module) checkBubblewrap(ctx context.Context) error {
|
||||
if m.Bubblewrap == nil {
|
||||
return nil
|
||||
}
|
||||
v, err := m.Bubblewrap(ctx)
|
||||
if why := bubblewrapProblem(v, err); why != "" {
|
||||
return &buildstep.Refusal{Text: why, Want: BubblewrapLeast + " or later", Have: v}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// shutOff is one command a step ran in bubblewrap: the bubblewrap words before "--", what it bound read-write, read-only
|
||||
// and beneath a throwaway layer, and the command with its environment.
|
||||
type shutOff struct {
|
||||
flags []string
|
||||
binds, roBinds, overlays []string
|
||||
env []string
|
||||
command string
|
||||
chdir string
|
||||
}
|
||||
|
||||
// bwrapArity is how many words each bubblewrap option the SDK writes takes.
|
||||
var bwrapArity = map[string]int{"--unshare-all": 0, "--die-with-parent": 0, "--new-session": 0, "--clearenv": 0,
|
||||
"--proc": 1, "--dev": 1, "--tmpfs": 1, "--dir": 1, "--chdir": 1, "--overlay-src": 1, "--tmp-overlay": 1,
|
||||
"--ro-bind": 2, "--ro-bind-try": 2, "--bind": 2, "--symlink": 2, "--setenv": 2}
|
||||
|
||||
// throughBubblewrap is the fake runner of a step's tests: a bubblewrap command is read back (recorded in seen) and its
|
||||
// command handed to run with the environment bubblewrap was told to set, the way it runs in the step; any other command
|
||||
// (the fetch's git, postgres --version) goes to run as it is.
|
||||
func throughBubblewrap(t *testing.T, run forgekey.Runner, seen *[]shutOff) forgekey.Runner {
|
||||
t.Helper()
|
||||
var mu sync.Mutex
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if name != buildstep.Bubblewrap {
|
||||
return run(ctx, env, name, args...)
|
||||
}
|
||||
var s shutOff
|
||||
i := 0
|
||||
for ; i < len(args) && args[i] != "--"; i++ {
|
||||
n, ok := bwrapArity[args[i]]
|
||||
if !ok || i+n >= len(args) {
|
||||
t.Errorf("bubblewrap was given %q, which the test does not know: %v", args[i], args)
|
||||
return forgekey.Ran{Status: 1, Err: "unknown bubblewrap option"}
|
||||
}
|
||||
s.flags = append(s.flags, args[i])
|
||||
switch args[i] {
|
||||
case "--bind":
|
||||
s.binds = append(s.binds, args[i+1])
|
||||
case "--ro-bind", "--ro-bind-try":
|
||||
s.roBinds = append(s.roBinds, args[i+1])
|
||||
case "--overlay-src":
|
||||
s.overlays = append(s.overlays, args[i+1])
|
||||
case "--setenv":
|
||||
s.env = append(s.env, args[i+1]+"="+args[i+2])
|
||||
case "--chdir":
|
||||
s.chdir = args[i+1]
|
||||
}
|
||||
i += n
|
||||
}
|
||||
if i+1 >= len(args) {
|
||||
t.Errorf("bubblewrap was given no command: %v", args)
|
||||
return forgekey.Ran{Status: 1, Err: "no command"}
|
||||
}
|
||||
s.command = args[i+1] + " " + strings.Join(args[i+2:], " ")
|
||||
mu.Lock()
|
||||
*seen = append(*seen, s)
|
||||
mu.Unlock()
|
||||
return run(ctx, s.env, args[i+1], args[i+2:]...)
|
||||
}
|
||||
}
|
||||
|
||||
// checkShutOff fails the test unless s ran with no network and none of the account: every namespace unshared, the
|
||||
// environment cleared, nothing of the home or the state bound, the caches beneath a throwaway layer unless write.
|
||||
func checkShutOff(t *testing.T, m *Module, s shutOff, write bool) {
|
||||
t.Helper()
|
||||
for _, f := range []string{"--unshare-all", "--clearenv", "--die-with-parent"} {
|
||||
if !slices.Contains(s.flags, f) {
|
||||
t.Errorf("%s ran without %s: %v", s.command, f, s.flags)
|
||||
}
|
||||
}
|
||||
for _, b := range append(append([]string{}, s.binds...), s.roBinds...) {
|
||||
if m.Home != "" && (b == m.Home || strings.HasPrefix(b, m.Home+"/")) {
|
||||
t.Errorf("%s was given the account's home: %s", s.command, b)
|
||||
}
|
||||
if strings.HasPrefix(b, "/run") {
|
||||
t.Errorf("%s was given the runtime directory: %s", s.command, b)
|
||||
}
|
||||
}
|
||||
for _, e := range s.env {
|
||||
if strings.HasPrefix(e, "HOME="+m.Home) && m.Home != "" {
|
||||
t.Errorf("%s was told the account's home: %s", s.command, e)
|
||||
}
|
||||
}
|
||||
caches := m.caches()
|
||||
for _, c := range caches {
|
||||
bound, layered := slices.Contains(s.binds, c), slices.Contains(s.overlays, c)
|
||||
switch {
|
||||
case write && (!bound || layered):
|
||||
t.Errorf("a trunk build's %s does not write the shared cache %s (bound %v, layered %v)", s.command, c, bound, layered)
|
||||
case !write && (bound || !layered):
|
||||
t.Errorf("%s may write the shared cache %s (bound %v, layered %v): only a trunk build does", s.command, c, bound, layered)
|
||||
}
|
||||
}
|
||||
if want := filepath.Join(m.Work, "ws1"); !slices.Contains(s.binds, want) {
|
||||
t.Errorf("%s was not given its workspace %s read-write: %v", s.command, want, s.binds)
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S2: every go test of the unit level runs shut off in bubblewrap: no network, none of the account, the caches
|
||||
// beneath a throwaway layer.
|
||||
func TestTheUnitLevelRunsEveryGoTestShutOff(t *testing.T) {
|
||||
m, states, ws, calls := workspaceFor(t, "failing")
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, m.Run, &seen)
|
||||
if _, st := runUnit(t, m, states, ws); st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
ran := 0
|
||||
for _, c := range *calls {
|
||||
if strings.HasPrefix(c.line, "go ") {
|
||||
ran++
|
||||
}
|
||||
}
|
||||
if ran == 0 || len(seen) != ran {
|
||||
t.Fatalf("%d go commands ran, %d of them shut off: %v", ran, len(seen), *calls)
|
||||
}
|
||||
for _, s := range seen {
|
||||
checkShutOff(t, m, s, false)
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S2: a try's tests: build_go_test's run pattern reaches go test as one -run word; a pattern that is a flag is refused.
|
||||
func TestTheRunPatternReachesGoTestAndAFlagIsRefused(t *testing.T) {
|
||||
m, states, ws, calls := workspaceFor(t, "failing")
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, m.Run, &seen)
|
||||
run := stepTool(t, m, "build_go_test")
|
||||
if _, err := run(map[string]any{"workspace": ws, "level": "unit", "run": "TestA|TestB"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
states.waitEnd(t)
|
||||
found := false
|
||||
for _, c := range *calls {
|
||||
found = found || strings.Contains(c.line, " -run TestA|TestB ")
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no go test was given -run TestA|TestB: %v", *calls)
|
||||
}
|
||||
for _, bad := range []string{"-exec=/bin/sh", "a b", "(unclosed"} {
|
||||
if runPatternOK(bad) {
|
||||
t.Errorf("the run pattern %q was taken", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S3 and S4: a candidate's go build runs shut off with the caches beneath a throwaway layer; a trunk build
|
||||
// (publish) writes them.
|
||||
func TestABuildRunsShutOffAndOnlyATrunkBuildWritesTheCaches(t *testing.T) {
|
||||
for _, publish := range []bool{false, true} {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "binary")
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, m.Run, &seen)
|
||||
run := stepTool(t, m, "build_go_build")
|
||||
args := map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "tools"}
|
||||
if publish {
|
||||
args["publish"] = true
|
||||
}
|
||||
if _, err := run(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepPassed {
|
||||
t.Fatalf("publish %v: the step ended %s: %s", publish, st.State, st.Reason)
|
||||
}
|
||||
if len(seen) != 1 || !strings.HasPrefix(seen[0].command, "go ") || !strings.Contains(seen[0].command, " build ") {
|
||||
t.Fatalf("publish %v: shut off %v, want the one go build", publish, seen)
|
||||
}
|
||||
checkShutOff(t, m, seen[0], publish)
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S3: the digest's go list runs shut off.
|
||||
func TestTheDigestListsPackagesShutOff(t *testing.T) {
|
||||
m, _ := digestFixture(t)
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, m.Run, &seen)
|
||||
digests(t, m, LevelUnit)
|
||||
if len(seen) == 0 {
|
||||
t.Fatal("the digest ran no command shut off")
|
||||
}
|
||||
for _, s := range seen {
|
||||
if !strings.HasPrefix(s.command, "go ") || !strings.Contains(s.command, " list ") {
|
||||
t.Errorf("the digest shut off %q, want go list", s.command)
|
||||
}
|
||||
checkShutOff(t, m, s, false)
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S5: the integration level runs its store, its bus and its tests in ONE shut-off process (this program again,
|
||||
// `level <spec>`), so they reach each other and nothing else; its answer is the level's result. Nothing of the level
|
||||
// runs outside but reading postgres's version.
|
||||
func TestTheIntegrationLevelRunsInOneShutOffProcess(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "failing")
|
||||
m.Executable = "/opt/bundle/mesh-build-go"
|
||||
want := TestResult{Level: LevelIntegration, Passed: true, Tests: 3, Packages: []PackageResult{}, Store: "postgres 18"}
|
||||
var outside []string
|
||||
var seen []shutOff
|
||||
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch {
|
||||
case name == "postgres" && len(args) == 1 && args[0] == "--version":
|
||||
return forgekey.Ran{Stdout: "postgres (PostgreSQL) 18.6\n"}
|
||||
case name == m.Executable && len(args) == 2 && args[0] == levelArg:
|
||||
var spec levelSpec
|
||||
if err := json.Unmarshal([]byte(args[1]), &spec); err != nil || spec.Workspace != ws || spec.Work != m.Work {
|
||||
t.Errorf("the level's spec %s: %v", args[1], err)
|
||||
}
|
||||
raw, _ := json.Marshal(levelAnswer{Result: &want})
|
||||
return forgekey.Ran{Stdout: string(raw) + "\n"}
|
||||
}
|
||||
outside = append(outside, name+" "+strings.Join(args, " "))
|
||||
return forgekey.Ran{Status: 1, Err: "not expected here"}
|
||||
}, &seen)
|
||||
run := stepTool(t, m, "build_go_test")
|
||||
if _, err := run(map[string]any{"workspace": ws, "level": LevelIntegration}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
var got TestResult
|
||||
if err := json.Unmarshal(st.Result, &got); err != nil || got.Tests != 3 || !got.Passed || got.Store != "postgres 18" {
|
||||
t.Errorf("the result %s (%v), want the shut-off process's", st.Result, err)
|
||||
}
|
||||
if len(outside) > 0 {
|
||||
t.Errorf("ran outside the shut-off process: %v", outside)
|
||||
}
|
||||
if len(seen) != 1 {
|
||||
t.Fatalf("shut off %d processes, want one: %v", len(seen), seen)
|
||||
}
|
||||
checkShutOff(t, m, seen[0], false)
|
||||
if !slices.Contains(seen[0].roBinds, m.Executable) || slices.Contains(seen[0].roBinds, filepath.Dir(m.Executable)) {
|
||||
t.Errorf("the program is bound %v; want the program alone, read-only", seen[0].roBinds)
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S1: a step is refused, and the module is unhealthy, while bubblewrap is missing or older than 0.10, naming it.
|
||||
func TestAnOldOrMissingBubblewrapRefusesTheStepsAndIsUnhealthy(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
version string
|
||||
err error
|
||||
ok bool
|
||||
}{{"0.13.0", nil, true}, {"0.10.0", nil, true}, {"0.9.0", nil, false}, {"", errors.New("bwrap: not found"), false}} {
|
||||
m := moduleWith(newStates(), "go1.27.1")
|
||||
m.Bubblewrap = func(context.Context) (string, error) { return c.version, c.err }
|
||||
err := m.Check(context.Background())
|
||||
h := m.Health(context.Background())
|
||||
raw, _ := json.Marshal(h)
|
||||
if c.ok {
|
||||
if err != nil || strings.Contains(string(raw), "unhealthy") {
|
||||
t.Errorf("bubblewrap %q: refused %v, health %s", c.version, err, raw)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var ref *buildstep.Refusal
|
||||
if !errors.As(err, &ref) || !strings.Contains(err.Error(), "bubblewrap") {
|
||||
t.Errorf("bubblewrap %q %v: Check answered %v, want a refusal naming bubblewrap", c.version, c.err, err)
|
||||
}
|
||||
if !strings.Contains(string(raw), "unhealthy") || !strings.Contains(string(raw), "bubblewrap") {
|
||||
t.Errorf("bubblewrap %q %v: health %s, want unhealthy naming bubblewrap", c.version, c.err, raw)
|
||||
}
|
||||
}
|
||||
for out, want := range map[string]string{"bubblewrap 0.13.0\n": "0.13.0", "bubblewrap 0.10": "0.10"} {
|
||||
if got, err := parseBubblewrapVersion(out); err != nil || got != want {
|
||||
t.Errorf("%q read as %q, %v", out, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stepTool is the Run of the named tool.
|
||||
func stepTool(t *testing.T, m *Module, name string) func(map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == name {
|
||||
return tool.Run
|
||||
}
|
||||
}
|
||||
t.Fatalf("no tool %s", name)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1309-S6 (ADR 0448's check): a test of the change, run by the unit level, reads no key and no login of the account,
|
||||
// resolves no name, reaches no socket of the machine (a TCP port on its loopback, a unix socket) and leaves the shared
|
||||
// caches unchanged. The probe is testdata/repos/probe.
|
||||
func TestARunStepReachesNoKeyLoginNameOrSocketAndLeavesTheCachesUnchanged(t *testing.T) {
|
||||
needTools(t, "the shut-off unit level", "go", "git", "bwrap")
|
||||
tree := repository(t, "probe")
|
||||
work, err := os.MkdirTemp("", "w")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(work) })
|
||||
if err := os.MkdirAll(filepath.Join(work, "ws1"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(tree, filepath.Join(work, "ws1", "tree")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
home := t.TempDir()
|
||||
key, login := filepath.Join(home, ".ssh", "mesh_ed25519"), filepath.Join(home, "broker")
|
||||
for _, f := range []string{key, login} {
|
||||
if err := os.MkdirAll(filepath.Dir(f), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(f, []byte("secret"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
tcp, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tcp.Close()
|
||||
sockDir := t.TempDir()
|
||||
unix, err := net.Listen("unix", filepath.Join(sockDir, "s"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer unix.Close()
|
||||
m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()}
|
||||
m.Run = StepRunner(home)
|
||||
for _, c := range m.caches() {
|
||||
if err := os.MkdirAll(c, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
res, err := m.runLevel(context.Background(), filepath.Join(work, "ws1", "tree"), levelOpts{workspace: "ws1", level: LevelUnit,
|
||||
bounds: Bounds{Run: UnitBound * 100}, env: []string{"PROBE_KEY=" + key, "PROBE_LOGIN=" + login,
|
||||
"PROBE_TCP=" + tcp.Addr().String(), "PROBE_SOCKET=" + filepath.Join(sockDir, "s"), "PROBE_CACHE=" + m.caches()[1]}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !res.Passed {
|
||||
t.Fatalf("the probe did not run: %+v", res)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(work, "ws1", "tree", "seen.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("the probe wrote nothing: %v", err)
|
||||
}
|
||||
var seen map[string]bool
|
||||
if err := json.Unmarshal(raw, &seen); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for what, reached := range seen {
|
||||
if reached && what != "wroteCache" {
|
||||
t.Errorf("the run step reached %s: %s", what, raw)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.caches()[1], "written-by-a-candidate")); err == nil {
|
||||
t.Errorf("a candidate's step changed the shared cache")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
// The integration level shut off (novox/hq ADR 0448, ADR 0470; issue 1309 S5): its store, its bus and its tests run in
|
||||
// one bubblewrap of their own, so the tests reach the level's postgres (a socket in the run's directory) and its bus (a
|
||||
// loopback port of the step's own network) and nothing of the machine. The process in it is this binary started again
|
||||
// as `mesh-build-go level <spec>`: it runs the level as integrationHere does and prints its answer as one line of JSON.
|
||||
// The tools bundle never runs a level in its own process, and a level never serves tools.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// levelArg is the first argument that makes this binary an integration level's process.
|
||||
const levelArg = "level"
|
||||
|
||||
// levelSpec is what the shut-off process is asked: the workspace, the packages and the -run pattern, the level's bounds.
|
||||
type levelSpec struct {
|
||||
Work string `json:"work"`
|
||||
Cache string `json:"cache"`
|
||||
Workspace string `json:"workspace"`
|
||||
Packages []string `json:"packages,omitempty"`
|
||||
Run string `json:"run,omitempty"`
|
||||
Setup time.Duration `json:"setup"`
|
||||
Bound time.Duration `json:"bound"`
|
||||
}
|
||||
|
||||
// levelAnswer is the one line the process prints: the level's result, or why it could not run.
|
||||
type levelAnswer struct {
|
||||
Result *TestResult `json:"result,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// executable is this program's path, bound read-only into the step: Executable when a test set it.
|
||||
func (m *Module) executable() (string, error) {
|
||||
if m.Executable != "" {
|
||||
return m.Executable, nil
|
||||
}
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.EvalSymlinks(exe)
|
||||
}
|
||||
|
||||
// integrationShutOff runs the integration level in one shut-off process and reads its answer.
|
||||
func (m *Module) integrationShutOff(ctx context.Context, workspace string, packages []string, pattern string) (*TestResult, error) {
|
||||
exe, err := m.executable()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the integration level's program: %w", err)
|
||||
}
|
||||
b := m.integrationBounds()
|
||||
spec, err := json.Marshal(levelSpec{Work: m.Work, Cache: m.Cache, Workspace: workspace, Packages: packages, Run: pattern, Setup: b.Setup, Bound: b.Run})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range m.caches() {
|
||||
if err := os.MkdirAll(c, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("the shared cache %s: %w", c, err)
|
||||
}
|
||||
}
|
||||
iso := m.isolation(workspace, false)
|
||||
iso.ReadOnly = append(iso.ReadOnly, exe) // the program alone, not the directory it lies in
|
||||
r := buildstep.Isolated(m.Run, iso)(ctx, nil, exe, levelArg, string(spec))
|
||||
var answer levelAnswer
|
||||
line := lastLine(r.Stdout)
|
||||
if err := json.Unmarshal([]byte(line), &answer); err != nil || (answer.Result == nil && answer.Error == "") {
|
||||
return nil, fmt.Errorf("the integration level's shut-off process answered nothing it says (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return nil, fmt.Errorf("%s", answer.Error)
|
||||
}
|
||||
return answer.Result, nil
|
||||
}
|
||||
|
||||
func lastLine(s string) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
return strings.TrimSpace(lines[len(lines)-1])
|
||||
}
|
||||
|
||||
// runLevelProcess is the shut-off process: args is the spec. It prints its answer and returns the exit status, 0 when it
|
||||
// printed one.
|
||||
func runLevelProcess(args []string) int {
|
||||
say := func(a levelAnswer) int {
|
||||
raw, _ := json.Marshal(a)
|
||||
fmt.Println(string(raw))
|
||||
return 0
|
||||
}
|
||||
if len(args) != 1 {
|
||||
fmt.Fprintln(os.Stderr, "usage: mesh-build-go level <spec>")
|
||||
return 2
|
||||
}
|
||||
var spec levelSpec
|
||||
if err := json.Unmarshal([]byte(args[0]), &spec); err != nil {
|
||||
return say(levelAnswer{Error: "the level's spec: " + err.Error()})
|
||||
}
|
||||
if !workspaceOK.MatchString(spec.Workspace) {
|
||||
return say(levelAnswer{Error: fmt.Sprintf("%q is not a workspace id", spec.Workspace)})
|
||||
}
|
||||
m := &Module{Inside: true, Recipe: Current(), Work: spec.Work, Cache: spec.Cache, Home: os.Getenv("HOME"),
|
||||
IntegrationBounds: Bounds{Setup: spec.Setup, Run: spec.Bound}, Say: func(string, ...any) {}}
|
||||
m.Run = StepRunner(m.Home)
|
||||
ctx := context.Background()
|
||||
major, err := m.checkPostgres(ctx)
|
||||
if err != nil {
|
||||
return say(levelAnswer{Error: err.Error()})
|
||||
}
|
||||
tree := filepath.Join(spec.Work, spec.Workspace, "tree")
|
||||
res, err := m.integrationHere(ctx, spec.Workspace, tree, spec.Packages, spec.Run, major)
|
||||
if err != nil {
|
||||
return say(levelAnswer{Error: err.Error()})
|
||||
}
|
||||
return say(levelAnswer{Result: res})
|
||||
}
|
||||
@@ -29,6 +29,9 @@ func main() {
|
||||
if len(os.Args) > 1 && os.Args[1] == busArg {
|
||||
os.Exit(runBus(os.Args[2:]))
|
||||
}
|
||||
if len(os.Args) > 1 && os.Args[1] == levelArg {
|
||||
os.Exit(runLevelProcess(os.Args[2:]))
|
||||
}
|
||||
ctx := context.Background()
|
||||
m := &Module{
|
||||
Node: strings.TrimSpace(os.Getenv("MESH_NODE")),
|
||||
@@ -47,12 +50,19 @@ func main() {
|
||||
m.Home = u.HomeDir
|
||||
}
|
||||
m.Run = StepRunner(m.Home)
|
||||
m.RegistryFile = m.Home + "/registry"
|
||||
m.Probe = ProbeGo(m.Home, m.Cache)
|
||||
m.Bubblewrap = ProbeBubblewrap
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
if m.Node != "" {
|
||||
go m.Start(ctx)
|
||||
}
|
||||
go startKeeper(ctx)
|
||||
// Every state it keeps, said on the bus before it serves (ADR 0495 §2; issue 1790).
|
||||
if err := keepStates(); err != nil {
|
||||
logf("announcing the states it keeps: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -64,7 +64,7 @@ func TestManifestIsValidAndDeclaresTheAccountTheCapabilityAndThePackages(t *test
|
||||
}
|
||||
}
|
||||
slices.Sort(packages)
|
||||
if want := []string{"git", "openssh", "postgresql"}; !slices.Equal(packages, want) {
|
||||
if want := []string{"bubblewrap", "git", "openssh", "postgresql"}; !slices.Equal(packages, want) {
|
||||
t.Errorf("packages = %v, want %v", packages, want)
|
||||
}
|
||||
}
|
||||
@@ -144,7 +144,34 @@ func TestGiteaReadsTheBuildModulesKeysAndTheCatalogueAcceptsThemTogether(t *test
|
||||
t.Errorf("gitea reads %v, lacks %s", gitea.Reads, want)
|
||||
}
|
||||
}
|
||||
if problems := catalogue.StateReadsNothingDeclares(all); len(problems) > 0 {
|
||||
t.Errorf("a read names a state its owner does not keep: %v", problems)
|
||||
}
|
||||
|
||||
// 1309-S1: every run step runs in bubblewrap, so the module declares its package (the lead's decision: each build module
|
||||
// that runs a step through it declares it), beside the user-namespaces capability bubblewrap needs.
|
||||
func TestTheModuleDeclaresBubblewrapAndTheCapabilityItNeeds(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
if r := resource(t, m, "bubblewrap"); r["type"] != "package" || r["package"] != "bubblewrap" {
|
||||
t.Errorf("the bubblewrap resource is %v, want the package bubblewrap", r)
|
||||
}
|
||||
if !slices.Contains(m.Capabilities, "user-namespaces") {
|
||||
t.Errorf("capabilities %v lack user-namespaces", m.Capabilities)
|
||||
}
|
||||
}
|
||||
|
||||
// 1587-S1 (ADR 0498 §2): the module requires the artifact store and is told its address in a file of its home, which no
|
||||
// run step is given (a run step has no network either, ADR 0448).
|
||||
func TestTheModuleRequiresTheArtifactStoreAndIsToldItsAddress(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
if !slices.Contains(m.Requires, "artifact-store") {
|
||||
t.Errorf("requires %v lack artifact-store", m.Requires)
|
||||
}
|
||||
r := resource(t, m, "registry")
|
||||
// The validator reads the template (files/registry) into the resource's content.
|
||||
if content, _ := r["content"].(string); r["type"] != "file" || r["path"] != "${dir:state}/registry" || !strings.Contains(content, "${seat:mesh-artifact-store:reach}") {
|
||||
t.Errorf("the registry resource is %v", r)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(moduleDir, "files", "registry"))
|
||||
if err != nil || strings.TrimSpace(string(raw)) != "${seat:mesh-artifact-store:reach}" {
|
||||
t.Errorf("files/registry is %q (%v), want the store seat's reach", raw, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +50,18 @@ type Module struct {
|
||||
IntegrationBounds Bounds
|
||||
// Now is the clock the levels' times are read from; time.Now when nil.
|
||||
Now func() time.Time
|
||||
// Bubblewrap reads the installed bubblewrap's version (ProbeBubblewrap): a step is refused, and the module is
|
||||
// unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing.
|
||||
Bubblewrap func(ctx context.Context) (string, error)
|
||||
// Inside is the integration level's own process, started shut off in bubblewrap by a test step (level.go): its
|
||||
// commands run as they are, never shut off a second time.
|
||||
Inside bool
|
||||
// RegistryFile holds the artifact store's address (host:port), placed by the manifest; Push puts a blob there
|
||||
// (buildstep.PushBlob when nil; a test's fake store).
|
||||
RegistryFile string
|
||||
Push func(ctx context.Context, registry, repository, digest string, body []byte) error
|
||||
// Executable is this program's path, which the integration level starts again shut off; os.Executable when empty.
|
||||
Executable string
|
||||
// StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own
|
||||
// program, StartBus in bus.go, when nil).
|
||||
StartBus func(ctx context.Context, dir string) (url string, stop func(), err error)
|
||||
@@ -69,7 +81,10 @@ func (m *Module) guard(installed string, probeErr error) buildstep.Guard {
|
||||
// CheckRound.
|
||||
func (m *Module) Check(ctx context.Context) error {
|
||||
installed, err := m.Probe(ctx)
|
||||
return m.guard(installed, err).Check(ctx)
|
||||
if err := m.guard(installed, err).Check(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.checkBubblewrap(ctx)
|
||||
}
|
||||
|
||||
// Start checks once, as the module starts, and then every CheckRound until ctx ends.
|
||||
@@ -95,6 +110,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
|
||||
if err := m.guard(installed, perr).Check(ctx); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
if err := m.checkBubblewrap(ctx); err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("%s installed, the version the module builds with (recipe %s)", installed, m.Recipe.Version()))
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ADR 0499 §4 (issue 1758): a fetch asked with the module's directory and its named paths leaves the workspace holding
|
||||
// only those, the manifest and the lock files; the rest of the commit is gone before any run step; a request naming
|
||||
// paths without a directory is refused before anything is fetched.
|
||||
func TestAFetchWithNamedPathsNarrowsTheWorkspace(t *testing.T) {
|
||||
f := newForge(t)
|
||||
_, head := f.repo("novox/narrow", map[string]string{
|
||||
"README.md": "the repository\n",
|
||||
"lib/go.mod": "module fixture/lib\n\ngo 1.25.0\n",
|
||||
"lib/module.json": `{"module":"lib","version":"1"}`,
|
||||
"lib/cmd/x/x.go": "package main\n\nfunc main() {}\n",
|
||||
"lib/docs/a.md": "not named\n",
|
||||
"other/go.mod": "module fixture/other\n\ngo 1.25.0\n",
|
||||
"other/o.go": "package other\n",
|
||||
})
|
||||
m, states := f.module(t)
|
||||
res, st := fetched(t, m, states, map[string]any{"repository": "novox/narrow", "commit": head, "dir": "lib", "paths": []any{"cmd"}})
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the fetch ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
tree := filepath.Join(m.Work, res.Workspace, "tree")
|
||||
for _, kept := range []string{"lib/go.mod", "lib/module.json", "lib/cmd/x/x.go"} {
|
||||
if _, err := os.Stat(filepath.Join(tree, kept)); err != nil {
|
||||
t.Errorf("%s was narrowed away: %v", kept, err)
|
||||
}
|
||||
}
|
||||
for _, gone := range []string{"README.md", "lib/docs/a.md", "other/o.go"} {
|
||||
if _, err := os.Stat(filepath.Join(tree, gone)); err == nil {
|
||||
t.Errorf("%s is still in the workspace", gone)
|
||||
}
|
||||
}
|
||||
if strings.Join(res.Modules, ",") != "lib" {
|
||||
t.Errorf("the fetch answered the modules %v, want lib alone", res.Modules)
|
||||
}
|
||||
_, st = fetched(t, m, states, map[string]any{"repository": "novox/narrow", "commit": head, "paths": []any{"cmd"}})
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "no module directory") {
|
||||
t.Errorf("paths without a directory: %s %q", st.State, st.Reason)
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,7 @@ func TestRealFetchAndBuildOfAMemberPinningAPullHead(t *testing.T) {
|
||||
}
|
||||
_, memberHead := f.repo("novox/member", member)
|
||||
m, states := f.module(t)
|
||||
withStore(t, m) // the build pushes its archive (ADR 0498)
|
||||
|
||||
res, st := fetched(t, m, states, map[string]any{"repository": "novox/member", "commit": memberHead})
|
||||
if st.State != delivery.StepPassed {
|
||||
@@ -49,6 +50,9 @@ func TestRealFetchAndBuildOfAMemberPinningAPullHead(t *testing.T) {
|
||||
if _, err := os.Stat(filepath.Join(m.Work, res.Workspace, "pinned", "dep@"+head, "beside", "dep", "go", "dep.go")); err != nil {
|
||||
t.Errorf("the pull request's repository was not placed beside the workspace: %v", err)
|
||||
}
|
||||
// The build runs shut off in bubblewrap (1309), which a build node has and build-agent's check container has not:
|
||||
// there the build half is not run, said loudly; the fetch half above, the pin's own part (issue 1148), always is.
|
||||
needTools(t, "the shut-off build of a member pinning a pull head", "bwrap")
|
||||
built, err := m.build(context.Background(), map[string]any{"workspace": res.Workspace, "module": ".", "artifact": "tools"})
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("the build: %w", err))
|
||||
|
||||
@@ -6,6 +6,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
@@ -104,9 +105,47 @@ func (m *Module) placeSiblings(ctx context.Context, root string, siblings []sibl
|
||||
placed = append(placed, Sibling{Repository: s.repository, Commit: s.commit, Dir: name})
|
||||
}
|
||||
sort.Slice(placed, func(i, j int) bool { return placed[i].Dir < placed[j].Dir })
|
||||
if len(placed) > 0 {
|
||||
// What the digest folds in (the lead's decision L5): the siblings this workspace's tests read, by commit.
|
||||
raw, err := json.Marshal(placed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, besideRecord), raw, 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return placed, nil
|
||||
}
|
||||
|
||||
// besideRecord is the file in a workspace's root naming the siblings the fetch placed beside it.
|
||||
const besideRecord = "beside.json"
|
||||
|
||||
// besideLines are the siblings a workspace's fetch placed, as the digest folds them in and echoes them: "<owner/repo>
|
||||
// <commit>", sorted; none when it placed none.
|
||||
func (m *Module) besideLines(workspace string) ([]string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(m.Work, workspace, besideRecord))
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var placed []Sibling
|
||||
if err := json.Unmarshal(raw, &placed); err != nil {
|
||||
return nil, fmt.Errorf("the workspace's record of its siblings: %w", err)
|
||||
}
|
||||
var out []string
|
||||
for _, s := range placed {
|
||||
if !repositoryOK.MatchString(s.Repository) || !commitOK.MatchString(s.Commit) {
|
||||
return nil, fmt.Errorf("the workspace's record of its siblings names %q at %q", s.Repository, s.Commit)
|
||||
}
|
||||
out = append(out, s.Repository+" "+s.Commit)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func firstSymlink(dir string) string {
|
||||
var found string
|
||||
filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
|
||||
|
||||
@@ -171,6 +171,7 @@ func TestRealFetchMergesConflictsAndPlainlyFetches(t *testing.T) {
|
||||
// sibling and passes with it; the sibling is a plain directory with no symlink in it; and a sibling that holds a
|
||||
// symlink fails the fetch naming it.
|
||||
func TestRealTestsReadTheSiblingBesideTheTree(t *testing.T) {
|
||||
needTools(t, "the shut-off unit level", "go", "git", "bwrap") // the tests run in bubblewrap (1309)
|
||||
f := newForge(t)
|
||||
files := map[string]string{}
|
||||
for _, n := range []string{"go.mod", "beside.go", "beside_test.go"} {
|
||||
|
||||
@@ -173,7 +173,7 @@ func TestSiblingsArePlacedBesideAndTheUnitLevelIsGivenTheirDirectory(t *testing.
|
||||
}
|
||||
|
||||
calls := &[]cmdRecord{}
|
||||
m.Run = capturedRunner(t, "failing", calls)
|
||||
m.Run = throughBubblewrap(t, capturedRunner(t, "failing", calls), &[]shutOff{})
|
||||
if err := os.MkdirAll(filepath.Join(m.Work, res.Workspace, "tree"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
// The push of a done build's archive (novox/hq ADR 0498 §1 §2; issue 1587): from this module's own process, after the
|
||||
// build step ended, with mesh-sdk's one client (buildstep.PushBlob), to the artifact store whose address the manifest
|
||||
// binds into RegistryFile. No build step reaches the store: a run step has no network (ADR 0448).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// registry is the artifact store's address, host:port, as the manifest bound it into RegistryFile.
|
||||
func (m *Module) registry() (string, error) {
|
||||
raw, err := os.ReadFile(m.RegistryFile)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the artifact store's address cannot be read: %w", err)
|
||||
}
|
||||
addr := strings.TrimSpace(string(raw))
|
||||
if addr == "" || strings.Contains(addr, "${") || strings.HasPrefix(addr, ":") || strings.HasSuffix(addr, ":") {
|
||||
return "", fmt.Errorf("the artifact store's address in %s is %q, not host:port", m.RegistryFile, addr)
|
||||
}
|
||||
return addr, nil
|
||||
}
|
||||
|
||||
// push puts body into the artifact store as a blob of repository (<module>/<artifact>) by its digest: Push when a test
|
||||
// set it, else buildstep.PushBlob.
|
||||
func (m *Module) push(ctx context.Context, repository, digest string, body []byte) error {
|
||||
addr, err := m.registry()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m.Push != nil {
|
||||
return m.Push(ctx, addr, repository, digest, body)
|
||||
}
|
||||
return buildstep.PushBlob(ctx, &http.Client{Timeout: 10 * time.Minute}, addr, repository, digest, body)
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// pushed are the blobs the fake artifact store took, by digest, for the tests that read an archive back.
|
||||
var pushed sync.Map
|
||||
|
||||
// fakeStore makes m push to a store of the test's own: its address in RegistryFile, each blob kept by repository and
|
||||
// digest; fail, when set, is the store's refusal.
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte // "<repository>@<digest>"
|
||||
fail error
|
||||
addr string
|
||||
}
|
||||
|
||||
func withStore(t *testing.T, m *Module) *fakeStore {
|
||||
t.Helper()
|
||||
s := &fakeStore{blobs: map[string][]byte{}}
|
||||
m.RegistryFile = filepath.Join(t.TempDir(), "registry")
|
||||
if err := os.WriteFile(m.RegistryFile, []byte("store.test:5000\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Push = func(ctx context.Context, registry, repository, digest string, body []byte) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.addr = registry
|
||||
if s.fail != nil {
|
||||
return s.fail
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if "sha256:"+hex.EncodeToString(sum[:]) != digest {
|
||||
t.Errorf("pushed %s with bytes of another digest", digest)
|
||||
}
|
||||
s.blobs[repository+"@"+digest] = append([]byte(nil), body...)
|
||||
pushed.Store(digest, s.blobs[repository+"@"+digest])
|
||||
return nil
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// archiveOf is the archive a build pushed, by its digest.
|
||||
func archiveOf(t *testing.T, res BuildResult) []byte {
|
||||
t.Helper()
|
||||
raw, ok := pushed.Load(res.Digest)
|
||||
if !ok {
|
||||
t.Fatalf("nothing was pushed under %s", res.Digest)
|
||||
}
|
||||
return raw.([]byte)
|
||||
}
|
||||
|
||||
// 1587-S2 (ADR 0498 §1): a done build answers a reference of the stated form whose tail is the build's digest, and the
|
||||
// store holds the archive there; a bundle and an archive alike.
|
||||
func TestADoneBuildAnswersTheReferenceOfTheArchiveItPushed(t *testing.T) {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
store := withStore(t, m)
|
||||
res, st := runBuild(t, m, states, "tools")
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
|
||||
}
|
||||
if want := "artifact-store://demo/tools/blobs/" + res.Digest; res.Reference != want {
|
||||
t.Errorf("reference %q, want %q", res.Reference, want)
|
||||
}
|
||||
if store.addr != "store.test:5000" || store.blobs["demo/tools@"+res.Digest] == nil {
|
||||
t.Errorf("the store at %q holds %d blobs, not demo/tools@%s", store.addr, len(store.blobs), res.Digest)
|
||||
}
|
||||
a, states2, _ := archiveModule(t, "")
|
||||
withStore(t, a)
|
||||
arch, st := runBuild(t, a, states2, "files")
|
||||
if st.State != delivery.StepPassed || arch.Reference != "artifact-store://demo/files/blobs/"+arch.Digest {
|
||||
t.Errorf("the archive artifact: %s %q, reference %q", st.State, st.Reason, arch.Reference)
|
||||
}
|
||||
}
|
||||
|
||||
// 1587-S3 (ADR 0498 §2): a push the store refuses fails the step `not stored: <artifact>: <the store's words>`, and so
|
||||
// does a store whose address the manifest did not place.
|
||||
func TestARefusedPushFailsTheStepNotStored(t *testing.T) {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
store := withStore(t, m)
|
||||
store.fail = errors.New("store.test:5000 refused the blob: 500 Internal Server Error disk full")
|
||||
if _, st := runBuild(t, m, states, "tools"); st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, delivery.NotStored+"tools: ") || !strings.Contains(st.Reason, "disk full") {
|
||||
t.Errorf("the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
m.RegistryFile = filepath.Join(t.TempDir(), "absent")
|
||||
if _, st := runBuild(t, m, states, "tools"); st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, delivery.NotStored+"tools: ") {
|
||||
t.Errorf("no address: the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// 1587-S4 (the lead's decision L9): the packed file is not left on the node once its push answered, a candidate's
|
||||
// included: the workspace holds no archive, pushed or refused.
|
||||
func TestThePackedFileLeavesTheNodeOnceItsPushAnswered(t *testing.T) {
|
||||
for _, fail := range []error{nil, errors.New("refused")} {
|
||||
m, states, _ := buildModule(t, manifestWith(goBundle), "the program")
|
||||
withStore(t, m).fail = fail
|
||||
runBuild(t, m, states, "tools")
|
||||
var archives []string
|
||||
filepath.WalkDir(m.Work, func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil && strings.HasSuffix(p, ".tar.gz") {
|
||||
archives = append(archives, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if len(archives) != 0 {
|
||||
t.Errorf("push answered %v: archives left on the node: %v", fail, archives)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/distribution/distribution/v3/configuration"
|
||||
"github.com/distribution/distribution/v3/registry/handlers"
|
||||
_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// realStore starts the artifact store's own program (the distribution registry, modules/distribution) inside this
|
||||
// test, with its in-memory storage, on a loopback port, and points m at it with the real push client. It is imported
|
||||
// by this integration test alone: the module's program does not carry it.
|
||||
func realStore(t *testing.T, m *Module) string {
|
||||
t.Helper()
|
||||
config := &configuration.Configuration{Storage: configuration.Storage{"inmemory": configuration.Parameters{}}}
|
||||
config.HTTP.Secret = "a throwaway store"
|
||||
config.Log.Level = "error"
|
||||
config.Log.AccessLog.Disabled = true
|
||||
srv := httptest.NewServer(handlers.NewApp(context.Background(), config))
|
||||
t.Cleanup(srv.Close)
|
||||
addr := strings.TrimPrefix(srv.URL, "http://")
|
||||
m.RegistryFile = filepath.Join(t.TempDir(), "registry")
|
||||
if err := os.WriteFile(m.RegistryFile, []byte(addr+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Push = nil
|
||||
return addr
|
||||
}
|
||||
|
||||
// resolve reads a done artifact's reference from the store, as the controller will: artifact-store://<module>/
|
||||
// <artifact>/blobs/<digest> is <store>/v2/<module>/<artifact>/blobs/<digest>.
|
||||
func resolve(t *testing.T, addr string, a delivery.BuiltArtifact) []byte {
|
||||
t.Helper()
|
||||
path, ok := strings.CutPrefix(a.Reference, "artifact-store://")
|
||||
if !ok {
|
||||
t.Fatalf("%s's reference %q is not the artifact store's", a.Name, a.Reference)
|
||||
}
|
||||
resp, err := http.Get("http://" + addr + "/v2/" + path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("the store answered %s for %s", resp.Status, a.Reference)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// 1587-S9 (ADR 0498's check): a done build's every artifact names a reference whose tail is its digest, and that
|
||||
// reference resolves in a real artifact store to bytes of that digest. The build is the real go's, shut off, of the
|
||||
// fixture testdata/repos/build, with an archive beside its bundle.
|
||||
func TestADoneBuildsArtifactsResolveInARealArtifactStore(t *testing.T) {
|
||||
needTools(t, "the build", "go", "bwrap")
|
||||
work := t.TempDir()
|
||||
tree := filepath.Join(work, "ws1", "tree")
|
||||
if err := os.MkdirAll(tree, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out, err := exec.Command("cp", "-r", filepath.Join("testdata", "repos", "build")+"/.", tree).CombinedOutput(); err != nil {
|
||||
t.Fatalf("%v %s", err, out)
|
||||
}
|
||||
manifest := `{"module":"fixture-build","version":"1","build":{"artifacts":[` +
|
||||
`{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/prog","binary":"prog-bin","loads":["prog-bin"]},` +
|
||||
`{"name":"files","kind":"archive","from":"cmd"}]}}`
|
||||
if err := os.WriteFile(filepath.Join(tree, "module.json"), []byte(manifest), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
home := t.TempDir()
|
||||
m := &Module{Home: home, Work: work, Cache: t.TempDir()}
|
||||
m.Run = StepRunner(home)
|
||||
addr := realStore(t, m)
|
||||
done := delivery.BuilderBuild{V: 1, State: delivery.BuildDone, Manifest: strings.Repeat("0", 64)}
|
||||
for _, a := range []struct{ name, kind string }{{"tools", "bundle"}, {"files", "archive"}} {
|
||||
out, err := m.build(context.Background(), map[string]any{"workspace": "ws1", "module": ".", "artifact": a.name})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", a.name, err)
|
||||
}
|
||||
r := out.(BuildResult)
|
||||
done.Artifacts = append(done.Artifacts, delivery.BuiltArtifact{Name: a.name, Kind: a.kind, Digest: r.Digest, Reference: r.Reference})
|
||||
}
|
||||
if _, err := delivery.EncodeBuilderBuild(done); err != nil {
|
||||
t.Errorf("the done key the builder would write is refused: %v", err)
|
||||
}
|
||||
if ok, missing := done.Referenced(); !ok {
|
||||
t.Errorf("the done key is not deliverable, missing %v", missing)
|
||||
}
|
||||
for _, a := range done.Artifacts {
|
||||
if !strings.HasSuffix(a.Reference, "/blobs/"+a.Digest) {
|
||||
t.Errorf("%s: the reference %q does not end in its digest %s", a.Name, a.Reference, a.Digest)
|
||||
}
|
||||
sum := sha256.Sum256(resolve(t, addr, a))
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != a.Digest {
|
||||
t.Errorf("%s resolves to %s, its digest is %s", a.Name, got, a.Digest)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,7 @@ func (m *Module) testTool() StepTool {
|
||||
"level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
|
||||
"packages": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "package patterns of each module, ./... when none"},
|
||||
"run": map[string]any{"type": "string", "description": "optional go test -run pattern: only the tests it matches run (a try's tests)"},
|
||||
},
|
||||
"required": []string{"workspace", "level"},
|
||||
},
|
||||
@@ -56,6 +57,10 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pattern, _ := args["run"].(string)
|
||||
if !runPatternOK(pattern) {
|
||||
return nil, fmt.Errorf("%q is not a go test -run pattern", pattern)
|
||||
}
|
||||
if !workspaceOK.MatchString(workspace) {
|
||||
return nil, fmt.Errorf("%q is not a workspace id", workspace)
|
||||
}
|
||||
@@ -65,9 +70,9 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
||||
}
|
||||
switch level {
|
||||
case LevelUnit:
|
||||
return m.runLevel(ctx, tree, levelOpts{level: LevelUnit, packages: packages, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)})
|
||||
return m.runLevel(ctx, tree, levelOpts{workspace: workspace, level: LevelUnit, packages: packages, run: pattern, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)})
|
||||
case LevelIntegration:
|
||||
return m.integration(ctx, workspace, tree, packages)
|
||||
return m.integration(ctx, workspace, tree, packages, pattern)
|
||||
}
|
||||
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
|
||||
}
|
||||
@@ -103,6 +108,10 @@ func packagesArg(v any) ([]string, error) {
|
||||
|
||||
// levelOpts are what a level asks of runLevel.
|
||||
type levelOpts struct {
|
||||
// workspace is the workspace's id: its commands run shut off in it (stepRunner).
|
||||
workspace string
|
||||
// run is go test's -run pattern, "" for every test.
|
||||
run string
|
||||
level string
|
||||
tags []string
|
||||
packages []string
|
||||
@@ -129,6 +138,10 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
run, err := m.stepRunner(o.workspace, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
began := m.clock()
|
||||
for _, d := range dirs {
|
||||
packages := byDir[d]
|
||||
@@ -139,8 +152,11 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
|
||||
if len(o.tags) > 0 {
|
||||
args = append(args, "-tags", strings.Join(o.tags, ","))
|
||||
}
|
||||
if o.run != "" {
|
||||
args = append(args, "-run", o.run)
|
||||
}
|
||||
args = append(args, packages...)
|
||||
r := m.Run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
|
||||
r := run(ctx, append(m.goEnvFor(filepath.Join(tree, d)), o.env...), "go", args...)
|
||||
rd := newReader(res, d)
|
||||
noise := rd.read(r.Stdout)
|
||||
rd.finish()
|
||||
@@ -202,3 +218,22 @@ func modulePath(gomod string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// runPatternOK says whether a -run pattern is one go test takes as a single argument: no flag, no space, no control
|
||||
// character, and short ("" is every test).
|
||||
func runPatternOK(p string) bool {
|
||||
if len(p) > 512 || strings.HasPrefix(p, "-") {
|
||||
return false
|
||||
}
|
||||
for _, r := range p {
|
||||
if r <= ' ' || r == 0x7f {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if p != "" {
|
||||
if _, err := regexp.Compile(strings.Split(p, "/")[0]); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -21,6 +21,9 @@ func TestMain(m *testing.M) {
|
||||
if len(os.Args) > 1 && os.Args[1] == busArg {
|
||||
os.Exit(runBus(os.Args[2:]))
|
||||
}
|
||||
if len(os.Args) > 1 && os.Args[1] == levelArg {
|
||||
os.Exit(runLevelProcess(os.Args[2:])) // and the integration level's shut-off process (level.go)
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
@@ -54,7 +57,7 @@ func repository(t *testing.T, name string) string {
|
||||
// test_test.go read the output captured from them, and this one proves the capture is what the real go says today. It
|
||||
// starts go test, so it carries the tag.
|
||||
func TestUnitLevelRunsTheRealGoOnTheFixtureRepositories(t *testing.T) {
|
||||
needTools(t, "the unit level on fixtures", "go", "git")
|
||||
needTools(t, "the unit level on fixtures", "go", "git", "bwrap")
|
||||
want := map[string]struct {
|
||||
passed, noTests bool
|
||||
failed string
|
||||
@@ -66,11 +69,18 @@ func TestUnitLevelRunsTheRealGoOnTheFixtureRepositories(t *testing.T) {
|
||||
"broken": {failed: "fixture/broken"},
|
||||
}
|
||||
for name, w := range want {
|
||||
tree := repository(t, name)
|
||||
home := t.TempDir()
|
||||
m := &Module{Home: home, Work: t.TempDir(), Cache: t.TempDir()}
|
||||
// The tree lies in a workspace of the work directory, as a fetch leaves it: the level runs shut off in it (1309).
|
||||
tree := filepath.Join(m.Work, "ws1", "tree")
|
||||
if err := os.MkdirAll(filepath.Dir(tree), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(repository(t, name), tree); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Run = StepRunner(home)
|
||||
res, err := m.runLevel(context.Background(), tree, levelOpts{level: LevelUnit, bounds: Bounds{Run: UnitBound}})
|
||||
res, err := m.runLevel(context.Background(), tree, levelOpts{workspace: "ws1", level: LevelUnit, bounds: Bounds{Run: UnitBound}})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
@@ -96,7 +106,7 @@ func processesNamed(text string) []string {
|
||||
// integration level passes, and afterwards no postgres or bus process of the run is left and its directory is gone.
|
||||
// It starts postgres and a bus, so it runs in the merge check and in a try, never on the way.
|
||||
func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing.T) {
|
||||
needTools(t, "the integration level", "go", "git", "initdb", "pg_ctl", "postgres")
|
||||
needTools(t, "the integration level", "go", "git", "initdb", "pg_ctl", "postgres", "bwrap")
|
||||
tree := repository(t, "store")
|
||||
// A short work directory: a unix socket's path is bounded at 108 bytes.
|
||||
work, err := os.MkdirTemp("/tmp", "w")
|
||||
@@ -113,7 +123,7 @@ func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing.
|
||||
home := t.TempDir()
|
||||
m := &Module{Recipe: Current(), Home: home, Work: work, Cache: t.TempDir()}
|
||||
m.Run = StepRunner(home)
|
||||
res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil)
|
||||
res, err := m.integration(context.Background(), "ws1", filepath.Join(work, "ws1", "tree"), nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("the integration level: %v", err)
|
||||
}
|
||||
@@ -134,7 +144,7 @@ func TestIntegrationLevelRunsAgainstARealStoreAndBusAndLeavesNothing(t *testing.
|
||||
// 1487/AC2: given a fixture Go module, when it is built twice, both digests are equal and the archive holds one
|
||||
// executable named as binaryName says. It runs the real go, so it carries the tag.
|
||||
func TestBuildTwiceGivesOneDigestAndOneExecutable(t *testing.T) {
|
||||
needTools(t, "the build", "go")
|
||||
needTools(t, "the build", "go", "bwrap")
|
||||
work := t.TempDir()
|
||||
tree := filepath.Join(work, "ws1", "tree")
|
||||
if err := os.MkdirAll(tree, 0o755); err != nil {
|
||||
@@ -151,6 +161,7 @@ func TestBuildTwiceGivesOneDigestAndOneExecutable(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
m := &Module{Home: home, Work: work, Cache: t.TempDir()}
|
||||
m.Run = StepRunner(home)
|
||||
withStore(t, m)
|
||||
args := map[string]any{"workspace": "ws1", "module": ".", "artifact": "tools"}
|
||||
first, err := m.build(context.Background(), args)
|
||||
if err != nil {
|
||||
@@ -164,11 +175,7 @@ func TestBuildTwiceGivesOneDigestAndOneExecutable(t *testing.T) {
|
||||
if a.Digest != b.Digest || a.Binary != "prog-bin" {
|
||||
t.Errorf("two builds of one tree: %+v and %+v", a, b)
|
||||
}
|
||||
raw, err := os.ReadFile(a.Path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||
zr, err := gzip.NewReader(bytes.NewReader(archiveOf(t, a)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ func workspaceFor(t *testing.T, captured string) (*Module, *keptStates, string,
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.27.1")
|
||||
m.Home, m.Work, m.Cache = t.TempDir(), t.TempDir(), t.TempDir()
|
||||
m.Run = capturedRunner(t, captured, calls)
|
||||
m.Run = throughBubblewrap(t, capturedRunner(t, captured, calls), &[]shutOff{}) // the step's go runs shut off (1309)
|
||||
if err := os.MkdirAll(filepath.Join(m.Work, "ws1", "tree"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -185,12 +185,12 @@ func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t *
|
||||
// A go test that said nothing a test says (it could not run) ends the step failed, not the level.
|
||||
func TestAGoTestThatCouldNotRunEndsTheStepFailed(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "empty")
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
m.Run = throughBubblewrap(t, func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if name == "go" {
|
||||
return forgekey.Ran{Status: 1, Stderr: "go: cannot find main module\n"}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "go.mod\n"}
|
||||
}
|
||||
}, &[]shutOff{})
|
||||
_, st := runUnit(t, m, states, ws)
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "go: cannot find main module") {
|
||||
t.Errorf("the step ended %s: %q", st.State, st.Reason)
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
module fixture/probe
|
||||
|
||||
go 1.25.0
|
||||
@@ -0,0 +1,34 @@
|
||||
package probe
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestProbe tries what a change's code must not reach, named by PROBE_* (the test that runs it sets them through
|
||||
// the environment of the step), and writes what it saw to seen.json beside it.
|
||||
func TestProbe(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
read := func(p string) bool { _, err := os.ReadFile(p); return err == nil }
|
||||
seen["key"] = read(os.Getenv("PROBE_KEY"))
|
||||
seen["login"] = read(os.Getenv("PROBE_LOGIN"))
|
||||
_, err := net.LookupHost("git.novox.be")
|
||||
seen["resolved"] = err == nil
|
||||
reach := func(network, addr string) bool {
|
||||
c, err := net.DialTimeout(network, addr, 2*time.Second)
|
||||
if err == nil {
|
||||
c.Close()
|
||||
}
|
||||
return err == nil
|
||||
}
|
||||
seen["tcp"] = reach("tcp", os.Getenv("PROBE_TCP"))
|
||||
seen["socket"] = reach("unix", os.Getenv("PROBE_SOCKET"))
|
||||
seen["wroteCache"] = os.WriteFile(os.Getenv("PROBE_CACHE")+"/written-by-a-candidate", []byte("x"), 0o644) == nil
|
||||
raw, _ := json.Marshal(seen)
|
||||
if err := os.WriteFile("seen.json", raw, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,293 @@
|
||||
package main
|
||||
|
||||
// Copied from mesh-controller origin/main internal/builder/xz_framing.go (issue 1590): the build agent's walk of an xz
|
||||
// file's framing before its reader decodes it. "The build seat" below is the build module now.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// xzBlocksMax and xzStreamsMax are the most blocks and the most concatenated streams an xz file may hold for
|
||||
// the build seat to read it.
|
||||
//
|
||||
// **Why a block count is limited at all.** The xz reader (github.com/therootcompany/xz v1.0.1) keeps one
|
||||
// filter decoder for every block that names a delta or BCJ filter, and frees them only when a stream ends:
|
||||
// its block counters are reset per stream, not per block. A stream of many tiny blocks therefore grows the
|
||||
// heap with its length: an `xz --block-size=1 --delta` file of 6 MB kept about 67 MiB live, which comes to
|
||||
// about 5.6 GiB at the 512 MiB download cap (review 68 on mesh-controller pull request 264, hq issue 345).
|
||||
//
|
||||
// **Why 1,024.** At about 340 bytes of decoder a block, 1,024 blocks keep at most about 0.35 MiB live. A
|
||||
// real release is far below it: Betterbird's archive is 1 stream with 1 block, and multi-threaded xz cuts
|
||||
// blocks of three times the dictionary (24 MiB at the default preset), so even 512 MiB unpacked in blocks
|
||||
// of 512 KiB stays within it. Streams are limited to 16 because a release is one stream; the decoders are
|
||||
// reused from one stream to the next, so the stream limit only bounds the work of this walk.
|
||||
const (
|
||||
xzBlocksMax = 1024
|
||||
xzStreamsMax = 16
|
||||
)
|
||||
|
||||
// xzWithinLimits walks an xz file's framing, without decompressing anything, and refuses one holding more
|
||||
// blocks or streams than the build seat reads.
|
||||
//
|
||||
// **The framing, not the index.** Each stream's index at its end lists its blocks, but the reader compares
|
||||
// the index with the blocks only after it has decoded them all, so a file whose index lies would still be
|
||||
// decoded block by block. This walk follows the same framing the reader follows: stream header, each block
|
||||
// header, the block's LZMA2 chunks (each chunk header carries its own compressed size, so a chunk is skipped
|
||||
// without being decoded), its padding and check, then the index and the footer. It counts the blocks the
|
||||
// reader would decode, and it also refuses an index whose count of blocks is not the count walked.
|
||||
func xzWithinLimits(file string) error {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
_, _, err = xzFraming(bufio.NewReader(f))
|
||||
if errors.Is(err, errXzTooManyBlocks) || errors.Is(err, errXzTooManyStreams) {
|
||||
return err
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("it is not an xz file this build seat reads: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
errXzTooManyBlocks = fmt.Errorf("it holds more than %d xz blocks, the most this build seat reads: "+
|
||||
"its xz reader keeps a filter decoder for every block (hq issue 345)", xzBlocksMax)
|
||||
errXzTooManyStreams = fmt.Errorf("it holds more than %d concatenated xz streams, the most this build "+
|
||||
"seat reads", xzStreamsMax)
|
||||
)
|
||||
|
||||
// xzFraming counts the streams and blocks of an xz file, checking its framing as it goes. It stops with
|
||||
// errXzTooManyBlocks or errXzTooManyStreams at the first block or stream over the limit.
|
||||
func xzFraming(r *bufio.Reader) (streams, blocks int, err error) {
|
||||
full := func(n int) ([]byte, error) {
|
||||
b := make([]byte, n)
|
||||
if _, err := io.ReadFull(r, b); err != nil {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
skip := func(n int) error {
|
||||
if got, err := r.Discard(n); err != nil || got != n {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
header, err := full(12)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: header: %v", streams+1, err)
|
||||
}
|
||||
if !bytes.Equal(header[:6], []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no xz stream header", streams+1)
|
||||
}
|
||||
if header[6] != 0 || header[7]&0xf0 != 0 ||
|
||||
crc32.ChecksumIEEE(header[6:8]) != binary.LittleEndian.Uint32(header[8:12]) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: a corrupt stream header", streams+1)
|
||||
}
|
||||
if streams++; streams > xzStreamsMax {
|
||||
return streams, blocks, errXzTooManyStreams
|
||||
}
|
||||
checkSize := 0
|
||||
if check := int(header[7]); check != 0 {
|
||||
checkSize = 4 << ((check - 1) / 3)
|
||||
}
|
||||
inStream := 0
|
||||
for {
|
||||
first, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no index: %v", streams, io.ErrUnexpectedEOF)
|
||||
}
|
||||
if first == 0 {
|
||||
break // the index indicator
|
||||
}
|
||||
rest, err := full(int(first)*4 + 3)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: header: %v", streams, inStream+1, err)
|
||||
}
|
||||
if err := xzBlockHeader(append([]byte{first}, rest...)); err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream+1, err)
|
||||
}
|
||||
inStream++
|
||||
if blocks++; blocks > xzBlocksMax {
|
||||
return streams, blocks, errXzTooManyBlocks
|
||||
}
|
||||
// The block's LZMA2 chunks, up to the end marker.
|
||||
size := 0
|
||||
for {
|
||||
control, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream, io.ErrUnexpectedEOF)
|
||||
}
|
||||
size++
|
||||
if control == 0 {
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case control == 1 || control == 2: // stored: a 16-bit size less one, then the bytes
|
||||
b, err := full(2)
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
n := int(binary.BigEndian.Uint16(b)) + 1
|
||||
if err := skip(n); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size += 2 + n
|
||||
case control >= 0x80: // LZMA: unpacked and packed sizes, the properties byte on a reset
|
||||
b, err := full(4)
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
n := int(binary.BigEndian.Uint16(b[2:])) + 1
|
||||
size += 4
|
||||
if control >= 0xc0 {
|
||||
if err := skip(1); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size++
|
||||
}
|
||||
if err := skip(n); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size += n
|
||||
default:
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: an LZMA2 chunk of kind %#x",
|
||||
streams, inStream, control)
|
||||
}
|
||||
}
|
||||
if pad := (4 - size%4) % 4; pad > 0 {
|
||||
b, err := full(pad)
|
||||
if err != nil || !bytes.Equal(b, make([]byte, pad)) {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: corrupt block padding", streams, inStream)
|
||||
}
|
||||
}
|
||||
if err := skip(checkSize); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
}
|
||||
// The index: its count of records, two numbers each, padding, CRC32.
|
||||
used := 1
|
||||
count, n, err := xzVLI(r)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
used += n
|
||||
if count != uint64(inStream) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: its index lists %d blocks, but it holds %d",
|
||||
streams, count, inStream)
|
||||
}
|
||||
for i := 0; i < 2*inStream; i++ {
|
||||
_, n, err := xzVLI(r)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
used += n
|
||||
}
|
||||
if err := skip((4-used%4)%4 + 4); err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
footer, err := full(12)
|
||||
if err != nil || footer[10] != 'Y' || footer[11] != 'Z' {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no stream footer", streams)
|
||||
}
|
||||
// Stream padding: zero bytes, a multiple of four, then the end or another stream.
|
||||
zeros := 0
|
||||
for {
|
||||
b, err := r.ReadByte()
|
||||
if err == io.EOF {
|
||||
if zeros%4 != 0 {
|
||||
return streams, blocks, errors.New("stream padding not a multiple of four bytes")
|
||||
}
|
||||
return streams, blocks, nil
|
||||
}
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
if b != 0 {
|
||||
_ = r.UnreadByte()
|
||||
break
|
||||
}
|
||||
zeros++
|
||||
}
|
||||
if zeros%4 != 0 {
|
||||
return streams, blocks, errors.New("stream padding not a multiple of four bytes")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// xzBlockHeader checks one block header: its CRC32, its flags, and that its last filter is LZMA2, the only
|
||||
// one whose chunks this walk can skip (and the only last filter the reader decodes).
|
||||
func xzBlockHeader(h []byte) error {
|
||||
size := len(h)
|
||||
if crc32.ChecksumIEEE(h[:size-4]) != binary.LittleEndian.Uint32(h[size-4:]) {
|
||||
return errors.New("a corrupt block header")
|
||||
}
|
||||
flags := h[1]
|
||||
if flags&0x3c != 0 {
|
||||
return errors.New("a block header with reserved flags")
|
||||
}
|
||||
body := bytes.NewReader(h[2 : size-4])
|
||||
if flags&0x40 != 0 {
|
||||
if _, _, err := xzVLI(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if flags&0x80 != 0 {
|
||||
if _, _, err := xzVLI(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var last uint64
|
||||
for i := 0; i <= int(flags&3); i++ {
|
||||
id, _, err := xzVLI(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
props, _, err := xzVLI(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if props > uint64(body.Len()) {
|
||||
return errors.New("a corrupt block header")
|
||||
}
|
||||
if _, err := body.Seek(int64(props), io.SeekCurrent); err != nil {
|
||||
return err
|
||||
}
|
||||
last = id
|
||||
}
|
||||
if last != 0x21 {
|
||||
return fmt.Errorf("a block whose last filter is %#x, not LZMA2", last)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// xzVLI reads one of xz's variable-length integers (at most 9 bytes, 7 bits each), and how many bytes it took.
|
||||
func xzVLI(r io.ByteReader) (uint64, int, error) {
|
||||
var v uint64
|
||||
for i := 0; i < 9; i++ {
|
||||
b, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return 0, i, io.ErrUnexpectedEOF
|
||||
}
|
||||
v |= uint64(b&0x7f) << (7 * i)
|
||||
if b&0x80 == 0 {
|
||||
if i > 0 && b == 0 {
|
||||
return 0, i + 1, errors.New("a number not in its shortest form")
|
||||
}
|
||||
return v, i + 1, nil
|
||||
}
|
||||
}
|
||||
return 0, 9, errors.New("a number longer than nine bytes")
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package main
|
||||
|
||||
// Copied from mesh-controller origin/main internal/builder/xz_framing_test.go (issue 1590), with xz_framing.go: the
|
||||
// build agent's walk of an xz file's framing, which this module now uses as the build agent does.
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// xzStream is one xz stream (no integrity check) of data cut into blocks of blockSize bytes, each block's
|
||||
// LZMA2 data stored uncompressed, each block a delta filter before LZMA2 when delta is set: the shape of
|
||||
// `xz --block-size=N --filters=delta,lzma2`, built without any xz tool. listed is how many blocks its index
|
||||
// lists (-1: as many as it holds), so a stream whose index lies can be built too.
|
||||
func xzStream(data []byte, blockSize int, delta bool, listed int) []byte {
|
||||
var out bytes.Buffer
|
||||
flags := []byte{0, 0}
|
||||
out.Write([]byte{0xfd, '7', 'z', 'X', 'Z', 0x00})
|
||||
out.Write(flags)
|
||||
out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(flags)))
|
||||
type record struct{ unpadded, uncompressed uint64 }
|
||||
var records []record
|
||||
for at := 0; at < len(data); at += blockSize {
|
||||
block := data[at:min(at+blockSize, len(data))]
|
||||
if delta {
|
||||
// Delta-encoded with a distance of one; the decoder's history starts empty at each block.
|
||||
encoded := make([]byte, len(block))
|
||||
for i := range block {
|
||||
encoded[i] = block[i]
|
||||
if i > 0 {
|
||||
encoded[i] -= block[i-1]
|
||||
}
|
||||
}
|
||||
block = encoded
|
||||
}
|
||||
header := []byte{2, 0} // 12 bytes
|
||||
if delta {
|
||||
header[1] = 1 // two filters
|
||||
header = append(header, 0x03, 0x01, 0x00)
|
||||
}
|
||||
header = append(header, 0x21, 0x01, 0x00) // LZMA2, a 4 KiB dictionary
|
||||
for len(header) < 8 {
|
||||
header = append(header, 0)
|
||||
}
|
||||
header = binary.LittleEndian.AppendUint32(header, crc32.ChecksumIEEE(header))
|
||||
out.Write(header)
|
||||
size := 0
|
||||
for i, chunk := 0, block; len(chunk) > 0; i++ {
|
||||
n := min(len(chunk), 1<<16)
|
||||
control := byte(2)
|
||||
if i == 0 {
|
||||
control = 1 // the first chunk of a block resets the dictionary
|
||||
}
|
||||
out.WriteByte(control)
|
||||
out.Write(binary.BigEndian.AppendUint16(nil, uint16(n-1)))
|
||||
out.Write(chunk[:n])
|
||||
size += 3 + n
|
||||
chunk = chunk[n:]
|
||||
}
|
||||
out.WriteByte(0)
|
||||
size++
|
||||
out.Write(make([]byte, (4-size%4)%4))
|
||||
records = append(records, record{uint64(len(header) + size), uint64(len(block))})
|
||||
}
|
||||
if listed >= 0 {
|
||||
for len(records) < listed {
|
||||
records = append(records, records[0])
|
||||
}
|
||||
records = records[:listed]
|
||||
}
|
||||
vli := func(b []byte, v uint64) []byte {
|
||||
for v >= 0x80 {
|
||||
b = append(b, byte(v)|0x80)
|
||||
v >>= 7
|
||||
}
|
||||
return append(b, byte(v))
|
||||
}
|
||||
index := vli([]byte{0}, uint64(len(records)))
|
||||
for _, r := range records {
|
||||
index = vli(vli(index, r.unpadded), r.uncompressed)
|
||||
}
|
||||
for len(index)%4 != 0 {
|
||||
index = append(index, 0)
|
||||
}
|
||||
index = binary.LittleEndian.AppendUint32(index, crc32.ChecksumIEEE(index))
|
||||
out.Write(index)
|
||||
footer := binary.LittleEndian.AppendUint32(nil, uint32(len(index)/4-1))
|
||||
footer = append(footer, flags...)
|
||||
out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(footer)))
|
||||
out.Write(footer)
|
||||
out.WriteString("YZ")
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// tarHolding is a tar holding betterbird/data with body.
|
||||
func tarHolding(t *testing.T, body []byte) []byte {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
w := tar.NewWriter(&out)
|
||||
if err := w.WriteHeader(&tar.Header{Name: "betterbird/data", Mode: 0o644, Size: int64(len(body)),
|
||||
Typeflag: tar.TypeReg}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write(body)
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func writeTemp(t *testing.T, body []byte) string {
|
||||
t.Helper()
|
||||
file := filepath.Join(t.TempDir(), "r.tar.xz")
|
||||
if err := os.WriteFile(file, body, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
func TestAnXzFileOfManyDeltaBlocksIsRefusedBeforeItIsDecoded(t *testing.T) {
|
||||
// Review 68 on pull request 264: `xz --block-size=1 --delta` of a 200 kB tar kept about 67 MiB live in
|
||||
// the reader, one delta decoder per block. Its index here lists one block, as a file crafted to pass a
|
||||
// check of the index alone would; the walk counts the blocks the reader would decode.
|
||||
body := make([]byte, 200_000)
|
||||
for i := range body {
|
||||
body[i] = byte(i * 7)
|
||||
}
|
||||
crafted := xzStream(tarHolding(t, body), 1, true, 1)
|
||||
path := writeTemp(t, crafted)
|
||||
var before, after runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
err := unpackUpstream(path, t.TempDir(), "betterbird")
|
||||
runtime.ReadMemStats(&after)
|
||||
if err == nil || !strings.Contains(err.Error(), "more than 1024 xz blocks") {
|
||||
t.Fatalf("a file of %d bytes in blocks of one byte was not refused for its blocks: %v", len(crafted), err)
|
||||
}
|
||||
if grew := after.TotalAlloc - before.TotalAlloc; grew > 4<<20 {
|
||||
t.Fatalf("%d bytes were allocated for a refused file", grew)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheXzBlockLimitIsExact(t *testing.T) {
|
||||
data := make([]byte, xzBlocksMax+1)
|
||||
streams, blocks, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data[:xzBlocksMax], 1, true, -1))))
|
||||
if err != nil || streams != 1 || blocks != xzBlocksMax {
|
||||
t.Fatalf("%d blocks: %d streams, %d blocks, %v", xzBlocksMax, streams, blocks, err)
|
||||
}
|
||||
if _, _, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data, 1, true, -1)))); !errors.Is(err, errXzTooManyBlocks) {
|
||||
t.Fatalf("%d blocks: %v", xzBlocksMax+1, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzFileOfTooManyStreamsIsRefused(t *testing.T) {
|
||||
var file []byte
|
||||
for i := 0; i <= xzStreamsMax; i++ {
|
||||
file = append(file, xzStream([]byte("x"), 1, false, -1)...)
|
||||
}
|
||||
err := unpackUpstream(writeTemp(t, file), t.TempDir(), "")
|
||||
if err == nil || !strings.Contains(err.Error(), "more than 16 concatenated xz streams") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzIndexThatDoesNotListItsBlocksIsRefused(t *testing.T) {
|
||||
file := xzStream(tarHolding(t, []byte("x")), 512, false, 1)
|
||||
err := unpackUpstream(writeTemp(t, file), t.TempDir(), "betterbird")
|
||||
if err == nil || !strings.Contains(err.Error(), "its index lists 1 blocks, but it holds 4") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// releaseXzSHA256 is the sha256 of testdata/upstream-release.tar.xz: the tar stream of upstream-release.tar.gz,
|
||||
// compressed with `xz -9e` (one stream, one block).
|
||||
const releaseXzSHA256 = "7e59472615d7d654d6f72e8275c83e36203da9db1b91372dcd38a3ff7984f997"
|
||||
|
||||
// pinnedModuleServing is pinnedModule for any fixture file, served at /<its name>.
|
||||
func pinnedModuleServing(t *testing.T, fixture, sha, extra string) (*Module, *keptStates) {
|
||||
t.Helper()
|
||||
body, err := os.ReadFile(filepath.Join("testdata", fixture))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/"+fixture {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Write(body)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
m, states, _ := archiveModule(t, pinned("https://example.com/"+fixture, sha, extra))
|
||||
client := srv.Client()
|
||||
client.Transport.(*http.Transport).DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, network, srv.Listener.Addr().String())
|
||||
}
|
||||
m.HTTP = client
|
||||
return m, states
|
||||
}
|
||||
|
||||
// Issue 1590: a pinned release compressed with xz (as betterbird's is) is unpacked as the build agent unpacks it, and
|
||||
// gives the build agent's archive: the same tar under gzip and under xz packs to one stream.
|
||||
func TestAPinnedXzArchiveIsUnpackedAsTheBuildAgentDoes(t *testing.T) {
|
||||
m, states := pinnedModuleServing(t, "upstream-release.tar.xz", releaseXzSHA256, `,"within":"release/app"`)
|
||||
res, st := runBuild(t, m, states, "files")
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
sameAsBuildAgent(t, res, buildAgentWithinStream, buildAgentWithinDigest)
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
${seat:mesh-artifact-store:reach}
|
||||
@@ -3,19 +3,81 @@ module meshbuildgo
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876
|
||||
github.com/distribution/distribution/v3 v3.1.2
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
github.com/therootcompany/xz v1.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/docker-credential-helpers v0.9.5 // indirect
|
||||
github.com/docker/go-events v0.0.0-20250808211157-605354379745 // indirect
|
||||
github.com/docker/go-metrics v0.1.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gorilla/handlers v1.5.2 // indirect
|
||||
github.com/gorilla/mux v1.8.1 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
|
||||
github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/prometheus/client_golang v1.23.2 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/otlptranslator v1.0.0 // indirect
|
||||
github.com/prometheus/procfs v0.20.1 // indirect
|
||||
github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 // indirect
|
||||
github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 // indirect
|
||||
github.com/redis/go-redis/v9 v9.7.3 // indirect
|
||||
github.com/sirupsen/logrus v1.10.2 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/bridges/prometheus v0.67.0 // indirect
|
||||
go.opentelemetry.io/contrib/exporters/autoexport v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.64.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/log v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/log v0.21.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.45.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/net v0.59.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
|
||||
google.golang.org/grpc v1.83.2 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
)
|
||||
|
||||
@@ -1,17 +1,67 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 h1:OKxU64kHN1yQ/GOa6LwoUwVmiQ74d83WMSZJRFPVm9c=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876 h1:dtiIJzrX7ey1wmsdzapiFgCKDllCph7/vxfUp1ib/t8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/distribution/distribution/v3 v3.1.2 h1:/Bv2YIqqSR00HiO49FT93hovy6r9IlrzwUDsJjCN5qg=
|
||||
github.com/distribution/distribution/v3 v3.1.2/go.mod h1:Oz0Drs2Tx3iTXzK3DhUD4otfTfqlCxlWVC7QJeXWBP0=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/docker-credential-helpers v0.9.5 h1:EFNN8DHvaiK8zVqFA2DT6BjXE0GzfLOZ38ggPTKePkY=
|
||||
github.com/docker/docker-credential-helpers v0.9.5/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c=
|
||||
github.com/docker/go-events v0.0.0-20250808211157-605354379745 h1:yOn6Ze6IbYI/KAw2lw/83ELYvZh6hvsygTVkD0dzMC4=
|
||||
github.com/docker/go-events v0.0.0-20250808211157-605354379745/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
|
||||
github.com/docker/go-metrics v0.1.0 h1:r76KPNpstz+IvQKSWpYegSkkyzex0V3A1ZGVx6bhGlY=
|
||||
github.com/docker/go-metrics v0.1.0/go.mod h1:PciI3sONtB051kXALN1JoIlpcu54E1FuPh+4DuqEzyw=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
|
||||
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE=
|
||||
github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
|
||||
github.com/hashicorp/golang-lru/arc/v2 v2.0.5 h1:l2zaLDubNhW4XO3LnliVj0GXO3+/CGNJAg1dcN2Fpfw=
|
||||
github.com/hashicorp/golang-lru/arc/v2 v2.0.5/go.mod h1:ny6zBSQZi2JxIeYcv7kt2sH2PXJtirBN7RDhRpxPkxU=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
@@ -22,10 +72,109 @@ github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
|
||||
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
|
||||
github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos=
|
||||
github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM=
|
||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||
github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho=
|
||||
github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U=
|
||||
github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc=
|
||||
github.com/redis/go-redis/extra/redisotel/v9 v9.0.5/go.mod h1:WZjPDy7VNzn77AAfnAfVjZNvfJTYfPetfZk5yoSTLaQ=
|
||||
github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
|
||||
github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM=
|
||||
github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA=
|
||||
github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo=
|
||||
github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
|
||||
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/bridges/prometheus v0.67.0 h1:dkBzNEAIKADEaFnuESzcXvpd09vxvDZsOjx11gjUqLk=
|
||||
go.opentelemetry.io/contrib/bridges/prometheus v0.67.0/go.mod h1:Z5RIwRkZgauOIfnG5IpidvLpERjhTninpP1dTG2jTl4=
|
||||
go.opentelemetry.io/contrib/exporters/autoexport v0.67.0 h1:4fnRcNpc6YFtG3zsFw9achKn3XgmxPxuMuqIL5rE8e8=
|
||||
go.opentelemetry.io/contrib/exporters/autoexport v0.67.0/go.mod h1:qTvIHMFKoxW7HXg02gm6/Wofhq5p3Ib/A/NNt1EoBSQ=
|
||||
go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
|
||||
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0 h1:WseeVYf5dJZTsyPiyW5L14k5qsSibqXAMTSiFEDiWr0=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0/go.mod h1:SiLZnQS6Qk2eCpvr2CH/XMAOa64TWGXxEZJZCpD2Lmc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.21.0 h1:fvNHGyo3CdRv/DQveXqhqBxnKTDyRaC5sMSQxilX/A0=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.21.0/go.mod h1:zyGrjRKL2B/6+Jc/m4/otPoZqV2MY9ZjC/aBraRO7zc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 h1:MdKucPl/HbzckWWEisiNqMPhRrAOQX8r4jTuGr636gk=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0/go.mod h1:RolT8tWtfHcjajEH5wFIZ4Dgh5jpPdFXYV9pTAk/qjc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 h1:w1K+pCJoPpQifuVpsKamUdn9U0zM3xUziVOqsGksUrY=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0/go.mod h1:HBy4BjzgVE8139ieRI75oXm3EcDN+6GhD88JT1Kjvxg=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.64.0 h1:g0LRDXMX/G1SEZtK8zl8Chm4K6GBwRkjPKE36LxiTYs=
|
||||
go.opentelemetry.io/otel/exporters/prometheus v0.64.0/go.mod h1:UrgcjnarfdlBDP3GjDIJWe6HTprwSazNjwsI+Ru6hro=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.21.0 h1:2lpf4hnrasYIsUyEXwnTZq5lsxrMm4T2Bwb06IctAZQ=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.21.0/go.mod h1:YWOW6h7jwApz9Pl76ie/izUsSPj0s2MdIlpqbPqaf3U=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0 h1:lSZHgNHfbmQTPfuTmWVkEu8J8qXaQwuV30pjCcAUvP8=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0 h1:s/1iRkCKDfhlh1JF26knRneorus8aOwVIDhvYx9WoDw=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0/go.mod h1:UI3wi0FXg1Pofb8ZBiBLhtMzgoTm1TYkMvn71fAqDzs=
|
||||
go.opentelemetry.io/otel/log v0.21.0 h1:SLsVDGmtyBrdw8/a2Z0bOIxou/+bN4z56GebH7T0LvA=
|
||||
go.opentelemetry.io/otel/log v0.21.0/go.mod h1:iReetQrZL9Wyg84cCkOoCmqDHS5RCFfyxC7J+r8fn8g=
|
||||
go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
|
||||
go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
|
||||
go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns=
|
||||
go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI=
|
||||
go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
|
||||
go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
|
||||
go.opentelemetry.io/otel/sdk/log v0.21.0 h1:QsE7XSR0ktQdKmRKGnR+f1ObGF32WG+7MER/P9KgmYc=
|
||||
go.opentelemetry.io/otel/sdk/log v0.21.0/go.mod h1:m9mApjCoD2/1QuKCAptjv+BrG9WKOvQLVdNx+iBldTo=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.21.0 h1:X+JBBgKlswCGYsmgL0CnoUUtlE//VB345c84jYAYkdQ=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.21.0/go.mod h1:HD1575K8e6sIFBBDd5tZB3t9DlMytWXq9FuR+Y4rfjE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
|
||||
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
|
||||
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
|
||||
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
|
||||
@@ -31,7 +31,8 @@
|
||||
],
|
||||
"requires": [
|
||||
"forge-account",
|
||||
"go-toolchain"
|
||||
"go-toolchain",
|
||||
"artifact-store"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
@@ -109,6 +110,14 @@
|
||||
"mode": "0644",
|
||||
"template": "files/gitconfig"
|
||||
},
|
||||
{
|
||||
"id": "registry",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/registry",
|
||||
"owner": "mesh-build-go",
|
||||
"mode": "0644",
|
||||
"template": "files/registry"
|
||||
},
|
||||
{
|
||||
"id": "git",
|
||||
"type": "package",
|
||||
@@ -119,6 +128,11 @@
|
||||
"type": "package",
|
||||
"package": "openssh"
|
||||
},
|
||||
{
|
||||
"id": "bubblewrap",
|
||||
"type": "package",
|
||||
"package": "bubblewrap"
|
||||
},
|
||||
{
|
||||
"id": "postgresql",
|
||||
"type": "package",
|
||||
|
||||
@@ -30,6 +30,10 @@ The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can re
|
||||
`<module>/<artifact>:<tag of its own>` (the first 12 hex digits of the digest unless the call gives one), waiting out
|
||||
a registry held still, and the key carries the digest the registry serves and the reference pinned by it. Without
|
||||
`publish` (a pull request) nothing is pushed.
|
||||
- `build_image_copy` copies an upstream artifact (`module`, `artifact`, `from`) registry to registry into the artifact store
|
||||
over the registry API, the whole index and every blob, by the digest the manifest pins (ADR 0498 §4), and verifies the
|
||||
store serves that digest. Answers the digest and the reference `<registry>/<module>/<artifact>@sha256:<digest>`. A
|
||||
`from` without a digest fails `not pinned:`. It needs no workspace and no buildah.
|
||||
- It refuses, naming why, an image whose build context is a whole repository (`context`) or whose `build.on` names
|
||||
another module's artifact (`unsupportedImage` in `build.go`): those forms belong to the images build-agent builds
|
||||
until it is removed at R4g.
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package main
|
||||
|
||||
// What this module keeps on the bus, announced from its code (novox/hq ADR 0495 §2 §5; issue 1790): every state it
|
||||
// keeps, each readable by others, since another module reads each today (the builder reads steps; gitea reads keys). A state of its own that it
|
||||
// did not name here is refused at the call (stdio.Keeps).
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// Kept are the states the module keeps.
|
||||
var Kept = []announce.State{
|
||||
{Name: "keys", Readable: true},
|
||||
{Name: "steps", Readable: true},
|
||||
}
|
||||
|
||||
// keepStates announces Kept as this module's (stdio.Keeps): main calls it before it serves.
|
||||
func keepStates() error { return stdio.Keeps(Kept...) }
|
||||
@@ -0,0 +1,63 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announcetest"
|
||||
)
|
||||
|
||||
// 1790 (ADR 0495 §5): served as the runtime serves it, the module announces on the bus every state it keeps, as its
|
||||
// code declares them. The runtime's part, publishing what the bundle asks to publish, is played by relay.
|
||||
func TestTheModuleAnnouncesEveryStateItKeeps(t *testing.T) {
|
||||
const module = "mesh-build-image"
|
||||
announcetest.Check(t, module, Kept, func(url string) func() {
|
||||
t.Setenv("MESH_MODULE", module)
|
||||
t.Setenv("MESH_NODE", "shanks")
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := keepStates(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inR, inW := io.Pipe()
|
||||
outR, outW := io.Pipe()
|
||||
go func() { _ = stdio.ServeOn(module, nil, inR, outW) }()
|
||||
go relay(t, module, nc, outR, inW)
|
||||
return func() { _ = inW.Close(); _ = outW.Close(); nc.Close() }
|
||||
})
|
||||
}
|
||||
|
||||
// relay is the runtime's side of the channel: each mesh/publish the bundle asks is published on its module's event
|
||||
// subject, mesh.mod.<module>.event.<key>, and answered.
|
||||
func relay(t *testing.T, module string, nc *nats.Conn, from io.Reader, to io.Writer) {
|
||||
sc := bufio.NewScanner(from)
|
||||
sc.Buffer(make([]byte, 1<<20), 1<<24)
|
||||
enc := json.NewEncoder(to)
|
||||
for sc.Scan() {
|
||||
var m struct {
|
||||
ID json.RawMessage `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params struct {
|
||||
Key string `json:"key"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
} `json:"params"`
|
||||
}
|
||||
if json.Unmarshal(sc.Bytes(), &m) != nil || m.Method != "mesh/publish" {
|
||||
continue
|
||||
}
|
||||
if err := nc.Publish("mesh.mod."+module+".event."+m.Params.Key, m.Params.Body); err != nil {
|
||||
t.Errorf("publishing %s: %v", m.Params.Key, err)
|
||||
}
|
||||
_ = nc.Flush()
|
||||
_ = enc.Encode(map[string]any{"id": m.ID, "result": map[string]any{}})
|
||||
}
|
||||
}
|
||||
@@ -74,6 +74,23 @@ func unsupportedImage(manifest catalogue.Manifest, a catalogue.Artifact) string
|
||||
return ""
|
||||
}
|
||||
|
||||
// qualifyImage names a base image in full, as the controller's parseReference reads it: a name without a registry host
|
||||
// is the public hub's (docker.io), a single name under `library/`. buildah, with this machine's registry configuration,
|
||||
// has no unqualified-search registry and refuses a short name (issue 1824); a name that has a host is kept.
|
||||
func qualifyImage(ref string) string {
|
||||
name := ref
|
||||
if at := strings.Index(name, "@"); at >= 0 {
|
||||
name = name[:at]
|
||||
}
|
||||
if slash := strings.Index(name, "/"); slash >= 0 && (strings.ContainsAny(name[:slash], ".:") || name[:slash] == "localhost") {
|
||||
return ref
|
||||
}
|
||||
if !strings.Contains(name, "/") {
|
||||
return "docker.io/library/" + ref
|
||||
}
|
||||
return "docker.io/" + ref
|
||||
}
|
||||
|
||||
// imageArgs are the --build-arg words of the images the manifest pins in build.on, in its order.
|
||||
func imageArgs(manifest catalogue.Manifest) []string {
|
||||
var out []string
|
||||
@@ -82,7 +99,7 @@ func imageArgs(manifest catalogue.Manifest) []string {
|
||||
}
|
||||
for _, on := range manifest.Build.On {
|
||||
if on.Image != "" && on.Arg != "" {
|
||||
out = append(out, "--build-arg", on.Arg+"="+on.Image)
|
||||
out = append(out, "--build-arg", on.Arg+"="+qualifyImage(on.Image))
|
||||
}
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
package main
|
||||
|
||||
// The copy step (novox/hq ADR 0498 §4; issue 1587): an upstream artifact is copied registry to registry into the
|
||||
// artifact store, whole (an index with every manifest it lists, every blob), by the digest its manifest pins. The copy
|
||||
// runs no code of a change, so it needs neither buildah nor an isolation; it is made over the registry API, which
|
||||
// keeps every digest as it is (a pull and a push through buildah may rewrite a manifest). An upstream without a
|
||||
// digest is not copied: `not pinned:`. A copy that does not end with the pinned digest in the store is `not stored:`.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// CopyTool is the copy step's tool.
|
||||
const CopyTool = "build_image_copy"
|
||||
|
||||
// CopyResult is what a passed copy step says: the pinned digest, now in the store, and the reference to it in the
|
||||
// form an image's is (<registry>/<module>/<artifact>@sha256:<digest>).
|
||||
type CopyResult struct {
|
||||
Artifact string `json:"artifact"`
|
||||
Digest string `json:"digest"`
|
||||
Reference string `json:"reference"`
|
||||
Pushed bool `json:"pushed"`
|
||||
}
|
||||
|
||||
var (
|
||||
nameOK = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*(/[a-z0-9][a-z0-9._-]*)*$`)
|
||||
pinnedOK = regexp.MustCompile(`@(sha256:[0-9a-f]{64})$`)
|
||||
manifestKind = strings.Join([]string{
|
||||
"application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
"application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json"}, ", ")
|
||||
)
|
||||
|
||||
func (m *Module) copyTool() StepTool {
|
||||
return StepTool{
|
||||
Name: CopyTool,
|
||||
Description: "Copy an upstream image, registry to registry, into the artifact store by the digest the manifest pins " +
|
||||
"(novox/hq ADR 0498 §4). Answers a step id at once; the step ends on its key with the digest in the store and the " +
|
||||
"reference to it. A reference without a digest fails `not pinned:`. Needs no workspace and runs no code of a change.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module that names the upstream artifact"},
|
||||
"artifact": map[string]any{"type": "string", "description": "the upstream artifact's name in the module's manifest"},
|
||||
"from": map[string]any{"type": "string", "description": "the image reference the manifest gives, pinned by digest, such as nats:2.11.17-alpine@sha256:..."},
|
||||
},
|
||||
"required": []string{"module", "artifact", "from"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.copy,
|
||||
}
|
||||
}
|
||||
|
||||
// upstreamRef is an image reference split the way a container runtime does: no host means the public hub.
|
||||
type upstreamRef struct {
|
||||
base, repository, digest string
|
||||
}
|
||||
|
||||
func parseUpstream(from string) (upstreamRef, error) {
|
||||
pin := pinnedOK.FindStringSubmatch(from)
|
||||
if pin == nil {
|
||||
return upstreamRef{}, fmt.Errorf("%q names no digest", from)
|
||||
}
|
||||
name := strings.TrimSuffix(from, "@"+pin[1])
|
||||
if colon := strings.LastIndex(name, ":"); colon > strings.LastIndex(name, "/") {
|
||||
name = name[:colon] // the tag is only what it was called; the digest names the image
|
||||
}
|
||||
if name == "" {
|
||||
return upstreamRef{}, fmt.Errorf("%q is not an image reference", from)
|
||||
}
|
||||
host, repository := "docker.io", name
|
||||
if slash := strings.Index(name, "/"); slash >= 0 && (strings.ContainsAny(name[:slash], ".:") || name[:slash] == "localhost") {
|
||||
host, repository = name[:slash], name[slash+1:]
|
||||
}
|
||||
if host == "docker.io" {
|
||||
host = "registry-1.docker.io"
|
||||
if !strings.Contains(repository, "/") {
|
||||
repository = "library/" + repository
|
||||
}
|
||||
}
|
||||
scheme := "https://"
|
||||
if strings.HasPrefix(host, "localhost") || strings.HasPrefix(host, "127.") {
|
||||
scheme = "http://"
|
||||
}
|
||||
return upstreamRef{base: scheme + host, repository: repository, digest: pin[1]}, nil
|
||||
}
|
||||
|
||||
func (m *Module) copy(ctx context.Context, args map[string]any) (any, error) {
|
||||
module, _ := args["module"].(string)
|
||||
name, _ := args["artifact"].(string)
|
||||
from, _ := args["from"].(string)
|
||||
if !nameOK.MatchString(module) || !nameOK.MatchString(name) {
|
||||
return nil, fmt.Errorf("%q / %q: a module and an artifact are named in lower case letters, digits, dots, dashes and underscores", module, name)
|
||||
}
|
||||
src, err := parseUpstream(from)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("not pinned: %s: %v; an upstream artifact is copied by the digest its manifest pins", name, err)
|
||||
}
|
||||
addr, err := m.registry()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("not stored: %s: %w", name, err)
|
||||
}
|
||||
repository := module + "/" + name
|
||||
c := &copier{m: m, src: src, dstBase: "http://" + addr, dstRepo: repository, done: map[string]bool{}}
|
||||
if err := m.waitForRegistry(ctx, addr, "copying "+from, func() error { return c.copyManifest(ctx, src.digest) }); err != nil {
|
||||
return nil, fmt.Errorf("not stored: %s: %w", name, err)
|
||||
}
|
||||
if err := c.verify(ctx); err != nil {
|
||||
return nil, fmt.Errorf("not stored: %s: %w", name, err)
|
||||
}
|
||||
return CopyResult{Artifact: name, Digest: src.digest, Reference: addr + "/" + repository + "@" + src.digest, Pushed: true}, nil
|
||||
}
|
||||
|
||||
// copier copies one image from its upstream registry into the store's repository.
|
||||
type copier struct {
|
||||
m *Module
|
||||
src upstreamRef
|
||||
dstBase string
|
||||
dstRepo string
|
||||
token string
|
||||
done map[string]bool
|
||||
}
|
||||
|
||||
func (c *copier) client() *http.Client {
|
||||
if c.m.HTTP != nil {
|
||||
return c.m.HTTP
|
||||
}
|
||||
return &http.Client{Timeout: 30 * time.Minute}
|
||||
}
|
||||
|
||||
// get reads from the upstream registry, answering one bearer challenge with an anonymous token.
|
||||
func (c *copier) get(ctx context.Context, path, accept string) (*http.Response, error) {
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.src.base+"/v2/"+c.src.repository+path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if accept != "" {
|
||||
req.Header.Set("Accept", accept)
|
||||
}
|
||||
if c.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
}
|
||||
resp, err := c.client().Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode == http.StatusUnauthorized && attempt == 0 {
|
||||
challenge := resp.Header.Get("Www-Authenticate")
|
||||
resp.Body.Close()
|
||||
if c.token, err = c.fetchToken(ctx, challenge); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("upstream answered %s for %s%s", resp.Status, c.src.repository, path)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
return nil, fmt.Errorf("upstream refused %s%s after a token", c.src.repository, path)
|
||||
}
|
||||
|
||||
var challengeParam = regexp.MustCompile(`(\w+)="([^"]*)"`)
|
||||
|
||||
func (c *copier) fetchToken(ctx context.Context, challenge string) (string, error) {
|
||||
if !strings.HasPrefix(strings.ToLower(challenge), "bearer ") {
|
||||
return "", fmt.Errorf("upstream asks for credentials (%q) and this copy holds none", challenge)
|
||||
}
|
||||
p := map[string]string{}
|
||||
for _, kv := range challengeParam.FindAllStringSubmatch(challenge, -1) {
|
||||
p[kv[1]] = kv[2]
|
||||
}
|
||||
realm := p["realm"]
|
||||
if realm == "" {
|
||||
return "", fmt.Errorf("upstream's challenge %q names no realm", challenge)
|
||||
}
|
||||
q := url.Values{"service": {p["service"]}, "scope": {"repository:" + c.src.repository + ":pull"}}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, realm+"?"+q.Encode(), nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := c.client().Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var tok struct {
|
||||
Token string `json:"token"`
|
||||
Access string `json:"access_token"`
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK || json.NewDecoder(resp.Body).Decode(&tok) != nil || (tok.Token == "" && tok.Access == "") {
|
||||
return "", fmt.Errorf("upstream's token service answered %s", resp.Status)
|
||||
}
|
||||
if tok.Token != "" {
|
||||
return tok.Token, nil
|
||||
}
|
||||
return tok.Access, nil
|
||||
}
|
||||
|
||||
// copyManifest copies the manifest of digest and everything it names, children before the manifest itself.
|
||||
func (c *copier) copyManifest(ctx context.Context, digest string) error {
|
||||
if c.done[digest] {
|
||||
return nil
|
||||
}
|
||||
resp, err := c.get(ctx, "/manifests/"+digest, manifestKind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
|
||||
return fmt.Errorf("upstream served %s for the manifest pinned as %s", got, digest)
|
||||
}
|
||||
var doc struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Config struct{ Digest string } `json:"config"`
|
||||
Layers []struct{ Digest string } `json:"layers"`
|
||||
Manifests []struct{ Digest string } `json:"manifests"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil {
|
||||
return fmt.Errorf("the manifest %s is not JSON: %w", digest, err)
|
||||
}
|
||||
for _, child := range doc.Manifests {
|
||||
if err := c.copyManifest(ctx, child.Digest); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
blobs := []string{}
|
||||
if doc.Config.Digest != "" {
|
||||
blobs = append(blobs, doc.Config.Digest)
|
||||
}
|
||||
for _, l := range doc.Layers {
|
||||
blobs = append(blobs, l.Digest)
|
||||
}
|
||||
for _, b := range blobs {
|
||||
if err := c.copyBlob(ctx, b); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
mediaType := resp.Header.Get("Content-Type")
|
||||
if mediaType == "" {
|
||||
mediaType = doc.MediaType
|
||||
}
|
||||
if err := c.putManifest(ctx, digest, mediaType, body); err != nil {
|
||||
return err
|
||||
}
|
||||
c.done[digest] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *copier) copyBlob(ctx context.Context, digest string) error {
|
||||
if !digestOK.MatchString(digest) {
|
||||
return fmt.Errorf("%q is not a blob digest", digest)
|
||||
}
|
||||
if c.done[digest] {
|
||||
return nil
|
||||
}
|
||||
head, err := http.NewRequestWithContext(ctx, http.MethodHead, c.dstBase+"/v2/"+c.dstRepo+"/blobs/"+digest, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r, err := c.client().Do(head); err != nil {
|
||||
return err
|
||||
} else {
|
||||
r.Body.Close()
|
||||
if r.StatusCode == http.StatusOK {
|
||||
c.done[digest] = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
post, err := http.NewRequestWithContext(ctx, http.MethodPost, c.dstBase+"/v2/"+c.dstRepo+"/blobs/uploads/", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pr, err := c.client().Do(post)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pr.Body.Close()
|
||||
loc, err := pr.Location()
|
||||
if pr.StatusCode != http.StatusAccepted || err != nil {
|
||||
return fmt.Errorf("the store answered %s to a blob upload of %s", pr.Status, digest)
|
||||
}
|
||||
q := loc.Query()
|
||||
q.Set("digest", digest)
|
||||
loc.RawQuery = q.Encode()
|
||||
blob, err := c.get(ctx, "/blobs/"+digest, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer blob.Body.Close()
|
||||
put, err := http.NewRequestWithContext(ctx, http.MethodPut, loc.String(), blob.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
put.ContentLength = blob.ContentLength
|
||||
put.Header.Set("Content-Type", "application/octet-stream")
|
||||
resp, err := c.client().Do(put)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
return fmt.Errorf("the store answered %s to the blob %s", resp.Status, digest)
|
||||
}
|
||||
c.done[digest] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *copier) putManifest(ctx context.Context, digest, mediaType string, body []byte) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.dstBase+"/v2/"+c.dstRepo+"/manifests/"+digest, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", mediaType)
|
||||
resp, err := c.client().Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("the store answered %s to the manifest %s", resp.Status, digest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verify reads back what the store serves for the pinned digest: only the store can say.
|
||||
func (c *copier) verify(ctx context.Context) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, c.dstBase+"/v2/"+c.dstRepo+"/manifests/"+c.src.digest, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Accept", manifestKind)
|
||||
resp, err := c.client().Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
if got := resp.Header.Get("Docker-Content-Digest"); resp.StatusCode != http.StatusOK || got != c.src.digest {
|
||||
return fmt.Errorf("the store answered %s with digest %q for the pinned %s", resp.Status, got, c.src.digest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Plain is a long step that needs nothing of the machine: no guard. The copy runs no buildah.
|
||||
func (m *Module) Plain(tool string, run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
|
||||
return func(args map[string]any) (any, error) {
|
||||
id, err := buildstep.Start(m.Ctx, m.Steps, m.Node, tool, func(ctx context.Context) (any, error) { return run(ctx, args) })
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, _ := delivery.StepKey(m.Node, id)
|
||||
return map[string]any{"step": id, "key": key, "state": delivery.StepRunning}, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func digestOf(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) }
|
||||
|
||||
// memRegistry is a registry in memory: blobs and manifests by digest, per repository.
|
||||
type memRegistry struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte // repo|digest
|
||||
manifests map[string][]byte // repo|digest
|
||||
types map[string]string
|
||||
puts []string
|
||||
srv *httptest.Server
|
||||
}
|
||||
|
||||
func newMem(t *testing.T) *memRegistry {
|
||||
r := &memRegistry{blobs: map[string][]byte{}, manifests: map[string][]byte{}, types: map[string]string{}}
|
||||
r.srv = httptest.NewServer(http.HandlerFunc(r.serve))
|
||||
t.Cleanup(r.srv.Close)
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *memRegistry) addr() string { return r.srv.Listener.Addr().String() }
|
||||
|
||||
func (r *memRegistry) serve(w http.ResponseWriter, req *http.Request) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
path := strings.TrimPrefix(req.URL.Path, "/v2/")
|
||||
if i := strings.Index(path, "/blobs/uploads/"); i >= 0 {
|
||||
if req.Method == http.MethodPost {
|
||||
w.Header().Set("Location", "/v2/"+path[:i]+"/blobs/uploads/u1")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
return
|
||||
}
|
||||
body, _ := io.ReadAll(req.Body)
|
||||
d := req.URL.Query().Get("digest")
|
||||
if digestOf(body) != d {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
r.blobs[path[:i]+"|"+d] = body
|
||||
r.puts = append(r.puts, "blob "+d)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
return
|
||||
}
|
||||
for _, kind := range []string{"blobs", "manifests"} {
|
||||
if i := strings.Index(path, "/"+kind+"/"); i >= 0 {
|
||||
key := path[:i] + "|" + path[i+len(kind)+2:]
|
||||
store := r.blobs
|
||||
if kind == "manifests" {
|
||||
store = r.manifests
|
||||
}
|
||||
if req.Method == http.MethodPut {
|
||||
body, _ := io.ReadAll(req.Body)
|
||||
if digestOf(body) != path[i+len(kind)+2:] {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
r.manifests[key] = body
|
||||
r.types[key] = req.Header.Get("Content-Type")
|
||||
r.puts = append(r.puts, "manifest "+path[i+len(kind)+2:])
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
return
|
||||
}
|
||||
body, ok := store[key]
|
||||
if !ok {
|
||||
http.NotFound(w, req)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Docker-Content-Digest", path[i+len(kind)+2:])
|
||||
w.Header().Set("Content-Type", r.types[key])
|
||||
if req.Method == http.MethodHead {
|
||||
return
|
||||
}
|
||||
w.Write(body)
|
||||
return
|
||||
}
|
||||
}
|
||||
http.NotFound(w, req)
|
||||
}
|
||||
|
||||
// upstreamImage puts a two-platform index (two manifests, each with a config and a layer) into r and answers the index digest.
|
||||
func upstreamImage(r *memRegistry, repo string) string {
|
||||
var children []map[string]string
|
||||
for _, tag := range []string{"amd64", "arm64"} {
|
||||
cfg, layer := []byte(`{"arch":"`+tag+`"}`), []byte("layer-"+tag)
|
||||
r.blobs[repo+"|"+digestOf(cfg)], r.blobs[repo+"|"+digestOf(layer)] = cfg, layer
|
||||
mf, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"config": map[string]string{"digest": digestOf(cfg)}, "layers": []map[string]string{{"digest": digestOf(layer)}}})
|
||||
r.manifests[repo+"|"+digestOf(mf)] = mf
|
||||
r.types[repo+"|"+digestOf(mf)] = "application/vnd.oci.image.manifest.v1+json"
|
||||
children = append(children, map[string]string{"digest": digestOf(mf)})
|
||||
}
|
||||
idx, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json", "manifests": children})
|
||||
d := digestOf(idx)
|
||||
r.manifests[repo+"|"+d] = idx
|
||||
r.types[repo+"|"+d] = "application/vnd.oci.image.index.v1+json"
|
||||
return d
|
||||
}
|
||||
|
||||
func runCopy(t *testing.T, m *Module, states *keptStates, args map[string]any) (CopyResult, delivery.Step) {
|
||||
t.Helper()
|
||||
if _, err := tool(t, m, CopyTool)(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res CopyResult
|
||||
if st.State == delivery.StepPassed {
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
|
||||
// ADR 0498 §4 / AC: the copy leaves the pinned digest in the store and answers it, with the whole index (every
|
||||
// manifest it lists, every blob), and runs no buildah.
|
||||
func TestTheCopyLeavesThePinnedDigestInTheStoreAndAnswersIt(t *testing.T) {
|
||||
up, store := newMem(t), newMem(t)
|
||||
digest := upstreamImage(up, "library/nats")
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
m.HTTP = nil
|
||||
reg := store.addr()
|
||||
writeFile(t, m.RegistryFile, reg+"\n")
|
||||
from := up.addr() + "/library/nats:2.11.17-alpine@" + digest
|
||||
res, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": from})
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
if res.Digest != digest || res.Reference != reg+"/nats/server@"+digest || !res.Pushed || res.Artifact != "server" {
|
||||
t.Errorf("result = %+v", res)
|
||||
}
|
||||
if _, ok := store.manifests["nats/server|"+digest]; !ok {
|
||||
t.Fatal("the index is not in the store")
|
||||
}
|
||||
if len(store.manifests) != 3 || len(store.blobs) != 4 {
|
||||
t.Errorf("store holds %d manifests and %d blobs, want 3 and 4", len(store.manifests), len(store.blobs))
|
||||
}
|
||||
if len(f.lines()) != 0 {
|
||||
t.Errorf("the copy ran commands: %v", f.lines())
|
||||
}
|
||||
// A second copy writes nothing new but the manifests.
|
||||
store.puts = nil
|
||||
if _, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": from}); st.State != delivery.StepPassed {
|
||||
t.Fatalf("again: %+v", st)
|
||||
}
|
||||
for _, p := range store.puts {
|
||||
if strings.HasPrefix(p, "blob ") {
|
||||
t.Errorf("a blob the store holds was sent again: %s", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0498 §4 / AC: an upstream without a digest fails `not pinned:`, and nothing is read from the upstream.
|
||||
func TestAnUpstreamWithoutADigestFailsNotPinned(t *testing.T) {
|
||||
up, store := newMem(t), newMem(t)
|
||||
upstreamImage(up, "library/nats")
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
writeFile(t, m.RegistryFile, store.addr()+"\n")
|
||||
_, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": up.addr() + "/library/nats:2.11.17-alpine"})
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "not pinned: server") {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
if len(store.puts) != 0 {
|
||||
t.Errorf("the store was written: %v", store.puts)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest the upstream serves under another digest than the pin is not copied; the failure is `not stored:`.
|
||||
func TestAnUpstreamServingAnotherManifestThanPinnedIsNotStored(t *testing.T) {
|
||||
up, store := newMem(t), newMem(t)
|
||||
d := upstreamImage(up, "library/nats")
|
||||
up.manifests["library/nats|"+d] = []byte(`{"tampered":true}`)
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
writeFile(t, m.RegistryFile, store.addr()+"\n")
|
||||
_, st := runCopy(t, m, states, map[string]any{"module": "nats", "artifact": "server", "from": up.addr() + "/library/nats@" + d})
|
||||
if st.State != delivery.StepFailed || !strings.HasPrefix(st.Reason, "not stored: server:") {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
if len(store.manifests) != 0 {
|
||||
t.Errorf("the store holds %v", store.manifests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAShortNameIsQualifiedAsTheHubsAndAHostIsKept(t *testing.T) {
|
||||
d := "sha256:" + strings.Repeat("a", 64)
|
||||
for from, want := range map[string]upstreamRef{
|
||||
"alpine@" + d: {"https://registry-1.docker.io", "library/alpine", d},
|
||||
"nats:2.11.17-alpine@" + d: {"https://registry-1.docker.io", "library/nats", d},
|
||||
"grafana/grafana:11@" + d: {"https://registry-1.docker.io", "grafana/grafana", d},
|
||||
"ghcr.io/org/app:1@" + d: {"https://ghcr.io", "org/app", d},
|
||||
"localhost:5000/x/y@" + d: {"http://localhost:5000", "x/y", d},
|
||||
} {
|
||||
got, err := parseUpstream(from)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("%s = %+v, %v; want %+v", from, got, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -47,6 +47,11 @@ func main() {
|
||||
go m.Start(ctx)
|
||||
}
|
||||
go startKeeper(ctx)
|
||||
// Every state it keeps, said on the bus before it serves (ADR 0495 §2; issue 1790).
|
||||
if err := keepStates(); err != nil {
|
||||
logf("announcing the states it keeps: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -154,9 +154,6 @@ func TestGiteaReadsTheImageBuildModulesKeys(t *testing.T) {
|
||||
t.Errorf("gitea reads %v, lacks %s", gitea.Reads, want)
|
||||
}
|
||||
}
|
||||
if problems := catalogue.StateReadsNothingDeclares(all); len(problems) > 0 {
|
||||
t.Errorf("a read names a state its owner does not keep: %v", problems)
|
||||
}
|
||||
}
|
||||
|
||||
// 1704b: over the real manifests, mesh-build-image where no module provides rootless-image-build is refused, the error
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
const pin = "@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00"
|
||||
|
||||
// 1824: a short base image name is qualified the way the controller's parseReference does (the public hub, `library/`
|
||||
// for a single name), because buildah has no unqualified-search registry here; a qualified name is left as it is.
|
||||
func TestAShortBaseImageNameIsQualifiedAsTheHubsAndAQualifiedOneIsKept(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"nats" + pin: "docker.io/library/nats" + pin,
|
||||
"nats:2.11.17-alpine" + pin: "docker.io/library/nats:2.11.17-alpine" + pin,
|
||||
"n8nio/n8n" + pin: "docker.io/n8nio/n8n" + pin,
|
||||
"docker.io/library/alpine" + pin: "docker.io/library/alpine" + pin,
|
||||
"ghcr.io/org/app:1" + pin: "ghcr.io/org/app:1" + pin,
|
||||
"localhost:5000/x/y" + pin: "localhost:5000/x/y" + pin,
|
||||
"registry-1.docker.io/library/a": "registry-1.docker.io/library/a",
|
||||
} {
|
||||
if got := qualifyImage(in); got != want {
|
||||
t.Errorf("%s = %s, want %s", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBuildArgsCarryTheQualifiedName(t *testing.T) {
|
||||
m := catalogue.Manifest{Build: &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "NATS_BASE", Image: "nats" + pin}}}}
|
||||
if got, want := imageArgs(m), []string{"--build-arg", "NATS_BASE=docker.io/library/nats" + pin}; !slices.Equal(got, want) {
|
||||
t.Errorf("%v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -25,14 +25,18 @@ type StepTool struct {
|
||||
}
|
||||
|
||||
// StepTools are the module's step tools.
|
||||
func (m *Module) StepTools() []StepTool { return []StepTool{m.fetchTool(), m.buildTool()} }
|
||||
func (m *Module) StepTools() []StepTool {
|
||||
return []StepTool{m.fetchTool(), m.buildTool(), m.copyTool()}
|
||||
}
|
||||
|
||||
// Tools is every tool the module serves: its step tools behind the guard, its read tools, its status and its health.
|
||||
func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
||||
tools := []stdio.Tool{}
|
||||
for _, s := range steps {
|
||||
t := stdio.Tool{Name: s.Name, Description: s.Description, Input: s.Input}
|
||||
if s.Long {
|
||||
if s.Name == CopyTool {
|
||||
t.Run = m.Plain(s.Name, s.Run) // a copy needs no buildah, so the buildah guard does not stand in front of it
|
||||
} else if s.Long {
|
||||
t.Run = m.Long(s.Name, s.Run)
|
||||
} else {
|
||||
t.Run = m.Short(s.Run)
|
||||
|
||||
@@ -3,7 +3,7 @@ module meshbuildimage
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876 h1:dtiIJzrX7ey1wmsdzapiFgCKDllCph7/vxfUp1ib/t8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
"tools": [
|
||||
"build_image_fetch",
|
||||
"build_image_build",
|
||||
"build_image_copy",
|
||||
"build_image_step",
|
||||
"build_image_steps",
|
||||
"build_image_status",
|
||||
|
||||
@@ -58,6 +58,15 @@ toolchain install: a long step answers at once with its id and ends on `<node>.<
|
||||
other `@novox/` packages install. A module with packages and no lock does not fail the fetch: its result lists it
|
||||
under `refused` with the reason, and a build of that bundle fails naming it (issue 1459). Every command runs through `forgekey.ExecRunnerClean(0)` with PATH, HOME, the locale and its own
|
||||
words, nothing of this process's environment. Workspaces older than a day are removed at each fetch.
|
||||
The arguments are mesh-sdk's `buildstep.FetchRequest` (ADR 0499 §4): with `dir` and `paths` (from the builder, never
|
||||
from a manifest) the workspace is narrowed to those paths, the manifest and the lock files before anything else. A
|
||||
member is built against the SDK version its own lock names (the lead's decision L10 of R4e, ADR 0500 §5-§6): the
|
||||
fetch places that version in the module's `node_modules/@novox/mesh-sdk`, from the group's package when the call
|
||||
names one (`package: {repository, commit, ref}`: fetched, its dependencies installed from its own lock, built and
|
||||
packed shut off, published nowhere) and its version and integrity are the lock's, else from the registry (integrity
|
||||
checked); a version the registry lacks fails `waits for: @novox/mesh-sdk@<version>` (the builder queues the key), or,
|
||||
with a group package of another version, fails naming both. A module whose lock names no SDK keeps the toolchain's.
|
||||
A module whose artifact is a `package` gets its dependencies, devDependencies included, from its own lock instead.
|
||||
- `build_node_test` {workspace, module, level} (issue 1491): the unit level is the type-check (the toolchain's tsc, the
|
||||
package's `tsconfig.json` or the build agent's flags over its .ts files) and the `test` script; the integration level
|
||||
is the `test:integration` script, the type-check being its setup. Bounds (the Go module's; ADR 0445 names none for
|
||||
@@ -70,4 +79,14 @@ toolchain install: a long step answers at once with its id and ends on `<node>.<
|
||||
toolchain's packages (not typescript or esbuild) copied to `stage-<artifact>/node_modules`: the build agent's depth
|
||||
(`/app/modules/module`, `/app/node_modules`), so esbuild's `../../node_modules/...` path comments, and the bundle's
|
||||
digest, equal its. Copies, not links, because the mesh makes no symlink. A bundle that keeps an external gets them in its own `node_modules`.
|
||||
The packed archive is written beside the tree as `<artifact>.tar.gz`.
|
||||
Once the build ended, this process pushes the packed archive to the artifact store (novox/hq ADR 0498 §1 §2, mesh-sdk's
|
||||
`buildstep.PushBlob`, the address in `${dir:state}/registry`) under `<module>/<artifact>` by its digest, a candidate's as a
|
||||
trunk build's, and answers `reference: artifact-store://<module>/<artifact>/blobs/<digest>`; a push that fails fails the
|
||||
step `not stored: <artifact>: <the store's words>`. The packed bytes never stay on the node. tsc and esbuild, like the
|
||||
type-check and the test script, run shut off in bubblewrap (ADR 0448): no network, none of the account, the toolchain
|
||||
read-only, the npm cache beneath a throwaway layer unless the step is told `publish` (a trunk build).
|
||||
A `package` artifact (ADR 0498 §5) is built by its own `build` script and packed with `npm pack --ignore-scripts`, both
|
||||
shut off; its digest is the packed file's. Only with `publish` is it published, by this process with the registry's
|
||||
write credential (never in a workspace or a step's mounts): a version the registry lacks is published, the same
|
||||
packed file already there is done without a publish, another file under that version fails `version taken:
|
||||
<name>@<version>`; the reference `<name>@<version>` is answered only then. A candidate answers its digest alone.
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package main
|
||||
|
||||
// What this module keeps on the bus, announced from its code (novox/hq ADR 0495 §2 §5; issue 1790): every state it
|
||||
// keeps, each readable by others, since another module reads each today (the builder, the checker and the indexer read toolchains; the builder and the checker read steps; gitea reads keys). A state of its own that it
|
||||
// did not name here is refused at the call (stdio.Keeps).
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// Kept are the states the module keeps.
|
||||
var Kept = []announce.State{
|
||||
{Name: "toolchains", Readable: true},
|
||||
{Name: "keys", Readable: true},
|
||||
{Name: "steps", Readable: true},
|
||||
}
|
||||
|
||||
// keepStates announces Kept as this module's (stdio.Keeps): main calls it before it serves.
|
||||
func keepStates() error { return stdio.Keeps(Kept...) }
|
||||
@@ -0,0 +1,63 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/announcetest"
|
||||
)
|
||||
|
||||
// 1790 (ADR 0495 §5): served as the runtime serves it, the module announces on the bus every state it keeps, as its
|
||||
// code declares them. The runtime's part, publishing what the bundle asks to publish, is played by relay.
|
||||
func TestTheModuleAnnouncesEveryStateItKeeps(t *testing.T) {
|
||||
const module = "mesh-build-node"
|
||||
announcetest.Check(t, module, Kept, func(url string) func() {
|
||||
t.Setenv("MESH_MODULE", module)
|
||||
t.Setenv("MESH_NODE", "shanks")
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := keepStates(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inR, inW := io.Pipe()
|
||||
outR, outW := io.Pipe()
|
||||
go func() { _ = stdio.ServeOn(module, nil, inR, outW) }()
|
||||
go relay(t, module, nc, outR, inW)
|
||||
return func() { _ = inW.Close(); _ = outW.Close(); nc.Close() }
|
||||
})
|
||||
}
|
||||
|
||||
// relay is the runtime's side of the channel: each mesh/publish the bundle asks is published on its module's event
|
||||
// subject, mesh.mod.<module>.event.<key>, and answered.
|
||||
func relay(t *testing.T, module string, nc *nats.Conn, from io.Reader, to io.Writer) {
|
||||
sc := bufio.NewScanner(from)
|
||||
sc.Buffer(make([]byte, 1<<20), 1<<24)
|
||||
enc := json.NewEncoder(to)
|
||||
for sc.Scan() {
|
||||
var m struct {
|
||||
ID json.RawMessage `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params struct {
|
||||
Key string `json:"key"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
} `json:"params"`
|
||||
}
|
||||
if json.Unmarshal(sc.Bytes(), &m) != nil || m.Method != "mesh/publish" {
|
||||
continue
|
||||
}
|
||||
if err := nc.Publish("mesh.mod."+module+".event."+m.Params.Key, m.Params.Body); err != nil {
|
||||
t.Errorf("publishing %s: %v", m.Params.Key, err)
|
||||
}
|
||||
_ = nc.Flush()
|
||||
_ = enc.Encode(map[string]any{"id": m.ID, "result": map[string]any{}})
|
||||
}
|
||||
}
|
||||
@@ -18,12 +18,9 @@ import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// launcherSuffix is what an entrypoint's launcher is called beside it: index.js is started as index.serve.mjs
|
||||
// (novox/hq ADR 0193).
|
||||
const launcherSuffix = ".serve.mjs"
|
||||
|
||||
// Out is where a bundle's compiled files go in the module's directory (builder toolchain.go:118).
|
||||
func Out(artifact string) string { return ".mesh-build/" + artifact }
|
||||
|
||||
@@ -32,7 +29,10 @@ const banner = "--banner:js=import { createRequire as __meshRequire } from 'node
|
||||
|
||||
// artifact is the part of a module.json artifact the build reads.
|
||||
type artifact struct {
|
||||
// module is the manifest's module name, which the artifact is stored under.
|
||||
module string
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
Entrypoints []string `json:"entrypoints"`
|
||||
External []string `json:"external"`
|
||||
@@ -46,9 +46,10 @@ type BuildResult struct {
|
||||
// Digest is "sha256:" and the hex of the archive's bytes (buildstep.Pack).
|
||||
Digest string `json:"digest"`
|
||||
Bytes int `json:"bytes"`
|
||||
// Archive is the packed file, in the workspace beside its tree.
|
||||
Archive string `json:"archive"`
|
||||
Files []string `json:"files"`
|
||||
// Reference is where the packed bundle is in the artifact store (novox/hq ADR 0498 §1):
|
||||
// artifact-store://<module>/<artifact>/blobs/<digest>. The packed file never stays on the node.
|
||||
Reference string `json:"reference"`
|
||||
Files []string `json:"files"`
|
||||
}
|
||||
|
||||
func (m *Module) buildTool() StepTool {
|
||||
@@ -63,6 +64,7 @@ func (m *Module) buildTool() StepTool {
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch step answered"},
|
||||
"module": map[string]any{"type": "string", "description": "the module's directory in the workspace, '.' when it is the root"},
|
||||
"artifact": map[string]any{"type": "string", "description": "the name of the bundle in the module's module.json"},
|
||||
"publish": map[string]any{"type": "boolean", "description": "true for a merged trunk commit alone: the build writes the shared cache; without it (a candidate) the cache is read beneath a throwaway layer"},
|
||||
},
|
||||
"required": []string{"workspace", "artifact"},
|
||||
},
|
||||
@@ -75,6 +77,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
name, _ := args["artifact"].(string)
|
||||
publish, _ := args["publish"].(bool)
|
||||
root, dir, err := m.moduleDirOf(workspace, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -83,6 +86,9 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.Kind == "package" {
|
||||
return m.buildPackage(ctx, workspace, module, root, dir, a, publish)
|
||||
}
|
||||
if why, err := lockProblem(dir, module); err != nil || why != "" {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("%s was refused by the fetch: %s", name, why)
|
||||
@@ -112,14 +118,14 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
tsc := []string{m.tsc(), "--module", "NodeNext", "--moduleResolution", "NodeNext", "--target", "ES2022",
|
||||
"--rootDir", ".", "--outDir", out}
|
||||
tsc = append(tsc, sourcesFor(a.Entrypoints, out, ".ts")...)
|
||||
if why := ranFailed(m.runIn(ctx, dir, nil, "node", tsc...)); why != "" {
|
||||
if why := ranFailed(m.runIn(ctx, root, publish, dir, nil, "node", tsc...)); why != "" {
|
||||
return nil, fmt.Errorf("tsc for %s: %s", a.Name, why)
|
||||
}
|
||||
launchers, err := writeLaunchers(filepath.Join(dir, out), a)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.bundle(ctx, dir, a, launchers); err != nil {
|
||||
if err := m.bundle(ctx, root, publish, dir, a, launchers); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bundleDir := filepath.Join(dir, final)
|
||||
@@ -132,9 +138,10 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("packing %s: %w", a.Name, err)
|
||||
}
|
||||
file := filepath.Join(root, a.Name+".tar.gz")
|
||||
if err := os.WriteFile(file, archive, 0o600); err != nil {
|
||||
return nil, err
|
||||
// Pushed from this process once the build ended, a candidate's as a trunk build's (ADR 0498 §1 §2): the step answers
|
||||
// the reference, or fails not stored. The packed bytes are never written to the node (the lead's decision L9).
|
||||
if err := m.push(ctx, a.module+"/"+a.Name, digest, archive); err != nil {
|
||||
return nil, fmt.Errorf("%s%s: %v", delivery.NotStored, a.Name, err)
|
||||
}
|
||||
files := []string{}
|
||||
_ = filepath.WalkDir(bundleDir, func(p string, d os.DirEntry, err error) error {
|
||||
@@ -145,12 +152,13 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
return nil
|
||||
})
|
||||
sort.Strings(files)
|
||||
return BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: digest, Bytes: len(archive), Archive: file, Files: files}, nil
|
||||
return BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: digest, Bytes: len(archive),
|
||||
Reference: buildstep.BlobReference(a.module, a.Name, digest), Files: files}, nil
|
||||
}
|
||||
|
||||
// bundle makes every entrypoint and every launcher ONE file, with esbuild (builder.go:1971-2039): an entrypoint is
|
||||
// bundled in place under its own name, a launcher keeps its name with the extension .mjs.
|
||||
func (m *Module) bundle(ctx context.Context, dir string, a artifact, launchers map[string]string) error {
|
||||
func (m *Module) bundle(ctx context.Context, root string, publish bool, dir string, a artifact, launchers map[string]string) error {
|
||||
out, final := Out(a.Name), Out(a.Name)+".bundled"
|
||||
if err := os.RemoveAll(filepath.Join(dir, final)); err != nil {
|
||||
return err
|
||||
@@ -177,7 +185,7 @@ func (m *Module) bundle(ctx context.Context, dir string, a artifact, launchers m
|
||||
return nil
|
||||
}
|
||||
args := append(append(append([]string{}, entries...), common...), extra...)
|
||||
if why := ranFailed(m.runIn(ctx, dir, nil, m.esbuild(), args...)); why != "" {
|
||||
if why := ranFailed(m.runIn(ctx, root, publish, dir, nil, m.esbuild(), args...)); why != "" {
|
||||
return fmt.Errorf("esbuild for %s: %s", a.Name, why)
|
||||
}
|
||||
return nil
|
||||
@@ -204,15 +212,12 @@ func (m *Module) bundle(ctx context.Context, dir string, a artifact, launchers m
|
||||
}
|
||||
|
||||
// writeLaunchers writes, beside every entrypoint, an executable that imports the entrypoint and serves what it
|
||||
// registered over MCP on stdio (builder.go:1926-1956; novox/hq ADR 0193). Its answer is each entrypoint's launcher,
|
||||
// relative to the bundle's root.
|
||||
// registered over MCP on stdio (builder.go:1926-1956; novox/hq ADR 0193). Which entrypoint gets which launcher is
|
||||
// mesh-sdk's one rule, buildstep.Launchers, which the controller reads a bundle by (R4e). Its answer is each
|
||||
// entrypoint's launcher, relative to the bundle's root.
|
||||
func writeLaunchers(root string, a artifact) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
for _, entry := range a.Entrypoints {
|
||||
if !strings.HasSuffix(entry, ".js") {
|
||||
continue
|
||||
}
|
||||
launcher := strings.TrimSuffix(entry, ".js") + launcherSuffix
|
||||
for entry, launcher := range buildstep.Launchers(a.Language, a.Entrypoints) {
|
||||
body := "#!/usr/bin/env node\n" +
|
||||
"// Written by the mesh's builder (novox/hq ADR 0193): serve what " + entry + " registers,\n" +
|
||||
"// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.\n" +
|
||||
@@ -260,7 +265,8 @@ func readArtifact(dir, name string) (artifact, error) {
|
||||
return artifact{}, fmt.Errorf("the module has no module.json: %w", err)
|
||||
}
|
||||
var m struct {
|
||||
Build struct {
|
||||
Module string `json:"module"`
|
||||
Build struct {
|
||||
Artifacts []artifact `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
@@ -271,12 +277,17 @@ func readArtifact(dir, name string) (artifact, error) {
|
||||
if a.Name != name {
|
||||
continue
|
||||
}
|
||||
if a.Kind == "package" && a.Language == "typescript" {
|
||||
a.module = m.Module
|
||||
return a, nil // a package is built by its own build script and packed (package.go), from no entrypoint
|
||||
}
|
||||
if a.Language != "typescript" {
|
||||
return artifact{}, fmt.Errorf("artifact %q is %q, and this module builds typescript", name, a.Language)
|
||||
}
|
||||
if len(a.Entrypoints) == 0 {
|
||||
return artifact{}, fmt.Errorf("artifact %q names no entrypoints", name)
|
||||
}
|
||||
a.module = m.Module
|
||||
return a, nil
|
||||
}
|
||||
return artifact{}, fmt.Errorf("module.json declares no artifact %q", name)
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
// registry for @novox, which reads without a credential) and makes two workspaces holding one fixture bundle.
|
||||
func realMachine(t *testing.T) (*Module, string) {
|
||||
t.Helper()
|
||||
for _, p := range []string{"node", "npm"} {
|
||||
for _, p := range []string{"node", "npm", "bwrap"} { // bwrap: tsc, esbuild and the scripts run shut off (1309)
|
||||
if _, err := exec.LookPath(p); err != nil {
|
||||
t.Skipf("NOT RUN: no %s installed where this check runs (the mesh's Go toolchain image holds none); proven live on a build node", p)
|
||||
}
|
||||
@@ -39,6 +39,7 @@ func realMachine(t *testing.T) (*Module, string) {
|
||||
Toolchain: &Toolchain{Paths: Paths{Home: home, Cache: cache}, Run: run, Registry: func() (Registry, error) {
|
||||
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil
|
||||
}}}
|
||||
withStore(t, m)
|
||||
if err := m.Toolchain.Ensure(context.Background(), m.Recipe.Version()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -89,3 +90,28 @@ func TestTheFixturePassesTheUnitLevel(t *testing.T) {
|
||||
t.Errorf("result = %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1869: a package whose test imports its own build output (as mesh-sdk's do) passes the unit level: the test step
|
||||
// emits it with the real tsc, shut off, before the test script runs.
|
||||
func TestAPackageWhoseTestsImportItsBuildPassesTheUnitLevel(t *testing.T) {
|
||||
if _, err := exec.LookPath("bwrap"); err != nil {
|
||||
t.Skip("NOT RUN: no bubblewrap installed where this check runs; every build node has it (the module declares it)")
|
||||
}
|
||||
m, work := realMachine(t)
|
||||
for p, body := range map[string]string{
|
||||
"modules/p/package.json": `{"name":"p","type":"module","scripts":{"build":"tsc","test":"node --test test/a.test.mjs","prepare":"tsc"}}`,
|
||||
"modules/p/tsconfig.json": `{"compilerOptions":{"module":"NodeNext","moduleResolution":"NodeNext","target":"ES2022","outDir":"dist","rootDir":"src"},"include":["src"]}`,
|
||||
"modules/p/src/index.ts": "export const answer: number = 42;\n",
|
||||
"modules/p/test/a.test.mjs": "import { test } from 'node:test';\nimport assert from 'node:assert';\nimport { answer } from '../dist/index.js';\n" +
|
||||
"test('the build is there', () => assert.equal(answer, 42));\n",
|
||||
} {
|
||||
write(t, filepath.Join(work, "w1", "tree", filepath.FromSlash(p)), body, 0o644)
|
||||
}
|
||||
out, err := m.test(context.Background(), map[string]any{"workspace": "w1", "module": "modules/p", "level": "unit"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r := out.(*TestResult); !r.Passed || !slices.Equal(r.Ran, []string{"build", "test"}) {
|
||||
t.Errorf("result = %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,12 +6,14 @@ import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
@@ -96,12 +98,10 @@ func TestTscAndEsbuildRunWithTheBuildAgentsFlags(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func readArchive(t *testing.T, path string) map[string]os.FileMode {
|
||||
// readArchive reads the archive a build pushed (ADR 0498: the packed file never stays on the node).
|
||||
func readArchive(t *testing.T, r BuildResult) map[string]os.FileMode {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := archiveOf(t, r)
|
||||
zr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -135,7 +135,7 @@ func TestTheBundleIsPackedWithItsLauncherAndPackageJson(t *testing.T) {
|
||||
if want := []string{"package.json", "tools/index.js", "tools/index.serve.mjs"}; !slices.Equal(r.Files, want) {
|
||||
t.Errorf("files = %v, want %v", r.Files, want)
|
||||
}
|
||||
modes := readArchive(t, r.Archive)
|
||||
modes := readArchive(t, r)
|
||||
if len(modes) != 3 || modes["tools/index.serve.mjs"]&0o111 == 0 {
|
||||
t.Errorf("archive = %v, want the three files with the launcher executable", modes)
|
||||
}
|
||||
@@ -143,7 +143,7 @@ func TestTheBundleIsPackedWithItsLauncherAndPackageJson(t *testing.T) {
|
||||
if err != nil || again.(BuildResult).Digest != r.Digest {
|
||||
t.Errorf("a second build: %v, digest %v, want %s", err, again, r.Digest)
|
||||
}
|
||||
launcher, _ := os.ReadFile(filepath.Join(filepath.Dir(r.Archive), "stage-tools", "modules", "module", ".mesh-build", "tools", "tools", "index.serve.mjs"))
|
||||
launcher, _ := os.ReadFile(filepath.Join(m.Work, "w1", "stage-tools", "modules", "module", ".mesh-build", "tools", "tools", "index.serve.mjs"))
|
||||
if !strings.Contains(string(launcher), `import { serveRegisteredOverStdio } from "@novox/mesh-sdk/stdio";`) || !strings.Contains(string(launcher), `await import("./index.js");`) {
|
||||
t.Errorf("launcher = %q", launcher)
|
||||
}
|
||||
@@ -181,7 +181,7 @@ func TestThePackagesAreCopiedBesideTheTreeAndIntoTheBundleOnlyForExternals(t *te
|
||||
t.Errorf("the toolchain's own %s was copied", p)
|
||||
}
|
||||
}
|
||||
for f := range readArchive(t, out.(BuildResult).Archive) {
|
||||
for f := range readArchive(t, out.(BuildResult)) {
|
||||
if strings.HasPrefix(f, "node_modules") {
|
||||
t.Errorf("a bundle without externals carries %s", f)
|
||||
}
|
||||
@@ -254,3 +254,16 @@ func TestTheBuildRunsWithThePackagesTwoLevelsAboveTheModule(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// R4e: the launchers a build writes are mesh-sdk's one rule (buildstep.Launchers), the names the controller reads a bundle
|
||||
// by; an entrypoint that is not JavaScript gets none.
|
||||
func TestTheLaunchersAreTheSDKsRule(t *testing.T) {
|
||||
a := artifact{Language: "typescript", Entrypoints: []string{"tools/index.js", "web/server.js", "assets/x.css"}}
|
||||
got, err := writeLaunchers(t.TempDir(), a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := buildstep.Launchers(a.Language, a.Entrypoints); !maps.Equal(got, want) || len(got) != 2 {
|
||||
t.Errorf("launchers %v, mesh-sdk's rule %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,18 +138,25 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
||||
return nil
|
||||
}
|
||||
if len(sources) > 0 {
|
||||
// A package that builds (a build or prepare script, which the fetch's --ignore-scripts never runs) is emitted
|
||||
// by tsc into the workspace, shut off like every run step, so its tests find what they import of its own output
|
||||
// (issue 1869: mesh-sdk's tests import ../dist); any other is type-checked without emitting.
|
||||
name, line := "type-check", m.typeCheck(dir, sources)
|
||||
if emit := m.emitBuild(dir, scripts); emit != nil {
|
||||
name, line = "build", emit
|
||||
}
|
||||
start := m.now()
|
||||
r := m.runIn(ctx, dir, nil, "node", m.typeCheck(dir, sources)...)
|
||||
r := m.runIn(ctx, root, false, dir, nil, "node", line...)
|
||||
check = m.now().Sub(start)
|
||||
if err := ran("type-check", r); err != nil {
|
||||
if err := ran(name, r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.Ran = append(res.Ran, "type-check")
|
||||
res.Ran = append(res.Ran, name)
|
||||
}
|
||||
if hasScript && len(failed) == 0 {
|
||||
start := m.now()
|
||||
env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm")}
|
||||
r := m.runIn(ctx, dir, env, "npm", scriptArgs(script)...)
|
||||
r := m.runIn(ctx, root, false, dir, env, "npm", scriptArgs(script)...)
|
||||
run = m.now().Sub(start)
|
||||
if err := ran(script, r); err != nil {
|
||||
return nil, err
|
||||
@@ -220,6 +227,18 @@ func (m *Module) typeCheck(dir string, sources []string) []string {
|
||||
"--target", "ES2022", "--rootDir", "."}, sources...)
|
||||
}
|
||||
|
||||
// emitBuild is tsc's line that builds a package as its tsconfig.json says (`tsc -p .`, emitting where it says), when the
|
||||
// package builds (a build or prepare script) and has a tsconfig.json; nil otherwise.
|
||||
func (m *Module) emitBuild(dir string, scripts map[string]string) []string {
|
||||
if scripts["build"] == "" && scripts["prepare"] == "" {
|
||||
return nil
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "tsconfig.json")); err != nil {
|
||||
return nil
|
||||
}
|
||||
return []string{m.tsc(), "-p", "."}
|
||||
}
|
||||
|
||||
// packageScripts are the scripts of the package.json in dir (none without one).
|
||||
func packageScripts(dir string) (map[string]string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -122,8 +123,9 @@ func TestTheStepsCommandsGetAnEnvironmentOfTheirOwn(t *testing.T) {
|
||||
t.Errorf("%s was given %s", joined(c), e)
|
||||
}
|
||||
}
|
||||
if len(c.env) < 3 || !strings.HasPrefix(c.env[0], "PATH=") || !strings.HasPrefix(c.env[1], "HOME=") {
|
||||
t.Errorf("%s env = %v, want PATH, HOME and the locale", c.name, c.env)
|
||||
// 1309: the HOME of a run step is its own, inside its /tmp, never the account's.
|
||||
if len(c.env) < 3 || !strings.HasPrefix(c.env[0], "PATH=") || !slices.Contains(c.env, "HOME=/tmp/home") || !slices.Contains(c.env, "LC_ALL=C") {
|
||||
t.Errorf("%s env = %v, want PATH, the step's own HOME and the locale", c.name, c.env)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The package.json of mesh-sdk's TypeScript package (feat/r4e-sdk 9cb4d59), whose tests import its own build output
|
||||
// (test/retire.test.ts imports ../dist/...): built by `tsc` (build and prepare), which a fetch with --ignore-scripts
|
||||
// never runs.
|
||||
const sdkPackageJSON = `{"name":"@novox/mesh-sdk","version":"0.1.15","type":"module","exports":{".":"./dist/index.js"},"files":["dist"],` +
|
||||
`"scripts":{"build":"tsc","test":"node --test --experimental-strip-types 'test/*.test.ts'","prepare":"tsc"}}`
|
||||
|
||||
// Issue 1869: a package that builds (a build or prepare script) is emitted by tsc into the workspace, inside the shut-off
|
||||
// run step, at both levels, before its tests, which import what it emits; the result names it "build". A package that
|
||||
// does not build is still type-checked without emitting.
|
||||
func TestAPackageThatBuildsIsEmittedBeforeItsTests(t *testing.T) {
|
||||
for _, level := range []string{"unit", "integration"} {
|
||||
m, f, work := newMachine(t)
|
||||
pkg := sdkPackageJSON
|
||||
if level == "integration" {
|
||||
pkg = strings.Replace(pkg, `"prepare":"tsc"`, `"prepare":"tsc","test:integration":"node --test test/it.mjs"`, 1)
|
||||
}
|
||||
workspaceWith(t, work, map[string]string{"package.json": pkg, "tsconfig.json": `{"compilerOptions":{"outDir":"dist"}}`,
|
||||
"src/index.ts": "export const a = 1;\n", "test/retire.test.ts": "import '../dist/index.js';\n"})
|
||||
out, err := m.test(context.Background(), testArgs(level))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := out.(*TestResult)
|
||||
if !slices.Contains(r.Ran, "build") || slices.Contains(r.Ran, "type-check") {
|
||||
t.Errorf("%s: ran %v, want the build and no separate type-check", level, r.Ran)
|
||||
}
|
||||
first := joined(f.calls[0])
|
||||
if strings.Contains(first, "--noEmit") || !strings.HasSuffix(first, "typescript/bin/tsc -p .") {
|
||||
t.Errorf("%s: the build ran %s, want tsc -p . emitting", level, first)
|
||||
}
|
||||
if len(f.shut) == 0 || !strings.Contains(f.shut[0].command, "tsc -p .") {
|
||||
t.Errorf("%s: the build did not run shut off", level)
|
||||
}
|
||||
}
|
||||
m, f, work := newMachine(t)
|
||||
workspaceWith(t, work, map[string]string{"package.json": `{"scripts":{"test":"node --test"}}`, "tsconfig.json": `{}`, "index.ts": "export const a = 1;\n"})
|
||||
if _, err := m.test(context.Background(), testArgs("unit")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first := joined(f.calls[0]); !strings.Contains(first, "--noEmit") {
|
||||
t.Errorf("a package that does not build ran %s, want the type-check", first)
|
||||
}
|
||||
}
|
||||
@@ -59,6 +59,18 @@ type FetchResult struct {
|
||||
Conflicts []string `json:"conflicts,omitempty"`
|
||||
Changed []string `json:"changed,omitempty"`
|
||||
Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"`
|
||||
// Group is the group's package the fetch packed and served to the TypeScript members (the call's `package`):
|
||||
// "<name>@<version> <integrity>".
|
||||
Group string `json:"group,omitempty"`
|
||||
}
|
||||
|
||||
// narrowedMark is the file beside a workspace's tree saying the fetch narrowed it to a module's named paths.
|
||||
const narrowedMark = "narrowed"
|
||||
|
||||
// narrowed says whether the fetch narrowed the workspace whose tree this is.
|
||||
func narrowed(tree string) bool {
|
||||
_, err := os.Stat(filepath.Join(filepath.Dir(tree), narrowedMark))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// Refusal is a module a fetch did not install the packages of.
|
||||
@@ -79,7 +91,12 @@ func (m *Module) fetchTool() StepTool {
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
||||
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
||||
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
|
||||
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
|
||||
"dir": buildstep.FetchRequestProperties()["dir"],
|
||||
"paths": buildstep.FetchRequestProperties()["paths"],
|
||||
"package": map[string]any{"type": "object", "description": "optional: the group's package member (ADR 0500 §5), packed here, publishing nothing, and served to each TypeScript member whose lock names its version, in place of the registry's",
|
||||
"properties": map[string]any{"repository": map[string]any{"type": "string"}, "commit": map[string]any{"type": "string"}, "ref": map[string]any{"type": "string"}},
|
||||
"required": []string{"repository", "commit"}},
|
||||
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
|
||||
},
|
||||
"required": []string{"repository", "commit"},
|
||||
},
|
||||
@@ -89,15 +106,16 @@ func (m *Module) fetchTool() StepTool {
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
commit, _ := args["commit"].(string)
|
||||
ref, _ := args["ref"].(string)
|
||||
base, _ := args["base"].(string)
|
||||
if base != "" && !commitOK.MatchString(base) {
|
||||
return nil, fmt.Errorf("%q is not a trunk commit (40 hex digits)", base)
|
||||
// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
|
||||
// named paths, the workspace is narrowed to them (buildstep.Narrow).
|
||||
req, err := buildstep.ParseFetchRequest(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
|
||||
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
|
||||
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
||||
group, err := groupPackageArg(args["package"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
id := buildstep.NewID()
|
||||
@@ -123,16 +141,49 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
return res, nil // a conflict: the tree is at the trunk, nothing of the head is installed
|
||||
}
|
||||
}
|
||||
dirs, err := m.typeScriptModules(ctx, tree)
|
||||
if len(req.Paths) > 0 {
|
||||
if err := buildstep.Narrow(tree, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Said beside the tree, so the listing of the tree's modules leaves out those narrowed away.
|
||||
if err := os.WriteFile(filepath.Join(root, narrowedMark), nil, 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// The group's package, packed here and served to its members in place of the registry's version (ADR 0500 §5).
|
||||
var served *packed
|
||||
if group != nil {
|
||||
p, err := m.packGroup(ctx, root, gitRun, group)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
served, res.Group = &p, p.Name+"@"+p.Version+" "+p.Integrity
|
||||
}
|
||||
dirs, packages, err := m.typeScriptModules(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, d := range dirs {
|
||||
res.Modules = append(res.Modules, d)
|
||||
did, refused, err := m.installOwn(ctx, root, filepath.Join(tree, filepath.FromSlash(d)), d)
|
||||
dir := filepath.Join(tree, filepath.FromSlash(d))
|
||||
if packages[d] {
|
||||
// A package is built from its own lock's dependencies, its devDependencies included (ADR 0498 §5).
|
||||
if err := m.installDev(ctx, root, dir, d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.Installed = append(res.Installed, d)
|
||||
continue
|
||||
}
|
||||
did, refused, err := m.installOwn(ctx, root, dir, d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if refused == "" {
|
||||
// The SDK the module's own lock names, from the group or the registry (the lead's decision L10).
|
||||
if err := m.placeSDK(ctx, dir, d, served); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if refused != "" {
|
||||
res.Refused = append(res.Refused, Refusal{Module: d, Reason: refused})
|
||||
}
|
||||
@@ -145,19 +196,22 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
|
||||
// typeScriptModules are the directories of the tree whose module.json declares a typescript artifact, relative to
|
||||
// it, in order.
|
||||
func (m *Module) typeScriptModules(ctx context.Context, tree string) ([]string, error) {
|
||||
func (m *Module) typeScriptModules(ctx context.Context, tree string) (dirs []string, packages map[string]bool, err error) {
|
||||
r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
return nil, nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
var dirs []string
|
||||
packages = map[string]bool{}
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if l = strings.TrimSpace(l); l == "" {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(l)))
|
||||
if os.IsNotExist(err) && narrowed(tree) {
|
||||
continue // outside a narrowed workspace's paths (buildstep.Narrow): no module of this build
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
var mj struct {
|
||||
Build struct {
|
||||
@@ -169,13 +223,15 @@ func (m *Module) typeScriptModules(ctx context.Context, tree string) ([]string,
|
||||
}
|
||||
for _, a := range mj.Build.Artifacts {
|
||||
if a.Language == "typescript" {
|
||||
dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
|
||||
d := filepath.ToSlash(filepath.Dir(l))
|
||||
dirs = append(dirs, d)
|
||||
packages[d] = a.Kind == "package"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(dirs)
|
||||
return dirs, nil
|
||||
return dirs, packages, nil
|
||||
}
|
||||
|
||||
// ownDependencies are the package.json's dependencies beyond the SDK and beyond what is build-time: the types and
|
||||
@@ -229,13 +285,6 @@ func (m *Module) installOwn(ctx context.Context, root, dir, rel string) (install
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if m.Toolchain == nil || m.Toolchain.Registry == nil {
|
||||
return false, "", fmt.Errorf("the module has no package registry to install %s from", rel)
|
||||
}
|
||||
reg, err := m.Toolchain.Registry()
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
// A scratch copy, so the module's package.json and lock are never rewritten (dependencies.go installSteps).
|
||||
scratch := filepath.Join(root, "npm-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel))
|
||||
if err := os.MkdirAll(scratch, 0o700); err != nil {
|
||||
@@ -255,10 +304,11 @@ func (m *Module) installOwn(ctx context.Context, root, dir, rel string) (install
|
||||
if err := os.WriteFile(filepath.Join(scratch, "package-lock.json"), lock, 0o600); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
// The registry is given through a config file of its own, never on the command line, and removed after.
|
||||
npmrc := filepath.Join(scratch, ".npmrc-install")
|
||||
if err := os.WriteFile(npmrc, []byte(reg.Npmrc()), 0o600); err != nil {
|
||||
return false, "", err
|
||||
// The registry is given through a config file of its own, never on the command line, outside the workspace, and
|
||||
// removed after (npmrcIn).
|
||||
npmrc, err := m.npmrcIn(root)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("installing the packages of %s: %w", rel, err)
|
||||
}
|
||||
env := append(m.baseEnv(), "npm_config_cache="+filepath.Join(m.Toolchain.Paths.Cache, "npm"))
|
||||
r := m.Run(ctx, env, "npm", "ci", "--omit=dev", "--omit=peer", "--ignore-scripts", "--no-bin-links",
|
||||
|
||||
@@ -27,7 +27,9 @@ type call struct {
|
||||
type fakeMachine struct {
|
||||
mu sync.Mutex
|
||||
calls []call
|
||||
now time.Time
|
||||
// shut are the commands that ran in bubblewrap, read back (isolate_test.go).
|
||||
shut []shutOff
|
||||
now time.Time
|
||||
// cost is how long a program takes, by the base name of its path; answer is what it prints and its status;
|
||||
// effect runs the program's effects on the disk.
|
||||
cost map[string]time.Duration
|
||||
@@ -74,7 +76,10 @@ func newMachine(t *testing.T) (*Module, *fakeMachine, string) {
|
||||
Toolchain: &Toolchain{Paths: Paths{Home: home, Cache: cache}, Registry: func() (Registry, error) {
|
||||
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/"}, nil
|
||||
}},
|
||||
Work: work, Run: f.run, Clock: f.clock, Ctx: context.Background()}
|
||||
Work: work, Clock: f.clock, Ctx: context.Background()}
|
||||
// A run step's commands run in bubblewrap (1309); the fake reads each back and runs its command.
|
||||
m.Run = throughBubblewrap(t, f.run, &f.shut)
|
||||
withStore(t, m) // a build pushes its bundle to a store of the test's own (ADR 0498)
|
||||
// The toolchain directory as npm ci leaves it, with no bin links.
|
||||
nm := filepath.Join(m.toolchainDir(), "node_modules")
|
||||
for _, p := range []string{"typescript/bin/tsc", "esbuild/bin/esbuild", "@esbuild/linux-x64/bin/esbuild",
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
package main
|
||||
|
||||
// The run steps shut off (novox/hq ADR 0448; issue 1309, lead decision L2 of R4e): the type-check, the package's test
|
||||
// script, tsc and esbuild run the change's code, so each runs through mesh-sdk's buildstep.Isolated, in bubblewrap: no
|
||||
// network, none of the account (its home, its ssh key, its registry secret, its bus login, the runtime directory), the
|
||||
// workspace read-write, the toolchain read-only, the shared npm cache read-only beneath a throwaway writable layer, or
|
||||
// read-write for a trunk build alone (publish). The fetch step (git, npm ci of the module's own packages) holds the key
|
||||
// and the network and runs none of the change's code, so it runs as before.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// BubblewrapLeast is the oldest bubblewrap a run step is shut off with: --overlay-src and --tmp-overlay, which keep the
|
||||
// shared cache unchanged by a candidate, came in 0.10.
|
||||
const BubblewrapLeast = "0.10"
|
||||
|
||||
// npmCache is the shared npm cache a package's test script reads.
|
||||
func (m *Module) npmCache() string { return filepath.Join(m.Toolchain.Paths.Cache, "npm") }
|
||||
|
||||
// isolation is how a run step in the workspace root is shut off.
|
||||
func (m *Module) isolation(root string, write bool) buildstep.Isolation {
|
||||
return buildstep.Isolation{
|
||||
Workspace: root,
|
||||
ReadOnly: []string{m.toolchainDir()},
|
||||
Caches: []string{m.npmCache()},
|
||||
WriteCaches: write,
|
||||
Env: []string{"PATH=" + pathOrDefault(), "LC_ALL=C"},
|
||||
}
|
||||
}
|
||||
|
||||
// stepRunner is the runner of a run step in the workspace root: buildstep.Isolated over m.Run, every command shut off;
|
||||
// write is a trunk build's alone.
|
||||
func (m *Module) stepRunner(root string, write bool) (forgekey.Runner, error) {
|
||||
// bubblewrap binds only a directory that is there.
|
||||
if err := os.MkdirAll(m.npmCache(), 0o700); err != nil {
|
||||
return nil, fmt.Errorf("the shared cache %s: %w", m.npmCache(), err)
|
||||
}
|
||||
return buildstep.Isolated(m.Run, m.isolation(root, write)), nil
|
||||
}
|
||||
|
||||
// ProbeBubblewrap reads the installed bubblewrap's version (`bwrap --version`: "bubblewrap 0.13.0").
|
||||
func ProbeBubblewrap(ctx context.Context) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
r := forgekey.ExecRunnerClean(0)(ctx, []string{"PATH=" + defaultPath}, buildstep.Bubblewrap, "--version")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("`%s --version` failed: %s", buildstep.Bubblewrap, firstLine(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
return parseBubblewrapVersion(r.Stdout)
|
||||
}
|
||||
|
||||
var bubblewrapVersion = regexp.MustCompile(`^bubblewrap (\d+)\.(\d+)(\.\d+)?\s*$`)
|
||||
|
||||
func parseBubblewrapVersion(out string) (string, error) {
|
||||
f := bubblewrapVersion.FindStringSubmatch(strings.TrimSpace(out))
|
||||
if f == nil {
|
||||
return "", fmt.Errorf("`%s --version` answered %q", buildstep.Bubblewrap, strings.TrimSpace(out))
|
||||
}
|
||||
return f[1] + "." + f[2] + f[3], nil
|
||||
}
|
||||
|
||||
// bubblewrapProblem says why the installed bubblewrap cannot shut a run step off, or "".
|
||||
func bubblewrapProblem(version string, err error) string {
|
||||
if err != nil {
|
||||
return "bubblewrap cannot be run, and every build and test step runs in it (ADR 0448): " + err.Error()
|
||||
}
|
||||
if !atLeast(version, BubblewrapLeast) {
|
||||
return fmt.Sprintf("bubblewrap %s installed, and a build or test step needs %s or later (ADR 0448)", version, BubblewrapLeast)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// atLeast says whether the dotted version v is least or later.
|
||||
func atLeast(v, least string) bool {
|
||||
a, b := strings.Split(v, "."), strings.Split(least, ".")
|
||||
for i := range b {
|
||||
var x int
|
||||
if i < len(a) {
|
||||
x, _ = strconv.Atoi(a[i])
|
||||
}
|
||||
y, _ := strconv.Atoi(b[i])
|
||||
if x != y {
|
||||
return x > y
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// checkBubblewrap refuses a step when bubblewrap cannot shut it off; nil Bubblewrap probes nothing.
|
||||
func (m *Module) checkBubblewrap(ctx context.Context) error {
|
||||
if m.Bubblewrap == nil {
|
||||
return nil
|
||||
}
|
||||
v, err := m.Bubblewrap(ctx)
|
||||
if why := bubblewrapProblem(v, err); why != "" {
|
||||
return &buildstep.Refusal{Text: why, Want: BubblewrapLeast + " or later", Have: v}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// shutOff is one command a step ran in bubblewrap: what it bound read-write, read-only and beneath a throwaway layer,
|
||||
// its bubblewrap flags, and the command.
|
||||
type shutOff struct {
|
||||
flags []string
|
||||
binds, roBinds, overlays []string
|
||||
env []string
|
||||
command string
|
||||
}
|
||||
|
||||
// bwrapArity is how many words each bubblewrap option the SDK writes takes.
|
||||
var bwrapArity = map[string]int{"--unshare-all": 0, "--die-with-parent": 0, "--new-session": 0, "--clearenv": 0,
|
||||
"--proc": 1, "--dev": 1, "--tmpfs": 1, "--dir": 1, "--chdir": 1, "--overlay-src": 1, "--tmp-overlay": 1,
|
||||
"--ro-bind": 2, "--ro-bind-try": 2, "--bind": 2, "--symlink": 2, "--setenv": 2}
|
||||
|
||||
// throughBubblewrap reads a bubblewrap command back (recorded in seen) and hands its command to run with the environment
|
||||
// bubblewrap sets (a name set twice keeps its last value, as bubblewrap's --setenv does); any other command goes to run
|
||||
// as it is.
|
||||
func throughBubblewrap(t *testing.T, run forgekey.Runner, seen *[]shutOff) forgekey.Runner {
|
||||
t.Helper()
|
||||
var mu sync.Mutex
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if name != buildstep.Bubblewrap {
|
||||
return run(ctx, env, name, args...)
|
||||
}
|
||||
var s shutOff
|
||||
var names []string
|
||||
values := map[string]string{}
|
||||
i := 0
|
||||
for ; i < len(args) && args[i] != "--"; i++ {
|
||||
n, ok := bwrapArity[args[i]]
|
||||
if !ok || i+n >= len(args) {
|
||||
t.Errorf("bubblewrap was given %q, which the test does not know: %v", args[i], args)
|
||||
return forgekey.Ran{Status: 1, Err: "unknown bubblewrap option"}
|
||||
}
|
||||
s.flags = append(s.flags, args[i])
|
||||
switch args[i] {
|
||||
case "--bind":
|
||||
s.binds = append(s.binds, args[i+1])
|
||||
case "--ro-bind", "--ro-bind-try":
|
||||
s.roBinds = append(s.roBinds, args[i+1])
|
||||
case "--overlay-src":
|
||||
s.overlays = append(s.overlays, args[i+1])
|
||||
case "--setenv":
|
||||
if _, set := values[args[i+1]]; !set {
|
||||
names = append(names, args[i+1])
|
||||
}
|
||||
values[args[i+1]] = args[i+2]
|
||||
}
|
||||
i += n
|
||||
}
|
||||
if i+1 >= len(args) {
|
||||
t.Errorf("bubblewrap was given no command: %v", args)
|
||||
return forgekey.Ran{Status: 1, Err: "no command"}
|
||||
}
|
||||
for _, n := range names {
|
||||
s.env = append(s.env, n+"="+values[n])
|
||||
}
|
||||
s.command = args[i+1] + " " + strings.Join(args[i+2:], " ")
|
||||
mu.Lock()
|
||||
*seen = append(*seen, s)
|
||||
mu.Unlock()
|
||||
return run(ctx, s.env, args[i+1], args[i+2:]...)
|
||||
}
|
||||
}
|
||||
|
||||
// checkShutOff fails the test unless s ran with no network and none of the account, the toolchain read-only, and the
|
||||
// shared cache beneath a throwaway layer unless write.
|
||||
func checkShutOff(t *testing.T, m *Module, root string, s shutOff, write bool) {
|
||||
t.Helper()
|
||||
for _, f := range []string{"--unshare-all", "--clearenv", "--die-with-parent"} {
|
||||
if !slices.Contains(s.flags, f) {
|
||||
t.Errorf("%s ran without %s", s.command, f)
|
||||
}
|
||||
}
|
||||
home := m.Toolchain.Paths.Home
|
||||
for _, b := range append(append([]string{}, s.binds...), s.roBinds...) {
|
||||
if b == home || strings.HasPrefix(b, home+"/") || strings.HasPrefix(b, "/run") {
|
||||
t.Errorf("%s was given %s: the account's home or the runtime directory", s.command, b)
|
||||
}
|
||||
}
|
||||
for _, e := range s.env {
|
||||
if e == "HOME="+home {
|
||||
t.Errorf("%s was told the account's home", s.command)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(s.roBinds, m.toolchainDir()) || slices.Contains(s.binds, m.toolchainDir()) {
|
||||
t.Errorf("%s was not given the toolchain read-only: %v %v", s.command, s.roBinds, s.binds)
|
||||
}
|
||||
c := m.npmCache()
|
||||
bound, layered := slices.Contains(s.binds, c), slices.Contains(s.overlays, c)
|
||||
if write != bound || write == layered {
|
||||
t.Errorf("%s: the shared cache bound %v, layered %v; a trunk build (%v) alone writes it", s.command, bound, layered, write)
|
||||
}
|
||||
if !slices.Contains(s.binds, root) {
|
||||
t.Errorf("%s was not given its workspace %s read-write: %v", s.command, root, s.binds)
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S7: the type-check and the test script run shut off, the cache beneath a throwaway layer.
|
||||
func TestTheTestStepRunsItsCommandsShutOff(t *testing.T) {
|
||||
m, f, work := newMachine(t)
|
||||
workspaceWith(t, work, map[string]string{"index.ts": `export const a = 1;`, "package.json": `{"scripts":{"test":"x"}}`})
|
||||
if _, err := m.test(context.Background(), testArgs("unit")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.shut) != 2 || len(f.calls) != 2 {
|
||||
t.Fatalf("shut off %d of %d commands, want the type-check and the script: %v", len(f.shut), len(f.calls), f.calls)
|
||||
}
|
||||
for _, s := range f.shut {
|
||||
checkShutOff(t, m, work+"/w1", s, false)
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S8: tsc and esbuild of a candidate's build run shut off with the cache beneath a throwaway layer; a trunk build
|
||||
// (publish) writes it.
|
||||
func TestTheBuildStepRunsShutOffAndOnlyATrunkBuildWritesTheCache(t *testing.T) {
|
||||
for _, publish := range []bool{false, true} {
|
||||
m, f, work := builtMachine(t, "")
|
||||
ws := "w1"
|
||||
args := buildArgs()
|
||||
if publish {
|
||||
args["publish"] = true
|
||||
}
|
||||
if _, err := m.build(context.Background(), args); err != nil {
|
||||
t.Fatalf("publish %v: %v", publish, err)
|
||||
}
|
||||
if len(f.shut) == 0 || len(f.shut) != len(f.calls) {
|
||||
t.Fatalf("publish %v: shut off %d of %d commands", publish, len(f.shut), len(f.calls))
|
||||
}
|
||||
for _, s := range f.shut {
|
||||
checkShutOff(t, m, work+"/"+ws, s, publish)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1309-S1: a step is refused, and the module is unhealthy, while bubblewrap is missing or older than 0.10.
|
||||
func TestAnOldOrMissingBubblewrapRefusesTheStepsAndIsUnhealthy(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
version string
|
||||
err error
|
||||
ok bool
|
||||
}{{"0.13.0", nil, true}, {"0.10", nil, true}, {"0.9.0", nil, false}, {"", errors.New("bwrap: not found"), false}} {
|
||||
m, _, _ := newMachine(t)
|
||||
m.Probe = func(context.Context) (string, error) { return "v22.20.0", nil }
|
||||
m.Toolchains = func(string, any) (uint64, error) { return 1, nil }
|
||||
m.Bubblewrap = func(context.Context) (string, error) { return c.version, c.err }
|
||||
err := m.Check(context.Background())
|
||||
raw, _ := json.Marshal(m.Health(context.Background()))
|
||||
if c.ok {
|
||||
if err != nil || strings.Contains(string(raw), "unhealthy") {
|
||||
t.Errorf("bubblewrap %q: refused %v, health %s", c.version, err, raw)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var ref *buildstep.Refusal
|
||||
if !errors.As(err, &ref) || !strings.Contains(err.Error(), "bubblewrap") {
|
||||
t.Errorf("bubblewrap %q %v: Check answered %v", c.version, c.err, err)
|
||||
}
|
||||
if !strings.Contains(string(raw), "unhealthy") || !strings.Contains(string(raw), "bubblewrap") {
|
||||
t.Errorf("bubblewrap %q %v: health %s", c.version, c.err, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// probeScript is a package's test script that tries what a change's code must not reach and writes what it saw.
|
||||
const probeScript = `import fs from "node:fs";
|
||||
import dns from "node:dns/promises";
|
||||
import net from "node:net";
|
||||
const [key, login, port, sock, cache] = process.argv.slice(2);
|
||||
const seen = {};
|
||||
const read = (p) => { try { fs.readFileSync(p); return true; } catch { return false; } };
|
||||
seen.key = read(key);
|
||||
seen.login = read(login);
|
||||
try { await dns.lookup("git.novox.be"); seen.resolved = true; } catch { seen.resolved = false; }
|
||||
const reach = (opts) => new Promise((ok) => { const c = net.connect(opts); c.on("connect", () => { c.destroy(); ok(true); }); c.on("error", () => ok(false)); });
|
||||
seen.tcp = await reach({ host: "127.0.0.1", port: Number(port) });
|
||||
seen.socket = await reach({ path: sock });
|
||||
try { fs.writeFileSync(cache + "/written-by-a-candidate", "x"); seen.wroteCache = true; } catch { seen.wroteCache = false; }
|
||||
fs.writeFileSync("seen.json", JSON.stringify(seen));
|
||||
`
|
||||
|
||||
// 1309-S9 (ADR 0448's check): a package's test script, run by the test step, reads no key and no login of the account,
|
||||
// resolves no name, reaches no socket of the machine (a TCP port on its loopback, a unix socket), and leaves the
|
||||
// shared cache unchanged.
|
||||
func TestARunStepReachesNoKeyLoginNameOrSocketAndLeavesTheCacheUnchanged(t *testing.T) {
|
||||
if _, err := exec.LookPath("bwrap"); err != nil {
|
||||
t.Skip("NOT RUN: no bubblewrap installed where this check runs; every build node has it (the module declares it)")
|
||||
}
|
||||
m, work := realMachine(t)
|
||||
home := m.Toolchain.Paths.Home
|
||||
key, login := filepath.Join(home, ".ssh", "mesh_ed25519"), filepath.Join(home, "broker")
|
||||
write(t, key, "a key", 0o600)
|
||||
write(t, login, "a login", 0o600)
|
||||
tcp, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tcp.Close()
|
||||
sockDir, err := os.MkdirTemp("", "sock")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(sockDir)
|
||||
sock := filepath.Join(sockDir, "s")
|
||||
unix, err := net.Listen("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer unix.Close()
|
||||
if err := os.MkdirAll(m.npmCache(), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
port := strconv.Itoa(tcp.Addr().(*net.TCPAddr).Port)
|
||||
dir := filepath.Join(work, "w1", "tree", "modules", "m")
|
||||
write(t, filepath.Join(dir, "probe.mjs"), probeScript, 0o644)
|
||||
write(t, filepath.Join(dir, "package.json"), `{"name":"m","type":"module","scripts":{"test":"node probe.mjs `+key+` `+login+` `+port+` `+sock+` `+m.npmCache()+`"}}`, 0o644)
|
||||
out, err := m.test(context.Background(), map[string]any{"workspace": "w1", "module": "modules/m", "level": "unit"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r := out.(*TestResult); !r.Passed {
|
||||
t.Fatalf("the probe did not run: %+v", r)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "seen.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("the probe wrote nothing: %v", err)
|
||||
}
|
||||
var seen map[string]bool
|
||||
if err := json.Unmarshal(raw, &seen); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for what, reached := range seen {
|
||||
if reached && what != "wroteCache" {
|
||||
t.Errorf("the run step reached %s: %s", what, raw)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.npmCache(), "written-by-a-candidate")); err == nil {
|
||||
t.Errorf("a candidate's step changed the shared cache")
|
||||
}
|
||||
}
|
||||
@@ -51,12 +51,19 @@ func main() {
|
||||
Run: forgekey.ExecRunnerClean(0),
|
||||
Ctx: ctx,
|
||||
Say: logf,
|
||||
Bubblewrap: ProbeBubblewrap,
|
||||
RegistryFile: filepath.Join(home, "registry"),
|
||||
}
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
if m.Node != "" {
|
||||
go m.Start(ctx)
|
||||
}
|
||||
go startKeeper(ctx)
|
||||
// Every state it keeps, said on the bus before it serves (ADR 0495 §2; issue 1790).
|
||||
if err := keepStates(); err != nil {
|
||||
logf("announcing the states it keeps: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user