Merge remote-tracking branch 'origin/main' into fix/403-fix-time
mesh/merge-gate head daf52b3aeb79 merged into trunk 4dbd4066ca54 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head daf52b3aeb79 merged into trunk 4dbd4066ca54 — pass: its merge-check.sh passed
mesh/delivery failed: its walk failed, carrying its own commit: a gate on a first machine (what it carried put back), a build, a machine
mesh/delivery-group group fix/403-fix-time failed: a member failed

# Conflicts:
#	internal/link/events.go
This commit is contained in:
jochen
2026-10-12 21:28:52 +02:00
68 changed files with 6816 additions and 111 deletions
+77 -6
View File
@@ -49,8 +49,10 @@ import (
//
// One check:
//
// 1. clones the repository at the pull request's head, and beside it the repositories its check
// reads — the controller the mesh runs, the catalogue, the host, the lab — each at the ref asked;
// 1. clones the repository at the pull request's head and **makes the merge of that head into the trunk's
// head**, the check's trunk commit, in its own clone (novox/hq ADR 0323, issue 341; check_merge.go): what
// is judged is what merging makes, never pushed. Beside it the repositories its check reads — the
// controller the mesh runs, the catalogue, the host, the lab — each at the ref asked;
// 2. reads the facts snapshot the controller keeps, and with it **the versions the mesh runs**: the
// store a check's tests stand on is the store's own release, and the bus the bus's;
// 3. raises a throwaway PostgreSQL and a throwaway bus of those versions, labelled with the ask so a
@@ -79,7 +81,8 @@ type CheckSpec struct {
Modules []string
New []string
Manifests []string
// Base is the branch the pull request merges into: what the gate compares the change against.
// Base is the branch the pull request merges into: what the gate compares the change against, and the trunk
// whose head the change is merged into before it is judged (novox/hq ADR 0323). Empty judges the head alone.
Base string
// Judge is who judges the gate: "" the controller the mesh runs, "self" the change's own, "validator"
// the running one with the change's validator.
@@ -104,6 +107,9 @@ type GroupHead struct {
Repository string
Ref string
Paths []string
// Base is the branch it merges into: the head is merged into that trunk's head before it is composed
// (novox/hq ADR 0323 §6). Empty composes the head alone, as a controller from before it asks.
Base string
}
// Gated is whether the change touches the mesh's graph, and so whether the gate runs.
@@ -139,6 +145,16 @@ type CheckVerdict struct {
Took time.Duration
Gate *Layer
Repo *Layer
// Head and Trunk are what was judged (novox/hq ADR 0323 §3): the pull request's head merged into the trunk
// commit, each as twelve characters. Trunk is empty for a check asked with no trunk, which judged the head
// alone. Merge is the merge commit made for the check, whole: only checked, never pushed or registered.
Head, Trunk, Merge string
// Beside are the commits of the repositories the check read beside its own, by the directory each was
// cloned under, as twelve characters: named, not judged against (ADR 0323 §3, issue 568).
Beside map[string]string
// Members are a delivery group's other heads as the composed check judged them, each with the trunk
// commit it was merged into (ADR 0323 §6).
Members []JudgedHead
}
// Layer is one layer of a check, judged.
@@ -281,6 +297,34 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
return CheckVerdict{}, err
}
tree := filepath.Join(root, name)
// **What merging makes is judged, not the head** (novox/hq ADR 0323, issue 341): the head merged into the
// trunk's head as this clone found it, in this clone alone. A head that does not merge cleanly fails here.
judged := CheckVerdict{Head: twelve(spec.Ref)}
paths := spec.Paths
if spec.Base != "" {
m, err := mergeIntoTrunk(ctx, run, tree, spec.Base, spec.Ref)
if err != nil {
return CheckVerdict{}, err
}
judged.Head, judged.Trunk = twelve(m.Head), twelve(m.Trunk)
if len(m.Conflicts) > 0 {
said := conflictWords(m)
say("check", "%s", said)
return CheckVerdict{Verdict: "fail", Summary: said, Report: said, Took: time.Since(began),
Gate: &Layer{Verdict: "fail", Summary: said}, Repo: &Layer{Verdict: "fail", Summary: said},
Head: judged.Head, Trunk: judged.Trunk}, nil
}
judged.Merge = m.Merge
if changed := changedByMerge(ctx, run, tree, m); len(changed) > 0 {
paths = changed
}
say("check", "head %s merged into trunk %s (%s): the merge is judged, and never pushed", judged.Head,
judged.Trunk, spec.Base)
}
named := func(v CheckVerdict) CheckVerdict {
v.Head, v.Trunk, v.Merge, v.Beside, v.Members = judged.Head, judged.Trunk, judged.Merge, judged.Beside, judged.Members
return v
}
// **The base branch's check judges the change** (novox/hq issue 310), never the one the change brings.
judging, err := TheCheckThatJudges(ctx, tree, spec.Base)
if err != nil {
@@ -297,7 +341,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
v := CheckVerdict{Verdict: "pass", Summary: noModule, Took: time.Since(began),
Gate: &Layer{Verdict: "pass", Summary: noModule}, Repo: &Layer{Verdict: "warning", Summary: noScript}}
say("check", "%s; %s", noModule, noScript)
return v, nil
return named(v), nil
}
for dir, b := range spec.Beside {
if dir == name || !safeName.MatchString(dir) {
@@ -306,6 +350,13 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
if err := clone(b.Repository, b.Ref, dir); err != nil {
return CheckVerdict{}, fmt.Errorf("beside it, %w", err)
}
// The commit each one read is named in the verdict (ADR 0323 §3): its move is issue 568's question.
if at := headAt(ctx, run, filepath.Join(root, dir)); at != "" {
if judged.Beside == nil {
judged.Beside = map[string]string{}
}
judged.Beside[dir] = at
}
say("check", "beside it %s at %s", dir, short(b.Ref))
}
@@ -322,6 +373,26 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
return CheckVerdict{}, fmt.Errorf("a head of the group, %w", err)
}
say("check", "with it, of its group, %s/%s at %s", g.Owner, g.Repo, short(g.Ref))
// Each member is composed as its merge into its own trunk's head (ADR 0323 §6).
member := JudgedHead{Owner: g.Owner, Repo: g.Repo, Head: twelve(g.Ref)}
if g.Base != "" {
m, err := mergeIntoTrunk(ctx, run, filepath.Join(root, dir), g.Base, g.Ref)
if err != nil {
return CheckVerdict{}, fmt.Errorf("a head of the group, %s/%s: %w", g.Owner, g.Repo, err)
}
member.Head, member.Trunk = twelve(m.Head), twelve(m.Trunk)
if len(m.Conflicts) > 0 {
said := fmt.Sprintf("%s/%s: %s", g.Owner, g.Repo, conflictWords(m))
say("check", "%s", said)
judged.Members = append(judged.Members, member)
return named(CheckVerdict{Verdict: "fail", Summary: said, Report: said, Took: time.Since(began),
Gate: &Layer{Verdict: "fail", Summary: said}, Repo: &Layer{Verdict: "fail", Summary: said}}), nil
}
if changed := changedByMerge(ctx, run, filepath.Join(root, dir), m); len(changed) > 0 {
g.Paths = changed
}
}
judged.Members = append(judged.Members, member)
heads = append(heads, map[string]any{"repository": g.Owner + "/" + g.Repo,
"tree": filepath.Join(root, dir), "changed": g.Paths})
}
@@ -433,7 +504,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
verdictFile := filepath.Join(root, "verdict.json")
env := []string{EnvFacts + "=" + factsFile, EnvGate + "=" + gate, EnvGateStore + "=" + storeURL,
EnvTestStore + "=" + storeURL, EnvTestBus + "=nats://" + bus, EnvRepository + "=" + spec.Owner + "/" + spec.Repo,
EnvChanged + "=" + strings.Join(spec.Paths, ","), EnvBeside + "=" + root,
EnvChanged + "=" + strings.Join(paths, ","), EnvBeside + "=" + root,
EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","),
EnvGroup + "=" + groupFile,
"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")}
@@ -501,7 +572,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began)
say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary,
strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second))
return v, nil
return named(v), nil
}
// ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the
+115
View File
@@ -0,0 +1,115 @@
package builder
import (
"context"
"fmt"
"strings"
)
// **A merge check judges the merge, not the head** (novox/hq ADR 0323, issue 341).
//
// On 2026-10-09 a node-engine pull request was checked green at its head and broke the trunk when it merged:
// a merge an hour earlier had added a field under the same JSON key on other lines, git merged the two edits
// without a conflict, and the result did not compile. The check had judged the head alone. So a check now
// takes the trunk the pull request merges into, its head at the moment the check clones it as the check's
// **trunk commit**, and makes in its own clone the merge of the head into that commit, as the forge's
// default merge would. The gate and the repository's own check judge that merge.
//
// **The merge is only checked** (ADR 0323 §2, ADR 0238 §1): it is a commit off the trunk, made in the
// check's own clone, never pushed to the forge and never registered; the clone is removed when the check
// ends. Nothing here writes to a remote.
//
// The forge keeps no merge reference for a pull request, only its head, so the builder makes it itself.
// TrunkMerge is the merge a check made: the head and the trunk commit it was merged into, whole, and the merge
// commit; or the files in conflict when the head does not merge cleanly.
type TrunkMerge struct {
Head, Trunk, Merge string
Conflicts []string
}
// JudgedHead is one head a check judged, with the trunk commit it merged that head into, each as twelve
// characters (ADR 0323 §3, §6).
type JudgedHead struct {
Owner, Repo string
Head, Trunk string
}
// mergeIdentity is who a check's merge is made as: it is never pushed, so it names nobody's account. No hook of the
// clone's or the build machine's git configuration runs, and no setting there changes the merge.
var mergeIdentity = []string{"-c", "user.name=mesh merge check", "-c", "user.email=merge-check@mesh.invalid",
"-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "-c", "merge.verifySignatures=false"}
// mergeIntoTrunk makes, in the clone at tree, the merge of head into the head of origin/<base> as it stands in
// that fresh clone: the trunk commit. A head that does not merge cleanly answers the files in conflict and leaves
// the tree at the trunk commit; an error is that the merge could not be tried at all.
func mergeIntoTrunk(ctx context.Context, run Runner, tree, base, head string) (TrunkMerge, error) {
var m TrunkMerge
revParse := func(ref string) (string, error) {
out, err := run(ctx, tree, "git", "rev-parse", "--verify", "--quiet", ref+"^{commit}")
return strings.TrimSpace(out), err
}
var err error
if m.Head, err = revParse(head); err != nil || m.Head == "" {
return m, fmt.Errorf("the head %s cannot be read in the check's clone: %v", head, err)
}
if m.Trunk, err = revParse("refs/remotes/origin/" + base); err != nil || m.Trunk == "" {
return m, fmt.Errorf("the trunk %s cannot be read, and a check judges the head merged into it "+
"(novox/hq ADR 0323): %v", base, err)
}
if _, err := run(ctx, tree, "git", "checkout", "--quiet", "--detach", m.Trunk); err != nil {
return m, fmt.Errorf("the trunk commit %s cannot be checked out: %w", twelve(m.Trunk), err)
}
message := fmt.Sprintf("Check the merge of %s into %s at %s (made for a merge check, never pushed)",
twelve(m.Head), base, twelve(m.Trunk))
args := append(append([]string{}, mergeIdentity...), "merge", "--no-ff", "--no-edit", "--no-verify", "--quiet", "-m", message, m.Head)
if _, err := run(ctx, tree, "git", args...); err != nil {
out, _ := run(ctx, tree, "git", "diff", "--name-only", "--diff-filter=U")
m.Conflicts = strings.Fields(out)
_, _ = run(ctx, tree, "git", "merge", "--abort")
if len(m.Conflicts) == 0 {
return m, fmt.Errorf("the head %s could not be merged into trunk %s: %w", twelve(m.Head), twelve(m.Trunk), err)
}
return m, nil
}
if m.Merge, err = revParse("HEAD"); err != nil {
return m, fmt.Errorf("the merge made for the check cannot be read: %w", err)
}
return m, nil
}
// changedByMerge is every path the merge changes against the trunk commit: what the gate's delivery plan is
// computed from (ADR 0323 §1). Nil when it cannot be read; the caller keeps the pull request's own list.
func changedByMerge(ctx context.Context, run Runner, tree string, m TrunkMerge) []string {
if m.Merge == "" || m.Trunk == "" {
return nil
}
out, err := run(ctx, tree, "git", "diff", "--name-only", "--no-renames", m.Trunk, m.Merge)
if err != nil {
return nil // empty-on-error: the caller keeps the pull request's own list of changed files, the safe one
}
return strings.Fields(out)
}
// conflictWords is a head that does not merge cleanly, said as its check's failure.
func conflictWords(m TrunkMerge) string {
return fmt.Sprintf("head %s does not merge cleanly into trunk %s: in conflict %s", twelve(m.Head), twelve(m.Trunk),
strings.Join(m.Conflicts, ", "))
}
// headAt is the commit a clone is checked out at, as twelve characters; empty when it cannot be read.
func headAt(ctx context.Context, run Runner, tree string) string {
out, err := run(ctx, tree, "git", "rev-parse", "--verify", "--quiet", "HEAD^{commit}")
if err != nil {
return ""
}
return twelve(strings.TrimSpace(out))
}
// twelve is a commit as a verdict names it: its first twelve characters.
func twelve(commit string) string {
if len(commit) > 12 {
return commit[:12]
}
return commit
}
+212
View File
@@ -0,0 +1,212 @@
package builder
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
// **A merge check judges the merge, not the head** (novox/hq ADR 0323, issue 341): the head merged into the trunk
// commit in the check's own clone, never pushed; a head that does not merge cleanly fails, naming the files; the
// verdict names the head, the trunk commit and the commits of the repositories read beside it.
// aTrunkAndABranch is a repository whose main moved after the branch was cut: main holds trunkFiles over the
// common files, the branch headFiles. It answers the repository, the branch's head and main's head.
func aTrunkAndABranch(t *testing.T, common, trunkFiles, headFiles map[string]string) (string, string, string) {
t.Helper()
dir := t.TempDir()
git := func(args ...string) string {
t.Helper()
cmd := exec.Command("git", args...)
cmd.Dir = dir
cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org",
"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org")
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
return strings.TrimSpace(string(out))
}
write := func(files map[string]string) {
for name, body := range files {
if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil {
t.Fatal(err)
}
}
git("add", "-A")
}
git("init", "--quiet", "-b", "main")
write(common)
git("commit", "--quiet", "-m", "common")
git("checkout", "--quiet", "-b", "change")
write(headFiles)
git("commit", "--quiet", "-m", "the change")
head := git("rev-parse", "HEAD")
git("checkout", "--quiet", "main")
write(trunkFiles)
git("commit", "--quiet", "-m", "the trunk moved")
return dir, head, git("rev-parse", "HEAD")
}
func TestACheckJudgesTheHeadMergedIntoTheTrunk(t *testing.T) {
// main's script passes only in a tree holding both the trunk's file and the head's: the merge.
repo, head, trunk := aTrunkAndABranch(t,
map[string]string{CheckScript: "test -f trunk-only && test -f head-only || { echo 'FAIL: not the merge'; exit 1; }\n"},
map[string]string{"trunk-only": "t"}, map[string]string{"head-only": "h"})
beside, besideHead := aCheckedRepository(t, map[string]string{"README": "the catalogue"})
registry := aFactsRegistry(t)
aRuntimeOnThisMachine(t)
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-341", Repository: repo, Ref: head, Owner: "novox",
Repo: "mesh-lab", Number: 1, Base: "main", Toolchain: "golang",
Beside: map[string]Beside{"mesh-catalog": {Repository: beside, Ref: besideHead}}}, t.TempDir(), registry,
GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Repo == nil || v.Repo.Verdict != "pass" {
t.Fatalf("the check did not judge the head merged into the trunk: %+v\n%s", v.Repo, v.Report)
}
if v.Head != head[:12] || v.Trunk != trunk[:12] {
t.Fatalf("the verdict names head %q and trunk %q, not %s and %s", v.Head, v.Trunk, head[:12], trunk[:12])
}
if v.Beside["mesh-catalog"] != besideHead[:12] {
t.Fatalf("the verdict names the catalogue read beside it as %q, not %s: %v", v.Beside["mesh-catalog"],
besideHead[:12], v.Beside)
}
if v.Merge == "" || v.Merge == head || v.Merge == trunk {
t.Fatalf("no merge of the head into the trunk was made for the check: %q", v.Merge)
}
}
func TestAHeadThatDoesNotMergeCleanlyFailsNamingTheFiles(t *testing.T) {
repo, head, trunk := aTrunkAndABranch(t,
map[string]string{"store.go": "package store\n\nconst Declared = 1\n", "other.go": "package store\n"},
map[string]string{"store.go": "package store\n\nconst Declared = 2\n"},
map[string]string{"store.go": "package store\n\nconst Declared = 3\n"})
// No artifact store and no container runtime: a head that does not merge is judged before either is needed.
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-341-conflict", Repository: repo, Ref: head,
Owner: "novox", Repo: "mesh-lab", Number: 2, Base: "main", Toolchain: "golang"}, t.TempDir(), "",
GitCredential{}, nil)
if err != nil {
t.Fatalf("a head that does not merge cleanly was not judged: %v", err)
}
want := "does not merge cleanly into trunk " + trunk[:12]
for name, l := range map[string]*Layer{"gate": v.Gate, "repository": v.Repo} {
if l == nil || l.Verdict != "fail" || !strings.Contains(l.Summary, want) || !strings.Contains(l.Summary, "store.go") {
t.Fatalf("its %s layer said %+v, not a failure saying %q and naming store.go", name, l, want)
}
}
if strings.Contains(v.Summary, "other.go") {
t.Fatalf("a file not in conflict is named: %s", v.Summary)
}
if v.Head != head[:12] || v.Trunk != trunk[:12] {
t.Fatalf("the failure names head %q and trunk %q", v.Head, v.Trunk)
}
}
// **The merge made for a check is only checked** (ADR 0323 §2): the forge never receives it.
func TestTheMergeMadeForACheckIsNeverPushed(t *testing.T) {
repo, head, trunk := aTrunkAndABranch(t, map[string]string{CheckScript: "true\n"},
map[string]string{"trunk-only": "t"}, map[string]string{"head-only": "h"})
registry := aFactsRegistry(t)
aRuntimeOnThisMachine(t)
refsBefore := gitOut(t, repo, "for-each-ref")
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-341-push", Repository: repo, Ref: head, Owner: "novox",
Repo: "mesh-lab", Number: 3, Base: "main", Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Merge == "" {
t.Fatal("the check made no merge of the head into the trunk")
}
if err := exec.Command("git", "-C", repo, "cat-file", "-e", v.Merge+"^{commit}").Run(); err == nil {
t.Fatalf("the merge %s made for the check reached the forge's repository", v.Merge)
}
if after := gitOut(t, repo, "for-each-ref"); after != refsBefore {
t.Fatalf("the check moved a ref of the forge's repository:\nbefore %s\nafter %s", refsBefore, after)
}
if got := gitOut(t, repo, "rev-parse", "main"); got != trunk {
t.Fatalf("the trunk moved to %s", got)
}
}
// A delivery group's composed check merges each member's head into its own trunk's head (ADR 0323 §6), and names
// both for each; a member that does not merge cleanly fails the composed check, naming it.
func TestAGroupsComposedCheckMergesEachMemberIntoItsTrunk(t *testing.T) {
repo, head, trunk := aTrunkAndABranch(t, map[string]string{CheckScript: "true\n"},
map[string]string{"trunk-only": "t"}, map[string]string{"head-only": "h"})
member, memberHead, memberTrunk := aTrunkAndABranch(t, map[string]string{"a": "1"},
map[string]string{"b": "trunk"}, map[string]string{"c": "head"})
registry := aFactsRegistry(t)
aRuntimeOnThisMachine(t)
v, err := Check(t.Context(), Command, CheckSpec{ID: "check-341-group", Repository: repo, Ref: head, Owner: "novox",
Repo: "mesh-controller", Number: 4, Base: "main", Toolchain: "golang", Group: []GroupHead{{Owner: "novox",
Repo: "mesh-host", Repository: member, Ref: memberHead, Base: "main"}}}, t.TempDir(), registry,
GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Head != head[:12] || v.Trunk != trunk[:12] {
t.Fatalf("the composed check names its first head %q into %q", v.Head, v.Trunk)
}
if len(v.Members) != 1 || v.Members[0].Head != memberHead[:12] || v.Members[0].Trunk != memberTrunk[:12] {
t.Fatalf("the composed check names its members %+v, not mesh-host %s into %s", v.Members, memberHead[:12],
memberTrunk[:12])
}
conflicting, conflictHead, _ := aTrunkAndABranch(t, map[string]string{"a": "1"}, map[string]string{"a": "2"},
map[string]string{"a": "3"})
v, err = Check(t.Context(), Command, CheckSpec{ID: "check-341-group-conflict", Repository: repo, Ref: head,
Owner: "novox", Repo: "mesh-controller", Number: 4, Base: "main", Toolchain: "golang", Group: []GroupHead{{
Owner: "novox", Repo: "mesh-host", Repository: conflicting, Ref: conflictHead, Base: "main"}}}, t.TempDir(),
registry, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if v.Verdict != "fail" || !strings.Contains(v.Summary, "novox/mesh-host") ||
!strings.Contains(v.Summary, "does not merge cleanly") {
t.Fatalf("a member that does not merge cleanly was answered %s: %s", v.Verdict, v.Summary)
}
}
func gitOut(t *testing.T, dir string, args ...string) string {
t.Helper()
out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput()
if err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
return strings.TrimSpace(string(out))
}
// No hook runs when a check makes its merge: not one in the clone, whatever the build machine's git configuration says.
func TestACheckMergeRunsNoHook(t *testing.T) {
repo, head, _ := aTrunkAndABranch(t, map[string]string{"a": "1"}, map[string]string{"b": "trunk"},
map[string]string{"c": "head"})
clone := filepath.Join(t.TempDir(), "clone")
if out, err := exec.Command("git", "clone", "--quiet", repo, clone).CombinedOutput(); err != nil {
t.Fatalf("%v\n%s", err, out)
}
ran := filepath.Join(t.TempDir(), "ran")
for _, hook := range []string{"pre-merge-commit", "commit-msg", "prepare-commit-msg", "post-merge"} {
if err := os.WriteFile(filepath.Join(clone, ".git", "hooks", hook),
[]byte("#!/bin/sh\necho "+hook+" >> "+ran+"\n"), 0o755); err != nil {
t.Fatal(err)
}
}
m, err := mergeIntoTrunk(t.Context(), Command, clone, "main", head)
if err != nil || m.Merge == "" {
t.Fatalf("the merge was not made: %+v, %v", m, err)
}
if said, err := os.ReadFile(ran); err == nil {
t.Fatalf("a hook ran when the check made its merge: %s", said)
}
}
+139
View File
@@ -0,0 +1,139 @@
package builder
import (
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
// TestReplay341 is the replay of novox/hq issue 341, written with only what the build agent had before its fix,
// so mesh-lab's prover can lay it over the commit before. On 2026-10-09 node-engine #64/#65 merged at 00:15 and
// added Applied.Declared, under the JSON key `declared`, to the store record. #66, checked at 00:49 at its head,
// had added its own Applied.Declared under the same key on other lines. Its check passed; git merged the two
// edits without a conflict, and the trunk no longer compiled: "Declared redeclared". Before the fix the check
// judged the head alone and passed. After it (novox/hq ADR 0323), the check judges the head merged into the
// trunk commit that held the other field, and fails saying the trunk does not compile.
//
// It runs the check whole, with a container runtime that runs each container's command on this machine
// (aRuntimeOnThisMachine): the repository's own check is `go build ./...`.
func TestReplay341(t *testing.T) {
origin, head := theNinthOfOctober(t)
registry := aFactsRegistry(t)
aRuntimeOnThisMachine(t)
v, err := Check(t.Context(), Command, CheckSpec{ID: "replay-341", Repository: origin, Ref: head, Owner: "novox",
Repo: "mesh-host", Number: 66, Base: "main", Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil)
if err != nil {
t.Fatalf("the check could not run: %v", err)
}
if v.Repo == nil || v.Repo.Verdict != "fail" {
t.Fatalf("node-engine #66 at its head was answered %+v: the merge it makes into the trunk that holds #65's "+
"field does not compile, and the check let it merge", v.Repo)
}
if said := v.Repo.Summary + "\n" + v.Repo.Failed + "\n" + v.Report; !strings.Contains(said, "Declared redeclared") {
t.Fatalf("the check failed without saying the trunk does not compile:\n%s", said)
}
}
// theNinthOfOctober is the node-engine's repository as it stood at 00:49 on 2026-10-09, in miniature: a trunk
// holding #65's Applied.Declared (a string, key `declared`), and #66's head, branched before it, holding its own
// Applied.Declared (a *FoundUnit, the same key) on other lines. Each compiles alone; their merge does not.
func theNinthOfOctober(t *testing.T) (string, string) {
t.Helper()
origin := t.TempDir()
git := func(args ...string) string {
t.Helper()
cmd := exec.Command("git", args...)
cmd.Dir = origin
cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org",
"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org")
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
return strings.TrimSpace(string(out))
}
write := func(name, body string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(filepath.Join(origin, name)), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(origin, name), []byte(body), 0o755); err != nil {
t.Fatal(err)
}
}
store := func(declared65, declared66 string) string {
return "package store\n\n// FoundUnit is a unit found on the machine.\ntype FoundUnit struct{ Name string }\n\n" +
"// Applied is what the node-engine applied.\ntype Applied struct {\n\tGeneration int `json:\"generation\"`\n" +
declared66 + "\tModules []string `json:\"modules\"`\n\tFiles []string `json:\"files\"`\n" +
"\tUnits []string `json:\"units\"`\n\tAt string `json:\"at\"`\n" + declared65 + "}\n"
}
git("init", "--quiet", "-b", "main")
write("go.mod", "module example.org/mesh-host\n\ngo 1.21\n")
write("internal/store/store.go", store("", ""))
write(CheckScript, "# mesh-check-toolchain: go\nset -e\nGOFLAGS= GOWORK=off GOTOOLCHAIN=local GOPROXY=off go build ./...\n")
git("add", "-A")
git("commit", "--quiet", "-m", "#63: the store record")
git("checkout", "--quiet", "-b", "adr-0269")
write("internal/store/store.go", store("", "\tDeclared *FoundUnit `json:\"declared\"`\n"))
git("commit", "--quiet", "-am", "#66: the unit a declaration names")
head := git("rev-parse", "HEAD")
git("checkout", "--quiet", "main")
write("internal/store/store.go", store("\tDeclared string `json:\"declared\"`\n", ""))
git("commit", "--quiet", "-am", "#65: the declaration applied")
return origin, head
}
// aRuntimeOnThisMachine puts a container runtime on PATH that runs a container's command on this machine, in the
// directory and with the environment the check gives it: a store and a bus that answer, a judge that is never
// built, and the repository's own check run as written. Only for a check whose code is the test's own.
func aRuntimeOnThisMachine(t *testing.T) {
t.Helper()
bus, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { bus.Close() })
go func() {
for {
c, err := bus.Accept()
if err != nil {
return
}
c.Close()
}
}()
dir := t.TempDir()
runtime := `#!/bin/sh
cmd="$1"; shift
case "$cmd" in
ps|rm|exec) exit 0 ;;
port) echo "` + bus.Addr().String() + `"; exit 0 ;;
run)
workdir=""; detached=""
while [ $# -gt 0 ]; do
case "$1" in
-d) detached=1; shift ;;
--rm) shift ;;
--workdir) workdir="$2"; shift 2 ;;
--env) export "$2"; shift 2 ;;
--network|--volume|--user|--label|--name|-p|-e) shift 2 ;;
*) break ;;
esac
done
if [ -n "$detached" ]; then echo replay-container; exit 0; fi
shift
if [ "$1" = "sh" ] && [ -n "$workdir" ]; then cd "$workdir" && exec "$@"; fi
echo "no such image on this machine" >&2
exit 1 ;;
esac
exit 1
`
if err := os.WriteFile(filepath.Join(dir, "docker"), []byte(runtime), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
}
Binary file not shown.
+19 -40
View File
@@ -14,13 +14,13 @@ import (
"io"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/therootcompany/xz"
)
// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316
@@ -183,7 +183,6 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
buffered := bufio.NewReader(f)
magic, _ := buffered.Peek(6)
var stream io.Reader
var wait func() error
switch {
case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}):
zipped, err := gzip.NewReader(buffered)
@@ -195,53 +194,33 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
case bytes.HasPrefix(magic, []byte("BZh")):
stream = bzip2.NewReader(buffered)
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
// Go's standard library reads no xz; the build seat's image carries xz (cmd/mesh-builder/Dockerfile).
if _, err := exec.LookPath("xz"); err != nil {
return errors.New("it is compressed with xz, and this build seat has no xz to read it")
// Read in Go (github.com/therootcompany/xz, a port of xz-embedded), never by a tool the build seat's image
// may not carry: the seat's image is the catalogue's build-agent recipe, which installs git and docker-cli
// alone, and a release pinned as .tar.xz failed there on 2026-10-12 (hq issue 345). xzDictMax is a real
// maximum: a block whose LZMA2 dictionary is larger is refused before its dictionary is allocated
// (ErrMemlimit). The reader keeps a filter decoder for every block of a stream, so the file's framing is
// walked first and a file of more blocks than xzBlocksMax is refused before anything is decoded
// (xz_framing.go). What the stream unpacks to is bounded by untar (maxUpstream, maxEntries).
if err := xzWithinLimits(file); err != nil {
return err
}
cmd := exec.CommandContext(ctx, "xz", "-dc")
cmd.Stdin = buffered
var stderr bytes.Buffer
cmd.Stderr = &stderr
out, err := cmd.StdoutPipe()
zipped, err := xz.NewReader(buffered, xzDictMax)
if err != nil {
return err
return fmt.Errorf("it is not an xz stream this build seat reads (a dictionary of %d MiB at most): %v",
xzDictMax>>20, err)
}
if err := cmd.Start(); err != nil {
return err
}
stream = out
wait = func() error {
if err := cmd.Wait(); err != nil {
return fmt.Errorf("xz could not read it: %v %s", err, strings.TrimSpace(stderr.String()))
}
return nil
}
defer func() {
if wait != nil {
_ = cmd.Process.Kill()
_ = cmd.Wait()
}
}()
stream = zipped
default:
return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept " +
"whole by naming it in `file`")
}
if err := untar(stream, into, within); err != nil {
return err
}
if wait != nil {
// Whatever follows the tar's end is read before xz is waited for: xz blocks on a full pipe otherwise.
if _, err := io.Copy(io.Discard, stream); err != nil {
return err
}
w := wait
wait = nil
return w()
}
return nil
return untar(stream, into, within)
}
// xzDictMax is the largest xz dictionary read: xz's own presets use at most 64 MiB (-9); a stream asking for
// more is refused rather than given the memory.
const xzDictMax = 64 << 20
// untar writes a tar stream's files and directories within `within` into a directory.
func untar(stream io.Reader, into, within string) error {
reader := tar.NewReader(stream)
+84
View File
@@ -6,8 +6,10 @@ import (
"compress/gzip"
"context"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"fmt"
"hash/crc32"
"io"
"net"
"net/http"
@@ -15,6 +17,7 @@ import (
"os"
"os/exec"
"path/filepath"
"runtime"
"sort"
"strings"
"testing"
@@ -383,3 +386,84 @@ func TestARedirectOffHttpsIsNotFollowed(t *testing.T) {
t.Fatalf("%v", err)
}
}
func TestAnXzReleaseIsReadWithoutAnyXzToolOnTheBuildSeat(t *testing.T) {
// The build seat's image (the catalogue's build-agent recipe) carries git and docker-cli alone: Betterbird's
// .tar.xz failed there on 2026-10-12 with "this build seat has no xz to read it" (hq issue 345). It is
// read in Go, with nothing on PATH.
t.Setenv("PATH", "")
body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
if err != nil {
t.Fatal(err)
}
v := &vendor{files: map[string][]byte{"/r.tar.xz": body}}
base := serveVendor(t, v)
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
URL: base + "/r.tar.xz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
if err != nil {
t.Fatal(err)
}
e := entriesOf(t, packed)
if len(e) != 2 || e["betterbird"] != "#!/bin/sh\n -rwxr-xr-x" || e["defaults/p.js"] != "pref(\"a\", 1);\n -rw-r--r--" {
t.Fatalf("%v", e)
}
}
// withDictionary is an xz stream whose first block's LZMA2 dictionary property is props, its header's CRC32
// recomputed: a stream asking for a dictionary of its own choosing.
func withDictionary(t *testing.T, stream []byte, props byte) []byte {
t.Helper()
out := append([]byte{}, stream...)
const at = 12 // the stream header
size := (int(out[at]) + 1) * 4
header := out[at : at+size]
flags := header[1]
i := 2
readVarint := func() {
for header[i]&0x80 != 0 {
i++
}
i++
}
if flags&0x40 != 0 {
readVarint()
}
if flags&0x80 != 0 {
readVarint()
}
if header[i] != 0x21 || header[i+1] != 0x01 {
t.Fatalf("the fixture's first filter is not LZMA2: % x", header)
}
header[i+2] = props
binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4]))
return out
}
func TestAnXzStreamAskingForTooLargeADictionaryIsRefused(t *testing.T) {
body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
if err != nil {
t.Fatal(err)
}
// 1 GiB (props 36) and 4 GiB less one (props 40): over the 64 MiB maximum, refused before allocation.
for _, props := range []byte{36, 40} {
crafted := withDictionary(t, body, props)
v := &vendor{files: map[string][]byte{"/r.tar.xz": crafted}}
base := serveVendor(t, v)
var before, after runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&before)
_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
URL: base + "/r.tar.xz", SHA256: sumHex(crafted), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
runtime.ReadMemStats(&after)
if err == nil || !strings.Contains(err.Error(), "dictionary") {
t.Errorf("props %d: a dictionary over the maximum was not refused: %v", props, err)
}
if grew := after.TotalAlloc - before.TotalAlloc; grew > 256<<20 {
t.Errorf("props %d: %d bytes were allocated for a refused stream", props, grew)
}
}
// The fixture as it is (a small dictionary) still reads.
if crafted := withDictionary(t, body, 0); len(crafted) == 0 {
t.Fatal("no fixture")
}
}
+290
View File
@@ -0,0 +1,290 @@
package builder
import (
"bufio"
"bytes"
"encoding/binary"
"errors"
"fmt"
"hash/crc32"
"io"
"os"
)
// xzBlocksMax and xzStreamsMax are the most blocks and the most concatenated streams an xz file may hold for
// the build seat to read it.
//
// **Why a block count is limited at all.** The xz reader (github.com/therootcompany/xz v1.0.1) keeps one
// filter decoder for every block that names a delta or BCJ filter, and frees them only when a stream ends:
// its block counters are reset per stream, not per block. A stream of many tiny blocks therefore grows the
// heap with its length: an `xz --block-size=1 --delta` file of 6 MB kept about 67 MiB live, which comes to
// about 5.6 GiB at the 512 MiB download cap (review 68 on mesh-controller pull request 264, hq issue 345).
//
// **Why 1,024.** At about 340 bytes of decoder a block, 1,024 blocks keep at most about 0.35 MiB live. A
// real release is far below it: Betterbird's archive is 1 stream with 1 block, and multi-threaded xz cuts
// blocks of three times the dictionary (24 MiB at the default preset), so even 512 MiB unpacked in blocks
// of 512 KiB stays within it. Streams are limited to 16 because a release is one stream; the decoders are
// reused from one stream to the next, so the stream limit only bounds the work of this walk.
const (
xzBlocksMax = 1024
xzStreamsMax = 16
)
// xzWithinLimits walks an xz file's framing, without decompressing anything, and refuses one holding more
// blocks or streams than the build seat reads.
//
// **The framing, not the index.** Each stream's index at its end lists its blocks, but the reader compares
// the index with the blocks only after it has decoded them all, so a file whose index lies would still be
// decoded block by block. This walk follows the same framing the reader follows: stream header, each block
// header, the block's LZMA2 chunks (each chunk header carries its own compressed size, so a chunk is skipped
// without being decoded), its padding and check, then the index and the footer. It counts the blocks the
// reader would decode, and it also refuses an index whose count of blocks is not the count walked.
func xzWithinLimits(file string) error {
f, err := os.Open(file)
if err != nil {
return err
}
defer f.Close()
_, _, err = xzFraming(bufio.NewReader(f))
if errors.Is(err, errXzTooManyBlocks) || errors.Is(err, errXzTooManyStreams) {
return err
}
if err != nil {
return fmt.Errorf("it is not an xz file this build seat reads: %v", err)
}
return nil
}
var (
errXzTooManyBlocks = fmt.Errorf("it holds more than %d xz blocks, the most this build seat reads: "+
"its xz reader keeps a filter decoder for every block (hq issue 345)", xzBlocksMax)
errXzTooManyStreams = fmt.Errorf("it holds more than %d concatenated xz streams, the most this build "+
"seat reads", xzStreamsMax)
)
// xzFraming counts the streams and blocks of an xz file, checking its framing as it goes. It stops with
// errXzTooManyBlocks or errXzTooManyStreams at the first block or stream over the limit.
func xzFraming(r *bufio.Reader) (streams, blocks int, err error) {
full := func(n int) ([]byte, error) {
b := make([]byte, n)
if _, err := io.ReadFull(r, b); err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err
}
return b, nil
}
skip := func(n int) error {
if got, err := r.Discard(n); err != nil || got != n {
return io.ErrUnexpectedEOF
}
return nil
}
for {
header, err := full(12)
if err != nil {
return streams, blocks, fmt.Errorf("stream %d: header: %v", streams+1, err)
}
if !bytes.Equal(header[:6], []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}) {
return streams, blocks, fmt.Errorf("stream %d: no xz stream header", streams+1)
}
if header[6] != 0 || header[7]&0xf0 != 0 ||
crc32.ChecksumIEEE(header[6:8]) != binary.LittleEndian.Uint32(header[8:12]) {
return streams, blocks, fmt.Errorf("stream %d: a corrupt stream header", streams+1)
}
if streams++; streams > xzStreamsMax {
return streams, blocks, errXzTooManyStreams
}
checkSize := 0
if check := int(header[7]); check != 0 {
checkSize = 4 << ((check - 1) / 3)
}
inStream := 0
for {
first, err := r.ReadByte()
if err != nil {
return streams, blocks, fmt.Errorf("stream %d: no index: %v", streams, io.ErrUnexpectedEOF)
}
if first == 0 {
break // the index indicator
}
rest, err := full(int(first)*4 + 3)
if err != nil {
return streams, blocks, fmt.Errorf("stream %d, block %d: header: %v", streams, inStream+1, err)
}
if err := xzBlockHeader(append([]byte{first}, rest...)); err != nil {
return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream+1, err)
}
inStream++
if blocks++; blocks > xzBlocksMax {
return streams, blocks, errXzTooManyBlocks
}
// The block's LZMA2 chunks, up to the end marker.
size := 0
for {
control, err := r.ReadByte()
if err != nil {
return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream, io.ErrUnexpectedEOF)
}
size++
if control == 0 {
break
}
switch {
case control == 1 || control == 2: // stored: a 16-bit size less one, then the bytes
b, err := full(2)
if err != nil {
return streams, blocks, err
}
n := int(binary.BigEndian.Uint16(b)) + 1
if err := skip(n); err != nil {
return streams, blocks, err
}
size += 2 + n
case control >= 0x80: // LZMA: unpacked and packed sizes, the properties byte on a reset
b, err := full(4)
if err != nil {
return streams, blocks, err
}
n := int(binary.BigEndian.Uint16(b[2:])) + 1
size += 4
if control >= 0xc0 {
if err := skip(1); err != nil {
return streams, blocks, err
}
size++
}
if err := skip(n); err != nil {
return streams, blocks, err
}
size += n
default:
return streams, blocks, fmt.Errorf("stream %d, block %d: an LZMA2 chunk of kind %#x",
streams, inStream, control)
}
}
if pad := (4 - size%4) % 4; pad > 0 {
b, err := full(pad)
if err != nil || !bytes.Equal(b, make([]byte, pad)) {
return streams, blocks, fmt.Errorf("stream %d, block %d: corrupt block padding", streams, inStream)
}
}
if err := skip(checkSize); err != nil {
return streams, blocks, err
}
}
// The index: its count of records, two numbers each, padding, CRC32.
used := 1
count, n, err := xzVLI(r)
if err != nil {
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
}
used += n
if count != uint64(inStream) {
return streams, blocks, fmt.Errorf("stream %d: its index lists %d blocks, but it holds %d",
streams, count, inStream)
}
for i := 0; i < 2*inStream; i++ {
_, n, err := xzVLI(r)
if err != nil {
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
}
used += n
}
if err := skip((4-used%4)%4 + 4); err != nil {
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
}
footer, err := full(12)
if err != nil || footer[10] != 'Y' || footer[11] != 'Z' {
return streams, blocks, fmt.Errorf("stream %d: no stream footer", streams)
}
// Stream padding: zero bytes, a multiple of four, then the end or another stream.
zeros := 0
for {
b, err := r.ReadByte()
if err == io.EOF {
if zeros%4 != 0 {
return streams, blocks, errors.New("stream padding not a multiple of four bytes")
}
return streams, blocks, nil
}
if err != nil {
return streams, blocks, err
}
if b != 0 {
_ = r.UnreadByte()
break
}
zeros++
}
if zeros%4 != 0 {
return streams, blocks, errors.New("stream padding not a multiple of four bytes")
}
}
}
// xzBlockHeader checks one block header: its CRC32, its flags, and that its last filter is LZMA2, the only
// one whose chunks this walk can skip (and the only last filter the reader decodes).
func xzBlockHeader(h []byte) error {
size := len(h)
if crc32.ChecksumIEEE(h[:size-4]) != binary.LittleEndian.Uint32(h[size-4:]) {
return errors.New("a corrupt block header")
}
flags := h[1]
if flags&0x3c != 0 {
return errors.New("a block header with reserved flags")
}
body := bytes.NewReader(h[2 : size-4])
if flags&0x40 != 0 {
if _, _, err := xzVLI(body); err != nil {
return err
}
}
if flags&0x80 != 0 {
if _, _, err := xzVLI(body); err != nil {
return err
}
}
var last uint64
for i := 0; i <= int(flags&3); i++ {
id, _, err := xzVLI(body)
if err != nil {
return err
}
props, _, err := xzVLI(body)
if err != nil {
return err
}
if props > uint64(body.Len()) {
return errors.New("a corrupt block header")
}
if _, err := body.Seek(int64(props), io.SeekCurrent); err != nil {
return err
}
last = id
}
if last != 0x21 {
return fmt.Errorf("a block whose last filter is %#x, not LZMA2", last)
}
return nil
}
// xzVLI reads one of xz's variable-length integers (at most 9 bytes, 7 bits each), and how many bytes it took.
func xzVLI(r io.ByteReader) (uint64, int, error) {
var v uint64
for i := 0; i < 9; i++ {
b, err := r.ReadByte()
if err != nil {
return 0, i, io.ErrUnexpectedEOF
}
v |= uint64(b&0x7f) << (7 * i)
if b&0x80 == 0 {
if i > 0 && b == 0 {
return 0, i + 1, errors.New("a number not in its shortest form")
}
return v, i + 1, nil
}
}
return 0, 9, errors.New("a number longer than nine bytes")
}
+207
View File
@@ -0,0 +1,207 @@
package builder
import (
"archive/tar"
"bufio"
"bytes"
"context"
"encoding/binary"
"errors"
"hash/crc32"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
)
// xzStream is one xz stream (no integrity check) of data cut into blocks of blockSize bytes, each block's
// LZMA2 data stored uncompressed, each block a delta filter before LZMA2 when delta is set: the shape of
// `xz --block-size=N --filters=delta,lzma2`, built without any xz tool. listed is how many blocks its index
// lists (-1: as many as it holds), so a stream whose index lies can be built too.
func xzStream(data []byte, blockSize int, delta bool, listed int) []byte {
var out bytes.Buffer
flags := []byte{0, 0}
out.Write([]byte{0xfd, '7', 'z', 'X', 'Z', 0x00})
out.Write(flags)
out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(flags)))
type record struct{ unpadded, uncompressed uint64 }
var records []record
for at := 0; at < len(data); at += blockSize {
block := data[at:min(at+blockSize, len(data))]
if delta {
// Delta-encoded with a distance of one; the decoder's history starts empty at each block.
encoded := make([]byte, len(block))
for i := range block {
encoded[i] = block[i]
if i > 0 {
encoded[i] -= block[i-1]
}
}
block = encoded
}
header := []byte{2, 0} // 12 bytes
if delta {
header[1] = 1 // two filters
header = append(header, 0x03, 0x01, 0x00)
}
header = append(header, 0x21, 0x01, 0x00) // LZMA2, a 4 KiB dictionary
for len(header) < 8 {
header = append(header, 0)
}
header = binary.LittleEndian.AppendUint32(header, crc32.ChecksumIEEE(header))
out.Write(header)
size := 0
for i, chunk := 0, block; len(chunk) > 0; i++ {
n := min(len(chunk), 1<<16)
control := byte(2)
if i == 0 {
control = 1 // the first chunk of a block resets the dictionary
}
out.WriteByte(control)
out.Write(binary.BigEndian.AppendUint16(nil, uint16(n-1)))
out.Write(chunk[:n])
size += 3 + n
chunk = chunk[n:]
}
out.WriteByte(0)
size++
out.Write(make([]byte, (4-size%4)%4))
records = append(records, record{uint64(len(header) + size), uint64(len(block))})
}
if listed >= 0 {
for len(records) < listed {
records = append(records, records[0])
}
records = records[:listed]
}
vli := func(b []byte, v uint64) []byte {
for v >= 0x80 {
b = append(b, byte(v)|0x80)
v >>= 7
}
return append(b, byte(v))
}
index := vli([]byte{0}, uint64(len(records)))
for _, r := range records {
index = vli(vli(index, r.unpadded), r.uncompressed)
}
for len(index)%4 != 0 {
index = append(index, 0)
}
index = binary.LittleEndian.AppendUint32(index, crc32.ChecksumIEEE(index))
out.Write(index)
footer := binary.LittleEndian.AppendUint32(nil, uint32(len(index)/4-1))
footer = append(footer, flags...)
out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(footer)))
out.Write(footer)
out.WriteString("YZ")
return out.Bytes()
}
// tarHolding is a tar holding betterbird/data with body.
func tarHolding(t *testing.T, body []byte) []byte {
t.Helper()
var out bytes.Buffer
w := tar.NewWriter(&out)
if err := w.WriteHeader(&tar.Header{Name: "betterbird/data", Mode: 0o644, Size: int64(len(body)),
Typeflag: tar.TypeReg}); err != nil {
t.Fatal(err)
}
w.Write(body)
if err := w.Close(); err != nil {
t.Fatal(err)
}
return out.Bytes()
}
func writeTemp(t *testing.T, body []byte) string {
t.Helper()
file := filepath.Join(t.TempDir(), "r.tar.xz")
if err := os.WriteFile(file, body, 0o644); err != nil {
t.Fatal(err)
}
return file
}
func TestAnXzFileOfAFewDeltaBlocksAndTwoStreamsIsRead(t *testing.T) {
// The streams this test builds are ones the reader decodes, so what the walk refuses below is the
// same framing the reader follows.
body := bytes.Repeat([]byte("betterbird "), 300)
whole := tarHolding(t, body)
half := len(whole) / 2
for name, file := range map[string][]byte{
"one stream of delta blocks": xzStream(whole, 512, true, -1),
"two streams with padding": append(append(xzStream(whole[:half], 700, true, -1), 0, 0, 0, 0),
xzStream(whole[half:], 100, false, -1)...),
} {
path := writeTemp(t, file)
if xzPath, err := exec.LookPath("xz"); err == nil {
if out, err := exec.Command(xzPath, "-t", path).CombinedOutput(); err != nil {
t.Fatalf("%s: xz itself refuses the built stream: %v %s", name, err, out)
}
}
into := t.TempDir()
if err := unpackUpstream(context.Background(), path, into, "betterbird"); err != nil {
t.Fatalf("%s: %v", name, err)
}
got, err := os.ReadFile(filepath.Join(into, "data"))
if err != nil || !bytes.Equal(got, body) {
t.Fatalf("%s: unpacked %d bytes, want %d: %v", name, len(got), len(body), err)
}
}
}
func TestAnXzFileOfManyDeltaBlocksIsRefusedBeforeItIsDecoded(t *testing.T) {
// Review 68 on pull request 264: `xz --block-size=1 --delta` of a 200 kB tar kept about 67 MiB live in
// the reader, one delta decoder per block. Its index here lists one block, as a file crafted to pass a
// check of the index alone would; the walk counts the blocks the reader would decode.
body := make([]byte, 200_000)
for i := range body {
body[i] = byte(i * 7)
}
crafted := xzStream(tarHolding(t, body), 1, true, 1)
path := writeTemp(t, crafted)
var before, after runtime.MemStats
runtime.GC()
runtime.ReadMemStats(&before)
err := unpackUpstream(context.Background(), path, t.TempDir(), "betterbird")
runtime.ReadMemStats(&after)
if err == nil || !strings.Contains(err.Error(), "more than 1024 xz blocks") {
t.Fatalf("a file of %d bytes in blocks of one byte was not refused for its blocks: %v", len(crafted), err)
}
if grew := after.TotalAlloc - before.TotalAlloc; grew > 4<<20 {
t.Fatalf("%d bytes were allocated for a refused file", grew)
}
}
func TestTheXzBlockLimitIsExact(t *testing.T) {
data := make([]byte, xzBlocksMax+1)
streams, blocks, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data[:xzBlocksMax], 1, true, -1))))
if err != nil || streams != 1 || blocks != xzBlocksMax {
t.Fatalf("%d blocks: %d streams, %d blocks, %v", xzBlocksMax, streams, blocks, err)
}
if _, _, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data, 1, true, -1)))); !errors.Is(err, errXzTooManyBlocks) {
t.Fatalf("%d blocks: %v", xzBlocksMax+1, err)
}
}
func TestAnXzFileOfTooManyStreamsIsRefused(t *testing.T) {
var file []byte
for i := 0; i <= xzStreamsMax; i++ {
file = append(file, xzStream([]byte("x"), 1, false, -1)...)
}
err := unpackUpstream(context.Background(), writeTemp(t, file), t.TempDir(), "")
if err == nil || !strings.Contains(err.Error(), "more than 16 concatenated xz streams") {
t.Fatalf("%v", err)
}
}
func TestAnXzIndexThatDoesNotListItsBlocksIsRefused(t *testing.T) {
file := xzStream(tarHolding(t, []byte("x")), 512, false, 1)
err := unpackUpstream(context.Background(), writeTemp(t, file), t.TempDir(), "betterbird")
if err == nil || !strings.Contains(err.Error(), "its index lists 1 blocks, but it holds 4") {
t.Fatalf("%v", err)
}
}
+9 -1
View File
@@ -645,6 +645,11 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
sealed = ForeseenSealed
*foreseen = append(*foreseen, m.Module+"/"+name)
}
if sealed == "" && m.OwnSecrets[name].IssuedBy == IssuedOutside {
// **Not given, and the mesh never makes it** (novox/hq issue 401, ADR 0228): no file, as for a
// family's member (ADR 0283), so the module reads its absence as not given.
continue
}
if sealed == "" {
// Declared and not made. Refused rather than skipped: a module whose own
// credential is silently absent starts, fails to authenticate, and the reason is
@@ -1002,7 +1007,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
// content is where a setting lands, so a placeholder may only exist once the setting
// has been put in — filling secrets first would look at content that is not yet what
// the machine receives.
if err := intoFile(copied, sealed, m.Module); err != nil {
if err := intoFile(copied, sealed, outsideNotGiven(m, with.Needed[m.Module]), m.Module); err != nil {
return nil, err
}
// **Substituted here, not on the machine.** A bound value is not secret — the mesh
@@ -2431,6 +2436,9 @@ func secretsReadBy(resource map[string]any, m Manifest, given map[string]string)
}
paths := map[string]string{}
for name, own := range m.OwnSecrets.Plain() {
if own.IssuedBy == IssuedOutside && given[name] == "" {
continue // not given, so not placed: nothing to restart on (novox/hq issue 401)
}
paths[name] = own.Path
}
for name, sealed := range given {
+1
View File
@@ -1859,6 +1859,7 @@ func ParseManifest(raw []byte) (Manifest, error) {
problems = append(problems, invokeProblems(m)...)
problems = append(problems, replacesProblems(m)...)
problems = append(problems, UnitSettingProblems(m)...)
problems = append(problems, OwnerSettingProblems(m)...)
problems = append(problems, endpointNameProblems(m)...)
problems = append(problems, RouteProblems(m)...)
for _, port := range m.Guards {
@@ -0,0 +1,106 @@
package catalogue
import (
"strings"
"testing"
)
// An own secret issued outside the mesh that nobody gave (novox/hq issue 401, ADR 0228): the mesh makes no value for it,
// so the composition places no file, a file of the module that names it says empty there, and nothing restarts on it.
// Given, it is placed and filled like any own secret.
func outsideManifest(t *testing.T) Manifest {
t.Helper()
m, err := ParseManifest([]byte(`{"module":"assistant","version":"1",
"own-secrets":{
"password":{"path":"/var/lib/assistant/password.secret","taken":"at-start"},
"vendor-key":{"path":"/var/lib/assistant/vendor-key.secret","taken":"at-start","issued-by":"outside"}},
"resources":[
{"id":"env","type":"file","path":"/var/lib/assistant/env","mode":"0600",
"content":"PASSWORD=${secret:password}\nVENDOR_KEY=${secret:vendor-key}\n"},
{"id":"server","type":"container","name":"assistant","image":"assistant:1",
"env-file":["/var/lib/assistant/env"],
"volumes":["/var/lib/assistant/vendor-key.secret:/run/vendor-key:ro"],
"secrets-in-environment":"reads its settings from the environment only"}]}`))
if err != nil {
t.Fatal(err)
}
return m
}
func resourceAt(out []map[string]any, path string) map[string]any {
for _, r := range out {
if r["path"] == path {
return r
}
}
return nil
}
func resourceNamed(out []map[string]any, name string) map[string]any {
for _, r := range out {
if r["name"] == name {
return r
}
}
return nil
}
func TestAnOutsideSecretNotGivenIsNoFileAndAFileNamingItSaysEmpty(t *testing.T) {
r := Resolution{Node: "server", Modules: []Manifest{outsideManifest(t)}}
out, err := r.Declaration(Rendering{Needed: map[string]map[string]string{"assistant": {"password": "sealed-password"}}})
if err != nil {
t.Fatalf("a module whose outside secret nobody gave did not compose: %v", err)
}
if got := resourceAt(out, "/var/lib/assistant/vendor-key.secret"); got != nil {
t.Fatalf("a secret nobody gave was placed: %v", got)
}
env := resourceAt(out, "/var/lib/assistant/env")
if env == nil {
t.Fatal("the module's own file is not composed")
}
if content := env["content"].(string); content != "PASSWORD=${secret:password}\nVENDOR_KEY=\n" {
t.Fatalf("the file names the secret nobody gave as %q", content)
}
secrets, _ := env["secrets"].(map[string]any)
if len(secrets) != 1 || secrets["password"] != "sealed-password" {
t.Fatalf("the file is filled from %v", secrets)
}
server := resourceNamed(out, "assistant")
if on := strings.Join(stringsIn(server["restart-on"]), " "); strings.Contains(on, "vendor-key") {
t.Fatalf("the container restarts on a secret nobody gave: %q", on)
}
}
func TestAnOutsideSecretGivenIsPlacedAndFilled(t *testing.T) {
r := Resolution{Node: "server", Modules: []Manifest{outsideManifest(t)}}
out, err := r.Declaration(Rendering{Needed: map[string]map[string]string{"assistant": {
"password": "sealed-password", "vendor-key": "sealed-vendor-key"}}})
if err != nil {
t.Fatal(err)
}
placed := resourceAt(out, "/var/lib/assistant/vendor-key.secret")
if placed == nil || placed["sealed"] != "sealed-vendor-key" {
t.Fatalf("the secret given is placed as %v", placed)
}
env := resourceAt(out, "/var/lib/assistant/env")
if content := env["content"].(string); content != "PASSWORD=${secret:password}\nVENDOR_KEY=${secret:vendor-key}\n" {
t.Fatalf("the file was changed: %q", content)
}
if secrets, _ := env["secrets"].(map[string]any); secrets["vendor-key"] != "sealed-vendor-key" {
t.Fatalf("the file is filled from %v", secrets)
}
server := resourceNamed(out, "assistant")
if on := strings.Join(stringsIn(server["restart-on"]), " "); !strings.Contains(on, "assistant."+NeedID("vendor-key")) {
t.Fatalf("the container does not restart on the secret given: %q", on)
}
}
// A secret the mesh makes is still refused when it was not made: only an outside one is left out.
func TestAnOwnSecretTheMeshMakesIsStillRefusedWhenNotMade(t *testing.T) {
r := Resolution{Node: "server", Modules: []Manifest{outsideManifest(t)}}
_, err := r.Declaration(Rendering{})
if nm, ok := err.(*NotMadeError); !ok || nm.Name != "password" {
t.Fatalf("a secret the mesh makes, not made: %v", err)
}
}
+25 -2
View File
@@ -156,7 +156,11 @@ func secretFilesOf(resources []map[string]any) map[string]bool {
// A name the module never declared is refused here rather than on the machine. The host would
// refuse it too — but it would do so having already been handed a declaration, which reads as the
// mesh sending something broken, and the name it could not find is a manifest's typo.
func intoFile(resource map[string]any, sealed map[string]string, module string) error {
//
// **An own secret issued outside the mesh that nobody gave is filled with nothing** (novox/hq issue 401): the mesh
// never makes one (ADR 0228), and a file is where the module reads it, so the place says empty — the module sees it
// was not given, where a value the mesh made looked given and wrong. notGiven names those secrets.
func intoFile(resource map[string]any, sealed map[string]string, notGiven map[string]bool, module string) error {
if fmt.Sprint(resource["type"]) != "file" {
return nil
}
@@ -171,6 +175,10 @@ func intoFile(resource map[string]any, sealed map[string]string, module string)
into := map[string]any{}
for _, name := range used {
value := sealed[name]
if value == "" && notGiven[name] {
content = strings.ReplaceAll(content, "${secret:"+name+"}", "")
continue
}
if value == "" {
return fmt.Errorf(
"%s has a file that says ${secret:%s}, and %s has no secret of its own by that "+
@@ -179,10 +187,25 @@ func intoFile(resource map[string]any, sealed map[string]string, module string)
}
into[name] = value
}
resource["secrets"] = into
resource["content"] = content
if len(into) > 0 {
resource["secrets"] = into
}
return nil
}
// outsideNotGiven is each own secret of a module issued outside the mesh that this composition holds no value for
// (novox/hq issue 401): what intoFile fills with nothing, and what nothing restarts on.
func outsideNotGiven(m Manifest, given map[string]string) map[string]bool {
out := map[string]bool{}
for name, own := range m.OwnSecrets.Plain() {
if own.IssuedBy == IssuedOutside && given[name] == "" {
out[name] = true
}
}
return out
}
// namesOr says what a module could have written, because the answer to "that name is wrong" is
// almost always one of two or three right ones.
func namesOr(sealed map[string]string) string {
+181
View File
@@ -0,0 +1,181 @@
package catalogue
import (
"strings"
"testing"
)
// A declared directory may be owned by the account its module runs as (novox/hq issue 279): a media
// manager runs as the puid and pgid its assignment sets per node, and a folder it must write was owned
// by the manifest's literal, so the app refused it on a machine whose identity was not the image's.
func recycler() Manifest {
return Manifest{Module: "sonarr",
Resources: []map[string]any{
{"id": "recycle-bin", "type": "directory", "path": "/srv/sonarr/recycle", "mode": "0755",
"owner": "${setting:puid}:${setting:pgid}"},
{"id": "identity", "type": "file", "path": "/srv/sonarr/identity.env", "mode": "0644",
"content": "PUID=${setting:puid}\nPGID=${setting:pgid}\n"},
},
}
}
func copied(r map[string]any) map[string]any {
out := map[string]any{}
for k, v := range r {
out[k] = v
}
return out
}
// The node's identity reaches the directory's owner, the node's layer over the mesh's, a number given
// as a string or as a JSON number alike.
func TestASettingOwnsADirectory(t *testing.T) {
m := recycler()
layers := []Layer{
{From: "the mesh", Values: map[string]any{"puid": "1000", "pgid": "1000"}},
{From: "this node", Values: map[string]any{"puid": float64(1001), "pgid": "2000"}},
}
r := copied(m.Resources[0])
if err := settingInto(r, layers, m.Module); err != nil {
t.Fatal(err)
}
if r["owner"] != "1001:2000" {
t.Fatalf("owned by %q, not 1001:2000", r["owner"])
}
user := copied(m.Resources[0])
user["owner"] = "${setting:puid}"
if err := settingInto(user, layers, m.Module); err != nil || user["owner"] != "1001" {
t.Fatalf("user alone: %v, %v", user["owner"], err)
}
}
// Composed for a machine, the way a push writes it: the directory reaches the node-engine owned by the
// number the node's settings name, never by the placeholder.
func TestAComposedMachineGetsTheOwnerTheSettingNames(t *testing.T) {
r := anAdoptedAnchor()
r.Modules = append(r.Modules, recycler())
with := anchorRendering(false)
with.Settings["sonarr"] = []Layer{{From: "anchor", Values: map[string]any{"puid": "1001", "pgid": "2000"}}}
composed, err := r.Compose(with)
if err != nil {
t.Fatal(err)
}
if why, left := composed.LeftOut["sonarr"]; left {
t.Fatalf("left out: %s", why)
}
got := byID(composed.Resources)["sonarr.recycle-bin"]
if got["owner"] != "1001:2000" {
t.Fatalf("composed owned by %v, not 1001:2000", got["owner"])
}
}
// Only a whole number owns a directory: an account's name, text, a sign, a fraction or a value past the
// kernel's ids is refused by name, and the directory is left as it was. So is a key nothing sets.
func TestAnOwnerSettingThatIsNoNumberIsRefused(t *testing.T) {
m := recycler()
for _, bad := range []any{"abc", "root", "", "-1", "+5", "1000:1000", "10 00", "1000\nx", float64(1.5),
float64(-3), "4294967295", "99999999999", true} {
r := copied(m.Resources[0])
err := settingInto(r, []Layer{{From: "ace", Values: map[string]any{"puid": bad, "pgid": "1000"}}}, m.Module)
if err == nil || !strings.Contains(err.Error(), `"puid"`) || !strings.Contains(err.Error(), "whole number") {
t.Errorf("%v: %v", bad, err)
}
if r["owner"] != m.Resources[0]["owner"] {
t.Errorf("%v: the owner was changed on refusal: %v", bad, r["owner"])
}
}
r := copied(m.Resources[0])
if err := settingInto(r, nil, m.Module); err == nil || !strings.Contains(err.Error(), "${setting:puid}") {
t.Errorf("nothing set: %v", err)
}
if err := JudgeSettings(m, []Layer{{From: "ace", Values: map[string]any{"puid": "media", "pgid": "1000"}}}, false); err == nil ||
!strings.Contains(err.Error(), "whole number") {
t.Errorf("a stored name is not refused when judged: %v", err)
}
}
// Where the manifest is read: an owner holding a setting is a whole placeholder or a whole number on
// each side of the colon, and every key is one the module asks for elsewhere or declares a default for.
func TestAnOwnerSettingIsJudgedWhereTheManifestIsRead(t *testing.T) {
ok := []string{"${setting:puid}:${setting:pgid}", "${setting:puid}", "${setting:puid}:1000", "1000:${setting:pgid}",
"1000:1000", "media:media"}
bad := []string{
"${setting:puid}x",
"x${setting:puid}",
"${setting:puid}:${setting:pgid}:1",
"${setting:puid}${setting:pgid}",
"media:${setting:pgid}",
"${setting:puid}:",
"${setting:other}:${setting:pgid}",
}
for _, owner := range append(ok, bad...) {
m := recycler()
m.Resources[0] = copied(m.Resources[0])
m.Resources[0]["owner"] = owner
err := parsed(t, m)
refused := err != nil && strings.Contains(err.Error(), "owner")
want := false
for _, b := range bad {
want = want || b == owner
}
if refused != want {
t.Errorf("%s: refused %v, want %v (%v)", owner, refused, want, err)
}
}
// A key with a default is declared, though nothing else asks for it.
m := recycler()
m.Resources = m.Resources[:1]
m.Resources[0] = copied(m.Resources[0])
m.Resources[0]["owner"] = "${setting:workers}"
m.Settings = map[string]SettingDeclaration{"workers": {Kind: KindPreference, Default: float64(4), Why: "a test"}}
if err := parsed(t, m); err != nil && strings.Contains(err.Error(), "owner") {
t.Errorf("a declared key was refused: %v", err)
}
// A file's owner takes no setting: the sweep says so.
f := recycler()
f.Resources[1] = copied(f.Resources[1])
f.Resources[1]["owner"] = "${setting:puid}"
if err := parsed(t, f); err == nil || !strings.Contains(err.Error(), "not filled in this field") {
t.Errorf("a file's owner setting passed: %v", err)
}
}
// A key a directory's owner asks for is a destination, so setting it is not called stray.
func TestASettingAnOwnerAsksForIsNotStray(t *testing.T) {
m := recycler()
m.Resources = m.Resources[:1]
stray := strings.Join(UnusedSettings(m, []Layer{{From: "ace", Values: map[string]any{"puid": "1001", "pgid": "2000"}}}), "; ")
if strings.Contains(stray, "puid") || strings.Contains(stray, "pgid") {
t.Errorf("called stray: %s", stray)
}
}
// An owner is written as the number parsed: "0001" and "00" reach the node-engine as 1 and 0, never as text.
// Uid 0 is allowed, because a key an owner asks for is the terminal's alone.
func TestAnOwnerSettingIsWrittenAsTheNumber(t *testing.T) {
m := recycler()
r := copied(m.Resources[0])
if err := settingInto(r, []Layer{{From: "ace", Values: map[string]any{"puid": "0001", "pgid": "00"}}}, m.Module); err != nil {
t.Fatal(err)
}
if r["owner"] != "1:0" {
t.Fatalf("owned by %q, not 1:0", r["owner"])
}
}
// Every key a directory's owner asks for is the controller's terminal's alone (novox/hq issue 279, issue 339):
// root chowns the folder to it, so it is terminal-only whatever else asks for it — a file nothing trusts, a
// default, or nothing at all.
func TestAnOwnersKeyIsTheTerminals(t *testing.T) {
m := recycler()
m.Resources[1] = copied(m.Resources[1])
m.Resources[1]["trusted"] = false
m.Settings = map[string]SettingDeclaration{"pgid": {Kind: KindPreference, Default: "1000", Why: "a test"}}
keys := strings.Join(TerminalKeys(m), ",")
for _, k := range []string{"puid", "pgid"} {
if !strings.Contains(","+keys+",", ","+k+",") {
t.Errorf("%s is not the terminal's: %s", k, keys)
}
}
}
+125 -3
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
)
@@ -15,7 +16,8 @@ import (
// (novox/hq issues 122, 134). ADR 0112 names the operator as one of the four providers; this is the
// operator answering.
//
// `${setting:<key>}` in a file's content, and in a service's unit name, is filled from the module's settings layers — the mesh's,
// `${setting:<key>}` in a file's content, in a service's unit name and in a directory's owner (a uid or
// gid only, issue 279) is filled from the module's settings layers — the mesh's,
// then this node's — the same layers a mergeable JSON file and a contribution already take, so
// `settings set <module>` is the one place a person's values go. **Refused when no layer sets it**,
// naming the key and the remedy: a definition that carried a default for a mail domain would be
@@ -59,6 +61,9 @@ func settingInto(resource map[string]any, layers []Layer, module string) error {
if fmt.Sprint(resource["type"]) == "service" {
return settingIntoUnit(resource, layers, module)
}
if fmt.Sprint(resource["type"]) == "directory" {
return settingIntoOwner(resource, layers, module)
}
if fmt.Sprint(resource["type"]) != "file" {
return nil
}
@@ -143,6 +148,107 @@ func settingIntoUnit(resource map[string]any, layers []Layer, module string) err
return nil
}
// A declared directory may be owned by the account its module runs as (novox/hq issue 279).
//
// A containerised module runs as the identity its assignment sets per node — its puid and pgid
// settings, written into the environment file its image reads (ADR 0112, issue 153) — and a directory
// its app must write, beyond the one its image re-owns at start, was owned by whatever the manifest
// spelt: right on a machine whose identity matched the image's default, refused by the app on one whose
// did not. So a directory's `owner` may name the settings that identity is: `${setting:<key>}` for the
// user alone, `${setting:<user>}:${setting:<group>}` for both, filled from the same layers at
// composition.
//
// **Whole numbers only.** An owner a setting fills is a uid or a gid — never an account's name, never
// text — so no value can put anything into the ownership the node-engine applies but a number; one that
// is not is refused at composition, naming the key, and the directory is left as it was. The shape is
// judged where the manifest is read (OwnerSettingProblems): each side of the colon a whole placeholder
// or a whole number, and every key one the module itself asks for elsewhere — in a file, a unit, a
// contribution or a served fact — or declares a default for, so the owner follows the identity its
// process is told it runs as, never a key that reaches nothing else.
// ownerSide is one side of an owner a setting may fill: a whole placeholder or a whole number.
var ownerSide = `(?:\$\{setting:[a-z0-9][a-z0-9_.-]*\}|[0-9]+)`
// ownerWithSetting is the only shape an owner holding a setting may have: user, or user:group.
var ownerWithSetting = regexp.MustCompile(`^` + ownerSide + `(?::` + ownerSide + `)?$`)
// wholeID is a value as a uid or a gid, and whether it is one: a whole number below the kernel's "no id"
// (2^32-1). Uid 0 is one: a key an owner asks for is the controller's terminal's alone (TerminalKeys).
func wholeID(v string) (uint64, bool) {
if v == "" || strings.TrimLeft(v, "0123456789") != "" {
return 0, false
}
n, err := strconv.ParseUint(v, 10, 32)
return n, err == nil && n < 1<<32-1
}
// OwnerSettingProblems refuses, where a manifest is read, a directory whose owner carries a setting
// in any shape but a whole placeholder or number on each side of the colon, or names a key the module
// asks for nowhere else and declares no default for.
func OwnerSettingProblems(m Manifest) []string {
var problems []string
asked := settingKeysAskedOutsideOwners(m)
for k := range m.Settings {
asked[k] = true
}
for _, r := range m.Resources {
owner, _ := r["owner"].(string)
if !strings.Contains(owner, "${setting:") {
continue
}
if fmt.Sprint(r["type"]) != "directory" {
// Said by the placeholder sweep: only a directory's owner takes a setting.
continue
}
if !ownerWithSetting.MatchString(owner) {
problems = append(problems, fmt.Sprintf("%s: the directory %v's owner %q carries a setting in a shape "+
"the mesh does not fill; an owner names the account by number only, each side of the colon a whole "+
"${setting:key} or a whole number (${setting:puid}:${setting:pgid}) (novox/hq issue 279)",
m.Module, r["id"], owner))
continue
}
for _, key := range settingsUsed(owner) {
if !asked[key] {
problems = append(problems, fmt.Sprintf("%s: the directory %v's owner says ${setting:%s}, and the "+
"module asks for %q nowhere else and declares no default for it: an owner follows the identity "+
"the module's process is told it runs as (its identity file's ${setting:%s}), never a key of its "+
"own (novox/hq issue 279)", m.Module, r["id"], key, key, key))
}
}
}
return problems
}
// settingIntoOwner fills ${setting:…} in a directory's owner, refusing — with the key and why, the
// resource left as it was — a key nothing sets and a value that is not a whole number.
func settingIntoOwner(resource map[string]any, layers []Layer, module string) error {
owner, ok := resource["owner"].(string)
if !ok {
return nil
}
for _, key := range settingsUsed(owner) {
value, set := settingValue(layers, key)
if !set {
return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf(
"%s has a directory %v whose owner says ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112): "+
"`settings set %s <file>` with {%q: …}%s",
module, resource["id"], key, key, module, key, orNoSettings(layers))}
}
v := plainly(value)
id, whole := wholeID(v)
if !whole {
return fmt.Errorf("%s: the setting %q is %q, which cannot own the directory %v: an owner a setting "+
"names is a uid or a gid, a whole number, never an account's name or other text (novox/hq issue 279)",
module, key, v, resource["id"])
}
// Written as the number parsed, so "0001" reaches the node-engine as 1, never as text.
owner = strings.ReplaceAll(owner, "${setting:"+key+"}", strconv.FormatUint(id, 10))
}
resource["owner"] = owner
return nil
}
func settingValue(layers []Layer, key string) (any, bool) {
var value any
set := false
@@ -171,9 +277,25 @@ func orNoSettings(layers []Layer) string {
return "; set today: " + strings.Join(keys, ", ")
}
// settingKeysUsedBy is every key a module's files, contributions and served facts ask for, so a
// setting that lands in one is not called stray.
// settingKeysUsedBy is every key a module's files, units, directory owners, contributions and served
// facts ask for, so a setting that lands in one is not called stray.
func settingKeysUsedBy(m Manifest) map[string]bool {
used := settingKeysAskedOutsideOwners(m)
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "directory" {
if owner, ok := r["owner"].(string); ok {
for _, k := range settingsUsed(owner) {
used[k] = true
}
}
}
}
return used
}
// settingKeysAskedOutsideOwners is every key a module's files, units, contributions and served facts
// ask for: what a directory's owner may follow (OwnerSettingProblems).
func settingKeysAskedOutsideOwners(m Manifest) map[string]bool {
used := map[string]bool{}
note := func(s string) {
for _, k := range settingsUsed(s) {
+12 -2
View File
@@ -33,6 +33,9 @@ import (
// 4. **A module's whole layer, when it has a mergeable file not marked `"trusted": false`** (novox/hq issue 340,
// TrustedMergeable). A mergeable file takes any key a layer sets, not only those its content names, so no list
// of keys covers it: an empty runtime configuration a provider reads would take a `url` of the caller's.
// 5. **Every setting a directory's owner asks for** (novox/hq issue 279): root chowns the directory to the number
// at the next push, so through a verb any caller could hand itself — or uid 0 — a module's folder. Whatever
// else asks for the key, a file marked `"trusted": false` included, the owner makes it the terminal's.
//
// Derived from the manifest, never listed by hand, so a provider or a trusted file added tomorrow is covered.
@@ -42,8 +45,8 @@ import (
const TrustedField = "trusted"
// TerminalKeys are the settings keys of a module that are set at the controller's terminal alone: places and
// accesses, every key its provisions serve and every setting a served value asks for, and every setting a file
// asks for unless it says `"trusted": false`. Places and accesses first, then the rest sorted.
// accesses, every key its provisions serve and every setting a served value asks for, every setting a file
// asks for unless it says `"trusted": false`, and every setting a directory's owner asks for. Places and accesses first, then the rest sorted.
func TerminalKeys(m Manifest) []string {
keys := map[string]bool{}
for _, served := range m.Serves {
@@ -57,6 +60,13 @@ func TerminalKeys(m Manifest) []string {
}
}
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "directory" {
owner, _ := r["owner"].(string)
for _, asked := range settingsUsed(owner) {
keys[asked] = true
}
continue
}
if fmt.Sprint(r["type"]) != "file" {
continue
}
+1 -1
View File
@@ -56,7 +56,7 @@ type filler struct {
// seatInto, machineInto and contributionsInto. A pass that comes to read another field adds it here,
// or the sweep refuses the placeholder it would have filled.
var fillers = []filler{
{namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}}},
{namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}, "directory": {"owner"}}},
{namespace: "dir", patterns: []*regexp.Regexp{dirRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}},
{namespace: "access", patterns: []*regexp.Regexp{accessRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}},
{namespace: "secret", patterns: []*regexp.Regexp{placeholder}, fields: map[string][]string{"file": {"content"}},
+8 -2
View File
@@ -116,9 +116,15 @@ var ControllerVerbs = []Verb{
"limit": "how many builds to list (default 20); not with log",
}, nil)},
{Name: "plans", Description: "What the last merges produced and where each stands (novox/hq ADR 0162): " +
"the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id.",
"the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id; " +
"or, given wait, one plan followed until it moves.",
Input: schema(map[string]string{
"id": "a plan's id (as `plans` lists them): that plan, tier by tier",
"id": "a plan's id (as `plans` lists them): that plan, tier by tier",
"wait": "a plan's id: wait until it moves, then answer it as data — the plan, and what ended the wait " +
"(moved, tier, done, or bound); ask again to follow it further (novox/hq issue 229)",
"until": "with wait: moved (any change, the default), tier (it reaches another tier) or done (it ends)",
"bound": "with wait: how long to wait at most, as 15s or seconds — 20s, the default, is the longest, " +
"inside what a call is answered within",
"stop": "a plan's id: stop it — what was asked still builds, nothing further is asked",
"close": "a plan's id: close a plan that will not move again, as failed by hand (novox/hq issue 254)",
"retry": "a failed plan's id: ask its failed builds again under new ids, and carry the plan on from that tier (novox/hq ADR 0219)",
+45
View File
@@ -0,0 +1,45 @@
package inventory
import (
"strings"
"testing"
"time"
)
// **A plan's id is a value, never SQL** (novox/hq issue 561): the id reaches PlanByID from a caller of the
// controller's plans verb, so an id holding a quote is read as a plain id — no error from the store, and no
// other plan answered for it.
func TestAPlanIdHoldingAQuoteIsReadAsAPlainId(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
p := Plan{ID: "plan-561", Repository: "novox/mesh-controller", Commit: "abc", Created: time.Now().UTC(),
State: PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*PlanModule{"a": {}}}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
for _, id := range []string{
"x' or '1'='1", // every plan, when pasted into the query
"plan-561' or id like '%", // the plan above, by a pattern
"it's", // a quote alone: a syntax error, when pasted
} {
got, err := inv.PlanByID(ctx, id)
if err == nil {
t.Fatalf("PlanByID(%q) answered %s, a plan of another id", id, got.ID)
}
if want := "no plan " + id; err.Error() != want {
t.Fatalf("PlanByID(%q) failed with %q, not as a plain id no plan has (%q)", id, err, want)
}
}
// A plan whose id holds a quote is found by it, and by nothing else.
quoted := p
quoted.ID, quoted.Revision = "plan-'561'", 0
if err := inv.SavePlan(ctx, &quoted); err != nil {
t.Fatal(err)
}
if got, err := inv.PlanByID(ctx, quoted.ID); err != nil || got.ID != quoted.ID {
t.Fatalf("the plan with a quote in its id was not read back: %v %+v", err, got.ID)
}
if recent, err := inv.RecentPlans(ctx, 1); err != nil || len(recent) != 1 || !strings.HasPrefix(recent[0].ID, "plan-") {
t.Fatalf("RecentPlans with a parameter: %v %+v", err, recent)
}
}
+4 -3
View File
@@ -623,7 +623,7 @@ func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, er
// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
return i.plans(ctx, fmt.Sprintf(`order by created desc limit %d`, limit))
return i.plans(ctx, `order by created desc limit $1`, limit)
}
// NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh
@@ -638,9 +638,10 @@ func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string
return plans[0], true, nil
}
// PlanByID is one plan.
// PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a
// caller of the plans verb.
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
plans, err := i.plans(ctx, `where id = '`+id+`'`)
plans, err := i.plans(ctx, `where id = $1`, id)
if err != nil {
return Plan{}, err
}
+31
View File
@@ -325,6 +325,8 @@ func (i *Inventory) ModuleSecretIfIssued(
return sealed, true, nil
}
// SecretForModule makes what is not held, so it is never asked for an own secret issued outside the mesh: the
// composition reads only a value a person gave for one (GivenOwnSecret; novox/hq issue 401, ADR 0228).
func (i *Inventory) SecretForModule(ctx context.Context, node, module, name string) (string, error) {
key, err := i.SealingKeyOf(ctx, node)
if err != nil {
@@ -655,6 +657,35 @@ func (i *Inventory) GivenMembers(ctx context.Context, node, module, family strin
return out, rows.Err()
}
// GivenOwnSecret is an own secret issued outside the mesh as a person gave it on a machine (novox/hq issue 401, ADR
// 0228): its value sealed to the machine and whether that is the key the machine holds now. Not given is false: a
// value the mesh made for it is not one a person gave, and is never read as one; a machine with no sealing key
// holds none either.
func (i *Inventory) GivenOwnSecret(ctx context.Context, node, module, name string) (GivenMember, bool, error) {
key, err := i.SealingKeyOf(ctx, node)
if err != nil || key == "" {
return GivenMember{}, false, err
}
record, err := i.NodeByName(ctx, node)
if err != nil {
return GivenMember{}, false, err
}
g := GivenMember{Name: name}
var against string
err = i.store.Pool().QueryRow(ctx,
`select sealed, node_key from module_secret
where node = $1 and module = $2 and name = $3 and origin = 'accepted'`,
record.ID, module, name).Scan(&g.Sealed, &against)
if errors.Is(err, pgx.ErrNoRows) {
return GivenMember{}, false, nil
}
if err != nil {
return GivenMember{}, false, err
}
g.Current = against == key
return g, true, nil
}
// GivenOwnSecrets is every own secret of a module a person gave on a machine, by name, with when (novox/hq ADR
// 0283): what the controller checks a module's wait for a secret against. A value the mesh made is not one.
func (i *Inventory) GivenOwnSecrets(ctx context.Context, node, module string) (map[string]time.Time, error) {
+11
View File
@@ -138,6 +138,9 @@ type GroupMember struct {
Repository string `json:"repository"`
Ref string `json:"ref"`
Paths []string `json:"paths,omitempty"`
// Base is the branch it merges into: the build seat merges the head into that trunk's head before the
// group is composed (novox/hq ADR 0323 §6). A build seat from before it composes the head alone.
Base string `json:"base,omitempty"`
}
// Who judges a merge check's gate.
@@ -164,6 +167,14 @@ type CheckOutcome struct {
// Gate and RepoCheck are its two layers; Verdict and Summary above are the gate's.
Gate *CheckLayer `json:"gate,omitempty"`
RepoCheck *CheckLayer `json:"repo-check,omitempty"`
// Head and Trunk are what was judged (novox/hq ADR 0323 §3): the pull request's head merged into the trunk
// commit, each as twelve characters. Trunk is empty from a build seat from before it, which judged the head
// alone. Beside are the commits of the repositories read beside it, by directory, named and not judged
// against (issue 568); Members a delivery group's other heads, each with the trunk commit it merged into.
Head string `json:"head,omitempty"`
Trunk string `json:"trunk,omitempty"`
Beside map[string]string `json:"beside,omitempty"`
Members []CheckedMember `json:"members,omitempty"`
}
// SourceOnSeat names a repository by the seat whose holder serves it and its path there.
+47 -10
View File
@@ -601,6 +601,24 @@ func Acknowledge(ctx context.Context) {
}
}
// LongestAnswer is the longest a handler that waits on purpose may hold its caller's one answer
// (AnswersWithin, novox/hq issue 229): inside the console's thirty seconds with room for the trip back,
// and inside the bus's window for an answer (broker.ResponseTTL). A variable so a test need not wait.
var LongestAnswer = 25 * time.Second
type answerWithinKey struct{}
// AnswersWithin asks that the call's caller be answered in full if the handler ends within d of the
// call's start, rather than within AnswerWithin: for a handler that waits on purpose, with a bound —
// `plans` waiting for a plan to move (novox/hq issue 229). Past d, or past LongestAnswer whichever is
// shorter, its caller is told it is running, as for any call. Never shortens the window; without effect
// outside a served call, and once the caller was answered.
func AnswersWithin(ctx context.Context, d time.Duration) {
if f, ok := ctx.Value(answerWithinKey{}).(func(time.Duration)); ok {
f(d)
}
}
// running is the interim answer: what a caller reads when the call has not finished.
func running(c *Call, within time.Duration, acknowledged bool, follow string) []byte {
why := fmt.Sprintf("it has not finished in %s", within)
@@ -645,6 +663,13 @@ func (l *CallLog) serveCallWithin(seat, verb string, args json.RawMessage, reply
acknowledged := make(chan struct{})
var once sync.Once
ctx = context.WithValue(ctx, answerNowKey{}, func() { once.Do(func() { close(acknowledged) }) })
longer := make(chan time.Duration, 1)
ctx = context.WithValue(ctx, answerWithinKey{}, func(d time.Duration) {
select {
case longer <- d:
default:
}
})
type outcome struct {
body []byte
@@ -703,17 +728,29 @@ func (l *CallLog) serveCallWithin(seat, verb string, args json.RawMessage, reply
}
}
}
timer := time.NewTimer(AnswerWithin)
within, start := AnswerWithin, time.Now()
timer := time.NewTimer(within)
defer timer.Stop()
select {
case o := <-done:
l.finish(c, o.body, o.failed, false)
say(o.body)
return
case <-acknowledged:
say(running(c, AnswerWithin, true, l.Follow))
case <-timer.C:
say(running(c, AnswerWithin, false, l.Follow))
waiting:
for {
select {
case o := <-done:
l.finish(c, o.body, o.failed, false)
say(o.body)
return
case <-acknowledged:
say(running(c, within, true, l.Follow))
break waiting
case d := <-longer:
// A handler waiting on purpose (AnswersWithin): its window is stretched, never past LongestAnswer.
if d = min(d, LongestAnswer); d > within {
within = d
timer.Reset(time.Until(start.Add(d)))
}
case <-timer.C:
say(running(c, within, false, l.Follow))
break waiting
}
}
o := <-done
l.finish(c, o.body, o.failed, true)
+55
View File
@@ -0,0 +1,55 @@
package link
import (
"context"
"encoding/json"
"testing"
"time"
)
// **A handler that waits on purpose is answered in full when it ends inside the window it asked for**
// (novox/hq issue 229): `plans` asked to wait for a plan holds its caller's one answer until the plan
// moves or its bound passes, rather than being answered "still running" at AnswerWithin.
func TestAHandlerThatWaitsOnPurposeIsAnsweredInFullInsideItsWindow(t *testing.T) {
shortWindow(t, 20*time.Millisecond)
l, a := NewCallLog(), newAnswers(t)
l.serveCall("mesh-controller", "plans", json.RawMessage(`{"wait":"plan-1"}`), "_INBOX.x.3",
func(ctx context.Context, _ json.RawMessage) (any, error) {
AnswersWithin(ctx, 200*time.Millisecond)
time.Sleep(80 * time.Millisecond) // past AnswerWithin, inside what it asked for
return "plan-1 moved", nil
}, a.respond, nil)
if got := a.only()["result"]; got != "plan-1 moved" {
t.Fatalf("a waiting handler inside its window was answered %v, not in full", got)
}
}
// What a handler may ask for is bounded: past LongestAnswer its caller is told it is running, as any
// call is, so no answer is ever later than the console waits.
func TestAWaitIsNeverLongerThanTheLongestAnswer(t *testing.T) {
shortWindow(t, 10*time.Millisecond)
was := LongestAnswer
LongestAnswer = 30 * time.Millisecond
t.Cleanup(func() { LongestAnswer = was })
l, a := NewCallLog(), newAnswers(t)
release := make(chan struct{})
finished := make(chan struct{})
go func() {
l.serveCall("mesh-controller", "plans", nil, "_INBOX.x.4", func(ctx context.Context, _ json.RawMessage) (any, error) {
AnswersWithin(ctx, time.Hour)
<-release
return "late", nil
}, a.respond, nil)
close(finished)
}()
select {
case <-a.sent:
case <-time.After(2 * time.Second):
t.Fatal("a handler asking for an hour held its caller past LongestAnswer")
}
if got, _ := a.only()["result"].(map[string]any); got["running"] != true {
t.Fatalf("past LongestAnswer the caller was answered %v, not that it is running", a.only())
}
close(release)
<-finished
}
+25 -8
View File
@@ -221,13 +221,17 @@ type SourceMoved struct {
// PullUpdated is what the forge announces when an open pull request's head moves — opened, or pushed
// to (novox/hq to-be 45 §9): what the controller asks the build seat to check before it merges.
type PullUpdated struct {
Owner string `json:"owner"`
Repo string `json:"repo"`
Number int `json:"number"`
Title string `json:"title,omitempty"`
Base string `json:"base"`
Head string `json:"head"`
Commit string `json:"head_sha"`
Owner string `json:"owner"`
Repo string `json:"repo"`
Number int `json:"number"`
Title string `json:"title,omitempty"`
Base string `json:"base"`
Head string `json:"head"`
Commit string `json:"head_sha"`
// BaseSHA is the base branch's head when the forge announced the head (novox/hq ADR 0323): the trunk commit a
// verdict the controller gives without the build seat — a change that runs no build — is judged against. Empty
// from a forge module from before it.
BaseSHA string `json:"base_sha,omitempty"`
CloneURL string `json:"clone_url"`
HTMLURL string `json:"html_url,omitempty"`
// Paths are the files the pull request changes; PathsTruncated says there were more.
@@ -278,14 +282,24 @@ type Checked struct {
// status from it — the group's verdict is its owner's, mesh-delivery, to say on each member's head.
Group string `json:"group,omitempty"`
Members []CheckedMember `json:"members,omitempty"`
// Trunk is the trunk commit the head was merged into and judged as, twelve characters (novox/hq ADR 0323
// §3): the verdict holds for that merge only, and goes out of date when the trunk moves. Empty from a build
// seat that judged the head alone. Beside are the commits of the other repositories the check read, by
// directory, named and not judged against (issue 568). Took is how long the check ran, for the merge tool
// to say how long the next one may take (ADR 0323 §5).
Trunk string `json:"trunk,omitempty"`
Beside map[string]string `json:"beside,omitempty"`
Took string `json:"took,omitempty"`
}
// CheckedMember is one head a group's composed check judged.
// CheckedMember is one head a group's composed check judged, and the trunk commit it was merged into (novox/hq
// ADR 0323 §6), twelve characters; empty from a build seat that composed the head alone.
type CheckedMember struct {
Owner string `json:"owner"`
Repo string `json:"repo"`
Number int `json:"number,omitempty"`
Commit string `json:"commit"`
Trunk string `json:"trunk,omitempty"`
}
// ChangePlan is what a change does to the mesh, computed from its diffset — a repository, the branch it
@@ -314,6 +328,9 @@ type ChangePlan struct {
// Class is the class of the slowest-budgeted module the change moves (novox/hq ADR 0282 §1, ADR 0313 §2):
// core, leaf, or none when it moves no module at all — a fix that moves nothing has no fix budget.
Class string `json:"class,omitempty"`
// Trunk is the trunk commit the head was merged into when it was judged (novox/hq ADR 0323 §3), twelve
// characters: set on the plan a verdict carries, which mesh-delivery keeps with its other inputs.
Trunk string `json:"trunk,omitempty"`
}
// MachinePlan is one machine's part of a change plan.