From 4632b6a50857ddc1b5b04850bc1661f4e4e99f4d Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 03:20:15 +0200 Subject: [PATCH 01/16] Judge the SDK conformance fixtures against the SDK the controller pins, never a desktop's checkout (issue 449) A check clones mesh-sdk beside the controller at the commit go.mod pins; elsewhere the test reads a copy captured at that commit, held to go.mod. A missing clone fails instead of skipping. --- cmd/mesh-controller/checks.go | 49 +++++++++++++++++-- cmd/mesh-controller/merge_gate_test.go | 28 +++++++++-- internal/beside/beside.go | 5 +- internal/link/conformance_test.go | 44 +++++++++++++++-- testdata/beside/CAPTURED | 7 ++- .../conformance/events/module-event.json | 44 +++++++++++++++++ 6 files changed, 161 insertions(+), 16 deletions(-) create mode 100644 testdata/beside/mesh-sdk/conformance/events/module-event.json diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index 2fdfbddd..4cbb75b3 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "path" + "regexp" + "runtime/debug" "slices" "sort" "strings" @@ -246,9 +248,11 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop if dir == "mesh-controller" { beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]} if e.Source.Seat != "" { - if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository, - "mesh-lab")}); err == nil { - beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]} + for _, sibling := range []string{"mesh-lab", "mesh-sdk"} { + if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository, + sibling)}); err == nil { + beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]} + } } } } @@ -271,17 +275,52 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop // catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge // builds from; and beside the controller its main, for a judge the running controller predates, and the // lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by -// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones. +// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones. Beside the +// controller also the SDK, at the commit the controller's go.mod pins (sdkPinned): its conformance fixtures +// are judged against the SDK the controller is built with, not one a desktop holds (novox/hq issue 449). func besideRefs(dir, running string) map[string]string { switch dir { case "mesh-catalog": return map[string]string{dir: "main"} case "mesh-controller": - return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"} + return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main", + "mesh-sdk": sdkPinned()} } return map[string]string{dir: running} } +// sdkModule is the Go module of the SDK the controller is built against. +const sdkModule = "git.novox.be/novox/mesh-sdk/go" + +// pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970. +var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`) + +// sdkPinned is the ref of the SDK repository this controller was built from, as its go.mod pins it and its +// build records it: a pseudo-version's commit, or a release's tag (the SDK tags its Go module under go/). +// "main" only for a binary that records no SDK version — a local replace — which a running controller is +// not (novox/hq issue 449). +func sdkPinned() string { + info, ok := debug.ReadBuildInfo() + if !ok { + return "main" + } + for _, dep := range info.Deps { + if dep.Path != sdkModule { + continue + } + if dep.Replace != nil { + dep = dep.Replace + } + if m := pseudoCommit.FindStringSubmatch(dep.Version); m != nil { + return m[1] + } + if strings.HasPrefix(dep.Version, "v") { + return "go/" + dep.Version + } + } + return "main" +} + // siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab. func siblingOf(repository, name string) string { if cut := strings.LastIndex(repository, "/"); cut >= 0 { diff --git a/cmd/mesh-controller/merge_gate_test.go b/cmd/mesh-controller/merge_gate_test.go index 49792733..00289ff2 100644 --- a/cmd/mesh-controller/merge_gate_test.go +++ b/cmd/mesh-controller/merge_gate_test.go @@ -390,9 +390,10 @@ func TestAWithheldPathIsStoodInForByAPath(t *testing.T) { // controller's ask and by the facts — and finds the repository it checks from its origin. func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) { for dir, refs := range map[string]map[string]string{ - "mesh-catalog": {"mesh-catalog": "main"}, - "mesh-host": {"mesh-host": "c0ffee"}, - "mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main"}, + "mesh-catalog": {"mesh-catalog": "main"}, + "mesh-host": {"mesh-host": "c0ffee"}, + "mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main", + "mesh-sdk": sdkPinnedByGoMod(t)}, } { got := besideRefs(dir, "c0ffee") for d, ref := range refs { @@ -412,3 +413,24 @@ func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) { } } } + +// sdkPinnedByGoMod is the SDK commit this tree's go.mod pins, read from the file rather than the build, so +// sdkPinned is held to what the repository says (novox/hq issue 449). +func sdkPinnedByGoMod(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "go.mod")) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(raw), "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == sdkModule { + if m := pseudoCommit.FindStringSubmatch(fields[1]); m != nil { + return m[1] + } + return "go/" + fields[1] + } + } + t.Fatalf("go.mod requires no %s", sdkModule) + return "" +} diff --git a/internal/beside/beside.go b/internal/beside/beside.go index 3b281765..b64f67cc 100644 --- a/internal/beside/beside.go +++ b/internal/beside/beside.go @@ -1,5 +1,5 @@ // Package beside is where a test finds another repository of the mesh it reads: the catalogue's manifests, -// the node-engine's genesis template (novox/hq issue 432). +// the node-engine's genesis template (novox/hq issue 432), the SDK's conformance fixtures (issue 449). // // A test used to read the checkout beside this one, `../../../mesh-catalog`, so its verdict depended on // whatever sat on the machine running it: a stale or dirty checkout failed it on a desktop, and where none @@ -10,7 +10,8 @@ // test judges against those clones, so agreement with the other repository is checked where // `mesh/repo-check` runs; a repository missing there fails the test, never skips it. Which clones a // check gets is chosen from the inventory: mesh-catalog by the source of the `nats` module, mesh-host -// by the source of `mesh-host`. In a mesh where either module has no source repository nothing is +// by the source of `mesh-host`, and mesh-sdk as the controller's sibling at the commit the controller's +// go.mod pins. In a mesh where either module has no source repository nothing is // cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the // change. And in a delivery group that holds a mesh-catalog pull request, these tests read the // catalogue's main, not the group's head. diff --git a/internal/link/conformance_test.go b/internal/link/conformance_test.go index 50f60dac..edc3f148 100644 --- a/internal/link/conformance_test.go +++ b/internal/link/conformance_test.go @@ -5,15 +5,21 @@ import ( "os" "path/filepath" "regexp" + "strings" "testing" "time" + + "github.com/novox/mesh-controller/internal/beside" ) // The Go implementation, held to the shared fixtures (novox/hq ADR 0074, design 19). // -// **Read from the sdk's conformance directory by sibling path**, the way the lab finds its -// siblings — deliberately not copied here. A fixture copied into each implementation is two -// fixtures, and two fixtures drift, which is the exact failure the suite exists to prevent. +// **Read from the SDK's own conformance directory, never copied by hand**: a fixture written into each +// implementation is two fixtures, and two fixtures drift, which is the failure the suite exists to prevent. +// Which SDK is read is named, never the one a desktop happens to hold beside this checkout (novox/hq issue +// 449): in a merge check the SDK the build seat clones beside it, at the commit this controller's go.mod +// pins, and a missing clone fails the test; elsewhere the copy captured in testdata/beside at that same +// commit (internal/beside, testdata/beside/CAPTURED), held to go.mod by TestTheCapturedSDKIsTheOneGoModPins. type fixture struct { Name string `json:"name"` Given struct { @@ -32,10 +38,11 @@ type fixture struct { func loadFixture(t *testing.T, name string) fixture { t.Helper() - path := filepath.Join("..", "..", "..", "mesh-sdk", "conformance", name) + path := filepath.Join(beside.Dir(t, "mesh-sdk"), "conformance", name) raw, err := os.ReadFile(path) if err != nil { - t.Skipf("the sdk's conformance fixtures are not beside this checkout: %v", err) + // Failed, never skipped: a skip here passed the suite with nothing judged. + t.Fatalf("the SDK's conformance fixture %s: %v", name, err) } var f fixture if err := json.Unmarshal(raw, &f); err != nil { @@ -44,6 +51,33 @@ func loadFixture(t *testing.T, name string) fixture { return f } +// The captured SDK is the one this controller is built against: when go.mod moves the SDK, the copy moves +// with it, or the tests away from a merge check judge an SDK the controller no longer uses (novox/hq issue +// 449). +func TestTheCapturedSDKIsTheOneGoModPins(t *testing.T) { + mod, err := os.ReadFile(filepath.Join("..", "..", "go.mod")) + if err != nil { + t.Fatal(err) + } + pinned := regexp.MustCompile(`(?m)^\s*git\.novox\.be/novox/mesh-sdk/go v\S+-([0-9a-f]{12})$`).FindSubmatch(mod) + if pinned == nil { + t.Fatal("go.mod pins no commit of git.novox.be/novox/mesh-sdk/go as a pseudo-version; say here how a " + + "release's tag is matched against testdata/beside/CAPTURED") + } + captured, err := os.ReadFile(filepath.Join(beside.Captured(), "CAPTURED")) + if err != nil { + t.Fatal(err) + } + at := regexp.MustCompile(`(?m)^mesh-sdk\s+([0-9a-f]{40})\s`).FindSubmatch(captured) + if at == nil { + t.Fatal("testdata/beside/CAPTURED names no commit for mesh-sdk") + } + if !strings.HasPrefix(string(at[1]), string(pinned[1])) { + t.Errorf("the SDK is captured at %s but go.mod pins %s: capture it again at the pinned commit, as "+ + "testdata/beside/CAPTURED says", at[1], pinned[1]) + } +} + // Every header the fixture requires is one this implementation actually sets. func TestTheGoEmitterSetsEveryRequiredHeader(t *testing.T) { f := loadFixture(t, "events/module-event.json") diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index 6b819622..17d63986 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,6 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog b9de001833b1b61b297182b8e3bcdae1cbdeace9 modules/*/module.json, modules/nats/Dockerfile mesh-host bd5cc6980419c1bd4824be6d58dfebd2381a9de3 examples/foundation-first-node-nats.lock +mesh-sdk f047d0a4a9702f5d7f3d7eadd3e62496311acc00 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: @@ -19,4 +20,8 @@ To move them, from this repository's root, with the two repositories checked out find testdata/beside -name module.json -exec mv {} {}.captured \; git -C ../mesh-host archive examples/foundation-first-node-nats.lock | tar -x -C testdata/beside/mesh-host -and write the commits here. +and write the commits here. The SDK is captured at the commit go.mod pins for git.novox.be/novox/mesh-sdk/go +(the last part of its pseudo-version; novox/hq issue 449), which internal/link's conformance test holds it to: + + rm -rf testdata/beside/mesh-sdk && mkdir -p testdata/beside/mesh-sdk + git -C ../mesh-sdk archive conformance/events | tar -x -C testdata/beside/mesh-sdk diff --git a/testdata/beside/mesh-sdk/conformance/events/module-event.json b/testdata/beside/mesh-sdk/conformance/events/module-event.json new file mode 100644 index 00000000..8de1169b --- /dev/null +++ b/testdata/beside/mesh-sdk/conformance/events/module-event.json @@ -0,0 +1,44 @@ +{ + "capability": "events", + "name": "a module emits an event", + "why": "The envelope is what two implementations can disagree about without either failing: a missing header, a header spelled differently, or a body nested where metadata belongs. None of those stop a mesh running; they stop it reacting.", + "given": { + "module": "shop", + "node": "one", + "key": "order.placed", + "body": { "id": "a1", "total": 12 }, + "headers": { + "x-event-id": "0123456789abcdef0123456789abcdef", + "x-source": "shop", + "x-node": "one", + "x-time": "2026-09-26T12:00:00Z", + "content-type": "application/json" + } + }, + "wire": { + "subject": "mesh.mod.shop.event.order.placed", + "requiredHeaders": ["x-event-id", "x-source", "x-node", "x-time", "content-type"], + "optionalHeaders": ["x-causation-id", "x-schema"], + "headerFormats": { + "x-time": "RFC3339", + "content-type": "application/json", + "x-event-id": "^[0-9a-f]{32}$" + }, + "payloadIs": "the body alone, not the envelope", + "keyRecoveredFrom": "the subject, after the .event. token" + }, + "refuses": [ + { + "what": "an envelope nested in the payload", + "why": "an implementation that publishes the whole envelope as the body passes all of its own tests and is unreadable to every other" + }, + { + "what": "a missing x-event-id", + "why": "delivery is at-least-once and only the emitter can say which of two messages is a redelivery" + }, + { + "what": "an x-source that differs from the subject's module", + "why": "the bus enforces the namespace, so a disagreement means the envelope is lying about its origin" + } + ] +} From 4f5fab81fe9e0b34005f7ab7f2c06c94ba6e9b34 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 03:25:27 +0200 Subject: [PATCH 02/16] Read the SDK fixtures at the pin of the tree under check, not the running controller's (issue 449 review) A pull request moving the SDK passed its check against the old SDK and then failed everywhere once it rolled out. In a merge check the test now reads the clone's history at the tree's own go.mod pin, and holds the captured copy to the clone byte for byte. --- cmd/mesh-controller/checks.go | 6 +- internal/link/conformance_sources_test.go | 128 ++++++++++++++++++ internal/link/conformance_test.go | 152 +++++++++++++++++++--- 3 files changed, 264 insertions(+), 22 deletions(-) create mode 100644 internal/link/conformance_sources_test.go diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index 4cbb75b3..bb78c718 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -276,8 +276,10 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop // builds from; and beside the controller its main, for a judge the running controller predates, and the // lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by // the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones. Beside the -// controller also the SDK, at the commit the controller's go.mod pins (sdkPinned): its conformance fixtures -// are judged against the SDK the controller is built with, not one a desktop holds (novox/hq issue 449). +// controller also the SDK, checked out at the commit the running controller's go.mod pins (sdkPinned), not +// one a desktop holds (novox/hq issue 449). The checkout only places the clone: the conformance test reads the +// fixtures at the pin of the tree under check, from the clone's history, so a pull request moving the SDK is +// judged against the SDK it moves to (internal/link/conformance_test.go). func besideRefs(dir, running string) map[string]string { switch dir { case "mesh-catalog": diff --git a/internal/link/conformance_sources_test.go b/internal/link/conformance_sources_test.go new file mode 100644 index 00000000..53e40484 --- /dev/null +++ b/internal/link/conformance_sources_test.go @@ -0,0 +1,128 @@ +package link + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Where the conformance fixtures are read from, held on a small SDK repository of the test's own: two +// commits, the clone checked out at the older as the build seat leaves it at the running controller's pin +// (novox/hq issue 449). The contents are the test's, because what is judged is which commit is read. +type sdkBed struct { + root, goMod, captured, capturedLog string + older, newer string +} + +const fixtureName = "events/module-event.json" + +func newSDKBed(t *testing.T) sdkBed { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Fatalf("git is needed to read the SDK at a pin, as a merge check does: %v", err) + } + b := sdkBed{root: t.TempDir()} + clone := filepath.Join(b.root, "mesh-sdk") + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", clone, "-c", "user.name=t", "-c", "user.email=t@t", + "-c", "commit.gpgsign=false"}, args...)...) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + write := func(dir, body string) { + t.Helper() + file := filepath.Join(dir, "conformance", filepath.FromSlash(fixtureName)) + if err := os.MkdirAll(filepath.Dir(file), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + git("init", "--quiet") + write(clone, "older\n") + git("add", "-A") + git("commit", "--quiet", "-m", "older") + b.older = git("rev-parse", "HEAD") + write(clone, "newer\n") + git("commit", "--quiet", "-am", "newer") + b.newer = git("rev-parse", "HEAD") + git("checkout", "--quiet", b.older) + + other := t.TempDir() + b.captured = filepath.Join(other, "mesh-sdk") + write(b.captured, "older\n") + b.capturedLog = filepath.Join(other, "CAPTURED") + if err := os.WriteFile(b.capturedLog, []byte("mesh-sdk "+b.older+" conformance/events\n"), 0o644); err != nil { + t.Fatal(err) + } + b.goMod = filepath.Join(other, "go.mod") + b.pin(t, b.newer) + return b +} + +func (b sdkBed) pin(t *testing.T, commit string) { + t.Helper() + mod := "module x\n\nrequire (\n\t" + sdkModule + " v0.1.11-0.20261009143344-" + commit[:12] + "\n)\n" + if err := os.WriteFile(b.goMod, []byte(mod), 0o644); err != nil { + t.Fatal(err) + } +} + +// A pull request moving the SDK is judged against the SDK it moves to: the clone is read at the tree's own +// pin, not at what it is checked out at, the running controller's. +func TestAMergeCheckReadsTheSDKAtTheTreesOwnPin(t *testing.T) { + b := newSDKBed(t) + raw, from, err := sdkFixture(b.root, b.goMod, b.captured, b.capturedLog, fixtureName) + if err != nil { + t.Fatal(err) + } + if string(raw) != "newer\n" { + t.Errorf("read %q from %s; the tree pins the newer commit", raw, from) + } +} + +// A captured copy that is not the SDK's at the commit CAPTURED names fails a merge check. +func TestAMergeCheckFailsACapturedCopyEditedByHand(t *testing.T) { + b := newSDKBed(t) + if err := os.WriteFile(filepath.Join(b.captured, "conformance", filepath.FromSlash(fixtureName)), + []byte("edited\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := sdkFixture(b.root, b.goMod, b.captured, b.capturedLog, fixtureName); err == nil || + !strings.Contains(err.Error(), "is not mesh-sdk's at") { + t.Errorf("a hand-edited captured copy was accepted: %v", err) + } +} + +// A pin the clone does not hold, or no clone at all, fails loudly and names what is missing. +func TestAMergeCheckFailsWithoutTheSDKAtItsPin(t *testing.T) { + b := newSDKBed(t) + b.pin(t, "0123456789ab0123456789ab0123456789ab0123") + if _, _, err := sdkFixture(b.root, b.goMod, b.captured, b.capturedLog, fixtureName); err == nil || + !strings.Contains(err.Error(), "0123456789ab") { + t.Errorf("a pin the clone lacks was not named: %v", err) + } + if _, _, err := sdkFixture(t.TempDir(), b.goMod, b.captured, b.capturedLog, fixtureName); err == nil || + !strings.Contains(err.Error(), "not beside this check") { + t.Errorf("a missing clone was not refused: %v", err) + } +} + +// Away from a merge check the captured copy is read, and nothing else. +func TestAwayFromACheckTheCapturedCopyIsRead(t *testing.T) { + b := newSDKBed(t) + raw, _, err := sdkFixture("", b.goMod, b.captured, b.capturedLog, fixtureName) + if err != nil || string(raw) != "older\n" { + t.Errorf("read %q, %v; the captured copy holds the older", raw, err) + } +} diff --git a/internal/link/conformance_test.go b/internal/link/conformance_test.go index edc3f148..66c7471e 100644 --- a/internal/link/conformance_test.go +++ b/internal/link/conformance_test.go @@ -1,8 +1,12 @@ package link import ( + "bytes" "encoding/json" + "fmt" "os" + "os/exec" + "path" "path/filepath" "regexp" "strings" @@ -16,10 +20,19 @@ import ( // // **Read from the SDK's own conformance directory, never copied by hand**: a fixture written into each // implementation is two fixtures, and two fixtures drift, which is the failure the suite exists to prevent. -// Which SDK is read is named, never the one a desktop happens to hold beside this checkout (novox/hq issue -// 449): in a merge check the SDK the build seat clones beside it, at the commit this controller's go.mod -// pins, and a missing clone fails the test; elsewhere the copy captured in testdata/beside at that same -// commit (internal/beside, testdata/beside/CAPTURED), held to go.mod by TestTheCapturedSDKIsTheOneGoModPins. +// Which SDK is read is the one this tree's go.mod pins, never the one a desktop happens to hold beside this +// checkout (novox/hq issue 449): +// +// - in a merge check, the fixture as it stands at that pin in the mesh-sdk clone the build seat places +// beside the check — read with `git show`, because the clone is checked out at the *running* +// controller's pin, and a pull request moving the SDK must be judged against the SDK it moves to, or it +// passes the check and fails everywhere after it rolls out. A missing clone or a pin the clone lacks +// fails the test; +// - elsewhere, the copy captured in testdata/beside at the commit testdata/beside/CAPTURED names, held to +// go.mod by TestTheCapturedSDKIsTheOneGoModPins. +// +// In a merge check the captured copy is also compared with the clone at the captured commit, byte for byte, +// so a hand-edited copy cannot pass for the SDK's. type fixture struct { Name string `json:"name"` Given struct { @@ -36,14 +49,24 @@ type fixture struct { } `json:"wire"` } +// sdkModule is the Go module of the SDK the controller is built against. +const sdkModule = "git.novox.be/novox/mesh-sdk/go" + +// The tree's go.mod, and the captured copy, as this package finds them. +var ( + goModFile = filepath.Join("..", "..", "go.mod") + capturedSDK = filepath.Join(beside.Captured(), "mesh-sdk") + capturedLog = filepath.Join(beside.Captured(), "CAPTURED") +) + func loadFixture(t *testing.T, name string) fixture { t.Helper() - path := filepath.Join(beside.Dir(t, "mesh-sdk"), "conformance", name) - raw, err := os.ReadFile(path) + raw, from, err := sdkFixture(os.Getenv(beside.Env), goModFile, capturedSDK, capturedLog, name) if err != nil { // Failed, never skipped: a skip here passed the suite with nothing judged. - t.Fatalf("the SDK's conformance fixture %s: %v", name, err) + t.Fatal(err) } + t.Logf("%s: %s", name, from) var f fixture if err := json.Unmarshal(raw, &f); err != nil { t.Fatalf("%s: %v", name, err) @@ -51,30 +74,119 @@ func loadFixture(t *testing.T, name string) fixture { return f } +// sdkFixture is one conformance fixture and where it was read: from the mesh-sdk clone in root (a merge +// check's MESH_CHECK_BESIDE) at the pin of goMod, or, with no root, from the captured copy. +func sdkFixture(root, goMod, captured, capturedLog, name string) ([]byte, string, error) { + file := path.Join("conformance", name) + if root == "" { + raw, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) + if err != nil { + return nil, "", fmt.Errorf("the captured SDK's %s: %w", file, err) + } + return raw, "the copy captured in testdata/beside (see CAPTURED); a merge check reads the SDK's clone " + + "at this tree's pin", nil + } + clone := filepath.Join(root, "mesh-sdk") + if _, err := os.Stat(clone); err != nil { + return nil, "", fmt.Errorf("mesh-sdk is not beside this check in %s=%s, so the SDK this controller "+ + "pins cannot be judged here: %w", beside.Env, root, err) + } + ref, err := sdkRef(goMod) + if err != nil { + return nil, "", err + } + raw, err := gitShow(clone, ref, file) + if err != nil { + return nil, "", fmt.Errorf("the mesh-sdk clone beside this check has no %s at %s, the SDK this tree's "+ + "go.mod pins: %w", file, ref, err) + } + // The captured copy is the SDK's own, never edited by hand: the clone at the captured commit says so. + at, err := capturedCommit(capturedLog) + if err != nil { + return nil, "", err + } + theirs, err := gitShow(clone, at, file) + if err != nil { + return nil, "", fmt.Errorf("the mesh-sdk clone has no %s at %s, the commit CAPTURED names: %w", file, at, err) + } + ours, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) + if err != nil { + return nil, "", fmt.Errorf("the captured SDK's %s: %w", file, err) + } + if !bytes.Equal(ours, theirs) { + return nil, "", fmt.Errorf("the captured %s is not mesh-sdk's at %s, the commit CAPTURED names: it was "+ + "edited or captured wrong; capture it again as CAPTURED says", file, at) + } + return raw, "mesh-sdk cloned beside this check, at " + ref + " (this tree's go.mod)", nil +} + +// gitShow is one file of a repository at a ref. safe.directory, because the clone is the build seat's and +// the test may run as another user. +func gitShow(repository, ref, file string) ([]byte, error) { + cmd := exec.Command("git", "-c", "safe.directory=*", "-C", repository, "show", ref+":"+file) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git show %s:%s: %v: %s", ref, file, err, strings.TrimSpace(stderr.String())) + } + return out, nil +} + +// pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970. +var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`) + +// sdkRef is the SDK repository's ref a go.mod pins: a pseudo-version's commit, or a release's tag (the SDK +// tags its Go module under go/). +func sdkRef(goMod string) (string, error) { + raw, err := os.ReadFile(goMod) + if err != nil { + return "", err + } + version := "" + for _, line := range strings.Split(string(raw), "\n") { + fields := strings.Fields(strings.TrimPrefix(strings.TrimSpace(line), "require ")) + if len(fields) >= 2 && fields[0] == sdkModule { + version = fields[1] + } + } + if m := pseudoCommit.FindStringSubmatch(version); m != nil { + return m[1], nil + } + if strings.HasPrefix(version, "v") { + return "go/" + version, nil + } + return "", fmt.Errorf("%s pins no version of %s that names a commit or a tag", goMod, sdkModule) +} + +// capturedCommit is the commit testdata/beside/CAPTURED names for mesh-sdk. +func capturedCommit(capturedLog string) (string, error) { + raw, err := os.ReadFile(capturedLog) + if err != nil { + return "", err + } + at := regexp.MustCompile(`(?m)^mesh-sdk\s+([0-9a-f]{40})\s`).FindSubmatch(raw) + if at == nil { + return "", fmt.Errorf("%s names no commit for mesh-sdk", capturedLog) + } + return string(at[1]), nil +} + // The captured SDK is the one this controller is built against: when go.mod moves the SDK, the copy moves // with it, or the tests away from a merge check judge an SDK the controller no longer uses (novox/hq issue // 449). func TestTheCapturedSDKIsTheOneGoModPins(t *testing.T) { - mod, err := os.ReadFile(filepath.Join("..", "..", "go.mod")) + pinned, err := sdkRef(goModFile) if err != nil { t.Fatal(err) } - pinned := regexp.MustCompile(`(?m)^\s*git\.novox\.be/novox/mesh-sdk/go v\S+-([0-9a-f]{12})$`).FindSubmatch(mod) - if pinned == nil { - t.Fatal("go.mod pins no commit of git.novox.be/novox/mesh-sdk/go as a pseudo-version; say here how a " + - "release's tag is matched against testdata/beside/CAPTURED") - } - captured, err := os.ReadFile(filepath.Join(beside.Captured(), "CAPTURED")) + at, err := capturedCommit(capturedLog) if err != nil { t.Fatal(err) } - at := regexp.MustCompile(`(?m)^mesh-sdk\s+([0-9a-f]{40})\s`).FindSubmatch(captured) - if at == nil { - t.Fatal("testdata/beside/CAPTURED names no commit for mesh-sdk") - } - if !strings.HasPrefix(string(at[1]), string(pinned[1])) { + if strings.HasPrefix(pinned, "go/") || !strings.HasPrefix(at, pinned) { t.Errorf("the SDK is captured at %s but go.mod pins %s: capture it again at the pinned commit, as "+ - "testdata/beside/CAPTURED says", at[1], pinned[1]) + "testdata/beside/CAPTURED says", at, pinned) } } From 9b6acf0ef53c797d15dc22ecff2628679e3d9b0c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 04:20:54 +0200 Subject: [PATCH 03/16] Read the captured SDK, saying so, when the seat placed no clone beside the check (issue 449) The seat runs the running controller's besideRefs, which clones no mesh-sdk until this change has rolled out, so a missing clone failing the test kept this change from ever passing its own check. A follow-up makes it a failure again after the rollout. --- internal/link/conformance_sources_test.go | 17 +++++++--- internal/link/conformance_test.go | 39 +++++++++++++++++++---- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/internal/link/conformance_sources_test.go b/internal/link/conformance_sources_test.go index 53e40484..06a0fcd7 100644 --- a/internal/link/conformance_sources_test.go +++ b/internal/link/conformance_sources_test.go @@ -104,7 +104,7 @@ func TestAMergeCheckFailsACapturedCopyEditedByHand(t *testing.T) { } } -// A pin the clone does not hold, or no clone at all, fails loudly and names what is missing. +// A pin the clone does not hold fails loudly and names it. func TestAMergeCheckFailsWithoutTheSDKAtItsPin(t *testing.T) { b := newSDKBed(t) b.pin(t, "0123456789ab0123456789ab0123456789ab0123") @@ -112,9 +112,18 @@ func TestAMergeCheckFailsWithoutTheSDKAtItsPin(t *testing.T) { !strings.Contains(err.Error(), "0123456789ab") { t.Errorf("a pin the clone lacks was not named: %v", err) } - if _, _, err := sdkFixture(t.TempDir(), b.goMod, b.captured, b.capturedLog, fixtureName); err == nil || - !strings.Contains(err.Error(), "not beside this check") { - t.Errorf("a missing clone was not refused: %v", err) +} + +// Until mesh-controller #220 has rolled out the seat places no mesh-sdk beside a check: the captured copy is +// read, and where it was read from says plainly that the clone did not judge it (novox/hq issue 449). +func TestAMergeCheckWithoutTheCloneSaysItReadTheCapturedCopy(t *testing.T) { + b := newSDKBed(t) + raw, from, err := sdkFixture(t.TempDir(), b.goMod, b.captured, b.capturedLog, fixtureName) + if err != nil || string(raw) != "older\n" { + t.Fatalf("read %q, %v; the captured copy holds the older", raw, err) + } + if !strings.HasPrefix(from, "NOT JUDGED AGAINST THE CLONE") || !strings.Contains(from, b.older) { + t.Errorf("the log does not say the clone was missing and which copy was read: %q", from) } } diff --git a/internal/link/conformance_test.go b/internal/link/conformance_test.go index 66c7471e..048ba4f3 100644 --- a/internal/link/conformance_test.go +++ b/internal/link/conformance_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" "testing" "time" @@ -26,8 +27,9 @@ import ( // - in a merge check, the fixture as it stands at that pin in the mesh-sdk clone the build seat places // beside the check — read with `git show`, because the clone is checked out at the *running* // controller's pin, and a pull request moving the SDK must be judged against the SDK it moves to, or it -// passes the check and fails everywhere after it rolls out. A missing clone or a pin the clone lacks -// fails the test; +// passes the check and fails everywhere after it rolls out. A pin the clone lacks fails the test; a +// missing clone reads the captured copy and says so loudly, until mesh-controller #220 has rolled out +// and the build seat clones mesh-sdk (then its follow-up makes a missing clone fail again); // - elsewhere, the copy captured in testdata/beside at the commit testdata/beside/CAPTURED names, held to // go.mod by TestTheCapturedSDKIsTheOneGoModPins. // @@ -67,6 +69,10 @@ func loadFixture(t *testing.T, name string) fixture { t.Fatal(err) } t.Logf("%s: %s", name, from) + if strings.HasPrefix(from, notJudged) { + // Said on the check's own output too, where `go test` without -v prints no log of a passing test. + saidNotJudged.Do(func() { fmt.Fprintln(os.Stderr, "internal/link conformance: "+from) }) + } var f fixture if err := json.Unmarshal(raw, &f); err != nil { t.Fatalf("%s: %v", name, err) @@ -74,22 +80,41 @@ func loadFixture(t *testing.T, name string) fixture { return f } +// notJudged opens what a merge check without the SDK's clone says; saidNotJudged says it once on stderr. +const notJudged = "NOT JUDGED AGAINST THE CLONE" + +var saidNotJudged sync.Once + // sdkFixture is one conformance fixture and where it was read: from the mesh-sdk clone in root (a merge // check's MESH_CHECK_BESIDE) at the pin of goMod, or, with no root, from the captured copy. func sdkFixture(root, goMod, captured, capturedLog, name string) ([]byte, string, error) { file := path.Join("conformance", name) - if root == "" { + readCaptured := func() ([]byte, error) { raw, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) if err != nil { - return nil, "", fmt.Errorf("the captured SDK's %s: %w", file, err) + return nil, fmt.Errorf("the captured SDK's %s: %w", file, err) } + return raw, nil + } + if root == "" { + raw, err := readCaptured() return raw, "the copy captured in testdata/beside (see CAPTURED); a merge check reads the SDK's clone " + - "at this tree's pin", nil + "at this tree's pin", err } clone := filepath.Join(root, "mesh-sdk") if _, err := os.Stat(clone); err != nil { - return nil, "", fmt.Errorf("mesh-sdk is not beside this check in %s=%s, so the SDK this controller "+ - "pins cannot be judged here: %w", beside.Env, root, err) + // A build seat asked by a controller from before mesh-controller #220 places no mesh-sdk beside the + // check, so #220 could never pass its own check if this failed. Until #220 has rolled out, the + // captured copy is read and the log says so loudly; the follow-up of novox/hq issue 449 makes this + // a failure again. + at, cerr := capturedCommit(capturedLog) + if cerr != nil { + return nil, "", cerr + } + raw, rerr := readCaptured() + return raw, fmt.Sprintf(notJudged+": the seat placed no mesh-sdk beside this check in "+ + "%s=%s (it does once mesh-controller #220 has rolled out); read the captured copy at %s", beside.Env, + root, at), rerr } ref, err := sdkRef(goMod) if err != nil { From 1747e339239ae8b5ad7ac57a3f296bed6cbbffd0 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 04:24:34 +0200 Subject: [PATCH 04/16] Name what failed in a repository check from its whole output, not its tail (issue 460) A long run's logs pushed the --- FAIL lines out of the last 200 lines the summary was named from, so a failed check could not say which test failed. Pick the lines that name a failure as the output streams, keep them whole at the end of the verdict's report, and say the whole output in the build's log. --- internal/builder/check.go | 115 ++- internal/builder/failures.go | 285 ++++++++ internal/builder/failures_test.go | 182 +++++ internal/builder/testdata/issue460/CAPTURED | 7 + .../testdata/issue460/a-build-error.txt | 5 + .../builder/testdata/issue460/a-data-race.txt | 31 + .../issue460/a-long-run-failing-early.txt | 609 ++++++++++++++++ .../builder/testdata/issue460/a-panic.txt | 18 + .../issue460/a-race-run-of-every-package.txt | 660 ++++++++++++++++++ 9 files changed, 1905 insertions(+), 7 deletions(-) create mode 100644 internal/builder/failures.go create mode 100644 internal/builder/failures_test.go create mode 100644 internal/builder/testdata/issue460/CAPTURED create mode 100644 internal/builder/testdata/issue460/a-build-error.txt create mode 100644 internal/builder/testdata/issue460/a-data-race.txt create mode 100644 internal/builder/testdata/issue460/a-long-run-failing-early.txt create mode 100644 internal/builder/testdata/issue460/a-panic.txt create mode 100644 internal/builder/testdata/issue460/a-race-run-of-every-package.txt diff --git a/internal/builder/check.go b/internal/builder/check.go index 773f6702..30e58f7e 100644 --- a/internal/builder/check.go +++ b/internal/builder/check.go @@ -146,6 +146,9 @@ type Layer struct { Verdict string Summary string Modules []string + // Failed is what the repository's own check printed that names what failed, picked from the whole of + // its output and said at the end of the verdict's report (novox/hq issue 460). Empty when it passed. + Failed string } // CheckScript is what a repository declares its own merge check as: run from its root, with the @@ -477,7 +480,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry } v.Verdict, v.Summary = v.Gate.Verdict, v.Gate.Summary - v.Report, v.Took = out.String(), time.Since(began) + v.Report, v.Took = withWhatFailed(out.String(), v.Repo), time.Since(began) say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary, strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second)) return v, nil @@ -515,17 +518,29 @@ func ownCheck(ctx context.Context, spec CheckSpec, parts []ScriptPart, tree stri say("check", "running its %s in the mesh's %s toolchain", part.Script, part.Toolchain) fmt.Fprintf(out, "--- its %s (%s toolchain)\n", part.Script, part.Toolchain) var own tail + var picked failures + logged := &toTheLog{say: say} cmd := command(image, part.Script) inItsOwnGroup(cmd) - w := io.MultiWriter(out, &own) + w := io.MultiWriter(out, &own, &picked, logged) cmd.Stdout, cmd.Stderr = w, w - switch err := cmd.Run(); { + err := cmd.Run() + logged.close(picked.String()) + switch { case timedOut(): - return &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", part.Script, CheckTimeout)} + return &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", part.Script, CheckTimeout), + Failed: picked.String()} case ctx.Err() != nil: return nil case err != nil: - return &Layer{Verdict: "fail", Summary: passedSoFar(ran) + "its " + part.Script + " failed: " + whatFailed(own.String())} + // Named from the whole run, not the tail: what failed may be thousands of lines from the end + // (novox/hq issue 460). + said := picked.said() + if said == "" { + said = whatFailed(own.String()) + } + return &Layer{Verdict: "fail", Summary: passedSoFar(ran) + "its " + part.Script + " failed: " + said, + Failed: picked.String()} } ran = append(ran, fmt.Sprintf("%s (%s)", part.Script, part.Toolchain)) } @@ -535,6 +550,84 @@ func ownCheck(ctx context.Context, spec CheckSpec, parts []ScriptPart, tree stri return &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed, each part in its toolchain: " + strings.Join(ran, ", ")} } +// withWhatFailed is a check's report with what its repository's own check names as failed said last, whole: +// the report's tail is the last reportLines lines, and the controller keeps the end of a report it cuts, so +// what failed travels whatever came before or after it (novox/hq issue 460). +func withWhatFailed(report string, repo *Layer) string { + if repo == nil || repo.Failed == "" { + return report + } + return report + "\n--- what failed, picked from the whole of its output (the whole is in the build's log)\n" + + repo.Failed +} + +// logLines bounds the lines of a repository's own check that go into the build's log. The bus keeps 10 000 +// messages per subject (EVENTS' MaxMsgsPerSubject), and a build's log is one subject: past that, its first +// lines would be lost. The build's own lines are a few hundred at most, so the check's output takes 8 000. +const logLines = 8000 + +// logLineBytes bounds one line of it, so a line that pastes a document does not fill a message. +const logLineBytes = 4 << 10 + +// toTheLog says each line a repository's own check prints in the build's log, as it prints it — the whole +// output kept for the build and read with `builds` and its id (novox/hq issue 460), where before only the +// verdict's tail of it was kept. Past logLines it says how many lines it left out, and the lines that name +// what failed after them. +type toTheLog struct { + say func(step, format string, args ...any) + partial []byte + said int + dropped int +} + +func (l *toTheLog) Write(p []byte) (int, error) { + l.partial = append(l.partial, p...) + for { + i := bytes.IndexByte(l.partial, '\n') + if i < 0 { + break + } + l.line(string(bytes.TrimRight(l.partial[:i], "\r"))) + l.partial = l.partial[i+1:] + } + if len(l.partial) > logLineBytes { + l.line(string(l.partial)) + l.partial = nil + } + return len(p), nil +} + +func (l *toTheLog) line(line string) { + if l.said >= logLines { + l.dropped++ + return + } + if len(line) > logLineBytes { + line = line[:logLineBytes] + fmt.Sprintf(" … (%d bytes more)", len(line)-logLineBytes) + } + l.said++ + l.say("output", "%s", line) +} + +// close says the last line, and, when lines were left out, how many and what failed. +func (l *toTheLog) close(failed string) { + if len(l.partial) > 0 { + l.line(string(l.partial)) + l.partial = nil + } + if l.dropped == 0 { + return + } + l.say("output", "… %d more line(s) of its output are not in this log, which keeps its first %d", l.dropped, logLines) + if failed == "" { + return + } + l.say("output", "--- what failed, picked from the whole of its output") + for _, line := range strings.Split(failed, "\n") { + l.say("output", "%s", line) + } +} + // passedSoFar is what of a check's parts passed before the one that did not, said first. func passedSoFar(ran []string) string { if len(ran) == 0 { @@ -1018,9 +1111,17 @@ func whatFailed(s string) string { return "not gofmt'd by the toolchain's gofmt: " + strings.Join(files, ", ") } } - for _, prefix := range []string{"--- FAIL:", "FAIL\t", "panic:"} { + // A panic, a data race or a build error outside any test says more than the package's bare FAIL line + // (novox/hq issue 460). + for _, said := range []func(string) bool{ + func(l string) bool { return strings.HasPrefix(l, "--- FAIL:") }, + func(l string) bool { return strings.HasPrefix(l, "panic:") || strings.HasPrefix(l, "fatal error:") }, + func(l string) bool { return l == "WARNING: DATA RACE" }, + goError.MatchString, + func(l string) bool { return strings.HasPrefix(l, "FAIL\t") }, + } { for _, line := range lines { - if line = strings.TrimSpace(line); strings.HasPrefix(line, prefix) { + if line = strings.TrimSpace(line); said(line) { return line } } diff --git a/internal/builder/failures.go b/internal/builder/failures.go new file mode 100644 index 00000000..3e5e0e12 --- /dev/null +++ b/internal/builder/failures.go @@ -0,0 +1,285 @@ +package builder + +import ( + "bytes" + "fmt" + "regexp" + "strings" +) + +// **What failed is picked from the whole of a check's output, as it streams** (novox/hq issue 460). +// +// A repository's own check kept only the last reportLines lines of what it printed, and named what failed +// from those. A long store-backed run prints its logs after the failure: the `--- FAIL:` line of mesh-controller +// #218 was pushed out of the tail by the package's own log lines, and the verdict said only +// "FAIL 268.072s" — the test that failed could not be named, and the fix was a guess. #220's failure +// came a few dozen lines before the end, inside the tail, and was named. The length of what came after the +// failure decided it, nothing else. +// +// So every line is looked at as it is written, and the lines that name what failed are kept whole, wherever +// in the output they are: each `--- FAIL:` line with its indented message lines, a `panic:` with its first +// frames, a `WARNING: DATA RACE` report, each `FAIL\t` line, a build error with its package, and +// the files gofmt lists. The rest is cut as before. + +// Bounds on what is picked, so a run that fails everywhere still travels in a verdict: the controller cuts a +// report to its last 60 KiB (maxCheckReport), and what failed is said at the report's end, so it stays inside. +const ( + // failureLines and failureBytes bound everything picked from one run. + failureLines = 400 + failureBytes = 32 << 10 + // failureLineBytes bounds one picked line: a message that pastes a whole document is cut, said. + failureLineBytes = 1 << 10 + // The lines kept after the line that opens each kind of block. + failMessageLines = 30 // a --- FAIL's messages + panicLines = 40 // a panic's first frames + raceLines = 80 // a data race report, to its closing rule + buildLines = 40 // a package's build errors + gofmtLines = 50 // the files gofmt lists + // failedNames bounds how many further failing tests the summary names after the first. + failedNames = 5 +) + +// goError is a compiler's or vet's line: a Go file, a line, and what is wrong there. +var goError = regexp.MustCompile(`^\S+\.go:\d+(:\d+)?: `) + +// raceRule is the line the race detector opens and closes its report with. +const raceRule = "==================" + +type failureBlock int + +const ( + noBlock failureBlock = iota + inFail + inPanic + inRace + inBuild + inGofmt +) + +// failures keeps the lines of a check's output that name what failed. It is an io.Writer, fed the same +// bytes as the report's tail. +type failures struct { + partial []byte + kept []string + size int + dropped int + + block failureBlock + indent int // a --- FAIL line's indentation: its messages are indented deeper + left int // lines the open block may still keep + headers []string + tests []string // every failing test's --- FAIL line, in order + lastRule bool // the line before was the race detector's rule, which opens its report +} + +func (f *failures) Write(p []byte) (int, error) { + f.partial = append(f.partial, p...) + for { + i := bytes.IndexByte(f.partial, '\n') + if i < 0 { + break + } + f.line(strings.TrimRight(string(f.partial[:i]), "\r")) + f.partial = f.partial[i+1:] + } + // A line with no end, longer than any line is kept, is judged by its start. + if len(f.partial) > failureLineBytes*4 { + f.line(string(f.partial)) + f.partial = nil + } + return len(p), nil +} + +// flush judges what is left of a last line with no newline. +func (f *failures) flush() { + if len(f.partial) > 0 { + f.line(strings.TrimRight(string(f.partial), "\r")) + f.partial = nil + } +} + +func indentOf(line string) int { + return len(line) - len(strings.TrimLeft(line, " \t")) +} + +func (f *failures) keep(line string) { + if f.size >= failureBytes || len(f.kept) >= failureLines { + f.dropped++ + return + } + if len(line) > failureLineBytes { + line = line[:failureLineBytes] + fmt.Sprintf(" … (%d bytes more)", len(line)-failureLineBytes) + } + f.kept = append(f.kept, line) + f.size += len(line) + 1 +} + +func (f *failures) open(block failureBlock, lines int) { + f.block, f.left = block, lines +} + +func (f *failures) line(line string) { + trimmed := strings.TrimSpace(line) + rule := trimmed == raceRule + + // A line that opens a block ends whatever block was open. + if f.opens(line, trimmed) { + f.lastRule = rule + return + } + switch f.block { + case inFail: + if trimmed != "" && indentOf(line) > f.indent && f.left > 0 { + f.left-- + f.keep(line) + f.lastRule = rule + return + } + case inPanic: + if f.left > 0 && !strings.HasPrefix(line, "FAIL") && !strings.HasPrefix(line, "ok \t") && + !strings.HasPrefix(line, "exit status") { + f.left-- + f.keep(line) + f.lastRule = rule + return + } + case inRace: + if f.left > 0 { + f.left-- + f.keep(line) + if rule { + f.block = noBlock + } + f.lastRule = rule + return + } + case inBuild: + if f.left > 0 && trimmed != "" && !strings.HasPrefix(line, "FAIL") && !strings.HasPrefix(line, "ok \t") && + !strings.HasPrefix(line, "? \t") { + f.left-- + f.keep(line) + f.lastRule = rule + return + } + case inGofmt: + if f.left > 0 && trimmed != "" { + f.left-- + f.keep(line) + f.lastRule = rule + return + } + } + f.block = noBlock + + switch { + case strings.HasPrefix(line, "# "): + // A package's name before its build errors — kept only when an error follows it. + f.headers = append(f.headers, line) + if len(f.headers) > 4 { + f.headers = f.headers[1:] + } + case len(f.headers) > 0 && goError.MatchString(line): + for _, h := range f.headers { + f.keep(h) + } + f.headers = nil + f.keep(line) + f.open(inBuild, buildLines) + case strings.HasPrefix(line, "FAIL\t"): + f.headers = nil + f.keep(line) + default: + f.headers = nil + } + f.lastRule = rule +} + +// opens keeps a line that opens a block of what failed, and opens the block. False for any other line. +func (f *failures) opens(line, trimmed string) bool { + switch { + case strings.HasPrefix(trimmed, "--- FAIL:"): + f.headers = nil + f.keep(line) + f.tests = append(f.tests, trimmed) + f.indent = indentOf(line) + f.open(inFail, failMessageLines) + case strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:"): + f.headers = nil + f.keep(line) + f.open(inPanic, panicLines) + case trimmed == "WARNING: DATA RACE": + f.headers = nil + if f.lastRule { + f.keep(raceRule) + } + f.keep(line) + f.open(inRace, raceLines) + case strings.HasPrefix(trimmed, "not gofmt'd:"): + f.headers = nil + f.keep(line) + f.open(inGofmt, gofmtLines) + default: + return false + } + return true +} + +// String is every line picked, in the order the run printed them, and how many more were left out. +func (f *failures) String() string { + f.flush() + s := strings.Join(f.kept, "\n") + if f.dropped > 0 { + s += fmt.Sprintf("\n… %d more line(s) naming what failed, past the %d lines or %d KiB kept", f.dropped, + failureLines, failureBytes>>10) + } + return s +} + +// said is what the repository layer's summary says failed: the first thing that failed, whole, then the +// further failing tests by name. Empty when nothing was picked. +func (f *failures) said() string { + picked := f.String() + if picked == "" { + return "" + } + first := whatFailed(picked) + var more []string + for _, t := range f.tests { + if t == first { + continue + } + // A subtest's parent fails with it; its name is in the subtest's. + name := strings.Fields(strings.TrimPrefix(t, "--- FAIL:")) + if len(name) > 0 { + more = append(more, name[0]) + } + } + more = withoutParents(more) + switch { + case len(more) == 0: + return first + case len(more) > failedNames: + return fmt.Sprintf("%s; and %s and %d more", first, strings.Join(more[:failedNames], ", "), + len(more)-failedNames) + default: + return first + "; and " + strings.Join(more, ", ") + } +} + +// withoutParents drops a test whose subtest is also named. +func withoutParents(names []string) []string { + var out []string + for _, n := range names { + parent := false + for _, m := range names { + if strings.HasPrefix(m, n+"/") { + parent = true + break + } + } + if !parent { + out = append(out, n) + } + } + return out +} diff --git a/internal/builder/failures_test.go b/internal/builder/failures_test.go new file mode 100644 index 00000000..e6278dfa --- /dev/null +++ b/internal/builder/failures_test.go @@ -0,0 +1,182 @@ +package builder + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// ownCheckOf runs a repository's own check whose script prints a captured output and fails, as the build +// seat runs it, and answers the layer and every line it said in the build's log. +func ownCheckOf(t *testing.T, printed string) (*Layer, []string) { + t.Helper() + tree := t.TempDir() + if err := os.WriteFile(filepath.Join(tree, "printed.txt"), []byte(printed), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, CheckScript), []byte("cat printed.txt\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + var out tail + var logged []string + layer := ownCheck(t.Context(), CheckSpec{Toolchain: "go-image"}, []ScriptPart{{"go", CheckScript}}, tree, &out, + func() bool { return false }, + func(_, script string) *exec.Cmd { + cmd := exec.CommandContext(t.Context(), "sh", script) + cmd.Dir = tree + return cmd + }, func(step, format string, args ...any) { + if step == "output" { + logged = append(logged, fmt.Sprintf(format, args...)) + } + }) + if layer == nil || layer.Verdict != "fail" { + t.Fatalf("a failing check answered %+v", layer) + } + return layer, logged +} + +func captured(t *testing.T, name string) string { + t.Helper() + body, err := os.ReadFile(filepath.Join("testdata", "issue460", name)) + if err != nil { + t.Fatal(err) + } + return string(body) +} + +// **novox/hq issue 460**: mesh-controller #218's check failed with "FAIL 268.072s" and no test +// name: the package's own logs, printed after its `--- FAIL:` lines, pushed them out of the last 200 lines, +// which was all the summary was named from. The first failing test is named, and the others after it, +// however much came after them. +func TestAFailureEarlyInALongRunIsStillNamed(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt")) + if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") { + t.Errorf("a failure 600 lines from the end is said as %q", layer.Summary) + } + if !strings.Contains(layer.Summary, "TestSubtests/the_hub") { + t.Errorf("the second failing test is not named: %q", layer.Summary) + } +} + +// One -race run of every package: the first failure is the earliest, not the last within the tail. +func TestARaceRunOfEveryPackageNamesItsFirstFailure(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-race-run-of-every-package.txt")) + if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") || + !strings.Contains(layer.Summary, "TestAMapIsNil") || !strings.Contains(layer.Summary, "TestACounterIsShared") { + t.Errorf("a run failing in four packages is said as %q", layer.Summary) + } +} + +// A package that does not build is named by its error, not by its bare "[build failed]". +func TestABuildErrorIsNamedByTheError(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-build-error.txt")) + if !strings.HasSuffix(layer.Summary, "failed: broken/broken.go:3:28: undefined: undefinedThing") { + t.Errorf("a build error is said as %q", layer.Summary) + } +} + +// The whole of what the check printed is in the build's log, line by line — where before none of it was. +func TestTheWholeOutputIsInTheBuildsLog(t *testing.T) { + printed := captured(t, "a-long-run-failing-early.txt") + _, logged := ownCheckOf(t, printed) + want := strings.Split(strings.TrimRight(printed, "\n"), "\n") + if len(logged) != len(want) { + t.Fatalf("the build's log holds %d line(s) of the %d printed", len(logged), len(want)) + } + for i := range want { + if logged[i] != want[i] { + t.Fatalf("line %d is logged as %q, printed as %q", i+1, logged[i], want[i]) + } + } +} + +// What failed is kept whole in the verdict: each --- FAIL with its messages, a panic with its first frames, +// a data race report from rule to rule — and none of the log lines around them. +func TestTheVerdictKeepsWhatFailedWhole(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt")) + want := "--- FAIL: TestTheEnvelopeIsPinned (0.00s)\n" + + " early_test.go:9: the envelope's subject is \"mesh.a\", not \"mesh.b\"\n" + + " early_test.go:10: a second line of the same failure\n" + + "--- FAIL: TestSubtests (0.00s)\n" + + " --- FAIL: TestSubtests/the_hub (0.00s)\n" + + " early_test.go:14: the hub does not compose\n" + + "FAIL\texample.com/cap/early\t" + if !strings.HasPrefix(layer.Failed, want) || strings.Contains(layer.Failed, "kept what home-server says") { + t.Errorf("the early failure is kept as:\n%s", layer.Failed) + } + + layer, _ = ownCheckOf(t, captured(t, "a-panic.txt")) + for _, line := range []string{"--- FAIL: TestAMapIsNil (0.00s)", "panic: assignment to entry in nil map", + "[running]:", "example.com/cap/panics.TestAMapIsNil(", "/src/cap/panics/panics_test.go:7", + "FAIL\texample.com/cap/panics\t"} { + if !strings.Contains(layer.Failed, line) { + t.Errorf("a panic is kept without %q:\n%s", line, layer.Failed) + } + } + + layer, _ = ownCheckOf(t, captured(t, "a-data-race.txt")) + race := captured(t, "a-data-race.txt") + report := race[:strings.Index(race, "--- FAIL:")] + if !strings.HasPrefix(layer.Failed, report) || + !strings.Contains(layer.Failed, "--- FAIL: TestACounterIsShared (0.00s)\n testing.go:1865: race detected") { + t.Errorf("a data race is kept as:\n%s", layer.Failed) + } + + layer, _ = ownCheckOf(t, captured(t, "a-build-error.txt")) + if layer.Failed != "# example.com/cap/broken\nbroken/broken.go:3:28: undefined: undefinedThing\n"+ + "FAIL\texample.com/cap/broken [build failed]" { + t.Errorf("a build error is kept as:\n%s", layer.Failed) + } +} + +// A run that fails everywhere is bounded in what it keeps, and what it keeps survives the controller's cut +// of a report to its last 60 KiB, said at the report's end. +func TestWhatFailedIsBoundedAndSurvivesTheReportsCut(t *testing.T) { + var b strings.Builder + for i := range 3000 { + fmt.Fprintf(&b, "--- FAIL: TestNumber%d (0.00s)\n x_test.go:1: %s\n", i, strings.Repeat("why ", 600)) + fmt.Fprintf(&b, "2026/10/11 01:30:03 a log line %d\n", i) + } + b.WriteString("FAIL\tx/everything\t1s\nFAIL\n") + layer, logged := ownCheckOf(t, b.String()) + if len(layer.Failed) > failureBytes+2*failureLineBytes || !strings.Contains(layer.Failed, "more line(s) naming what failed") { + t.Errorf("what failed in a run that fails everywhere is %d bytes, ending %q", len(layer.Failed), + layer.Failed[max(0, len(layer.Failed)-200):]) + } + if !strings.Contains(layer.Summary, "--- FAIL: TestNumber0 (0.00s); and TestNumber1, ") || + !strings.Contains(layer.Summary, "and 2994 more") { + t.Errorf("a run failing 3000 tests is said as %q", layer.Summary) + } + + var out tail + out.Write([]byte(b.String())) + report := withWhatFailed(out.String(), layer) + const maxCheckReport = 60 << 10 // as the controller cuts it, from the front + if len(report) > maxCheckReport { + report = report[len(report)-maxCheckReport:] + } + if !strings.Contains(report, "--- what failed") || !strings.Contains(report, "--- FAIL: TestNumber0 (0.00s)") { + t.Error("what failed did not survive the controller's cut of the report") + } + + // The build's log keeps its first logLines lines, says how many it left out, then what failed. + if len(logged) < logLines+2 || logged[logLines] != "… 1002 more line(s) of its output are not in this log, which keeps its first 8000" || + logged[logLines+1] != "--- what failed, picked from the whole of its output" || + logged[logLines+2] != "--- FAIL: TestNumber0 (0.00s)" { + t.Errorf("the log past its bound says %q", logged[logLines:min(len(logged), logLines+3)]) + } +} + +// A passing check says nothing of what failed. +func TestAPassingCheckReportsAsBefore(t *testing.T) { + if got := withWhatFailed("ok\tx\t1s", &Layer{Verdict: "pass"}); got != "ok\tx\t1s" { + t.Errorf("a passing report became %q", got) + } + if got := withWhatFailed("r", nil); got != "r" { + t.Errorf("no repository layer became %q", got) + } +} diff --git a/internal/builder/testdata/issue460/CAPTURED b/internal/builder/testdata/issue460/CAPTURED new file mode 100644 index 00000000..84084363 --- /dev/null +++ b/internal/builder/testdata/issue460/CAPTURED @@ -0,0 +1,7 @@ +Captured 2026-10-11 from real `go test` runs (go1.27.1, linux/amd64) of a throwaway module whose +packages fail each way a repository's own check fails: a test failing early in a run its own logs +then fill (early: 600 log lines after the failures), a panic, a data race under -race, a build error, +and one -race run of every package. Only the module's directory was rewritten to /src/cap. + +They are the output shapes of novox/hq issue 460: mesh-controller #218's check printed its +`--- FAIL:` line before the store-backed package's logs, which pushed it out of the report's tail. diff --git a/internal/builder/testdata/issue460/a-build-error.txt b/internal/builder/testdata/issue460/a-build-error.txt new file mode 100644 index 00000000..5e37bdd0 --- /dev/null +++ b/internal/builder/testdata/issue460/a-build-error.txt @@ -0,0 +1,5 @@ +# example.com/cap/broken +broken/broken.go:3:28: undefined: undefinedThing +FAIL example.com/cap/broken [build failed] +ok example.com/cap/fine (cached) +FAIL diff --git a/internal/builder/testdata/issue460/a-data-race.txt b/internal/builder/testdata/issue460/a-data-race.txt new file mode 100644 index 00000000..e56cf202 --- /dev/null +++ b/internal/builder/testdata/issue460/a-data-race.txt @@ -0,0 +1,31 @@ +================== +WARNING: DATA RACE +Read at 0x00c000018398 by goroutine 8: + example.com/cap/race.TestACounterIsShared.func1() + /src/cap/race/race_test.go:13 +0x7b + +Previous write at 0x00c000018398 by goroutine 9: + example.com/cap/race.TestACounterIsShared.func1() + /src/cap/race/race_test.go:13 +0x8d + +Goroutine 8 (running) created at: + example.com/cap/race.TestACounterIsShared() + /src/cap/race/race_test.go:13 +0x78 + testing.tRunner() + /usr/lib/go/src/testing/testing.go:2193 +0x21c + testing.(*T).Run.gowrap1() + /usr/lib/go/src/testing/testing.go:2258 +0x38 + +Goroutine 9 (finished) created at: + example.com/cap/race.TestACounterIsShared() + /src/cap/race/race_test.go:13 +0x78 + testing.tRunner() + /usr/lib/go/src/testing/testing.go:2193 +0x21c + testing.(*T).Run.gowrap1() + /usr/lib/go/src/testing/testing.go:2258 +0x38 +================== +--- FAIL: TestACounterIsShared (0.00s) + testing.go:1865: race detected during execution of test +FAIL +FAIL example.com/cap/race 0.008s +FAIL diff --git a/internal/builder/testdata/issue460/a-long-run-failing-early.txt b/internal/builder/testdata/issue460/a-long-run-failing-early.txt new file mode 100644 index 00000000..b58ac0be --- /dev/null +++ b/internal/builder/testdata/issue460/a-long-run-failing-early.txt @@ -0,0 +1,609 @@ +--- FAIL: TestTheEnvelopeIsPinned (0.00s) + early_test.go:9: the envelope's subject is "mesh.a", not "mesh.b" + early_test.go:10: a second line of the same failure +--- FAIL: TestSubtests (0.00s) + --- FAIL: TestSubtests/the_hub (0.00s) + early_test.go:14: the hub does not compose +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d0 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d1 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d2 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d3 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d4 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d5 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d6 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d7 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d8 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d9 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d10 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d11 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d12 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d13 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d14 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d15 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d16 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d17 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d18 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d19 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d20 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d21 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d22 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d23 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d24 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d25 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d26 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d27 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d28 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d29 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d30 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d31 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d32 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d33 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d34 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d35 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d36 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d37 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d38 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d39 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d40 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d41 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d42 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d43 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d44 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d45 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d46 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d47 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d48 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d49 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d50 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d51 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d52 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d53 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d54 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d55 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d56 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d57 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d58 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d59 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d60 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d61 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d62 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d63 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d64 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d65 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d66 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d67 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d68 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d69 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d70 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d71 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d72 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d73 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d74 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d75 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d76 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d77 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d78 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d79 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d80 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d81 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d82 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d83 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d84 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d85 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d86 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d87 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d88 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d89 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d90 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d91 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d92 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d93 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d94 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d95 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d96 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d97 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d98 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d99 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d100 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d101 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d102 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d103 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d104 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d105 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d106 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d107 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d108 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d109 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d110 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d111 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d112 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d113 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d114 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d115 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d116 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d117 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d118 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d119 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d120 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d121 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d122 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d123 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d124 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d125 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d126 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d127 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d128 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d129 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d130 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d131 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d132 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d133 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d134 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d135 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d136 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d137 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d138 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d139 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d140 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d141 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d142 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d143 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d144 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d145 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d146 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d147 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d148 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d149 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d150 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d151 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d152 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d153 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d154 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d155 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d156 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d157 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d158 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d159 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d160 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d161 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d162 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d163 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d164 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d165 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d166 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d167 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d168 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d169 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d170 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d171 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d172 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d173 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d174 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d175 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d176 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d177 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d178 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d179 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d180 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d181 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d182 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d183 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d184 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d185 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d186 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d187 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d188 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d189 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d190 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d191 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d192 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d193 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d194 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d195 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d196 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d197 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d198 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d199 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d200 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d201 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d202 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d203 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d204 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d205 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d206 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d207 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d208 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d209 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d210 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d211 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d212 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d213 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d214 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d215 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d216 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d217 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d218 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d219 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d220 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d221 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d222 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d223 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d224 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d225 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d226 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d227 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d228 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d229 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d230 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d231 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d232 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d233 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d234 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d235 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d236 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d237 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d238 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d239 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d240 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d241 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d242 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d243 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d244 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d245 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d246 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d247 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d248 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d249 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d250 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d251 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d252 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d253 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d254 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d255 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d256 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d257 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d258 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d259 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d260 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d261 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d262 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d263 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d264 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d265 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d266 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d267 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d268 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d269 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d270 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d271 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d272 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d273 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d274 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d275 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d276 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d277 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d278 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d279 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d280 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d281 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d282 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d283 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d284 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d285 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d286 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d287 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d288 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d289 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d290 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d291 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d292 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d293 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d294 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d295 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d296 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d297 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d298 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d299 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d300 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d301 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d302 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d303 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d304 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d305 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d306 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d307 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d308 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d309 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d310 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d311 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d312 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d313 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d314 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d315 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d316 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d317 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d318 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d319 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d320 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d321 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d322 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d323 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d324 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d325 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d326 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d327 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d328 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d329 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d330 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d331 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d332 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d333 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d334 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d335 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d336 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d337 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d338 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d339 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d340 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d341 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d342 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d343 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d344 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d345 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d346 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d347 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d348 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d349 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d350 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d351 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d352 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d353 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d354 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d355 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d356 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d357 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d358 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d359 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d360 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d361 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d362 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d363 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d364 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d365 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d366 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d367 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d368 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d369 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d370 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d371 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d372 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d373 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d374 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d375 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d376 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d377 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d378 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d379 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d380 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d381 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d382 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d383 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d384 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d385 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d386 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d387 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d388 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d389 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d390 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d391 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d392 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d393 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d394 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d395 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d396 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d397 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d398 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d399 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d400 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d401 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d402 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d403 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d404 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d405 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d406 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d407 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d408 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d409 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d410 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d411 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d412 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d413 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d414 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d415 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d416 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d417 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d418 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d419 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d420 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d421 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d422 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d423 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d424 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d425 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d426 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d427 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d428 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d429 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d430 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d431 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d432 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d433 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d434 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d435 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d436 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d437 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d438 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d439 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d440 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d441 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d442 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d443 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d444 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d445 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d446 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d447 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d448 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d449 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d450 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d451 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d452 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d453 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d454 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d455 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d456 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d457 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d458 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d459 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d460 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d461 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d462 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d463 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d464 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d465 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d466 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d467 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d468 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d469 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d470 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d471 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d472 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d473 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d474 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d475 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d476 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d477 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d478 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d479 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d480 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d481 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d482 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d483 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d484 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d485 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d486 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d487 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d488 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d489 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d490 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d491 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d492 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d493 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d494 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d495 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d496 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d497 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d498 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d499 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d500 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d501 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d502 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d503 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d504 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d505 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d506 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d507 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d508 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d509 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d510 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d511 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d512 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d513 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d514 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d515 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d516 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d517 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d518 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d519 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d520 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d521 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d522 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d523 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d524 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d525 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d526 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d527 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d528 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d529 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d530 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d531 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d532 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d533 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d534 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d535 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d536 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d537 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d538 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d539 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d540 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d541 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d542 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d543 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d544 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d545 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d546 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d547 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d548 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d549 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d550 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d551 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d552 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d553 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d554 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d555 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d556 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d557 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d558 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d559 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d560 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d561 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d562 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d563 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d564 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d565 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d566 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d567 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d568 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d569 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d570 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d571 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d572 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d573 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d574 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d575 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d576 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d577 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d578 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d579 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d580 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d581 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d582 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d583 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d584 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d585 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d586 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d587 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d588 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d589 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d590 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d591 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d592 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d593 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d594 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d595 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d596 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d597 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d598 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d599 +FAIL +FAIL example.com/cap/early 0.003s +FAIL diff --git a/internal/builder/testdata/issue460/a-panic.txt b/internal/builder/testdata/issue460/a-panic.txt new file mode 100644 index 00000000..ad18a6ac --- /dev/null +++ b/internal/builder/testdata/issue460/a-panic.txt @@ -0,0 +1,18 @@ +--- FAIL: TestAMapIsNil (0.00s) +panic: assignment to entry in nil map [recovered, repanicked] + +goroutine 18 [running]: +testing.tRunner.func1.2({0x6b6810, 0x6ee000}) + /usr/lib/go/src/testing/testing.go:2123 +0x232 +testing.tRunner.func1() + /usr/lib/go/src/testing/testing.go:2126 +0x329 +panic({0x6b6810?, 0x6ee000?}) + /usr/lib/go/src/runtime/panic.go:859 +0x125 +example.com/cap/panics.TestAMapIsNil(0x1e97d59dc248?) + /src/cap/panics/panics_test.go:7 +0x28 +testing.tRunner(0x1e97d59dc248, 0x6d41f8) + /usr/lib/go/src/testing/testing.go:2193 +0xea +created by testing.(*T).Run in goroutine 1 + /usr/lib/go/src/testing/testing.go:2258 +0x4d4 +FAIL example.com/cap/panics 0.004s +FAIL diff --git a/internal/builder/testdata/issue460/a-race-run-of-every-package.txt b/internal/builder/testdata/issue460/a-race-run-of-every-package.txt new file mode 100644 index 00000000..fa816310 --- /dev/null +++ b/internal/builder/testdata/issue460/a-race-run-of-every-package.txt @@ -0,0 +1,660 @@ +# example.com/cap/broken +broken/broken.go:3:28: undefined: undefinedThing +FAIL example.com/cap/broken [build failed] +--- FAIL: TestTheEnvelopeIsPinned (0.00s) + early_test.go:9: the envelope's subject is "mesh.a", not "mesh.b" + early_test.go:10: a second line of the same failure +--- FAIL: TestSubtests (0.00s) + --- FAIL: TestSubtests/the_hub (0.00s) + early_test.go:14: the hub does not compose +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d0 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d1 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d2 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d3 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d4 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d5 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d6 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d7 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d8 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d9 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d10 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d11 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d12 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d13 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d14 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d15 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d16 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d17 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d18 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d19 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d20 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d21 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d22 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d23 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d24 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d25 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d26 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d27 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d28 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d29 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d30 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d31 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d32 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d33 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d34 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d35 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d36 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d37 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d38 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d39 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d40 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d41 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d42 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d43 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d44 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d45 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d46 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d47 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d48 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d49 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d50 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d51 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d52 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d53 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d54 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d55 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d56 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d57 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d58 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d59 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d60 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d61 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d62 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d63 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d64 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d65 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d66 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d67 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d68 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d69 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d70 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d71 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d72 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d73 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d74 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d75 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d76 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d77 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d78 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d79 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d80 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d81 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d82 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d83 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d84 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d85 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d86 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d87 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d88 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d89 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d90 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d91 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d92 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d93 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d94 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d95 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d96 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d97 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d98 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d99 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d100 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d101 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d102 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d103 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d104 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d105 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d106 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d107 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d108 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d109 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d110 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d111 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d112 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d113 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d114 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d115 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d116 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d117 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d118 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d119 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d120 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d121 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d122 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d123 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d124 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d125 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d126 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d127 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d128 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d129 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d130 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d131 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d132 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d133 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d134 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d135 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d136 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d137 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d138 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d139 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d140 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d141 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d142 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d143 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d144 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d145 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d146 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d147 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d148 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d149 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d150 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d151 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d152 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d153 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d154 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d155 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d156 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d157 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d158 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d159 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d160 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d161 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d162 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d163 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d164 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d165 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d166 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d167 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d168 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d169 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d170 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d171 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d172 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d173 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d174 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d175 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d176 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d177 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d178 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d179 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d180 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d181 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d182 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d183 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d184 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d185 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d186 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d187 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d188 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d189 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d190 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d191 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d192 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d193 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d194 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d195 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d196 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d197 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d198 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d199 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d200 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d201 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d202 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d203 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d204 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d205 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d206 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d207 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d208 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d209 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d210 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d211 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d212 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d213 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d214 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d215 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d216 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d217 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d218 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d219 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d220 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d221 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d222 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d223 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d224 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d225 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d226 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d227 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d228 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d229 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d230 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d231 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d232 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d233 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d234 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d235 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d236 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d237 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d238 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d239 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d240 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d241 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d242 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d243 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d244 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d245 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d246 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d247 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d248 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d249 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d250 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d251 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d252 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d253 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d254 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d255 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d256 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d257 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d258 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d259 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d260 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d261 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d262 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d263 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d264 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d265 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d266 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d267 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d268 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d269 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d270 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d271 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d272 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d273 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d274 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d275 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d276 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d277 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d278 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d279 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d280 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d281 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d282 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d283 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d284 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d285 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d286 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d287 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d288 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d289 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d290 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d291 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d292 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d293 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d294 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d295 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d296 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d297 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d298 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d299 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d300 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d301 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d302 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d303 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d304 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d305 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d306 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d307 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d308 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d309 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d310 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d311 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d312 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d313 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d314 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d315 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d316 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d317 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d318 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d319 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d320 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d321 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d322 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d323 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d324 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d325 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d326 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d327 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d328 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d329 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d330 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d331 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d332 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d333 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d334 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d335 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d336 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d337 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d338 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d339 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d340 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d341 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d342 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d343 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d344 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d345 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d346 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d347 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d348 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d349 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d350 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d351 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d352 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d353 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d354 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d355 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d356 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d357 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d358 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d359 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d360 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d361 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d362 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d363 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d364 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d365 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d366 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d367 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d368 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d369 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d370 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d371 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d372 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d373 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d374 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d375 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d376 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d377 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d378 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d379 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d380 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d381 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d382 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d383 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d384 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d385 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d386 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d387 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d388 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d389 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d390 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d391 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d392 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d393 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d394 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d395 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d396 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d397 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d398 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d399 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d400 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d401 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d402 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d403 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d404 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d405 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d406 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d407 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d408 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d409 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d410 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d411 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d412 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d413 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d414 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d415 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d416 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d417 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d418 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d419 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d420 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d421 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d422 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d423 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d424 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d425 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d426 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d427 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d428 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d429 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d430 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d431 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d432 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d433 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d434 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d435 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d436 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d437 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d438 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d439 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d440 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d441 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d442 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d443 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d444 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d445 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d446 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d447 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d448 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d449 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d450 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d451 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d452 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d453 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d454 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d455 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d456 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d457 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d458 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d459 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d460 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d461 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d462 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d463 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d464 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d465 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d466 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d467 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d468 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d469 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d470 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d471 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d472 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d473 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d474 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d475 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d476 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d477 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d478 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d479 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d480 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d481 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d482 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d483 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d484 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d485 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d486 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d487 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d488 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d489 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d490 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d491 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d492 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d493 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d494 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d495 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d496 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d497 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d498 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d499 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d500 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d501 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d502 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d503 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d504 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d505 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d506 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d507 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d508 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d509 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d510 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d511 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d512 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d513 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d514 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d515 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d516 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d517 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d518 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d519 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d520 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d521 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d522 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d523 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d524 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d525 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d526 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d527 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d528 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d529 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d530 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d531 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d532 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d533 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d534 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d535 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d536 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d537 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d538 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d539 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d540 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d541 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d542 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d543 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d544 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d545 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d546 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d547 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d548 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d549 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d550 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d551 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d552 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d553 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d554 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d555 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d556 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d557 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d558 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d559 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d560 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d561 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d562 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d563 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d564 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d565 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d566 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d567 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d568 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d569 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d570 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d571 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d572 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d573 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d574 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d575 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d576 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d577 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d578 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d579 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d580 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d581 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d582 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d583 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d584 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d585 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d586 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d587 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d588 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d589 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d590 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d591 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d592 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d593 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d594 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d595 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d596 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d597 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d598 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d599 +FAIL +FAIL example.com/cap/early 0.013s +ok example.com/cap/fine (cached) +--- FAIL: TestAMapIsNil (0.00s) +panic: assignment to entry in nil map [recovered, repanicked] + +goroutine 35 [running]: +testing.tRunner.func1.2({0x7c6e60, 0x806640}) + /usr/lib/go/src/testing/testing.go:2123 +0x419 +testing.tRunner.func1() + /usr/lib/go/src/testing/testing.go:2126 +0x65f +panic({0x7c6e60?, 0x806640?}) + /usr/lib/go/src/runtime/panic.go:859 +0x125 +example.com/cap/panics.TestAMapIsNil(0xc0001d6248?) + /src/cap/panics/panics_test.go:7 +0x32 +testing.tRunner(0xc0001d6248, 0x7e5f78) + /usr/lib/go/src/testing/testing.go:2193 +0x21d +created by testing.(*T).Run in goroutine 1 + /usr/lib/go/src/testing/testing.go:2258 +0xb13 +FAIL example.com/cap/panics 0.011s +================== +WARNING: DATA RACE +Read at 0x00c000018398 by goroutine 9: + example.com/cap/race.TestACounterIsShared.func1() + /src/cap/race/race_test.go:13 +0x7b + +Previous write at 0x00c000018398 by goroutine 8: + example.com/cap/race.TestACounterIsShared.func1() + /src/cap/race/race_test.go:13 +0x8d + +Goroutine 9 (running) created at: + example.com/cap/race.TestACounterIsShared() + /src/cap/race/race_test.go:13 +0x78 + testing.tRunner() + /usr/lib/go/src/testing/testing.go:2193 +0x21c + testing.(*T).Run.gowrap1() + /usr/lib/go/src/testing/testing.go:2258 +0x38 + +Goroutine 8 (finished) created at: + example.com/cap/race.TestACounterIsShared() + /src/cap/race/race_test.go:13 +0x78 + testing.tRunner() + /usr/lib/go/src/testing/testing.go:2193 +0x21c + testing.(*T).Run.gowrap1() + /usr/lib/go/src/testing/testing.go:2258 +0x38 +================== +--- FAIL: TestACounterIsShared (0.00s) + testing.go:1865: race detected during execution of test +FAIL +FAIL example.com/cap/race 0.009s +FAIL From 9bed7d6398e2872624b59ae361604d86a4779cfe Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 05:25:29 +0200 Subject: [PATCH 05/16] broker: grant each credential the tool calls that name it as the caller, and no other (hq issue 365) Every grant to call a tool is granted again under the call kind, with the credential's own bus user as the last token, and every grant to answer one is granted for calls naming any caller: the caller of a tool call becomes a fact the bus enforces, as ADR 0259 section 3 made the asker of an ask. A kind of its own because every existing tool grant is a wildcard that would match any caller appended. The old tool grants stay for one release so nothing loses its way to a tool (hq issue 464); the controller's own grants move with that issue, since they are also the installer's first user list. --- internal/broker/callers.go | 117 ++++++++++++++++++++++++ internal/broker/callers_live_test.go | 100 +++++++++++++++++++++ internal/broker/callers_test.go | 119 +++++++++++++++++++++++++ internal/broker/invokes_test.go | 5 +- internal/broker/nats.go | 17 ++++ internal/broker/nats_test.go | 6 +- internal/broker/testdata/composed.conf | 6 +- 7 files changed, 362 insertions(+), 8 deletions(-) create mode 100644 internal/broker/callers.go create mode 100644 internal/broker/callers_live_test.go create mode 100644 internal/broker/callers_test.go diff --git a/internal/broker/callers.go b/internal/broker/callers.go new file mode 100644 index 00000000..24392b58 --- /dev/null +++ b/internal/broker/callers.go @@ -0,0 +1,117 @@ +package broker + +import ( + "regexp" + "strings" +) + +// A tool call names its caller (novox/hq issue 365, by ADR 0259 §3's precedent for asks). +// +// mesh.mod..call.[.]. +// mesh.seat..call.[.]. +// +// The last token is the bus user that published the call, and each user is granted these subjects with its own +// name there and no other — the caller is a fact the server enforces, and the tool runtime hands it to the +// module from the subject a call arrived on (mesh-tools node-tools internal/bus caller.go holds the same shape). +// +// **A kind of its own, `call`, not the `tool` subject with a token appended.** Every grant to call a tool is a +// wildcard over the `tool` kind — `.tool..*` for the instance on any machine, `mesh.mod.*.tool.>` for every +// tool — and either would match a `tool` subject with any caller appended, so a caller could name another. +// Under `call` nothing is granted but the caller-named subjects. No stream's filter covers it: a tool call is +// never persisted (design 25 §3). +// +// **Derived from the `tool` grants, in one place** (callerNamed): wherever a principal may call or answer a +// tool, it may call it in its own name, or answer it naming any caller — so no kind of principal is left +// unable to call in its own name, and a new grant to call is one in both shapes by construction. +// +// The `tool` grants stay beside these for one release, so every caller and every runtime moves without a gap: +// their retirement is hq issue 464. +const CallKind = "call" + +var userName = regexp.MustCompile(`^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$`) + +// CallerToken is a bus user's name as the last token of a call it publishes: each dot written `~`, which no part +// of a user's name may hold (safeSubject), so the token names that user and no other. "" for a name that is not +// a user's. +func CallerToken(user string) string { + if !userName.MatchString(user) { + return "" + } + return strings.ReplaceAll(user, ".", "~") +} + +// toolGrant splits a grant on the `tool` kind — `mesh.mod..tool.` or `mesh.seat..tool.`, any +// part possibly a wildcard — at the kind; ok is false for any other subject. +func toolGrant(subject string) (head, rest string, ok bool) { + parts := strings.SplitN(subject, ".", 5) + if len(parts) != 5 || parts[0] != "mesh" || (parts[1] != "mod" && parts[1] != "seat") || parts[3] != "tool" || + parts[2] == "" || parts[4] == "" { + return "", "", false + } + return parts[0] + "." + parts[1] + "." + parts[2], parts[4], true +} + +// CalledSubject is where a call to a tool subject goes naming its caller; "" when the subject is not a tool's +// or the user is not a bus user. +func CalledSubject(subject, user string) string { + head, rest, ok := toolGrant(subject) + token := CallerToken(user) + if !ok || token == "" || strings.ContainsAny(rest, "*>") { + return "" + } + return head + "." + CallKind + "." + rest + "." + token +} + +// CalledPattern is what a holder answering a tool subject also subscribes, to hear the calls that name their +// caller: the same address under the `call` kind, any caller last. "" for a subject that is not a tool's. +func CalledPattern(subject string) string { + head, rest, ok := toolGrant(subject) + if !ok || strings.ContainsAny(rest, "*>") { + return "" + } + return head + "." + CallKind + "." + rest + ".*" +} + +// calledPublish is a grant to call on the `tool` kind, as the same grant in the caller's own name: its last +// token the caller's, and nothing that reaches past it. A `>` is every tail a call carries — the tool alone or +// the tool and the machine — so it becomes both, each ending in the caller. +func calledPublish(grant, token string) []string { + head, rest, ok := toolGrant(grant) + if !ok || token == "" { + return nil + } + base := head + "." + CallKind + "." + switch { + case rest == ">": + return []string{base + "*." + token, base + "*.*." + token} + case strings.HasSuffix(rest, ".>"): + return []string{base + strings.TrimSuffix(rest, ">") + "*." + token} + } + return []string{base + rest + "." + token} +} + +// calledSubscribe is a grant to answer on the `tool` kind, as the same grant for the calls naming any caller. +func calledSubscribe(grant string) []string { + head, rest, ok := toolGrant(grant) + if !ok { + return nil + } + base := head + "." + CallKind + "." + if strings.HasSuffix(rest, ">") { + return []string{base + rest} + } + return []string{base + rest + ".*"} +} + +// callerNamed adds, beside a principal's grants on the `tool` kind, the same grants under `call`: to publish in +// its own name, to subscribe naming anybody. +func callerNamed(user string, pub, sub []string) ([]string, []string) { + token := CallerToken(user) + for _, g := range pub { + pub = append(pub, calledPublish(g, token)...) + } + for _, g := range sub { + sub = append(sub, calledSubscribe(g)...) + } + return unique(pub), unique(sub) +} diff --git a/internal/broker/callers_live_test.go b/internal/broker/callers_live_test.go new file mode 100644 index 00000000..cacb0424 --- /dev/null +++ b/internal/broker/callers_live_test.go @@ -0,0 +1,100 @@ +package broker + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "golang.org/x/crypto/bcrypt" +) + +// **On a real server, as composed** (novox/hq issue 365): a machine's runtime calls in its own name, and the +// server refuses it a call naming another — the grant, not the runtime, is what makes the caller a fact. +func TestAServerComposedFromTheGrantsRefusesACallNamingAnother(t *testing.T) { + hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost) + if err != nil { + t.Fatal(err) + } + ledger := Seat{Name: "issue-tracker", Scope: "mesh", Serves: []string{"open"}} + accounts, err := ComposeAccounts([]Principal{ + {Kind: KindNodeTools, Node: "novox", Module: RuntimeModule, PasswordHash: string(hash), + Carries: []Declared{{Module: "mesh-issues", Holds: []Seat{ledger}}}}, + {Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule, PasswordHash: string(hash)}, + }) + if err != nil { + t.Fatal(err) + } + conf := filepath.Join(t.TempDir(), "bus.conf") + if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+ + `"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(conf) + if err != nil { + t.Fatalf("the composed accounts do not parse: %v", err) + } + opts.NoLog, opts.NoSigs = true, true + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + defer s.Shutdown() + + holder, err := nats.Connect(s.ClientURL(), nats.UserInfo("novox.node-tools", "pw")) + if err != nil { + t.Fatal(err) + } + defer holder.Close() + heard := make(chan string, 4) + sub, err := holder.Subscribe("mesh.seat.issue-tracker.call.open.*", func(m *nats.Msg) { + heard <- m.Subject + _ = m.Respond([]byte(`{"result":{}}`)) + }) + if err != nil { + t.Fatal(err) + } + _ = holder.Flush() + if !sub.IsValid() { + t.Fatal("the holder may not hear the caller-named calls to its seat") + } + + refusals := make(chan error, 4) + caller, err := nats.Connect(s.ClientURL(), nats.UserInfo("shanks.node-tools", "pw"), + nats.CustomInboxPrefix("_INBOX.shanks.node-tools"), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err })) + if err != nil { + t.Fatal(err) + } + defer caller.Close() + if _, err := caller.Request("mesh.seat.issue-tracker.call.open.shanks~node-tools", []byte(`{}`), 3*time.Second); err != nil { + t.Fatalf("a call in the caller's own name was not answered: %v", err) + } + if got := <-heard; got != "mesh.seat.issue-tracker.call.open.shanks~node-tools" { + t.Fatalf("heard %s", got) + } + if err := caller.Publish("mesh.seat.issue-tracker.call.open.novox~node-tools", []byte(`{}`)); err != nil { + t.Fatal(err) + } + _ = caller.Flush() + select { + case err := <-refusals: + if !errors.Is(err, nats.ErrPermissionViolation) { + t.Errorf("the server said %v, want a permissions violation", err) + } + case <-time.After(3 * time.Second): + t.Error("the server did not refuse a call naming another caller") + } + select { + case got := <-heard: + t.Errorf("a call naming another reached the holder: %s", got) + case <-time.After(200 * time.Millisecond): + } +} diff --git a/internal/broker/callers_test.go b/internal/broker/callers_test.go new file mode 100644 index 00000000..f11cac30 --- /dev/null +++ b/internal/broker/callers_test.go @@ -0,0 +1,119 @@ +package broker + +import ( + "strings" + "testing" +) + +// **A tool call names its caller, and the bus lets each user name itself alone** (novox/hq issue 365, by ADR +// 0259 §3's precedent for asks): wherever a principal may call a tool, it may call it on the caller-named +// subject — kind `call`, its own bus user last, dots written `~` — and on no subject naming anybody else. +func TestEachCredentialMayCallOnlyInItsOwnName(t *testing.T) { + ledger := Seat{Name: "node-desk", Scope: "node", Serves: []string{"who"}} + tracker := Seat{Name: "issue-tracker", Scope: "mesh", Serves: []string{"open"}} + for _, c := range []struct { + p Principal + may []string + mayNot []string + subject []string // what it subscribes, to answer calls naming any caller + }{ + {p: Principal{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}}, + may: []string{"mesh.mod.ledger.call.who.person~jochen", "mesh.mod.ledger.call.who.anchor.person~jochen", + "mesh.seat.issue-tracker.call.open.person~jochen", "mesh.seat.node-desk.call.who.anchor.person~jochen"}, + mayNot: []string{"mesh.mod.ledger.call.who.shanks~node-tools", "mesh.mod.ledger.call.who.anchor.controller", + "mesh.seat.issue-tracker.call.open.person~somebody", "mesh.mod.ledger.call.who.person"}}, + {p: Principal{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule, + Carries: []Declared{{Module: "ledger", Holds: []Seat{ledger, tracker}}}}, + may: []string{"mesh.mod.ledger.call.who.shanks~node-tools", "mesh.seat.issue-tracker.call.open.shanks~node-tools"}, + mayNot: []string{"mesh.mod.ledger.call.who.novox~node-tools", "mesh.seat.issue-tracker.call.open.controller"}, + subject: []string{"mesh.mod.ledger.call.who.novox~node-tools", "mesh.seat.node-desk.call.who.shanks.person~jochen", "mesh.seat.issue-tracker.call.open.controller"}}, + {p: Principal{Kind: KindModule, Node: "two", Module: "shop", Invokes: []string{"ledger.who", "seat:issue-tracker.open"}}, + may: []string{"mesh.mod.ledger.call.who.two~shop", "mesh.mod.ledger.call.who.anchor.two~shop", + "mesh.seat.issue-tracker.call.open.two~shop"}, + mayNot: []string{"mesh.mod.ledger.call.other.two~shop", "mesh.mod.ledger.call.who.one~shop", + "mesh.seat.issue-tracker.call.open.one~telegram"}}, + {p: Principal{Kind: KindNode, Node: "one", Checks: []string{"ledger.health"}}, + may: []string{"mesh.mod.ledger.call.health.one.node~one"}, + mayNot: []string{"mesh.mod.ledger.call.health.two.node~one", "mesh.mod.ledger.call.health.one.node~two"}}, + {p: Principal{Kind: KindModule, Node: "one", Module: "ledger", Holds: []Seat{ledger, tracker}}, + subject: []string{"mesh.mod.ledger.call.who.person~jochen", "mesh.mod.ledger.call.who.one.controller", + "mesh.seat.node-desk.call.who.one.two~shop", "mesh.seat.issue-tracker.call.open.two~shop"}}, + } { + perms, err := PermissionsFor(c.p) + if err != nil { + t.Fatalf("%s: %v", c.p.Username(), err) + } + for _, s := range c.may { + if !MayPublish(perms, s) { + t.Errorf("%s may not call %s, in its own name", c.p.Username(), s) + } + } + for _, s := range c.mayNot { + if MayPublish(perms, s) { + t.Errorf("%s may call %s, naming somebody else", c.p.Username(), s) + } + } + for _, s := range c.subject { + if !MaySubscribe(perms, s) { + t.Errorf("%s does not hear %s, a call to what it serves", c.p.Username(), s) + } + } + } +} + +// The subjects that name no caller stay granted beside the caller-named ones for one release, so a caller +// moves over without a gap (their retirement: hq issue 464). +func TestTheSubjectsThatNameNoCallerStayGrantedForOneRelease(t *testing.T) { + perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "shop", Invokes: []string{"ledger.who"}}) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{"mesh.mod.ledger.tool.who", "mesh.mod.ledger.tool.who.anchor"} { + if !MayPublish(perms, s) { + t.Errorf("the old subject %s is no longer granted", s) + } + } +} + +// The desk's hidden prompt is the controller's alone on the caller-named subjects too (the review of +// 2026-10-09, M4): a grant of every tool does not reach it there either. +func TestTheDesksPromptIsTheControllersAloneUnderCallToo(t *testing.T) { + for _, p := range []Principal{{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}}, + {Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule}} { + perms, err := PermissionsFor(p) + if err != nil { + t.Fatal(err) + } + token := CallerToken(p.Username()) + for _, s := range []string{"mesh.seat.node-launcher.call.secret.shanks." + token, + "mesh.mod.shell.call.node-launcher.secret.shanks." + token, "mesh.mod.shell.call.node-launcher.secret." + token} { + if MayPublish(perms, s) { + t.Errorf("%s may publish %s, the desk's hidden prompt", p.Username(), s) + } + } + } +} + +// The controller's grants are the installer's first user list too, so they move with hq issue 464: this release +// it calls and serves on the subjects that name no caller alone, and nobody may call in its name. +func TestTheControllerKeepsTheSubjectsThatNameNoCallerThisRelease(t *testing.T) { + perms, err := PermissionsFor(Principal{Kind: KindController}) + if err != nil { + t.Fatal(err) + } + for _, s := range append(perms.Publish, perms.Subscribe...) { + if strings.Contains(s, "."+CallKind+".") { + t.Errorf("the controller is granted %s, which the installer's first user list does not carry", s) + } + } + for _, p := range []Principal{{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}}, + {Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule}} { + perms, err := PermissionsFor(p) + if err != nil { + t.Fatal(err) + } + if MayPublish(perms, "mesh.mod.ledger.call.who.controller") || MayPublish(perms, "mesh.seat.mesh-controller.call.status.controller") { + t.Errorf("%s may call in the controller's name", p.Username()) + } + } +} diff --git a/internal/broker/invokes_test.go b/internal/broker/invokes_test.go index 3d1e86ca..c22c12e8 100644 --- a/internal/broker/invokes_test.go +++ b/internal/broker/invokes_test.go @@ -42,8 +42,9 @@ func TestInvokingGrantsNothingButTheCall(t *testing.T) { if strings.Contains(p, ".event.") { t.Errorf("a module that only invokes may publish %q, an event it never declared", p) } - // A role's tools are tools (ADR 0132); a role's work queue and events are not. - if strings.HasPrefix(p, "mesh.seat.") && !strings.Contains(p, ".tool.") { + // A role's tools are tools (ADR 0132), named by their caller or not (novox/hq issue 365); a role's work + // queue and events are not. + if strings.HasPrefix(p, "mesh.seat.") && !strings.Contains(p, ".tool.") && !strings.Contains(p, ".call.") { t.Errorf("a module that only invokes may publish %q, a seat it neither holds nor uses", p) } } diff --git a/internal/broker/nats.go b/internal/broker/nats.go index d46caf7f..659a9cc8 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -212,6 +212,10 @@ func ControllerOnly() []string { for _, base := range []string{"mesh.seat." + v.Seat + ".tool." + v.Verb, "mesh.mod.*.tool." + v.Seat + "." + v.Verb} { out = append(out, base, base+".*") } + // And where a call names its caller (novox/hq issue 365): any machine, any caller. + for _, base := range []string{"mesh.seat." + v.Seat + "." + CallKind + "." + v.Verb, "mesh.mod.*." + CallKind + "." + v.Seat + "." + v.Verb} { + out = append(out, base+".>") + } } return out } @@ -875,6 +879,19 @@ func PermissionsFor(p Principal) (Permissions, error) { pub = append(pub, "$JS.ACK."+consumerStream(p)+"."+consumerDurable(p)+".>") } + // **And every tool grant again, naming its caller** (novox/hq issue 365): a call in this principal's own + // name, and an answer to a call naming anybody. The `tool` grants above stay for one release beside these, + // so callers and runtimes move without a gap; their retirement is hq issue 464. + // + // **Not the controller's, this release.** Its grants are also the installer's first user list + // (TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose), judged against the node-engine the mesh + // runs, so a change to them waits on a node-engine delivery. It calls and serves on the subjects that name + // no caller meanwhile — a caller-named call to its seat reaches nobody and is asked again on those at once — + // and moves with issue 464. + if p.Kind != KindController { + pub, sub = callerNamed(p.Username(), pub, sub) + } + sort.Strings(pub) sort.Strings(sub) // One writer per piece of state (novox/hq to-be 45 §1): a grant that would make a second is diff --git a/internal/broker/nats_test.go b/internal/broker/nats_test.go index 23d3f8d3..bfd52cfb 100644 --- a/internal/broker/nats_test.go +++ b/internal/broker/nats_test.go @@ -240,9 +240,9 @@ func TestAPersonReachesNothingButTools(t *testing.T) { perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", Invokes: []string{"*"}, PasswordHash: "x"}) for _, p := range perms.Publish { - // A tool call, or asking what answers (novox/hq ADR 0197) — a question every service - // answers about itself, which claims nothing and controls nothing. - if !strings.Contains(p, ".tool.") && !strings.HasPrefix(p, "$SRV.") { + // A tool call — named by its caller or not (novox/hq issue 365) — or asking what answers (novox/hq + // ADR 0197), a question every service answers about itself, which claims nothing and controls nothing. + if !strings.Contains(p, ".tool.") && !strings.Contains(p, ".call.") && !strings.HasPrefix(p, "$SRV.") { t.Errorf("a person may publish %q, which is not a tool call", p) } } diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 0919b2b8..ea22d3c5 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -43,17 +43,17 @@ accounts { } } { user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: { publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] } - subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] } + subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.call.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] } allow_responses: { max: 1, ttl: "1m" } } } { user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: { publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.audit"] } - subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] } + subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.call.>", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] } allow_responses: { max: 1, ttl: "1m" } } } { user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: { publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.shop", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] } - subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.tool.>"] } + subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.call.>", "mesh.mod.shop.tool.>"] } allow_responses: { max: 1, ttl: "1m" } } } ] From ebca7816bf0817bdba189479da21ae065f49a66c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 05:47:40 +0200 Subject: [PATCH 06/16] inventory: a person's one tool is granted naming that person as the caller too (hq issue 365) --- internal/inventory/people_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/inventory/people_test.go b/internal/inventory/people_test.go index 7df4b15c..5919607e 100644 --- a/internal/inventory/people_test.go +++ b/internal/inventory/people_test.go @@ -44,15 +44,17 @@ func TestAPersonMayCallToolsAndNothingElse(t *testing.T) { } // The one tool, both ways it is addressed (novox/hq ADR 0159): to whichever instance // answers, and to the instance on one machine. Nothing else. - // And asking what answers (novox/hq ADR 0197), which claims nothing and calls nothing. + // And asking what answers (novox/hq ADR 0197), which claims nothing and calls nothing. Each way again + // naming ada as the caller and nobody else (novox/hq issue 365). var tools []string for _, s := range perms.Publish { if !strings.HasPrefix(s, "$SRV.") { tools = append(tools, s) } } - if len(tools) != 2 || tools[0] != "mesh.mod.mesh-catalog.tool.catalog_tools" || - tools[1] != "mesh.mod.mesh-catalog.tool.catalog_tools.*" { + want := []string{"mesh.mod.mesh-catalog.call.catalog_tools.*.person~ada", "mesh.mod.mesh-catalog.call.catalog_tools.person~ada", + "mesh.mod.mesh-catalog.tool.catalog_tools", "mesh.mod.mesh-catalog.tool.catalog_tools.*"} + if strings.Join(tools, " ") != strings.Join(want, " ") { t.Errorf("ada may publish %v, which should be the one tool, both ways addressed, and nothing else", perms.Publish) } for _, s := range perms.Publish { From 094d3d5bc6c704bcaa451c44df90113544f1921b Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 10:51:41 +0200 Subject: [PATCH 07/16] Wait out a registry held still, for up to five minutes, instead of failing the build (issue 457) The store's nightly collection stops the registry for about a minute and a half; builds in that window failed on connection refused and failed their whole plans. A refusal is now waited for with a growing pause, said in the build's log, and still fails the build past the bound. --- internal/builder/registry.go | 12 +- internal/builder/registry_wait.go | 131 +++++++++++++++++++++ internal/builder/registry_wait_test.go | 154 +++++++++++++++++++++++++ 3 files changed, 294 insertions(+), 3 deletions(-) create mode 100644 internal/builder/registry_wait.go create mode 100644 internal/builder/registry_wait_test.go diff --git a/internal/builder/registry.go b/internal/builder/registry.go index 219b4fe5..4f7ecd11 100644 --- a/internal/builder/registry.go +++ b/internal/builder/registry.go @@ -52,7 +52,11 @@ func (r Registry) PublishImage(ctx context.Context, localTag, repository string) if _, err := r.Run(ctx, "", "docker", "tag", localTag, remote); err != nil { return "", err } - if _, err := r.Run(ctx, "", "docker", "push", remote); err != nil { + // The push waits for a registry held still, as every request to it does (novox/hq issue 457). + if err := waitForRegistry(ctx, r.Address, "docker push "+remote, func() error { + _, err := r.Run(ctx, "", "docker", "push", remote) + return err + }); err != nil { return "", err } out, err := r.Run(ctx, "", "docker", "inspect", "--format", "{{index .RepoDigests 0}}", remote) @@ -169,11 +173,13 @@ func (r Registry) has(ctx context.Context, url string, accept ...string) (bool, return response.StatusCode == http.StatusOK, nil } +// client is the client for the registry and for upstream, whose requests to the registry wait out a +// registry held still (novox/hq issue 457). func (r Registry) client() *http.Client { if r.HTTP != nil { - return r.HTTP + return waiting(r.HTTP, r.Address) } - return http.DefaultClient + return waiting(http.DefaultClient, r.Address) } // separator is whether the upload location already carries a query. diff --git a/internal/builder/registry_wait.go b/internal/builder/registry_wait.go new file mode 100644 index 00000000..010622a0 --- /dev/null +++ b/internal/builder/registry_wait.go @@ -0,0 +1,131 @@ +package builder + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "syscall" + "time" +) + +// A build waits out a registry that refuses connections, for a bounded time (novox/hq issue 457). +// +// **Why waiting, and why here.** The store's nightly collection holds the registry still for its run — +// about a minute and a half, measured on 2026-10-11 — and a build that reached the registry in that +// window failed on "connection refused", and its whole delivery plan with it: a plan failed for a +// pause the mesh itself scheduled. The other design weighed was the collection telling the controller +// it holds the registry, and the controller holding build asks while it runs. Waiting here is smaller +// and covers more: it is local to the one place that talks to the registry, needs no new message +// between modules, and also carries a build over any other short outage — a registry restarted by its +// own update, say. A refusal is the one error waited for: nothing was sent, so trying again cannot +// do anything twice, and it is what a registry that is stopped answers. +// +// **Bounded, and loud past the bound.** registryWait is longer than the collection holds the registry +// (five minutes against about one and a half), so the pause the mesh schedules is always waited out, +// and a registry that is really down still fails the build — saying how long it was refused — rather +// than holding a build machine for ever. Every wait is said in the build's log, with why, and so is +// the registry answering again. + +var ( + // registryWait is how long a build waits for a registry that refuses, per call that found it so. + registryWait = 5 * time.Minute + // registryFirstPause is the first pause between tries; each pause doubles, up to registryMostPause. + registryFirstPause = time.Second +) + +// registryMostPause is the longest pause between two tries: short enough that a build goes on within +// seconds of the registry answering again. +const registryMostPause = 10 * time.Second + +// refused is whether an error is a connection refused: from a dial here, or as a command such as docker +// said it in its output. +func refused(err error) bool { + return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused")) +} + +// waitForRegistry runs try, and while it fails because the registry at address refuses connections, +// tries again with a growing pause until registryWait has passed. what names the call, for the log. +func waitForRegistry(ctx context.Context, address, what string, try func() error) error { + err := try() + if !refused(err) { + return err + } + started := time.Now() + pause := registryFirstPause + tell("registry", "%s: refused; the build waits for the registry at %s, for up to %s — it is held still while "+ + "the store's nightly collection runs, about a minute and a half (novox/hq issue 457)", + what, address, registryWait) + for { + left := registryWait - time.Since(started) + if left <= 0 { + tell("registry", "%s: the registry at %s still refuses after %s; the build fails", what, address, + time.Since(started).Round(time.Second)) + return fmt.Errorf("the registry at %s refused every connection for %s, longer than its nightly "+ + "collection holds it still, so it is down, not paused: %w", + address, time.Since(started).Round(time.Millisecond), err) + } + wait := min(pause, left) + select { + case <-ctx.Done(): + return fmt.Errorf("stopped while waiting for the registry at %s: %w (last: %v)", address, ctx.Err(), err) + case <-time.After(wait): + } + pause = min(pause*2, registryMostPause) + if err = try(); !refused(err) { + tell("registry", "%s: the registry at %s answers again after %s; the build goes on", what, address, + time.Since(started).Round(time.Millisecond)) + return err + } + } +} + +// waitingTransport waits for the registry on every request to it, and on none to anywhere else: the +// same client copies from upstream registries, whose refusals are theirs to answer. +type waitingTransport struct { + base http.RoundTripper + address string +} + +func (t waitingTransport) RoundTrip(request *http.Request) (*http.Response, error) { + if request.URL.Host != t.address { + return t.base.RoundTrip(request) + } + var response *http.Response + tries := 0 + err := waitForRegistry(request.Context(), t.address, request.Method+" "+request.URL.Path, func() error { + attempt := request + if tries > 0 && request.Body != nil && request.Body != http.NoBody { + // A body is sent again only when it can be read again; one that cannot is not retried. + if request.GetBody == nil { + return fmt.Errorf("%s %s cannot be sent again: its body cannot be read twice", request.Method, request.URL) + } + body, err := request.GetBody() + if err != nil { + return err + } + attempt = request.Clone(request.Context()) + attempt.Body = body + } + tries++ + var err error + response, err = t.base.RoundTrip(attempt) + return err + }) + return response, err +} + +// waiting is a client like c whose requests to the registry wait for it. +func waiting(c *http.Client, address string) *http.Client { + if _, already := c.Transport.(waitingTransport); already { + return c + } + copied := *c + base := c.Transport + if base == nil { + base = http.DefaultTransport + } + copied.Transport = waitingTransport{base: base, address: address} + return &copied +} diff --git a/internal/builder/registry_wait_test.go b/internal/builder/registry_wait_test.go new file mode 100644 index 00000000..8bb9be85 --- /dev/null +++ b/internal/builder/registry_wait_test.go @@ -0,0 +1,154 @@ +package builder + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "net" + "net/http" + "strings" + "sync" + "testing" + "time" +) + +// A registry held still for a while (novox/hq issue 457): the store's nightly collection stops it for +// about a minute and a half, and a build in that window was failed, and its whole plan with it, for a +// pause the mesh itself scheduled. A build waits it out — for a bound longer than the collection +// holds it — says so in its log, and still fails, loudly, past the bound. + +// heldStill is a registry address that refuses every connection until it starts answering after +// pause, or never when pause is negative. +func heldStill(t *testing.T, f *fakeRegistry, pause time.Duration) string { + t.Helper() + handler := f.serve(t).Config.Handler + reserved, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + address := reserved.Addr().String() + reserved.Close() // refused from here on: nothing listens + if pause < 0 { + return address + } + server := &http.Server{Handler: handler} + go func() { + time.Sleep(pause) + l, err := net.Listen("tcp", address) + if err != nil { + t.Errorf("cannot answer at %s again: %v", address, err) + return + } + _ = server.Serve(l) + }() + t.Cleanup(func() { _ = server.Close() }) + return address +} + +// saying collects what a build says, as the build machine's per-build Said does. +func saying(t *testing.T) func() []string { + t.Helper() + var mu sync.Mutex + var lines []string + was := Said + Said = func(step, message string) { + mu.Lock() + defer mu.Unlock() + lines = append(lines, step+": "+message) + } + t.Cleanup(func() { Said = was }) + return func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), lines...) + } +} + +// waitingFor shortens the bound and the first pause, so a test waits for milliseconds. +func waitingFor(t *testing.T, bound time.Duration) { + t.Helper() + wasBound, wasFirst := registryWait, registryFirstPause + registryWait, registryFirstPause = bound, 20*time.Millisecond + t.Cleanup(func() { registryWait, registryFirstPause = wasBound, wasFirst }) +} + +func TestABuildWaitsForARegistryHeldStillAndGoesOn(t *testing.T) { + waitingFor(t, 5*time.Second) + said := saying(t) + f := &fakeRegistry{} + r := Registry{Address: heldStill(t, f, 400*time.Millisecond)} + body := []byte("a theme") + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + + if _, err := r.PublishArchive(context.Background(), "shell/config", body, digest); err != nil { + t.Fatalf("a registry refusing for 400ms failed the build: %v", err) + } + if string(f.blobs[digest]) != "a theme" { + t.Fatalf("the registry holds %q", f.blobs[digest]) + } + log := strings.Join(said(), "\n") + if !strings.Contains(log, "waits for the registry at "+r.Address) || !strings.Contains(log, "nightly collection") { + t.Errorf("the build's log does not say it waited for the registry, and why:\n%s", log) + } + if !strings.Contains(log, "answers again") { + t.Errorf("the build's log does not say the registry came back:\n%s", log) + } +} + +func TestABuildFailsLoudlyOnARegistryRefusingPastTheBound(t *testing.T) { + waitingFor(t, 300*time.Millisecond) + said := saying(t) + f := &fakeRegistry{} + r := Registry{Address: heldStill(t, f, -1)} + body := []byte("a theme") + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + + started := time.Now() + _, err := r.PublishArchive(context.Background(), "shell/config", body, digest) + if err == nil { + t.Fatal("a registry that never answered published the archive") + } + if !strings.Contains(err.Error(), "refused every connection for") || !strings.Contains(err.Error(), "connection refused") { + t.Errorf("the failure does not say it waited and was refused throughout: %v", err) + } + if waited := time.Since(started); waited < 300*time.Millisecond { + t.Errorf("failed after %s, before the bound", waited) + } + if log := strings.Join(said(), "\n"); !strings.Contains(log, "waits for the registry") { + t.Errorf("the build's log does not say it waited:\n%s", log) + } +} + +func TestAnImagePushWaitsForARegistryHeldStill(t *testing.T) { + waitingFor(t, 5*time.Second) + said := saying(t) + pushes := 0 + run := func(_ context.Context, _ string, name string, args ...string) (string, error) { + switch args[0] { + case "push": + pushes++ + if pushes < 3 { + return "", errorString("docker push: dial tcp 127.0.0.1:5000: connect: connection refused") + } + case "inspect": + return "127.0.0.1:5000/m/server@sha256:abc\n", nil + } + return "", nil + } + r := Registry{Address: "127.0.0.1:5000", Run: run} + if _, err := r.PublishImage(context.Background(), "local", "m/server"); err != nil { + t.Fatalf("a push refused twice failed the build: %v", err) + } + if pushes != 3 { + t.Errorf("pushed %d times, want 3", pushes) + } + if log := strings.Join(said(), "\n"); !strings.Contains(log, "waits for the registry") { + t.Errorf("the build's log does not say it waited:\n%s", log) + } +} + +type errorString string + +func (e errorString) Error() string { return string(e) } From 7758301444c99eb1163377ae53fccd13f75d393d Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 10:51:41 +0200 Subject: [PATCH 08/16] Ask every failed build of the tier on a retry, not only the first (issue 457) An outcome arriving after its plan failed is kept only in the build records, so a retry read from the plan alone asked one failed build and the records then failed the plan again on the next. Settle the tier from the records first. --- cmd/mesh-controller/plan_retry.go | 22 ++++++++ cmd/mesh-controller/plan_retry_tier_test.go | 57 +++++++++++++++++++++ cmd/mesh-controller/release_plan.go | 51 ++++++++++-------- 3 files changed, 110 insertions(+), 20 deletions(-) create mode 100644 cmd/mesh-controller/plan_retry_tier_test.go diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index a8377bc1..10ed9c45 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -288,6 +288,15 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { "those walks answer them; a newer merge, or `rebuild `, builds again", p.ID) } } + // Settled from the build records before anything is judged (novox/hq issue 457): a build of the tier + // that failed after the plan did is as failed as the one that failed it. + if p.Tier < len(p.Tiers) { + recorded, byID, err := recordsOfAsked(ctx, inv, &p, p.Tiers[p.Tier]) + if err != nil { + return "", err + } + toRetry(&p, recorded, byID) + } if err := retryRefusal(p, plans); err != nil { return "", err } @@ -525,3 +534,16 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again func sendAgain(s *inventory.PlanModule) { s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", "" } + +// toRetry is the modules a retry asks again: every one of the tier that failed, the plan's state +// settled from the build records first (novox/hq issue 457). A plan fails on the first failure in its +// tier, and an outcome arriving after that finds no open plan to answer — it is kept only in the build +// records. Read from the plan alone, a retry asked only the build that failed first, and the records +// then failed the plan again on the next: each failed build of a tier took a retry of its own. What +// still runs is left asked, and its outcome is the plan's once the retry sets it building. +func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[string]inventory.Build) []string { + if p.Tier < len(p.Tiers) { + settleFromRecords(p, p.Tiers[p.Tier], recorded, byID) + } + return failedIn(*p) +} diff --git a/cmd/mesh-controller/plan_retry_tier_test.go b/cmd/mesh-controller/plan_retry_tier_test.go new file mode 100644 index 00000000..7efd44a8 --- /dev/null +++ b/cmd/mesh-controller/plan_retry_tier_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "reflect" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// A retry asks every failed build of the tier, not one (novox/hq issue 457). Seen 2026-10-11: gitea +// and plex both failed in tier 0 while the registry was held still; gitea's failure failed the plan, +// and plex's, arriving after, found no open plan and was kept only in the build records. The first +// retry asked gitea alone, the records then failed the plan again on plex, and a second retry asked +// plex. +func TestARetryAsksEveryFailedBuildOfTheTier(t *testing.T) { + asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC) + failedAt := asked.Add(2 * time.Minute) + p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0, + Tiers: [][]string{{"gitea", "plex"}}, Note: "gitea failed to build in tier 0", + Modules: map[string]*inventory.PlanModule{ + "gitea": {State: "failed", AskedAt: &asked, Build: "build-gitea", Why: "cannot reach the registry"}, + "plex": {State: "asked", AskedAt: &asked, Build: "build-plex"}, + }} + byID := map[string]inventory.Build{ + "build-plex": {ID: "build-plex", Module: "plex", At: failedAt, Failed: "cannot reach the registry"}, + } + + if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"gitea", "plex"}) { + t.Fatalf("a retry of a tier where gitea and plex failed asks %v", got) + } +} + +// A build of the tier still running when the plan failed is not asked again: its outcome is the plan's +// once the retry sets it building, and one that is recorded built is taken as built. +func TestARetryLeavesABuildThatRunsOrWorked(t *testing.T) { + asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC) + builtAt := asked.Add(3 * time.Minute) + p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0, + Tiers: [][]string{{"a", "b", "c"}}, + Modules: map[string]*inventory.PlanModule{ + "a": {State: "failed", AskedAt: &asked, Build: "build-a", Why: "broken"}, + "b": {State: "asked", AskedAt: &asked, Build: "build-b"}, + "c": {State: "asked", AskedAt: &asked, Build: "build-c"}, + }} + byID := map[string]inventory.Build{"build-c": {ID: "build-c", Module: "c", At: builtAt, Commit: "c0ffee"}} + + if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"a"}) { + t.Fatalf("asks %v, want a alone", got) + } + if s := p.Modules["c"]; s.State != "built" || s.Commit != "c0ffee" { + t.Errorf("c, recorded built, is %+v", s) + } + if s := p.Modules["b"]; s.State != "asked" { + t.Errorf("b, still building, is %+v", s) + } +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 76fbd95b..7adc089c 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -649,26 +649,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, // the controller in its first tier: the build that produced the new one is recorded, and the // plan never hears it. The record is the fact; a build recorded after the ask is that tier's // outcome, whoever was listening. - recorded := map[string][]inventory.Build{} - byID := map[string]inventory.Build{} - for _, m := range tier { - if s := p.Modules[m]; s != nil && s.State == "asked" { - builds, err := inv.Builds(ctx, m, 5) - if err != nil { - return false, err - } - recorded[m] = builds - // Its own ask's record, by id — found even when the outcome named no module (ADR 0219). - if s.Build != "" { - b, found, err := inv.BuildByID(ctx, s.Build) - if err != nil { - return false, err - } - if found { - byID[s.Build] = b - } - } - } + recorded, byID, err := recordsOfAsked(ctx, inv, p, tier) + if err != nil { + return false, err } if settleFromRecords(p, tier, recorded, byID) { return true, nil @@ -1776,6 +1759,34 @@ func splitList(s string) []string { return out } +// recordsOfAsked is what settleFromRecords reads: for every module of the tier still `asked`, its last +// builds and the record of its own ask, by id. +func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, tier []string) ( + map[string][]inventory.Build, map[string]inventory.Build, error) { + recorded := map[string][]inventory.Build{} + byID := map[string]inventory.Build{} + for _, m := range tier { + if s := p.Modules[m]; s != nil && s.State == "asked" { + builds, err := inv.Builds(ctx, m, 5) + if err != nil { + return nil, nil, err + } + recorded[m] = builds + // Its own ask's record, by id — found even when the outcome named no module (ADR 0219). + if s.Build != "" { + b, found, err := inv.BuildByID(ctx, s.Build) + if err != nil { + return nil, nil, err + } + if found { + byID[s.Build] = b + } + } + } + } + return recorded, byID, nil +} + // settleFromRecords marks every module of the tier still `asked` built — or failed — from a build // recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214). // Newest first, as Builds answers: the first record after the ask is the outcome of that ask. From 83ce19b9c0141624445728d43a2cd969227a56e6 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 11:05:30 +0200 Subject: [PATCH 09/16] Say a registry came back only when the call worked, and retry from toRetry's set (issue 457 review) The streaming blob PUT is left unwaited, with why, since its body cannot be read twice and the POST before it already waited. --- cmd/mesh-controller/plan_retry.go | 11 ++++++----- internal/builder/mirror.go | 5 +++++ internal/builder/registry_wait.go | 10 ++++++++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index 10ed9c45..df28926e 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -289,21 +289,23 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { } } // Settled from the build records before anything is judged (novox/hq issue 457): a build of the tier - // that failed after the plan did is as failed as the one that failed it. + // that failed after the plan did is as failed as the one that failed it. What toRetry says is the + // failed set every step below works from. + var failed []string if p.Tier < len(p.Tiers) { recorded, byID, err := recordsOfAsked(ctx, inv, &p, p.Tiers[p.Tier]) if err != nil { return "", err } - toRetry(&p, recorded, byID) + failed = toRetry(&p, recorded, byID) } if err := retryRefusal(p, plans); err != nil { return "", err } - if again := unjudgedAtGate(p); len(failedIn(p)) == 0 && len(again) > 0 { + if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 { return retryTierWhole(ctx, open, &p, again) } - if len(failedIn(p)) == 0 { + if len(failed) == 0 { return retryRollouts(ctx, open, &p) } entries, err := inv.Catalogued(ctx) @@ -314,7 +316,6 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { for _, e := range entries { byName[e.Manifest.Module] = e } - failed := failedIn(p) var asked []string for _, m := range failed { askModule(ctx, &p, m, byName) diff --git a/internal/builder/mirror.go b/internal/builder/mirror.go index 8840e3d5..4a2da87e 100644 --- a/internal/builder/mirror.go +++ b/internal/builder/mirror.go @@ -429,6 +429,11 @@ func (r Registry) copyBlob(ctx context.Context, src *source, where upstream, dig if response.ContentLength > 0 { put.ContentLength = response.ContentLength } + // **Not waited for if refused** (novox/hq issue 457): the body streams from upstream and cannot be + // read twice, so a registry held still between the POST above and this PUT fails the copy with + // "its body cannot be read twice" rather than waiting. Accepted: the POST a moment before already + // waited the registry out, so the window is the length of one upstream fetch, and the build fails + // loudly, to be asked again, rather than buffering every base blob in memory. done, err := r.client().Do(put) if err != nil { return fmt.Errorf("cannot upload blob %s: %w", digest, err) diff --git a/internal/builder/registry_wait.go b/internal/builder/registry_wait.go index 010622a0..f2b10bae 100644 --- a/internal/builder/registry_wait.go +++ b/internal/builder/registry_wait.go @@ -74,8 +74,14 @@ func waitForRegistry(ctx context.Context, address, what string, try func() error } pause = min(pause*2, registryMostPause) if err = try(); !refused(err) { - tell("registry", "%s: the registry at %s answers again after %s; the build goes on", what, address, - time.Since(started).Round(time.Millisecond)) + // Said as it is: the registry answering is only the build going on when the call worked. + if err == nil { + tell("registry", "%s: the registry at %s answers again after %s; the build goes on", what, address, + time.Since(started).Round(time.Millisecond)) + } else { + tell("registry", "%s: the registry at %s no longer refuses after %s, and answered with: %v", what, + address, time.Since(started).Round(time.Millisecond), err) + } return err } } From c494ed03a7c1c60a17109aa9a09c5b6c997b704f Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 11:20:10 +0200 Subject: [PATCH 10/16] Keep the forge credential out of what a check's container sees, and redact what a check publishes (issue 462) The toolchain container mounts the workspace as HOME, so the credential kept there, and a clone's recorded userinfo, were readable by any pull request; its output is kept on the bus for days. The credential now lives in a private directory outside the workspace only while cloning, clones record their URL without userinfo, and every line said to the build's log and the verdict passes a redactor copied from the journal tool (sharing it: issue 471). --- internal/builder/builder.go | 60 ++++++- internal/builder/builder_test.go | 34 ++-- internal/builder/check.go | 45 +++-- internal/builder/check_credential_test.go | 200 ++++++++++++++++++++++ internal/builder/redact.go | 191 +++++++++++++++++++++ 5 files changed, 497 insertions(+), 33 deletions(-) create mode 100644 internal/builder/check_credential_test.go create mode 100644 internal/builder/redact.go diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 2c38e005..3f8a3ba2 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "os" @@ -159,13 +160,11 @@ func build(ctx context.Context, run Runner, publish Publisher, } // The credential is a file git reads, never an argument: a URL carrying a password in argv // would be readable by anything that can list processes for as long as a clone runs. - credentials := "" - if forge.URL != "" { - credentials = filepath.Join(workspace, "git-credentials") - if err := os.WriteFile(credentials, []byte(forge.URL+"\n"), 0o600); err != nil { - return Result{}, err - } + credentials, forget, err := storeCredential(workspace, forge) + if err != nil { + return Result{}, err } + defer forget() tree := filepath.Join(workspace, "source") if err := os.RemoveAll(tree); err != nil { return Result{}, err @@ -467,6 +466,55 @@ func contextURL(from catalogue.ArtifactContext, seats map[string]string) (string return strings.TrimRight(base, "/") + "/" + strings.TrimSuffix(strings.Trim(from.Repository, "/"), ".git") + ".git", nil } +// storeCredential writes the forge credential where git's credential store reads it, and the function that +// removes it again; "" and nothing to remove when the builder holds none. +// +// **Never inside the workspace** (novox/hq issue 462): a merge check's toolchain container mounts the +// workspace as its HOME, so a credential kept there — even one a build left behind — is readable by any pull +// request's merge-check.sh, and what it prints is kept on the bus. So it lives in a directory of its own +// outside the workspace, made private, and a credential an older builder left in the workspace is removed. +func storeCredential(workspace string, forge GitCredential) (string, func(), error) { + if err := os.Remove(filepath.Join(workspace, "git-credentials")); err != nil && !errors.Is(err, os.ErrNotExist) { + return "", nil, fmt.Errorf("a credential left in the workspace cannot be removed: %w", err) + } + if forge.URL == "" { + return "", func() {}, nil + } + dir, err := os.MkdirTemp("", "mesh-forge-credential-") + if err != nil { + return "", nil, err + } + forget := func() { os.RemoveAll(dir) } + if withinDir(workspace, dir) { + forget() + return "", nil, fmt.Errorf("the temporary directory %s is inside the workspace %s, which a check's "+ + "container mounts: the forge credential is not written where it could read it", dir, workspace) + } + path := filepath.Join(dir, "git-credentials") + if err := os.WriteFile(path, []byte(forge.URL+"\n"), 0o600); err != nil { + forget() + return "", nil, err + } + return path, forget, nil +} + +// withinDir is whether path is dir or under it, both made absolute and resolved. +func withinDir(dir, path string) bool { + d, err1 := filepath.Abs(dir) + p, err2 := filepath.Abs(path) + if err1 != nil || err2 != nil { + return true + } + if r, err := filepath.EvalSymlinks(d); err == nil { + d = r + } + if r, err := filepath.EvalSymlinks(p); err == nil { + p = r + } + rel, err := filepath.Rel(d, p) + return err != nil || rel == "." || filepath.IsLocal(rel) +} + // cloneWith is a git invocation that may offer a stored credential. // // The first `-c credential.helper=` clears every helper the environment might carry, so exactly diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index da362f2c..afe57572 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -438,30 +438,34 @@ func TestABuildOffersTheForgesCredentialThroughGitsOwnStore(t *testing.T) { if err != nil { t.Fatal(err) } - stored := filepath.Join(workspace, "git-credentials") clone := r.ran[0] - if !strings.Contains(clone, "credential.helper=store --file="+stored) { - t.Fatalf("the clone does not name the credential store: %s", clone) - } + stored := storeNamedIn(t, clone) for _, line := range r.ran { if strings.Contains(line, "sw0rdfi5h") { t.Fatalf("the secret is in a command line, readable by anything that can list processes: %s", line) } } - raw, err := os.ReadFile(stored) - if err != nil { - t.Fatal(err) + // Outside the workspace a check's container mounts, and gone once the build is (novox/hq issue 462); what + // it held while git read it is checked with the check's clones (TestACheckContainerSeesNoForgeCredential). + if withinDir(workspace, stored) { + t.Fatalf("the credential store %s is inside the workspace %s", stored, workspace) } - if strings.TrimSpace(string(raw)) != "http://mesh_novox_builder:sw0rdfi5h@forge.invalid:20000" { - t.Fatalf("the store does not hold the credential as given: %q", raw) + if _, err := os.Stat(stored); !os.IsNotExist(err) { + t.Fatalf("the credential store outlives the build: %v", err) } - info, err := os.Stat(stored) - if err != nil { - t.Fatal(err) + if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) { + t.Fatal("a credential file is left in the workspace") } - if info.Mode().Perm() != 0o600 { - t.Fatalf("the credential file is readable beyond its owner: %v", info.Mode()) +} + +// storeNamedIn is the credential store a git command line offers. +func storeNamedIn(t *testing.T, line string) string { + t.Helper() + _, after, ok := strings.Cut(line, "credential.helper=store --file=") + if !ok { + t.Fatalf("the clone does not name the credential store: %s", line) } + return strings.Fields(after)[0] } // Without a credential, a clone is exactly the invocation it always was, and no credential file @@ -506,7 +510,7 @@ func TestAContextCloneCarriesTheSameCredentialStore(t *testing.T) { if err != nil { t.Fatal(err) } - stored := filepath.Join(workspace, "git-credentials") + stored := storeNamedIn(t, r.ran[0]) var contextClone string for _, line := range r.ran { if strings.Contains(line, "clone") && strings.Contains(line, "source.git") { diff --git a/internal/builder/check.go b/internal/builder/check.go index 30e58f7e..fb154a6b 100644 --- a/internal/builder/check.go +++ b/internal/builder/check.go @@ -229,7 +229,13 @@ func ScriptToolchain(script []byte) string { // Check runs one merge check. An error is that it could not run; the verdict is then "error". func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry string, forge GitCredential, log Log) (CheckVerdict, error) { - say := logging(log) + // Everything a check says goes to the build's log, which the bus keeps: said redacted (novox/hq issue 462). + redact := redactorFor(forge) + say := logging(func(step, message string) { + if log != nil { + log(step, redact.redact(message)) + } + }) began := time.Now() ctx, stop := context.WithTimeout(ctx, CheckTimeout) defer stop() @@ -242,20 +248,27 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry return CheckVerdict{}, err } defer os.RemoveAll(root) - credentials := "" - if forge.URL != "" { - credentials = filepath.Join(workspace, "git-credentials") - if err := os.WriteFile(credentials, []byte(forge.URL+"\n"), 0o600); err != nil { - return CheckVerdict{}, err - } + // The forge credential lives outside the workspace the check's containers mount, and only while the + // check clones (novox/hq issue 462). + credentials, forget, err := storeCredential(workspace, forge) + if err != nil { + return CheckVerdict{}, err } + defer forget() clone := func(repository, ref, dir string) error { if _, err := run(ctx, root, "git", cloneWith(credentials, "clone", "--quiet", repository, dir)...); err != nil { - return fmt.Errorf("cannot clone %s: %w", repository, err) + return fmt.Errorf("cannot clone %s: %s", redact.redact(repository), redact.redact(err.Error())) + } + // Git records the URL cloned from as given, in the clone's .git/config the container reads: one + // carrying userinfo is recorded without it (novox/hq issue 462). + if bare, ok := withoutUserinfo(repository); ok { + if _, err := run(ctx, filepath.Join(root, dir), "git", "remote", "set-url", "origin", bare); err != nil { + return fmt.Errorf("the clone of %s keeps its credential: %w", bare, err) + } } if ref != "" { if _, err := run(ctx, filepath.Join(root, dir), "git", "checkout", "--quiet", ref); err != nil { - return fmt.Errorf("%s has no %s: %w", repository, ref, err) + return fmt.Errorf("%s has no %s: %w", redact.redact(repository), ref, err) } } return nil @@ -323,6 +336,9 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry } } + // Every clone is made: the credential is gone before anything of the check runs (novox/hq issue 462). + forget() + // The facts, and the versions they say the mesh runs. if registry == "" { return CheckVerdict{}, errors.New("no artifact store to read the facts snapshot from") @@ -479,8 +495,11 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry } } + // What the check printed travels in the verdict to the forge and the controller: redacted as the log is. + v.Gate.Summary, v.Repo.Summary, v.Repo.Failed = redact.redact(v.Gate.Summary), redact.redact(v.Repo.Summary), + redact.redact(v.Repo.Failed) v.Verdict, v.Summary = v.Gate.Verdict, v.Gate.Summary - v.Report, v.Took = withWhatFailed(out.String(), v.Repo), time.Since(began) + v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began) say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary, strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second)) return v, nil @@ -606,7 +625,9 @@ func (l *toTheLog) line(line string) { line = line[:logLineBytes] + fmt.Sprintf(" … (%d bytes more)", len(line)-logLineBytes) } l.said++ - l.say("output", "%s", line) + // Redacted by its shape before the bus keeps it (novox/hq issue 462); Check's own say adds the secrets + // the builder knows. + l.say("output", "%s", redactor{}.redact(line)) } // close says the last line, and, when lines were left out, how many and what failed. @@ -624,7 +645,7 @@ func (l *toTheLog) close(failed string) { } l.say("output", "--- what failed, picked from the whole of its output") for _, line := range strings.Split(failed, "\n") { - l.say("output", "%s", line) + l.say("output", "%s", redactor{}.redact(line)) } } diff --git a/internal/builder/check_credential_test.go b/internal/builder/check_credential_test.go new file mode 100644 index 00000000..f61a6474 --- /dev/null +++ b/internal/builder/check_credential_test.go @@ -0,0 +1,200 @@ +package builder + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io/fs" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/facts" +) + +// **A check's container never sees the forge credential** (novox/hq issue 462): the toolchain container +// mounts the workspace as HOME, so a credential kept there — or a clone's .git/config carrying one — is +// readable by any pull request's merge-check.sh, and printed, kept on the bus for days. + +const ( + forgeSecret = "sw0rdfi5h-forge" + forgeURL = "http://mesh_novox_builder:" + forgeSecret + "@forge.invalid:20000" + besideSecret = "b3side-t0ken" +) + +// aFactsRegistry is an artifact store holding the facts snapshot, and nothing else. +func aFactsRegistry(t *testing.T) string { + t.Helper() + body, err := json.Marshal(facts.Facts{Format: facts.Format, Taken: time.Now().UTC(), + Versions: facts.Versions{Bus: "2.11.17", Store: "17.11"}, Machines: []facts.Machine{{Name: "abcdef", Length: 6}}}) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + manifest, _ := json.Marshal(map[string]any{"schemaVersion": 2, "layers": []map[string]any{ + {"mediaType": facts.MediaType, "digest": digest, "size": len(body)}}}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "/manifests/"): + w.Write(manifest) + case strings.HasSuffix(r.URL.Path, "/blobs/"+digest): + w.Write(body) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return strings.TrimPrefix(srv.URL, "http://") +} + +// bareURL is a URL with its userinfo left out. +func bareURL(t *testing.T, raw string) string { + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + u.User = nil + return u.String() +} + +func TestACheckContainerSeesNoForgeCredential(t *testing.T) { + repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"}) + besideRepo, besideHead := aCheckedRepository(t, map[string]string{"README": "beside"}) + // A clone source that carries userinfo, as a forge's clone URL may: git records it as given in the + // clone's .git/config, which the container reads. + besideURL := "file://beside-user:" + besideSecret + "@" + besideRepo + workspace := t.TempDir() + + var stores []string + reached := false + var leaks []string + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + switch name { + case "git": + for _, a := range args { + if f, ok := strings.CutPrefix(a, "credential.helper=store --file="); ok { + stores = append(stores, f) + raw, err := os.ReadFile(f) + if err != nil || strings.TrimSpace(string(raw)) != forgeURL { + t.Errorf("git is offered a store that does not hold the credential as given: %q, %v", raw, err) + } + if info, err := os.Stat(f); err == nil && info.Mode().Perm() != 0o600 { + t.Errorf("the credential store is readable beyond its owner: %v", info.Mode()) + } + } + } + if len(args) >= 2 && args[len(args)-3] == "--quiet" && hasString(args, "clone") { + source := args[len(args)-2] + if u, err := url.Parse(source); err == nil && u.User != nil { + // Git cannot reach a file:// URL with userinfo; clone it without, then record it as git + // would have: as given. + clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1]) + if out, err := Command(ctx, dir, "git", clone...); err != nil { + return out, err + } + return Command(ctx, filepath.Join(dir, args[len(args)-1]), "git", "remote", "set-url", "origin", source) + } + } + return Command(ctx, dir, name, args...) + case "docker": + if !reached { + reached = true + // The first container: everything the workspace holds is what the toolchain container sees. + filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + raw, _ := os.ReadFile(path) + s := string(raw) + if strings.Contains(s, forgeSecret) || strings.Contains(s, besideSecret) || + d.Name() == "git-credentials" || strings.Contains(s, "credential.helper") { + leaks = append(leaks, path) + } + return nil + }) + for _, f := range stores { + if _, err := os.Stat(f); !errors.Is(err, os.ErrNotExist) { + leaks = append(leaks, f+" (still there when the first container runs)") + } + if rel, err := filepath.Rel(workspace, f); err == nil && !strings.HasPrefix(rel, "..") { + leaks = append(leaks, f+" (inside the workspace the container mounts)") + } + } + } + if len(args) > 0 && args[0] == "ps" { + return "", nil + } + return "", errors.New("no container runtime in this test") + } + return "", errors.New("unexpected command " + name) + } + // A credential an older builder left in the workspace is removed too. + if err := os.WriteFile(filepath.Join(workspace, "git-credentials"), []byte(forgeURL+"\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err := Check(t.Context(), run, CheckSpec{ID: "check-462", Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-controller", Number: 1, Toolchain: "golang", Beside: map[string]Beside{ + "mesh-catalog": {Repository: besideURL, Ref: besideHead}}}, workspace, aFactsRegistry(t), + GitCredential{URL: forgeURL}, nil) + if err == nil { + t.Fatal("the check ran past its first container in a test with none") + } + if !reached { + t.Fatalf("the check never reached its first container: %v", err) + } + if len(stores) == 0 { + t.Fatal("no clone was offered the forge credential") + } + if len(leaks) > 0 { + t.Fatalf("the check's container sees the credential:\n%s", strings.Join(leaks, "\n")) + } +} + +// Every line a repository's own check prints is published to the build's log redacted. +func TestACheckLinePublishedToTheLogIsRedacted(t *testing.T) { + var said []string + say := func(step, format string, args ...any) { + if step == "output" && len(args) > 0 { + said = append(said, args[0].(string)) + } + } + var out tail + layer := ownCheck(t.Context(), CheckSpec{Toolchain: "golang"}, []ScriptPart{{Toolchain: "go", Script: CheckScript}}, + t.TempDir(), &out, func() bool { return false }, func(string, string) *exec.Cmd { + return exec.CommandContext(t.Context(), "sh", "-c", "echo cloning http://mesh_builder:t0ps3cret-forge@forge.invalid/novox/x.git; "+ + "echo token ghp_abcdefghijklmnopqrstuvwxyz0123456789") + }, say) + if layer == nil || layer.Verdict != "pass" { + t.Fatalf("the check answered %+v\n%s", layer, out.String()) + } + joined := strings.Join(said, "\n") + if strings.Contains(joined, "t0ps3cret-forge") || strings.Contains(joined, "ghp_abcdef") { + t.Fatalf("a secret the check printed is published to the build's log:\n%s", joined) + } + if !strings.Contains(joined, "http://mesh_builder:[redacted: a password in a URI]@forge.invalid/novox/x.git") { + t.Fatalf("the line is not said with what was there named:\n%s", joined) + } +} + +func TestTheRedactorHidesTheForgeCredentialAndShapes(t *testing.T) { + r := redactorFor(GitCredential{URL: forgeURL}) + for in, want := range map[string]string{ + "the secret alone: " + forgeSecret: "the secret alone: [redacted: the forge credential]", + "go test ./... ok": "go test ./... ok", + "--password hunter22 and done": "--password [redacted: the word after --password] and done", + "commit 3b6b54a0c1d2e3f4a5b6c7d8e9f0": "commit 3b6b54a0c1d2e3f4a5b6c7d8e9f0", + } { + if got := r.redact(in); got != want { + t.Errorf("%q redacted as %q, want %q", in, got, want) + } + } +} diff --git a/internal/builder/redact.go b/internal/builder/redact.go new file mode 100644 index 00000000..84205614 --- /dev/null +++ b/internal/builder/redact.go @@ -0,0 +1,191 @@ +package builder + +// **Every line of a check's output is redacted before it is kept** (novox/hq issue 462). +// +// A repository's own check prints into the build's log, which the bus keeps for days and anyone who may read +// its events reads, and into the verdict, which the forge shows on the pull request. Software prints what it +// was given — a URL carrying a password, a token in a flag — and a pull request may print on purpose. +// So a line is said only after every secret the builder knows (the forge credential's password) and every +// value whose shape says it is one is replaced by a mark naming what was there, as the journal verb does. +// +// Copied from the journal tool's redactor (mesh-catalog, modules/systemd/cmd/systemd-tools/secrets.go, +// itself a copy of the docker module's), narrowed to a line's shapes, with the token shapes a check's output +// may carry added. A third copy: sharing them through mesh-sdk is novox/hq issue 471. + +import ( + "net/url" + "regexp" + "strings" +) + +// secretName is a variable name that says its value is a secret. +var secretName = regexp.MustCompile(`(?i)(pass(word|wd|phrase)?|secret|token|api_?key|private_?key|access_?key|credential|auth)`) + +// notAValue is a name that says its value is where a secret is, not the secret: a file or a path. +var notAValue = regexp.MustCompile(`(?i)(_FILE|FILE|_PATH|_DIR)$`) + +// uriPassword is a URI carrying a password in its userinfo: scheme://user:password@. +var uriPassword = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*://[^\s/:@'"]*:([^\s/@'"]+)@`) + +// tokenShaped are tokens recognised by their own prefix, whatever surrounds them: a forge's or a host's +// access token, a JSON web token, a NATS seed. +var tokenShaped = []struct { + name string + re *regexp.Regexp +}{ + {"an access token", regexp.MustCompile(`\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[abpr]-[A-Za-z0-9-]{10,}|sk-ant-[A-Za-z0-9_-]{20,})`)}, + {"a JSON web token", regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+`)}, + {"a NATS seed", regexp.MustCompile(`\bS[ACNOU][A-Z2-7]{56}\b`)}, +} + +// masked is a password a program already hid: ***, xxx, , [REDACTED]. +var masked = regexp.MustCompile(`^(\*+|x+|X+|<[^>]*>|\[[^\]]*\]|%2A+)$`) + +// ordinary is a value under a secret's name that is not one: a path, an address, a number, a switch. +var ordinary = regexp.MustCompile(`^(/.*|[A-Za-z][A-Za-z0-9+.-]*://.*|[0-9.]+[a-z]?|(?i:true|false|yes|no|on|off|none|null))$`) + +// leastSecret is the shortest value compared as a secret: a shorter one matches ordinary words. +const leastSecret = 6 + +// passwordFlags take a secret as their next word, or after `=`, whatever the program. +var passwordFlags = map[string]bool{ + "-P": true, "--password": true, "--pass": true, "--passwd": true, "--secret": true, "--secret-key": true, + "--token": true, "--api-key": true, "--apikey": true, "--auth": true, +} + +// knownSecret is one value the builder holds, by the name it is said under. +type knownSecret struct { + Name string + Value string +} + +// redactor hides the secrets it knows and those a line's shapes say are secrets. +type redactor struct{ known []knownSecret } + +// redactorFor knows the forge credential's password, and its user's name with it, in every form git or a +// program may print them. +func redactorFor(forge GitCredential) redactor { + var r redactor + if forge.URL == "" { + return r + } + for _, m := range uriPassword.FindAllStringSubmatch(forge.URL, -1) { + r.add("the forge credential", m[1]) + if dec, err := url.PathUnescape(m[1]); err == nil && dec != m[1] { + r.add("the forge credential", dec) + } + } + return r +} + +func (r *redactor) add(name, value string) { + if len(value) < leastSecret || masked.MatchString(value) { + return + } + for _, k := range r.known { + if k.Value == value { + return + } + } + r.known = append(r.known, knownSecret{name, value}) +} + +// redact is a text with every known secret, every value its shape says is one, and every password inside a +// URI replaced by a mark naming what was there. Line by line: a shape is judged within its line. +func (r redactor) redact(text string) string { + if !strings.ContainsAny(text, "\n") { + return r.line(text) + } + lines := strings.Split(text, "\n") + for i, l := range lines { + lines[i] = r.line(l) + } + return strings.Join(lines, "\n") +} + +func (r redactor) line(line string) string { + replace := func(s knownSecret) { + for _, f := range forms(s.Value) { + line = strings.ReplaceAll(line, f, "[redacted: "+s.Name+"]") + } + } + for _, s := range r.known { + replace(s) + } + for _, s := range shaped(line) { + replace(s) + } + line = uriPassword.ReplaceAllStringFunc(line, func(m string) string { + sub := uriPassword.FindStringSubmatch(m) + if masked.MatchString(sub[1]) || strings.HasPrefix(sub[1], "[redacted") { + return m + } + return strings.TrimSuffix(m, sub[1]+"@") + "[redacted: a password in a URI]@" + }) + for _, t := range tokenShaped { + line = t.re.ReplaceAllString(line, "[redacted: "+t.name+"]") + } + return line +} + +// forms are the ways a value may appear printed: as given, and URL-encoded. +func forms(value string) []string { + out := []string{value} + for _, f := range []string{url.QueryEscape(value), url.PathEscape(value)} { + if f != value && !hasString(out, f) { + out = append(out, f) + } + } + return out +} + +func hasString(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +// shaped are the values a line carries by their shape: the word after a password flag, or the value of one +// given with `=`, and a NAME=value whose name says secret. +func shaped(line string) []knownSecret { + var out []knownSecret + add := func(name, value string) { + value = strings.Trim(value, `"',;`) + if len(value) < leastSecret || masked.MatchString(value) || ordinary.MatchString(value) || + strings.HasPrefix(value, "[redacted") { + return + } + out = append(out, knownSecret{name, value}) + } + words := strings.Fields(line) + for i, w := range words { + if flag, value, ok := strings.Cut(w, "="); ok && strings.HasPrefix(flag, "-") { + if passwordFlags[flag] { + add("the value of "+flag, value) + } + continue + } + if name, value, ok := strings.Cut(w, "="); ok && name != "" && secretName.MatchString(name) && + !notAValue.MatchString(name) && !strings.ContainsAny(name, "/:") { + add("the value of "+name, value) + continue + } + if i+1 < len(words) && passwordFlags[w] { + add("the word after "+w, words[i+1]) + } + } + return out +} + +// withoutUserinfo is a URL with its userinfo left out, and whether it carried any. +func withoutUserinfo(raw string) (string, bool) { + u, err := url.Parse(raw) + if err != nil || u.User == nil { + return raw, false + } + u.User = nil + return u.String(), true +} From 7bd04342b6cc60c062f3674a462b275bc4fc4fb5 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 11:23:12 +0200 Subject: [PATCH 11/16] Pin the node-engine at its pull request's generation refusal, so the validator reads a declaration's generation (hq issue 234) --- go.mod | 4 +-- go.sum | 8 ++--- .../novox/mesh-sdk/go/asks/asks.go | 17 ++++++++++- .../internal/declaration/declaration.go | 30 ++++++++++++++++++- vendor/modules.txt | 6 ++-- 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 067a2f55..cdf300d5 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 @@ -35,4 +35,4 @@ require ( // committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and // fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without // `go mod vendor` fails loudly, at once, everywhere. -replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812 +replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f diff --git a/go.sum b/go.sum index c7f30d1a..e7436cbb 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ -git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812 h1:pzVzwF5VMWaTECxu8+Pd1dNoOHNEm7upC5wPadQTkBw= -git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812/go.mod h1:K3/xEzVgmrNKLMV2vv4M80MwmPnQNXqvQ4C5Jj0fJT4= -git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 h1:9tFDQsgmI+4X7/BpZGXIr+HemPKE7YddYGqWV0lINAI= -git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f h1:8N5OW2mdTNIck2pe4EciTYX5NsrPsHrTLENGNIWYNTU= +git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go index 81737ac4..008e69af 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go @@ -154,7 +154,9 @@ func canonical(b *strings.Builder, v string) { // // It is over the ask's named fields in a fixed order, each written canonically, and the expiry as UTC // RFC 3339 to the nanosecond — never over a language's encoding of the struct, so a field added to Ask -// later changes no digest until it is added here, on purpose. +// later changes no digest until it is added here, on purpose. Whole and Details (novox/hq issue 383) follow +// the options only when the ask gives either: an ask without them digests as it did before they existed, so +// a router and an asker of different builds still agree on every such ask. func (a Ask) Digest() string { var b strings.Builder b.WriteString("novox.ask.v1\n") @@ -168,6 +170,10 @@ func (a Ask) Digest() string { canonical(&b, v) } } + if a.Whole != "" || a.Details != "" { + canonical(&b, a.Whole) + canonical(&b, a.Details) + } sum := sha256.Sum256([]byte(b.String())) return "sha256:" + hex.EncodeToString(sum[:]) } @@ -190,6 +196,15 @@ type Ask struct { // About is what the ask is about (a condition's key): a newer ask about it replaces the older. About string `json:"about,omitempty"` Urgent bool `json:"urgent,omitempty"` + // Whole is the explanation with its exact values whole, shown in place of Explanation on a channel kind + // that proves who answers (verified-sender) and carries the ask's answers (novox/hq issue 383): what the + // person approves — a mount point, a share, a private address — must be readable where they approve it. + // The asker withholds a value shaped like a secret in it, and the router refuses the ask when one is left; + // Explanation stays under the whole content rule everywhere else. Empty, Explanation is shown everywhere. + Whole string `json:"whole,omitempty"` + // Details is what the Details answer on the ask's message shows, line by line under the content rule: a + // fingerprint, how to read the proposal whole at the terminal. Empty, an ask's own message offers no Details. + Details string `json:"details,omitempty"` } // The bounds of an ask (novox/hq ADR 0234 §8, ADR 0259 §4). diff --git a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go index ea03cf57..50db892e 100644 --- a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go +++ b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go @@ -1378,6 +1378,20 @@ type Declaration struct { // what it wrote for it — its resources are absent from the declaration, and absence would // otherwise read as removal. LeftOut []string + + // Generation is the assignment generation this declaration was composed from: the controller's + // counter, raised in the same transaction as every change to what is assigned where (novox/hq + // issue 234). The sequence orders arrival and cannot tell a later send that carries an older view + // of the assignments; this can. A node-engine refuses a declaration composed from a generation + // older than the highest it applied — unless it is a put-back — because applying it would + // undeclare what the mesh still assigns. Zero is a declaration from a controller that claims none, + // and is applied as before. + Generation int64 + + // PutBack says this declaration is a gate putting a machine back (novox/hq issue 234): it carries + // the generation of what it puts back, which may be older than what the machine applied, and is + // not refused for it. + PutBack bool } // LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the @@ -1542,6 +1556,11 @@ type envelope struct { Epoch int64 `json:"epoch,omitempty"` // LeftOut is optional on the wire too, and absent when nothing was left out (ADR 0163). LeftOut []string `json:"left_out,omitempty"` + // Generation and PutBack are optional on the wire too (novox/hq issue 234): absent is a controller + // that claims no generation. **An older host refuses these keys**, decoding strictly; a controller + // sends them only to a host whose reports say `reads_generation`. + Generation int64 `json:"generation,omitempty"` + PutBack bool `json:"put_back,omitempty"` } func parse(raw []byte, allowActions bool) (*Declaration, error) { @@ -1559,8 +1578,17 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { } d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence, - Epoch: env.Epoch, LeftOut: env.LeftOut} + Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack} var problems []string + if env.Generation < 0 { + // Below zero would read as "none claimed" and pass every refusal of an older generation. + problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+ + "mesh assigns", env.Generation)) + } + if env.PutBack && allowActions { + // A put-back is a gate's, sent by the mesh; a carried bundle puts nothing back. + problems = append(problems, "a carried bundle says it is a put-back, and only the mesh's gate can say that") + } if env.Sequence < 0 || env.Epoch < 0 { // Below zero is no order any controller assigns, and read as "none claimed" it would let the // declaration past every refusal of what is older. diff --git a/vendor/modules.txt b/vendor/modules.txt index 486530b9..bb6a448e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 +# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c ## explicit; go 1.22 git.novox.be/novox/mesh-sdk/go/asks # github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op @@ -78,7 +78,7 @@ github.com/nats-io/nkeys # github.com/nats-io/nuid v1.0.1 ## explicit github.com/nats-io/nuid -# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812 +# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f ## explicit; go 1.26.0 github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/rootsearch @@ -135,4 +135,4 @@ golang.org/x/text/width # golang.org/x/time v0.15.0 ## explicit; go 1.25.0 golang.org/x/time/rate -# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812 +# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f From 5c4fa43f8babb6d191dd82b1ceb0f6b88d46a8d9 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 11:24:58 +0200 Subject: [PATCH 12/16] Leave no package-registry credential or clone userinfo in the workspace a check mounts (issue 462) A build wrote .npmrc into its source tree and never removed it, and its clone recorded the URL's userinfo; a later check's container mounts the workspace as HOME. The .npmrc and the tree now go when the build ends, clones record their URL without userinfo, and a check first removes any .npmrc an older builder left. --- internal/builder/builder.go | 53 ++++++++++++++++++++++ internal/builder/check.go | 11 +++-- internal/builder/check_credential_test.go | 55 ++++++++++++++++++++++- internal/builder/packages_test.go | 24 +++++++--- 4 files changed, 129 insertions(+), 14 deletions(-) diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 3f8a3ba2..e142ef29 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "io/fs" "os" "os/exec" "path" @@ -169,6 +170,9 @@ func build(ctx context.Context, run Runner, publish Publisher, if err := os.RemoveAll(tree); err != nil { return Result{}, err } + // Removed when the build ends, whatever happens: the tree holds the .npmrc a build may be handed, in the + // workspace a later check's container mounts as its HOME (novox/hq issue 462). + defer os.RemoveAll(tree) // A fresh clone every time rather than a fetch into a tree that is already there. A build // that reuses a working tree can succeed because of something a previous build left behind, // and that is a build nobody can reproduce. @@ -176,6 +180,9 @@ func build(ctx context.Context, run Runner, publish Publisher, say("clone", "FAILED: %v", err) return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err) } + if err := recordedWithoutUserinfo(ctx, run, tree, repository); err != nil { + return Result{}, err + } say("clone", "done") if ref != "" { if _, err := run(ctx, tree, "git", "checkout", "--quiet", ref); err != nil { @@ -247,6 +254,8 @@ func build(ctx context.Context, run Runner, publish Publisher, if err := os.WriteFile(npmrcPath, []byte(content), 0o600); err != nil { return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err) } + // Only for as long as the build: a later check's container mounts this workspace (novox/hq issue 462). + defer os.Remove(npmrcPath) say("packages", "resolving %s from the mesh's package registry", npmrc.Scope) src.notPinned("it resolves packages from the mesh's registry at build time") } @@ -440,6 +449,9 @@ func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentia if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", url, dir)...); err != nil { return "", fmt.Errorf("cannot clone %s: %w", url, err) } + if err := recordedWithoutUserinfo(ctx, run, dir, url); err != nil { + return "", err + } if from.Ref != "" { if _, err := run(ctx, dir, "git", "checkout", "--quiet", from.Ref); err != nil { return "", fmt.Errorf("%s has no %s: %w", from.Repository, from.Ref, err) @@ -498,6 +510,47 @@ func storeCredential(workspace string, forge GitCredential) (string, func(), err return path, forget, nil } +// recordedWithoutUserinfo makes a clone record the URL it came from without userinfo: git keeps it as given in +// the clone's .git/config, inside the workspace a check's container mounts (novox/hq issue 462). +func recordedWithoutUserinfo(ctx context.Context, run Runner, clone, repository string) error { + bare, carried := withoutUserinfo(repository) + if !carried { + return nil + } + if _, err := run(ctx, clone, "git", "remote", "set-url", "origin", bare); err != nil { + return fmt.Errorf("the clone of %s keeps its credential: %w", bare, err) + } + return nil +} + +// forgetLeftCredentials removes what an earlier build or an older builder left in the workspace that holds +// a credential: the forge's git-credentials, and every .npmrc a build wrote into a tree it cloned. Run +// before a check, whose container mounts the workspace as its HOME (novox/hq issue 462). The Go caches are +// not walked: no build writes a credential there, and they hold more files than everything else. +func forgetLeftCredentials(workspace string) error { + if err := os.Remove(filepath.Join(workspace, "git-credentials")); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return filepath.WalkDir(workspace, func(p string, d fs.DirEntry, err error) error { + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + if d.IsDir() && p != workspace && (d.Name() == "go-cache" || d.Name() == "go-modules") && + filepath.Dir(p) == workspace { + return filepath.SkipDir + } + if !d.IsDir() && d.Name() == ".npmrc" { + if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("a package-registry credential left at %s cannot be removed: %w", p, err) + } + } + return nil + }) +} + // withinDir is whether path is dir or under it, both made absolute and resolved. func withinDir(dir, path string) bool { d, err1 := filepath.Abs(dir) diff --git a/internal/builder/check.go b/internal/builder/check.go index fb154a6b..50b3a476 100644 --- a/internal/builder/check.go +++ b/internal/builder/check.go @@ -255,16 +255,15 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry return CheckVerdict{}, err } defer forget() + if err := forgetLeftCredentials(workspace); err != nil { + return CheckVerdict{}, err + } clone := func(repository, ref, dir string) error { if _, err := run(ctx, root, "git", cloneWith(credentials, "clone", "--quiet", repository, dir)...); err != nil { return fmt.Errorf("cannot clone %s: %s", redact.redact(repository), redact.redact(err.Error())) } - // Git records the URL cloned from as given, in the clone's .git/config the container reads: one - // carrying userinfo is recorded without it (novox/hq issue 462). - if bare, ok := withoutUserinfo(repository); ok { - if _, err := run(ctx, filepath.Join(root, dir), "git", "remote", "set-url", "origin", bare); err != nil { - return fmt.Errorf("the clone of %s keeps its credential: %w", bare, err) - } + if err := recordedWithoutUserinfo(ctx, run, filepath.Join(root, dir), repository); err != nil { + return err } if ref != "" { if _, err := run(ctx, filepath.Join(root, dir), "git", "checkout", "--quiet", ref); err != nil { diff --git a/internal/builder/check_credential_test.go b/internal/builder/check_credential_test.go index f61a6474..7fc52aa9 100644 --- a/internal/builder/check_credential_test.go +++ b/internal/builder/check_credential_test.go @@ -28,6 +28,7 @@ const ( forgeSecret = "sw0rdfi5h-forge" forgeURL = "http://mesh_novox_builder:" + forgeSecret + "@forge.invalid:20000" besideSecret = "b3side-t0ken" + npmSecret = "npm-s3cret-t0ken" ) // aFactsRegistry is an artifact store holding the facts snapshot, and nothing else. @@ -116,7 +117,8 @@ func TestACheckContainerSeesNoForgeCredential(t *testing.T) { raw, _ := os.ReadFile(path) s := string(raw) if strings.Contains(s, forgeSecret) || strings.Contains(s, besideSecret) || - d.Name() == "git-credentials" || strings.Contains(s, "credential.helper") { + strings.Contains(s, npmSecret) || d.Name() == "git-credentials" || d.Name() == ".npmrc" || + strings.Contains(s, "credential.helper") { leaks = append(leaks, path) } return nil @@ -137,10 +139,20 @@ func TestACheckContainerSeesNoForgeCredential(t *testing.T) { } return "", errors.New("unexpected command " + name) } - // A credential an older builder left in the workspace is removed too. + // A credential an older builder left in the workspace is removed too: the forge's, and the .npmrc a + // build wrote into the tree it cloned. if err := os.WriteFile(filepath.Join(workspace, "git-credentials"), []byte(forgeURL+"\n"), 0o600); err != nil { t.Fatal(err) } + for _, dir := range []string{"source/x", "context-server"} { + if err := os.MkdirAll(filepath.Join(workspace, dir), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(workspace, dir, ".npmrc"), + []byte("//forge.invalid/api/packages/novox/npm/:_authToken="+npmSecret+"\n"), 0o600); err != nil { + t.Fatal(err) + } + } _, err := Check(t.Context(), run, CheckSpec{ID: "check-462", Repository: repo, Ref: head, Owner: "novox", Repo: "mesh-controller", Number: 1, Toolchain: "golang", Beside: map[string]Beside{ "mesh-catalog": {Repository: besideURL, Ref: besideHead}}}, workspace, aFactsRegistry(t), @@ -198,3 +210,42 @@ func TestTheRedactorHidesTheForgeCredentialAndShapes(t *testing.T) { } } } + +// A build's clone of a URL carrying userinfo records it without, so no build leaves a credential in +// workspace/source/.git/config while it runs either (novox/hq issue 462); the tree itself is gone when the +// build ends. +func TestABuildsCloneRecordsNoUserinfo(t *testing.T) { + repo, _ := aCheckedRepository(t, map[string]string{ManifestName: `{"module":"plain","version":"1"}`}) + source := "file://build-user:" + besideSecret + "@" + repo + workspace := t.TempDir() + tree := filepath.Join(workspace, "source") + var configs []string + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + if name == "git" && hasString(args, "clone") && args[len(args)-2] == source { + clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1]) + if out, err := Command(ctx, dir, "git", clone...); err != nil { + return out, err + } + return Command(ctx, args[len(args)-1], "git", "remote", "set-url", "origin", source) + } + if name == "git" && len(args) > 0 && args[0] == "rev-parse" { + raw, _ := os.ReadFile(filepath.Join(tree, ".git", "config")) + configs = append(configs, string(raw)) + } + return Command(ctx, dir, name, args...) + } + if _, err := Build(t.Context(), run, &recorded{}, source, "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + if len(configs) == 0 { + t.Fatal("the build never read its clone") + } + for _, c := range configs { + if strings.Contains(c, besideSecret) || strings.Contains(c, "build-user") { + t.Fatalf("the build's clone records the credential it was cloned with:\n%s", c) + } + } + if _, err := os.Stat(tree); !os.IsNotExist(err) { + t.Fatalf("the build's tree outlives the build: %v", err) + } +} diff --git a/internal/builder/packages_test.go b/internal/builder/packages_test.go index a87f19fd..7cccbf16 100644 --- a/internal/builder/packages_test.go +++ b/internal/builder/packages_test.go @@ -70,7 +70,16 @@ func TestNpmrcDisabledUntilThereIsARegistry(t *testing.T) { func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) { r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY .npmrc ./", "files/x": "y"}) n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} - if _, err := Build(context.Background(), r.run, r, + npmrc := filepath.Join(workspace, "source", ".npmrc") + inContext := false + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + if name == "docker" && len(args) > 0 && args[0] == "build" { + _, err := os.Stat(npmrc) + inContext = err == nil + } + return r.run(ctx, dir, name, args...) + } + if _, err := Build(context.Background(), run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil { t.Fatalf("the build failed: %v", err) } @@ -90,11 +99,14 @@ func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) { if !strings.Contains(build, "--network host") { t.Fatalf("the build was not given the host network to reach the registry: %s", build) } - // The .npmrc is written into the build context (the source tree), where a Dockerfile COPYs it. - tree := filepath.Join(workspace, "source") - npmrc := filepath.Join(tree, ".npmrc") - if _, err := os.Stat(npmrc); err != nil { - t.Fatalf("the credential was not written into the build context: %v", err) + // The .npmrc is written into the build context (the source tree), where a Dockerfile COPYs it, and + // removed with the tree when the build ends: a later check's container mounts the workspace (novox/hq + // issue 462). + if !inContext { + t.Fatal("the credential was not in the build context when the image was built") + } + if _, err := os.Stat(npmrc); !os.IsNotExist(err) { + t.Fatalf("the credential outlives the build in the workspace: %v", err) } } From 72fde0ee1ab933d192727ccbad73444797e6de33 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 11:30:16 +0200 Subject: [PATCH 13/16] Capture the SDK's conformance fixtures at the commit the node-engine's pin moved it to (hq issue 449's rule) --- testdata/beside/CAPTURED | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index 17d63986..f922ff40 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog b9de001833b1b61b297182b8e3bcdae1cbdeace9 modules/*/module.json, modules/nats/Dockerfile mesh-host bd5cc6980419c1bd4824be6d58dfebd2381a9de3 examples/foundation-first-node-nats.lock -mesh-sdk f047d0a4a9702f5d7f3d7eadd3e62496311acc00 conformance/events/module-event.json +mesh-sdk 33917f91ac3cc213b879b32f3c70e2aa70395f98 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: From 313efa826cec458a1d1c848eadc8e7e84b802707 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 11:30:16 +0200 Subject: [PATCH 14/16] Say in every declaration the assignment generation it came from, and record every send (hq issue 234) A declaration newer in sequence than every other named four fewer modules than the assignments held, a machine applied it, and nothing the mesh kept said who sent it or from what view. The store now raises an assignment generation in the same transaction as every assignment change (a trigger, so a cascade counts too); a send reads it before composing, carries it to engines that said they read it, marks a gate's put-back, and is recorded with its sequence, sender, generation and modules. The would-send is stamped with what was last sent, so nothing reads behind for it; a refusal is kept on the send it refused and raised as S20 naming the sender; plan says what the machine was last told. --- cmd/mesh-controller/facts.go | 4 + cmd/mesh-controller/gate.go | 5 +- cmd/mesh-controller/generation.go | 195 +++++++++++++++++ cmd/mesh-controller/generation_test.go | 107 ++++++++++ cmd/mesh-controller/held_back_test.go | 2 +- cmd/mesh-controller/issue_204_test.go | 2 +- cmd/mesh-controller/plan.go | 8 +- cmd/mesh-controller/push.go | 58 ++++- cmd/mesh-controller/release.go | 3 + cmd/mesh-controller/sendable.go | 25 +++ cmd/mesh-controller/signals.go | 14 ++ cmd/mesh-controller/signals_test.go | 8 + cmd/mesh-controller/status_summary.go | 5 + cmd/mesh-controller/watchdogs.go | 5 + ...ation-says-the-generation-it-came-from.sql | 64 ++++++ internal/inventory/sends.go | 202 ++++++++++++++++++ internal/inventory/sends_test.go | 138 ++++++++++++ internal/link/enrolment.go | 7 + internal/link/generation_test.go | 33 +++ internal/link/protocol.go | 16 ++ 20 files changed, 889 insertions(+), 12 deletions(-) create mode 100644 cmd/mesh-controller/generation.go create mode 100644 cmd/mesh-controller/generation_test.go create mode 100644 internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql create mode 100644 internal/inventory/sends.go create mode 100644 internal/inventory/sends_test.go create mode 100644 internal/link/generation_test.go diff --git a/cmd/mesh-controller/facts.go b/cmd/mesh-controller/facts.go index d24cc913..c1b6c891 100644 --- a/cmd/mesh-controller/facts.go +++ b/cmd/mesh-controller/facts.go @@ -514,6 +514,10 @@ func composedAndValidated(ctx context.Context, open *stores, node string, gens m if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil { return sendable{}, nil, err } + // And the generation it was last sent, as the would-send is (novox/hq issue 234). + if declared.Generation, declared.PutBack, err = open.inventory.SentGeneration(ctx, record.ID); err != nil { + return sendable{}, nil, err + } body, err := declared.Body() if err != nil { return sendable{}, nil, err diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 36eefc4d..d7a71972 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -994,7 +994,8 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str } return } - sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, putBack: true}), open, + g.Machines) if err != nil { notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ "sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0])) @@ -1087,7 +1088,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll } inv := open.inventory sort.Strings(b.machines) - sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, putBack: true}), open, b.machines) var back []string for _, r := range b.pending { if err != nil { diff --git a/cmd/mesh-controller/generation.go b/cmd/mesh-controller/generation.go new file mode 100644 index 00000000..f42d3fff --- /dev/null +++ b/cmd/mesh-controller/generation.go @@ -0,0 +1,195 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The assignment generation a declaration was composed from, and the record of every send (novox/hq +// issue 234). +// +// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the +// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and +// cannot tell a later send that carries an older view of the assignments. So a declaration now carries +// the generation of the assignments it was composed from — a counter the store raises in the same +// transaction as every assignment change — and a machine refuses one older than it applied, unless it is +// a gate's put-back. And every send is written down with who sent it, from which generation and naming +// which modules: the controller logged no send then, and which process sent the stale declaration could +// not be read back from anything the mesh kept. + +// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from. +const kindOlderGeneration = "older-generation" + +// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is. +const generationRefusalsSaid = advisoryQuiet + +// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, the +// epoch and generation when the machine reads them, and the put-back mark beside a generation — and keeps +// the generation and acting epoch for the record of the send whether or not the machine is sent them. +func (o order) stamp(d *sendable) { + d.Sequence, d.Epoch = o.sequence, o.epoch + d.composedFrom, d.actingEpoch, d.putBackSend = o.generation, o.acting, o.putBack + d.Generation, d.PutBack = 0, false + if o.readsGeneration && o.generation > 0 { + d.Generation, d.PutBack = o.generation, o.putBack + } +} + +// sentRecord is what the record of a send keeps beside its digest. +type sentRecord struct { + sequence int64 + epoch uint64 + generation int64 + putBack bool + // told is the generation and put-back mark on the wire, which the would-send is stamped with. + toldGeneration int64 + toldPutBack bool + sender string + modules []string +} + +// sentRecordOf is a composed send's record: its sender is the caller this process acts for. +func sentRecordOf(ctx context.Context, d sendable) sentRecord { + return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, putBack: d.putBackSend, + toldGeneration: d.Generation, toldPutBack: d.PutBack, sender: senderOf(callerOf(ctx)), modules: d.modules} +} + +// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the +// caller the controller ran this command for, or the account at the shell. +func callerOf(ctx context.Context) string { + if c := link.CallerIn(ctx); c != "" { + return c + } + return link.Caller() +} + +// senderOf is a send's sender in words: who asked, and the process and build that composed it — the +// answer issue 234 could not find, because two controllers and a one-shot push were all sending then. +func senderOf(caller string) string { + host, _ := os.Hostname() + return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version) +} + +// namedModules are the modules a declaration names: its machine's set, less what was left out of it. +func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string { + out := make([]string, 0, len(plan.Modules)) + for _, m := range plan.Modules { + if _, left := leftOut[m.Module]; !left { + out = append(out, m.Module) + } + } + sort.Strings(out) + return out +} + +// heardGenerationRefusal keeps a machine's refusal of a send for its generation on the send it refused, +// so S20 names its sender, and says it (novox/hq issue 234). +// +// **And raises the mesh's counter past what the machine applied, when the refused send was composed from +// the counter as it stands**: then the sender's view was not stale — the counter is behind the machine, +// which is a store put back from a backup — and every send after would be refused for ever. A stale +// sender's generation is below the counter, and the counter is left alone for it. +func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) { + r := report.OlderGeneration + if r == nil || inv == nil || report.Node == "" { + return + } + node, err := inv.NodeByName(ctx, report.Node) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be read: %v\n", + report.Node, err) + return + } + send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied) + switch { + case err != nil: + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+ + "kept: %v\n", report.Node, report.Sequence, err) + case !found: + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d (generation %d; it applied %d), which the mesh "+ + "has no record of sending\n", report.Node, report.Sequence, r.Generation, r.Applied) + default: + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+ + "and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied) + } + now, err := inv.AssignmentGeneration(ctx) + if err != nil || r.Generation < now || r.Applied < now { + return + } + raised, err := inv.RaiseAssignmentGeneration(ctx, r.Applied) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), and "+ + "could not be raised: %v\n", now, report.Node, r.Applied, err) + return + } + fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — a store "+ + "put back from a backup — and is raised to %d, so the next send is not refused\n", now, report.Node, r.Applied, + raised) +} + +// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was +// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal. +func watchGenerationRefusals(f *signalFacts) []conditions.Observation { + seen := map[string]bool{} + var out []conditions.Observation + for _, s := range f.refusedSends { + if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] { + continue + } + seen[s.NodeName] = true + out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration, + Machine: s.NodeName, Severity: conditions.Warning, + Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+ + "%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+ + "past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, + modulesWords(s.Modules)), + Said: fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)), + Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName), + Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+ + "refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName), + Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)}) + } + return out +} + +// modulesWords is a send's modules as a sentence says them. +func modulesWords(modules []string) string { + if len(modules) == 0 { + return "no module the mesh recorded" + } + return strings.Join(modules, ", ") +} + +// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234): +// nothing when the mesh has not recorded a send to it. +func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error { + s, found, err := inv.LastSend(ctx, node) + if err != nil || !found { + return err + } + generation := "no generation recorded" + if s.Generation > 0 { + generation = fmt.Sprintf("assignment generation %d", s.Generation) + } + kind := "sent" + if s.PutBack { + kind = "put back" + } + fmt.Fprintf(w, "%s was last %s sequence %d at %s by %s, composed from %s, naming %s\n", node, kind, + s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules)) + if s.RefusedAt != nil { + fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n", + s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied) + } + return nil +} diff --git a/cmd/mesh-controller/generation_test.go b/cmd/mesh-controller/generation_test.go new file mode 100644 index 00000000..f794760c --- /dev/null +++ b/cmd/mesh-controller/generation_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// The assignment generation a declaration was composed from (novox/hq issue 234). + +func bodyKeys(t *testing.T, s sendable) map[string]any { + t.Helper() + raw, err := s.Body() + if err != nil { + t.Fatal(err) + } + var keys map[string]any + if err := json.Unmarshal(raw, &keys); err != nil { + t.Fatal(err) + } + return keys +} + +// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older +// node-engine decodes strictly and refuses an unknown key, whole. And the put-back mark only with it. +func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) { + resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}} + reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57, putBack: true} + var told sendable + told.Resources = resources + reads.stamp(&told) + keys := bodyKeys(t, told) + if keys["generation"] != float64(40) || keys["put_back"] != true || keys["sequence"] != float64(12) { + t.Fatalf("a machine that reads a generation was sent %v", keys) + } + if told.composedFrom != 40 || told.actingEpoch != 57 { + t.Errorf("the send does not keep what it was composed from for its record: %+v", told) + } + + older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57, putBack: true} + var untold sendable + untold.Resources = resources + older.stamp(&untold) + keys = bodyKeys(t, untold) + if _, there := keys["generation"]; there { + t.Fatalf("a machine whose node-engine never said it reads a generation was sent one: %v", keys) + } + if _, there := keys["put_back"]; there { + t.Fatalf("a machine whose node-engine never said it reads a generation was sent a put-back mark: %v", keys) + } + if untold.composedFrom != 40 { + t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold) + } + + // An ordinary send carries no put-back mark at all: the body is what it was apart from the generation. + var ordinary sendable + ordinary.Resources = resources + ordinaryOrder := reads + ordinaryOrder.putBack = false + ordinaryOrder.stamp(&ordinary) + if _, there := bodyKeys(t, ordinary)["put_back"]; there { + t.Fatal("an ordinary send says it is a put-back") + } +} + +// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the +// process and build that composed it. +func TestASendNamesItsSender(t *testing.T) { + said := senderOf("g14.node-tools, through the mesh-controller seat") + if !strings.Contains(said, "g14.node-tools") || !strings.Contains(said, "pid ") || !strings.Contains(said, "build ") { + t.Fatalf("the sender reads %q", said) + } +} + +// refusedSend is a send a machine refused for its generation at a moment. +func refusedSend(at time.Time) inventory.Send { + return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40, + Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)", + Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at} +} + +// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation, +// against which the machine applied, and its sequence — the facts that took a morning to look for. +func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) { + now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) + f := calm(now) + f.refusedSends = []inventory.Send{refusedSend(now.Add(-time.Minute))} + got := watchGenerationRefusals(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + o := got[0] + if o.Key() != "machine.anchor.older-generation" || o.Machine != "anchor" { + t.Errorf("raised as %s about %q", o.Key(), o.Machine) + } + for _, want := range []string{"a one-shot push by jochen", "generation 38", "generation 40", "sequence 12"} { + if !strings.Contains(o.Summary, want) { + t.Errorf("the summary does not say %q: %s", want, o.Summary) + } + } + if o.Headline == "" || o.Explanation == "" || o.Resolved == "" { + t.Errorf("the condition is not worded for the operator: %+v", o) + } +} diff --git a/cmd/mesh-controller/held_back_test.go b/cmd/mesh-controller/held_back_test.go index bcb0fe94..f9d2ba06 100644 --- a/cmd/mesh-controller/held_back_test.go +++ b/cmd/mesh-controller/held_back_test.go @@ -128,7 +128,7 @@ func (r *recordedDelivery) grant(context.Context, []readyNode) error { return ni func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) { r.declared = append(r.declared, s.node) - return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch) + return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared)) } // aResolver is a module built from a repository, at a commit, with something on the machine that diff --git a/cmd/mesh-controller/issue_204_test.go b/cmd/mesh-controller/issue_204_test.go index feda54d8..1779292a 100644 --- a/cmd/mesh-controller/issue_204_test.go +++ b/cmd/mesh-controller/issue_204_test.go @@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) { } cancel() // the sender is going away: its context is cancelled between the send and the record body := []byte(`{"declaration":1,"resources":[]}`) - digest, err := recordSent(ctx, inv, "anchor", body, nil, 0) + digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"}) if err != nil { // NodeByName on the cancelled context may itself refuse; the record must still be possible // through the detached context, so look the node up again on a live one. diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 82f0fbb4..2ec829b1 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -420,7 +420,8 @@ func declarationWith(ctx context.Context, open *stores, node string, out := sendable{Resources: composed.Resources, Adoption: adoption, Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers, LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld, - unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced} + unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced, + modules: namedModules(plan, composed.LeftOut)} // And which build of each module it carries, for the send to record (novox/hq issue 259, ADR // 0221). Read only on the send path: a question about what would be sent records nothing. if choosing == Allocating { @@ -1290,6 +1291,11 @@ func planCommand(ctx context.Context, args []string) error { return nil } + // What the machine was last told, by whom and from which assignment generation (novox/hq issue 234), + // beside what it would be told now. A record that cannot be read is said, not taken for none. + if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil { + fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err) + } // Which modules a push would leave out, and why — said before the plan, since the plan is of // what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan` // without --json allocates nothing. diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 9779b442..be33434b 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -356,10 +356,18 @@ func declare(ctx context.Context, args []string) error { // the mesh did not compose it, so a push that does not name this machine treats it as held. // The epoch it carried, if a person wrote one in, is what the machine heard. var carried struct { - Epoch uint64 `json:"epoch"` + Epoch uint64 `json:"epoch"` + Sequence int64 `json:"sequence"` + Generation int64 `json:"generation"` + PutBack bool `json:"put_back"` } _ = json.Unmarshal(raw, &carried) - if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch); err != nil { + // And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it + // named are not known, since the mesh did not compose it. + if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence, + epoch: carried.Epoch, generation: carried.Generation, putBack: carried.PutBack, + toldGeneration: carried.Generation, toldPutBack: carried.PutBack, + sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil { return err } fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw)) @@ -798,7 +806,7 @@ func composeEach(names []string, allot func(node string) (order, error), refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) continue } - declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch + numbered.stamp(&declared) if len(declared.Resources) == 0 { // Sent, not skipped (novox/hq issue 127). A node whose declaration composes to // nothing may have HELD something before — the broker opening a placement gave it, @@ -1016,7 +1024,8 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri } // After it is away, not before. A digest recorded for something that failed to send would make // the machine look current for a declaration it never received. - digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch) + digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch, + sentRecordOf(ctx, s.declared)) if err != nil { return "", err } @@ -1228,7 +1237,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) continue } - declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch + numbered.stamp(&declared) reportLeftOut(name, declared) sending = append(sending, readyNode{name, declared}) } @@ -1393,6 +1402,11 @@ func wouldSendFrom(ctx context.Context, open *stores, if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil { return nil, err } + // And the generation and put-back mark it was last sent, for the same reason (novox/hq issue 234): + // an assignment elsewhere in the mesh is not a change of this machine. + if declared.Generation, declared.PutBack, err = open.inventory.SentGeneration(ctx, n.ID); err != nil { + return nil, err + } body, err := declared.Body() if err != nil { return nil, err @@ -1531,6 +1545,14 @@ var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.e type order struct { sequence int64 epoch uint64 + // generation is the assignment generation read before the composition, and readsGeneration whether the + // machine's node-engine said it reads one (novox/hq issue 234); acting is the lease epoch the sender acts + // under, whether or not the machine is sent it — both kept for the record of the send. + generation int64 + readsGeneration bool + acting uint64 + // putBack is a gate's put-back (sendScope.putBack), read from the send's context. + putBack bool } // allot takes the next sequence for a machine — the number its next declaration carries — under the @@ -1544,6 +1566,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e if err != nil { return order{}, err } + acting := epoch if epoch > 0 { reads, err := inv.ReadsEpoch(ctx, record.ID) if err != nil { @@ -1553,11 +1576,24 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e epoch = 0 } } + // **The generation is read here, before the composition reads a single assignment** (novox/hq issue + // 234). A generation only grows, so one read before is never newer than the view composed after it: + // the declaration may claim a generation older than its content, never newer — and a claim newer than + // the content is exactly the stale send the machine must be able to refuse. + generation, err := inv.AssignmentGeneration(ctx) + if err != nil { + return order{}, err + } + readsGeneration, err := inv.ReadsGeneration(ctx, record.ID) + if err != nil { + return order{}, err + } seq, err := inv.NextSequence(ctx, record.ID) if err != nil { return order{}, err } - return order{sequence: seq, epoch: epoch}, nil + return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration, + acting: acting, putBack: scopeOf(ctx).putBack}, nil } // recordSent writes down what a machine was just sent, and returns the digest. @@ -1572,7 +1608,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e // And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known: // what tells a machine held back by a policy or a plan from one a push left behind. func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte, - builds map[string]string, epoch uint64) (string, error) { + builds map[string]string, epoch uint64, sent sentRecord) (string, error) { kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() record, err := inv.NodeByName(kept, node) @@ -1583,6 +1619,14 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil { return "", err } + // And the send itself, who sent it and from which generation (novox/hq issue 234): what the machine + // was last told, by whom, is read back from here — and what the would-send is stamped with. + if err := inv.RecordSend(kept, inventory.Send{Node: record.ID, Sequence: sent.sequence, + Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, PutBack: sent.putBack, + ToldGeneration: sent.toldGeneration, ToldPutBack: sent.toldPutBack, Digest: digest, + Modules: sent.modules}); err != nil { + return "", fmt.Errorf("%s was sent its declaration, and the send could not be recorded: %w", node, err) + } // And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217). // Never a reason to fail a send that is already away: a summary that cannot be kept means the // next comparison has nothing to hold against, which is how every machine starts. diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 56b702c8..31c1ef87 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -50,6 +50,9 @@ type sendScope struct { modules map[string]bool // person is a person's act: it carries what it carries. person bool + // putBack is a gate putting machines back after a failed send (novox/hq issue 234): its declarations + // say so, and a machine does not refuse them for the generation they carry. + putBack bool } type sendScopeKey struct{} diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go index 2311522a..cbc89943 100644 --- a/cmd/mesh-controller/sendable.go +++ b/cmd/mesh-controller/sendable.go @@ -27,6 +27,25 @@ type sendable struct { // said it reads one is sent none, because an older node-engine refuses a key it does not know, whole // (link/order.go, the contract). Epoch uint64 + // Generation is the assignment generation it was composed from (novox/hq issue 234): a machine that + // applied a later one refuses it, so a send composed from a view of the assignments the mesh has moved + // past cannot undeclare what is still assigned. Zero is not sent at all — every machine whose + // node-engine has not said it reads one is sent none, for the reason the epoch is not (an older + // node-engine refuses a key it does not know, whole). + Generation int64 + // PutBack marks a gate's put-back (novox/hq issue 234): it carries the generation of what it puts + // back and is not refused for it. Sent only beside a generation. + PutBack bool + // composedFrom is the generation read before this declaration was composed, and actingEpoch the + // lease epoch its sender acted under — whether or not the machine is sent either — for the record of + // the send; never on the wire. + composedFrom int64 + actingEpoch uint64 + // putBackSend is whether a gate sent it to put the machine back, for the record; PutBack is the mark + // on the wire, only beside a generation. + putBackSend bool + // modules are the modules this declaration names, for the record of the send; never on the wire. + modules []string // Adoption is nil for a converged node, and then the body is byte for byte what it was before // adoption existed: an older host parses the envelope strictly and would refuse the key. Adoption *adoptionEnvelope @@ -94,6 +113,12 @@ func (s sendable) Body() ([]byte, error) { if len(s.LeftOut) > 0 { envelope["left_out"] = s.LeftOut } + if s.Generation > 0 { + envelope["generation"] = s.Generation + if s.PutBack { + envelope["put_back"] = true + } + } // An empty declaration is deliberate here — the node owns nothing the mesh put there // (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness // is meant, so a truncated or mis-composed body is never mistaken for "own nothing". diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index ab3e8a93..34a6781c 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -226,6 +226,20 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed }) }}, + {Row: "S20", Signal: "a send refused for the assignment generation it was composed from", + Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)", + Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " + + "machine applied; cleared an hour after the last", + Kind: kindOlderGeneration, Severity: conditions.Warning, Phase: 2, + needs: func(f *signalFacts) error { return f.refusedSendsErr }, watch: watchGenerationRefusals, + newest: func(f *signalFacts) time.Time { + return newestOf(f.refusedSends, func(s inventory.Send) time.Time { + if s.RefusedAt == nil { + return time.Time{} + } + return *s.RefusedAt + }) + }}, {Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan", Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)", Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " + diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index c69a9d02..9abf6ce8 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -178,6 +178,14 @@ var suppressions = map[string]suppression{ commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}} }, }, + // A send refused by its machine for the generation it was composed from (novox/hq issue 234): said at + // once, naming its sender, for an hour after. Inside: the last such refusal was more than an hour ago. + "S20": { + inside: func(f *signalFacts) { + f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-61 * time.Minute))} + }, + past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} }, + }, // Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not. "S15": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/status_summary.go b/cmd/mesh-controller/status_summary.go index 7974abd4..a81a0b8e 100644 --- a/cmd/mesh-controller/status_summary.go +++ b/cmd/mesh-controller/status_summary.go @@ -196,6 +196,11 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e if report.Ordered() { link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now) } + // A send refused for the generation it was composed from is kept on the send it refused, so S20 names + // its sender (novox/hq issue 234). + if report.OlderGeneration != nil { + heardGenerationRefusal(ctx, l.Enrolment.Inventory, report) + } // What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its // probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey. if report.Superseded == "" && report.Rekey == nil && report.Node != "" { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 35c2c951..7ccb352d 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -55,6 +55,10 @@ type signalFacts struct { waits []waitFacts // batches are the batches not yet cut (S18, S19, novox/hq ADR 0276). batches []batchFacts + // refusedSends are the sends a machine refused within the hour for the generation they were composed + // from, each naming its sender (S20, novox/hq issue 234). + refusedSends []inventory.Send + refusedSendsErr error loop loopFacts loopErr error @@ -364,6 +368,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { } } f.handActs, f.handActsErr = w.gatherHandActs(ctx, now) + f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid)) f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last() return f } diff --git a/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql b/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql new file mode 100644 index 00000000..07a182d3 --- /dev/null +++ b/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql @@ -0,0 +1,64 @@ +-- A declaration says the assignment generation it was composed from, and every send is recorded +-- (novox/hq issue 234). +-- +-- On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the +-- assignments held, and the machine undeclared all four. The sequence orders arrival; it cannot tell a +-- later send that carries an older view of the assignments. And nothing the mesh kept said which process +-- sent it, from what view: the node row holds only the digest of the last send. + +-- The assignment generation: one counter for the mesh, raised in the same transaction as every change to +-- what is assigned where. A trigger, not a line in each writer: an assignment is also taken by a node's +-- removal (on delete cascade) and renamed with its module (on update cascade), which no writer in Go +-- sees, and a rule kept only by the writers that remember it is the rule this issue found broken. Seeded +-- at 1, so every declaration composed after this claims a generation and none claims zero ("none"). +create table assignment_generation ( + one boolean primary key default true check (one), + generation bigint not null +); +insert into assignment_generation (one, generation) values (true, 1); + +create function raise_assignment_generation() returns trigger language plpgsql as $$ +begin + update assignment_generation set generation = generation + 1; + return null; +end +$$; + +-- Per row, so a statement that changes nothing (an assignment repeated, `on conflict do nothing`) raises +-- nothing; a statement that changes several raises once per row, which only ever moves it forward. +create trigger assignment_generation_raised + after insert or update or delete on assignment + for each row execute function raise_assignment_generation(); + +-- What a machine was last sent of it: the generation, and whether that send was a gate's put-back, so +-- what the mesh WOULD send is stamped the same way and reads as byte for byte what it DID send when +-- nothing else changed (as sent_epoch, migration 0068). Null and false for a send without. +alter table node add column sent_generation bigint; +alter table node add column sent_put_back boolean not null default false; +-- Whether the machine's node-engine said it reads a generation (its reports' `reads_generation`): until +-- it has, it is sent none, because an older node-engine refuses a key it does not know, whole. +alter table node add column reads_generation boolean not null default false; + +-- Every send: the machine, its sequence, who sent it — the lease epoch the sender acted under and the +-- caller, process and build — the generation it was composed from, whether it was a put-back, its +-- digest and the modules it named. Written after the declaration is away, as the node row's record is. +-- A refusal by the machine of a send for its generation is kept on the send it refused, so the +-- condition it raises names the sender from here. The newest 200 per machine are kept. +create table declaration_send ( + id bigserial primary key, + node uuid not null references node (id) on delete cascade, + sequence bigint, + epoch bigint, + sender text not null, + generation bigint, + put_back boolean not null default false, + digest text not null, + modules text[] not null default '{}', + sent_at timestamptz not null default now(), + refused_at timestamptz, + -- refused_applied is the generation the machine said it had applied when it refused this send. + refused_applied bigint +); +create index declaration_send_node on declaration_send (node, id desc); +create index declaration_send_sequence on declaration_send (node, sequence); +create index declaration_send_refused on declaration_send (refused_at) where refused_at is not null; diff --git a/internal/inventory/sends.go b/internal/inventory/sends.go new file mode 100644 index 00000000..085a2ac9 --- /dev/null +++ b/internal/inventory/sends.go @@ -0,0 +1,202 @@ +package inventory + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5" +) + +// The assignment generation and the record of every send (novox/hq issue 234, migration 0094). +// +// A declaration carries the generation of the assignments it was composed from, and a machine refuses one +// composed from an older generation than it applied — the sequence orders arrival and cannot tell a later +// send carrying an older view. And every send is written down with who sent it, from which generation and +// naming which modules, so what a machine was last told, by whom, is read back rather than guessed at. + +// sendsKept is how many sends are kept per machine: the newest. Enough to read back a morning of pushes +// on a busy machine; the record is for finding who sent what, not an archive. +const sendsKept = 200 + +// AssignmentGeneration is the mesh's assignment generation now: raised by the store itself in the same +// transaction as every change to what is assigned where (a trigger on the assignment table). +func (i *Inventory) AssignmentGeneration(ctx context.Context) (int64, error) { + var g int64 + if err := i.store.Pool().QueryRow(ctx, `select generation from assignment_generation`).Scan(&g); err != nil { + return 0, fmt.Errorf("reading the assignment generation: %w", err) + } + return g, nil +} + +// RaiseAssignmentGeneration raises the generation past one a machine applied, and answers it: one more +// than that, or what it already was when it is past it. It never lowers it. +// +// For one case only: a machine refused a send composed from the generation the mesh holds now, so the +// machine applied a higher one than the mesh's counter — a store put back from a backup. Every send after +// would be refused for ever; raised past it, the next send carries a generation the machine takes, composed +// from the assignments the store holds now. A stale sender never meets this: its generation is below the +// counter, and the counter is not touched for it. +func (i *Inventory) RaiseAssignmentGeneration(ctx context.Context, past int64) (int64, error) { + var g int64 + err := i.store.Pool().QueryRow(ctx, + `update assignment_generation set generation = greatest(generation, $1 + 1) returning generation`, past).Scan(&g) + if err != nil { + return 0, fmt.Errorf("raising the assignment generation past %d: %w", past, err) + } + return g, nil +} + +// SentGeneration is the generation a machine was last sent and whether that send was a put-back, by its +// id; zero for one sent without. What the mesh WOULD send is stamped with these, so it reads as byte for +// byte what it DID send when nothing else changed. +func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, bool, error) { + var g *int64 + var putBack bool + if err := i.store.Pool().QueryRow(ctx, `select sent_generation, sent_put_back from node where id = $1`, id). + Scan(&g, &putBack); err != nil { + return 0, false, fmt.Errorf("reading the generation %s was last sent: %w", id, err) + } + if g == nil { + return 0, putBack, nil + } + return *g, putBack, nil +} + +// ReadsGeneration says a machine's node-engine said it reads a generation in a declaration, by its id. +func (i *Inventory) ReadsGeneration(ctx context.Context, id string) (bool, error) { + var reads bool + if err := i.store.Pool().QueryRow(ctx, `select reads_generation from node where id = $1`, id).Scan(&reads); err != nil { + return false, fmt.Errorf("reading whether %s reads a generation: %w", id, err) + } + return reads, nil +} + +// RecordReadsGeneration keeps what a machine's latest report said of reading a generation. +func (i *Inventory) RecordReadsGeneration(ctx context.Context, id string, reads bool) error { + _, err := i.store.Pool().Exec(ctx, `update node set reads_generation = $2 where id = $1`, id, reads) + return err +} + +// Send is one declaration sent to a machine, as the mesh records it. +type Send struct { + // Node is the machine's id; NodeName its name, filled where it is read back. + Node string + NodeName string + Sequence int64 + // Epoch is the lease epoch its sender acted under, zero for one that acted under none. + Epoch int64 + // Sender is who sent it, in words: the caller, and the process and build that composed it. + Sender string + // Generation is the assignment generation it was composed from, zero when not known; PutBack whether a + // gate sent it to put the machine back. + Generation int64 + PutBack bool + // ToldGeneration and ToldPutBack are what the machine was told of them on the wire: zero and false for a + // machine whose node-engine has not said it reads a generation. What the would-send is stamped with. + ToldGeneration int64 + ToldPutBack bool + Digest string + // Modules are the modules it named. + Modules []string + SentAt time.Time + // RefusedAt is when the machine refused it for its generation, nil when it did not; RefusedApplied the + // generation the machine said it had applied then. + RefusedAt *time.Time + RefusedApplied int64 +} + +// RecordSend writes one send down, and what the machine was last sent of its generation, in one +// transaction; the oldest beyond the newest 200 for the machine are let go. +func (i *Inventory) RecordSend(ctx context.Context, s Send) error { + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(ctx) }() + if _, err := tx.Exec(ctx, `update node set sent_generation = $2, sent_put_back = $3 where id = $1`, + s.Node, nullIfZero(s.ToldGeneration), s.ToldPutBack); err != nil { + return err + } + modules := s.Modules + if modules == nil { + modules = []string{} + } + if _, err := tx.Exec(ctx, + `insert into declaration_send (node, sequence, epoch, sender, generation, put_back, digest, modules) + values ($1, $2, $3, $4, $5, $6, $7, $8)`, + s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.PutBack, + s.Digest, modules); err != nil { + return err + } + if _, err := tx.Exec(ctx, + `delete from declaration_send where node = $1 and id not in + (select id from declaration_send where node = $1 order by id desc limit $2)`, s.Node, sendsKept); err != nil { + return err + } + return tx.Commit(ctx) +} + +// sendColumns are a send's columns as scanSends reads them. +const sendColumns = `s.node, n.name, coalesce(s.sequence, 0), coalesce(s.epoch, 0), s.sender, coalesce(s.generation, 0), + s.put_back, s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0)` + +func scanSends(rows pgx.Rows) ([]Send, error) { + defer rows.Close() + var out []Send + for rows.Next() { + var s Send + if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.PutBack, + &s.Digest, &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + +// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. +func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) { + rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` + from declaration_send s join node n on n.id = s.node + where n.name = $1 order by s.id desc limit 1`, name) + if err != nil { + return Send{}, false, err + } + sends, err := scanSends(rows) + if err != nil || len(sends) == 0 { + return Send{}, false, err + } + return sends[0], true, nil +} + +// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and +// answers that send, sender and all; false when no send of that sequence was recorded — sent before the +// record, or by a hand the mesh did not see. The newest of that sequence when there are several: a +// declaration a person sent by hand may repeat one. +func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied int64) (Send, bool, error) { + rows, err := i.store.Pool().Query(ctx, `with refused as ( + update declaration_send set refused_at = now(), refused_applied = $3 + where id = (select id from declaration_send where node = $1 and sequence = $2 order by id desc limit 1) + returning *) + select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied) + if err != nil { + return Send{}, false, err + } + sends, err := scanSends(rows) + if err != nil || len(sends) == 0 { + return Send{}, false, err + } + return sends[0], true, nil +} + +// RefusedSendsSince is every send refused for its generation since a moment, newest first. +func (i *Inventory) RefusedSendsSince(ctx context.Context, since time.Time) ([]Send, error) { + rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` + from declaration_send s join node n on n.id = s.node + where s.refused_at >= $1 order by s.refused_at desc`, since) + if err != nil { + return nil, err + } + return scanSends(rows) +} diff --git a/internal/inventory/sends_test.go b/internal/inventory/sends_test.go new file mode 100644 index 00000000..9f5dfe6b --- /dev/null +++ b/internal/inventory/sends_test.go @@ -0,0 +1,138 @@ +package inventory + +import ( + "slices" + "testing" + "time" +) + +// The assignment generation and the record of every send (novox/hq issue 234). On 2026-10-04 a declaration +// newer in sequence than every other named four fewer modules than the assignments held, a machine applied +// it, and nothing the mesh kept could say who sent it or from what view. + +func generationNow(t *testing.T, inv *Inventory) int64 { + t.Helper() + g, err := inv.AssignmentGeneration(t.Context()) + if err != nil { + t.Fatal(err) + } + return g +} + +// **Every change to what is assigned where raises the generation, in its own transaction** — an +// assignment, an unassignment, and one taken by the machine's removal, which no writer in Go makes — and +// a repeated assignment, which changes nothing, does not. +func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres", "web") + ctx := t.Context() + start := generationNow(t, inv) + if start < 1 { + t.Fatalf("the generation starts at %d; zero is \"none claimed\" on the wire", start) + } + if _, err := inv.Assign(ctx, node, "postgres"); err != nil { + t.Fatal(err) + } + assigned := generationNow(t, inv) + if assigned <= start { + t.Fatalf("an assignment left the generation at %d", assigned) + } + if _, err := inv.Assign(ctx, node, "postgres"); err != nil { + t.Fatal(err) + } + if again := generationNow(t, inv); again != assigned { + t.Errorf("an assignment repeated, which changed nothing, moved the generation from %d to %d", assigned, again) + } + if err := inv.Unassign(ctx, node, "postgres"); err != nil { + t.Fatal(err) + } + unassigned := generationNow(t, inv) + if unassigned <= assigned { + t.Fatalf("an unassignment left the generation at %d", unassigned) + } + if _, err := inv.Assign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + before := generationNow(t, inv) + if _, err := inv.RemoveNodeForTest(ctx, node); err != nil { + t.Fatal(err) + } + if removed := generationNow(t, inv); removed <= before { + t.Errorf("a machine's removal took its assignments and left the generation at %d", removed) + } +} + +// **The generation never goes down, and is raised past what a machine applied when the mesh's own counter +// is behind it** — a store put back from a backup — so the next send is not refused for ever. +func TestTheGenerationIsRaisedPastWhatAMachineApplied(t *testing.T) { + inv, _ := aNodeWithModules(t) + now := generationNow(t, inv) + if raised, err := inv.RaiseAssignmentGeneration(t.Context(), now+40); err != nil || raised != now+41 { + t.Fatalf("raised past %d to %d (%v)", now+40, raised, err) + } + if raised, err := inv.RaiseAssignmentGeneration(t.Context(), 2); err != nil || raised != now+41 { + t.Fatalf("a raise below the counter moved it to %d (%v)", raised, err) + } +} + +// **Every send is recorded: its sequence, its sender, the generation it came from and the modules it +// named** — and the machine's last send is what `plan` reads, the would-send is stamped with its +// generation and put-back mark, and a refusal is kept on the send it refused, naming its sender. +func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + first := Send{Node: record.ID, Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)", + Generation: 40, Digest: "d11", Modules: []string{"docker", "pacman", "sudo"}} + if err := inv.RecordSend(ctx, first); err != nil { + t.Fatal(err) + } + stale := Send{Node: record.ID, Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", + Generation: 38, ToldGeneration: 38, Digest: "d12", Modules: []string{"docker"}} + if err := inv.RecordSend(ctx, stale); err != nil { + t.Fatal(err) + } + last, found, err := inv.LastSend(ctx, node) + if err != nil || !found { + t.Fatalf("the last send is not kept: %v", err) + } + if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || + !slices.Equal(last.Modules, []string{"docker"}) || last.SentAt.IsZero() { + t.Fatalf("the last send reads %+v", last) + } + generation, putBack, err := inv.SentGeneration(ctx, record.ID) + if err != nil || generation != 38 || putBack { + t.Fatalf("what the machine was last sent of it reads %d, %v (%v)", generation, putBack, err) + } + + refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40) + if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 { + t.Fatalf("the refusal is not kept on the send it refused: %+v, %v, %v", refused, found, err) + } + since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour)) + if err != nil || len(since) != 1 || since[0].NodeName != node || since[0].Sequence != 12 { + t.Fatalf("the refused sends within the hour read %+v (%v)", since, err) + } + if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40); err != nil || found { + t.Errorf("a refusal of a send never recorded was found: %v, %v", found, err) + } +} + +// **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer. +func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) { + inv, node := aNodeWithModules(t) + record, err := inv.NodeByName(t.Context(), node) + if err != nil { + t.Fatal(err) + } + for _, reads := range []bool{false, true, false} { + if err := inv.RecordReadsGeneration(t.Context(), record.ID, reads); err != nil { + t.Fatal(err) + } + if got, err := inv.ReadsGeneration(t.Context(), record.ID); err != nil || got != reads { + t.Fatalf("recorded %v, read %v (%v)", reads, got, err) + } + } +} diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index 6c3cab6b..de36001f 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -447,6 +447,13 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err if err := e.Inventory.RecordReadsEpoch(ctx, node.ID, report.ReadsEpoch()); err != nil { return false, err } + // And whether it reads a generation (novox/hq issue 234), the same way: only from a node-engine that + // orders its reports, so a one-shot report (a rekey) does not say an engine stopped reading one. + if report.Ordered() { + if err := e.Inventory.RecordReadsGeneration(ctx, node.ID, report.ReadsGeneration); err != nil { + return false, err + } + } if report.Ordered() { account := AccountOf(report) news, err = e.Inventory.RecordOrderedDoing(ctx, node.ID, doing, diff --git a/internal/link/generation_test.go b/internal/link/generation_test.go new file mode 100644 index 00000000..7a25283b --- /dev/null +++ b/internal/link/generation_test.go @@ -0,0 +1,33 @@ +package link + +import ( + "encoding/json" + "testing" +) + +// A report's word on the assignment generation (novox/hq issue 234), as mesh-host's report writes it: +// `reads_generation` on every report of a node-engine that reads one, and `older_generation` on a refusal +// of a declaration composed from an older generation than the machine applied. +func TestTheGenerationOnTheWire(t *testing.T) { + raw := []byte(`{"node":"anchor","refused":"older generation","declared":"d12","epoch":57,"sequence":12,` + + `"report_sequence":8,"reads_generation":true,"older_generation":{"generation":38,"applied":40}}`) + var r Report + if err := json.Unmarshal(raw, &r); err != nil { + t.Fatal(err) + } + if !r.ReadsGeneration || r.OlderGeneration == nil || *r.OlderGeneration != (GenerationRefusal{38, 40}) || + r.Sequence != 12 { + t.Fatalf("a node-engine's generation refusal reads as %+v", r) + } + // Not a refusal by the lease's epoch: S13 counts those, and this one is raised naming its sender. + if r.StaleRefusalOf() { + t.Errorf("a generation refusal reads as a stale writer's") + } + var older Report + if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":3}`), &older); err != nil { + t.Fatal(err) + } + if older.ReadsGeneration || older.OlderGeneration != nil { + t.Fatalf("a node-engine that never said it reads a generation reads as one that does: %+v", older) + } +} diff --git a/internal/link/protocol.go b/internal/link/protocol.go index b64a9a72..63349ece 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -197,6 +197,15 @@ type Report struct { // RefusedOlder is how many declarations the node-engine has refused as older, ever, on every report: // a refusal whose own report was lost is still counted from the next. RefusedOlder int64 `json:"refused_older,omitempty"` + // ReadsGeneration says the node-engine reads a declaration's assignment generation and its put-back + // mark (novox/hq issue 234), on every report it makes: the mesh sends them only to a machine that said + // so, because an older node-engine refuses a key it does not know, whole. Mirrors mesh-host + // internal/link/messages.go. + ReadsGeneration bool `json:"reads_generation,omitempty"` + // OlderGeneration is set on a report refusing a declaration composed from an older assignment + // generation than the machine applied (novox/hq issue 234). The refused declaration is the report's + // own Declared and Order, and its sender is read from the record of sends by that sequence. + OlderGeneration *GenerationRefusal `json:"older_generation,omitempty"` // Held is what an adopted node found and is keeping as it was until its module is taken // (novox/hq ADR 0100). Without it an adopted node reads as converged. @@ -605,3 +614,10 @@ type HealthSaid struct { Node string `json:"node"` Health Health `json:"health"` } + +// GenerationRefusal is a declaration refused for the assignment generation it was composed from: that +// generation, and the highest the machine applied (novox/hq issue 234; mesh-host's GenerationRefusal). +type GenerationRefusal struct { + Generation int64 `json:"generation"` + Applied int64 `json:"applied"` +} From 5dd0e3c056418e6aef6202d8b3e65758f89beadb Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 11:55:27 +0200 Subject: [PATCH 15/16] Raise S20 for every generation refusal, ask for a push when the counter was behind, and write a send's generation with its digest (hq issue 234 review) A refusal of a send the mesh never recorded went only to stderr; it is now kept with its sender said to be unknown, and raised. A counter behind the machine (a store put back) is raised and the condition names push, since the receive loop must not push. The node's digest and generation are one transaction, and a send record that fails is said loudly without failing the send. The trigger ignores an update that changes nothing, and the put-back mark is dropped: a put-back is composed afresh with the current generation. --- cmd/mesh-controller/facts.go | 2 +- cmd/mesh-controller/gate.go | 5 +- cmd/mesh-controller/generation.go | 119 +++++++----- cmd/mesh-controller/generation_test.go | 83 ++++++-- cmd/mesh-controller/push.go | 38 ++-- cmd/mesh-controller/release.go | 3 - cmd/mesh-controller/sendable.go | 9 - ...ation-says-the-generation-it-came-from.sql | 32 ++-- internal/inventory/sends.go | 181 +++++++++++++----- internal/inventory/sends_test.go | 97 ++++++++-- 10 files changed, 394 insertions(+), 175 deletions(-) diff --git a/cmd/mesh-controller/facts.go b/cmd/mesh-controller/facts.go index c1b6c891..50b891cf 100644 --- a/cmd/mesh-controller/facts.go +++ b/cmd/mesh-controller/facts.go @@ -515,7 +515,7 @@ func composedAndValidated(ctx context.Context, open *stores, node string, gens m return sendable{}, nil, err } // And the generation it was last sent, as the would-send is (novox/hq issue 234). - if declared.Generation, declared.PutBack, err = open.inventory.SentGeneration(ctx, record.ID); err != nil { + if declared.Generation, err = open.inventory.SentGeneration(ctx, record.ID); err != nil { return sendable{}, nil, err } body, err := declared.Body() diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index d7a71972..36eefc4d 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -994,8 +994,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str } return } - sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, putBack: true}), open, - g.Machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines) if err != nil { notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ "sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0])) @@ -1088,7 +1087,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll } inv := open.inventory sort.Strings(b.machines) - sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, putBack: true}), open, b.machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines) var back []string for _, r := range b.pending { if err != nil { diff --git a/cmd/mesh-controller/generation.go b/cmd/mesh-controller/generation.go index f42d3fff..15f79cbb 100644 --- a/cmd/mesh-controller/generation.go +++ b/cmd/mesh-controller/generation.go @@ -33,15 +33,20 @@ const kindOlderGeneration = "older-generation" // generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is. const generationRefusalsSaid = advisoryQuiet -// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, the -// epoch and generation when the machine reads them, and the put-back mark beside a generation — and keeps -// the generation and acting epoch for the record of the send whether or not the machine is sent them. +// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and +// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the +// record of the send whether or not the machine is sent them. +// +// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its +// declaration is allotted here like any other and carries the generation as it stands — never older than +// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never +// needs to send one. func (o order) stamp(d *sendable) { d.Sequence, d.Epoch = o.sequence, o.epoch - d.composedFrom, d.actingEpoch, d.putBackSend = o.generation, o.acting, o.putBack - d.Generation, d.PutBack = 0, false + d.composedFrom, d.actingEpoch = o.generation, o.acting + d.Generation = 0 if o.readsGeneration && o.generation > 0 { - d.Generation, d.PutBack = o.generation, o.putBack + d.Generation = o.generation } } @@ -50,18 +55,17 @@ type sentRecord struct { sequence int64 epoch uint64 generation int64 - putBack bool - // told is the generation and put-back mark on the wire, which the would-send is stamped with. + // toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine + // whose node-engine has not said it reads one. toldGeneration int64 - toldPutBack bool sender string modules []string } // sentRecordOf is a composed send's record: its sender is the caller this process acts for. func sentRecordOf(ctx context.Context, d sendable) sentRecord { - return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, putBack: d.putBackSend, - toldGeneration: d.Generation, toldPutBack: d.PutBack, sender: senderOf(callerOf(ctx)), modules: d.modules} + return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, + toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules} } // callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the @@ -92,13 +96,19 @@ func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string return out } -// heardGenerationRefusal keeps a machine's refusal of a send for its generation on the send it refused, -// so S20 names its sender, and says it (novox/hq issue 234). +// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender +// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that +// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the +// mesh cannot attribute is the louder fact, not a quieter one. // // **And raises the mesh's counter past what the machine applied, when the refused send was composed from -// the counter as it stands**: then the sender's view was not stale — the counter is behind the machine, -// which is a store put back from a backup — and every send after would be refused for ever. A stale -// sender's generation is below the counter, and the counter is left alone for it. +// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind +// the machine, which is a store put back from a backup — and every send after would be refused for ever. A +// stale sender's generation is below the counter, and the counter is left alone for it. +// +// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that +// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185. +// S20 names `push ` for that case instead. func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) { r := report.OlderGeneration if r == nil || inv == nil || report.Node == "" { @@ -106,35 +116,44 @@ func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, repor } node, err := inv.NodeByName(ctx, report.Node) if err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be read: %v\n", - report.Node, err) + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+ + "read, so it is not raised: %v\n", report.Node, err) return } - send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied) - switch { - case err != nil: - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+ - "kept: %v\n", report.Node, report.Sequence, err) - case !found: - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d (generation %d; it applied %d), which the mesh "+ - "has no record of sending\n", report.Node, report.Sequence, r.Generation, r.Applied) - default: - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+ - "and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied) + var raised int64 + if now, err := inv.AssignmentGeneration(ctx); err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+ + "read: %v\n", report.Node, err) + } else if counterBehind(*r, now) { + if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+ + "and could not be raised: %v\n", now, report.Node, r.Applied, err) + raised = 0 + } else { + fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+ + "a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n", + now, report.Node, r.Applied, raised, report.Node) + } } - now, err := inv.AssignmentGeneration(ctx) - if err != nil || r.Generation < now || r.Applied < now { - return + send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised) + if err == nil && !found { + send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence, + Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied, + CounterRaisedTo: raised}) } - raised, err := inv.RaiseAssignmentGeneration(ctx, r.Applied) if err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), and "+ - "could not be raised: %v\n", now, report.Node, r.Applied, err) + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+ + "kept, so it is not raised: %v\n", report.Node, report.Sequence, err) return } - fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — a store "+ - "put back from a backup — and is raised to %d, so the next send is not refused\n", now, report.Node, r.Applied, - raised) + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+ + "and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied) +} + +// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the +// refused send carried the counter as it stands now, and the machine applied more than that. +func counterBehind(r link.GenerationRefusal, now int64) bool { + return r.Generation >= now && r.Applied > r.Generation } // watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was @@ -147,7 +166,7 @@ func watchGenerationRefusals(f *signalFacts) []conditions.Observation { continue } seen[s.NodeName] = true - out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration, + o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration, Machine: s.NodeName, Severity: conditions.Warning, Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+ "%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+ @@ -157,7 +176,20 @@ func watchGenerationRefusals(f *signalFacts) []conditions.Observation { Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName), Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+ "refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName), - Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)}) + Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)} + if s.CounterRaisedTo > 0 { + // Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and + // was raised; nothing has sent the machine its declaration since, so a person is asked to. + o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+ + "the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+ + "put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now", + s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo, + s.NodeName) + o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+ + "update and kept what it had. The record is repaired; a push sends the update again.", + s.NodeName, s.NodeName, s.NodeName) + } + out = append(out, o) } return out } @@ -181,12 +213,7 @@ func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, n if s.Generation > 0 { generation = fmt.Sprintf("assignment generation %d", s.Generation) } - kind := "sent" - if s.PutBack { - kind = "put back" - } - fmt.Fprintf(w, "%s was last %s sequence %d at %s by %s, composed from %s, naming %s\n", node, kind, - s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules)) + fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules)) if s.RefusedAt != nil { fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n", s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied) diff --git a/cmd/mesh-controller/generation_test.go b/cmd/mesh-controller/generation_test.go index f794760c..a2372bcc 100644 --- a/cmd/mesh-controller/generation_test.go +++ b/cmd/mesh-controller/generation_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" ) // The assignment generation a declaration was composed from (novox/hq issue 234). @@ -25,45 +26,35 @@ func bodyKeys(t *testing.T, s sendable) map[string]any { } // **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older -// node-engine decodes strictly and refuses an unknown key, whole. And the put-back mark only with it. +// node-engine decodes strictly and refuses an unknown key, whole. No put-back mark is ever sent: a gate +// composes its put-back afresh, with the generation as it stands. func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) { resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}} - reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57, putBack: true} + reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57} var told sendable told.Resources = resources reads.stamp(&told) keys := bodyKeys(t, told) - if keys["generation"] != float64(40) || keys["put_back"] != true || keys["sequence"] != float64(12) { + if keys["generation"] != float64(40) || keys["sequence"] != float64(12) { t.Fatalf("a machine that reads a generation was sent %v", keys) } + if _, there := keys["put_back"]; there { + t.Fatalf("a put-back mark was sent: %v", keys) + } if told.composedFrom != 40 || told.actingEpoch != 57 { t.Errorf("the send does not keep what it was composed from for its record: %+v", told) } - older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57, putBack: true} + older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57} var untold sendable untold.Resources = resources older.stamp(&untold) - keys = bodyKeys(t, untold) - if _, there := keys["generation"]; there { - t.Fatalf("a machine whose node-engine never said it reads a generation was sent one: %v", keys) - } - if _, there := keys["put_back"]; there { - t.Fatalf("a machine whose node-engine never said it reads a generation was sent a put-back mark: %v", keys) + if _, there := bodyKeys(t, untold)["generation"]; there { + t.Fatal("a machine whose node-engine never said it reads a generation was sent one") } if untold.composedFrom != 40 { t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold) } - - // An ordinary send carries no put-back mark at all: the body is what it was apart from the generation. - var ordinary sendable - ordinary.Resources = resources - ordinaryOrder := reads - ordinaryOrder.putBack = false - ordinaryOrder.stamp(&ordinary) - if _, there := bodyKeys(t, ordinary)["put_back"]; there { - t.Fatal("an ordinary send says it is a put-back") - } } // **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the @@ -79,7 +70,7 @@ func TestASendNamesItsSender(t *testing.T) { func refusedSend(at time.Time) inventory.Send { return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40, Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)", - Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at} + Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at, Recorded: true} } // **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation, @@ -105,3 +96,53 @@ func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) { t.Errorf("the condition is not worded for the operator: %+v", o) } } + +// **A refusal of a send the mesh has no record of is raised too, its sender said to be unknown** — never +// only a line on stderr. +func TestARefusalOfAnUnrecordedSendIsRaisedSayingTheSenderIsUnknown(t *testing.T) { + now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) + f := calm(now) + s := refusedSend(now.Add(-time.Minute)) + s.Recorded, s.Sender, s.Modules = false, "a sender the mesh has no record of (no send of this sequence was recorded)", nil + f.refusedSends = []inventory.Send{s} + got := watchGenerationRefusals(f) + if len(got) != 1 || !strings.Contains(got[0].Summary, "no record of") { + t.Fatalf("an unattributed refusal was not raised as one: %+v", got) + } +} + +// **When the refusal showed the mesh's counter behind the machine, the condition asks for a push** — it was +// raised, and nothing has sent the machine its declaration since — and does not say there is nothing to do. +func TestACounterBehindTheMachineAsksForAPush(t *testing.T) { + now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) + f := calm(now) + s := refusedSend(now.Add(-time.Minute)) + s.Generation, s.RefusedApplied, s.CounterRaisedTo = 12, 40, 41 + f.refusedSends = []inventory.Send{s} + got := watchGenerationRefusals(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + if !strings.Contains(got[0].Summary, "`push anchor`") || !strings.Contains(got[0].Explanation, "push anchor") || + strings.Contains(got[0].Explanation, "Nothing for you to do") { + t.Errorf("a counter behind the machine does not ask for a push: %s / %s", got[0].Summary, got[0].Explanation) + } +} + +// **Only a refusal of the counter as it stands is the counter behind**: a stale sender's generation is below +// it, and the counter is left alone for that. +func TestOnlyARefusalOfTheCounterAsItStandsRaisesIt(t *testing.T) { + for _, c := range []struct { + refused, applied, now int64 + behind bool + }{ + {refused: 12, applied: 40, now: 12, behind: true}, // the store was put back: the mesh says 12, the machine had 40 + {refused: 38, applied: 40, now: 41, behind: false}, // a stale sender: the counter is already past + {refused: 38, applied: 40, now: 40, behind: false}, // a stale sender: the counter is where the machine is + } { + r := link.GenerationRefusal{Generation: c.refused, Applied: c.applied} + if got := counterBehind(r, c.now); got != c.behind { + t.Errorf("refused %d, applied %d, counter %d: behind %v, want %v", c.refused, c.applied, c.now, got, c.behind) + } + } +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index be33434b..9f3c1434 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -359,14 +359,12 @@ func declare(ctx context.Context, args []string) error { Epoch uint64 `json:"epoch"` Sequence int64 `json:"sequence"` Generation int64 `json:"generation"` - PutBack bool `json:"put_back"` } _ = json.Unmarshal(raw, &carried) // And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it // named are not known, since the mesh did not compose it. if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence, - epoch: carried.Epoch, generation: carried.Generation, putBack: carried.PutBack, - toldGeneration: carried.Generation, toldPutBack: carried.PutBack, + epoch: carried.Epoch, generation: carried.Generation, toldGeneration: carried.Generation, sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil { return err } @@ -1402,9 +1400,9 @@ func wouldSendFrom(ctx context.Context, open *stores, if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil { return nil, err } - // And the generation and put-back mark it was last sent, for the same reason (novox/hq issue 234): - // an assignment elsewhere in the mesh is not a change of this machine. - if declared.Generation, declared.PutBack, err = open.inventory.SentGeneration(ctx, n.ID); err != nil { + // And the generation it was last sent, for the same reason (novox/hq issue 234): an assignment + // elsewhere in the mesh is not a change of this machine. + if declared.Generation, err = open.inventory.SentGeneration(ctx, n.ID); err != nil { return nil, err } body, err := declared.Body() @@ -1551,8 +1549,6 @@ type order struct { generation int64 readsGeneration bool acting uint64 - // putBack is a gate's put-back (sendScope.putBack), read from the send's context. - putBack bool } // allot takes the next sequence for a machine — the number its next declaration carries — under the @@ -1593,7 +1589,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e return order{}, err } return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration, - acting: acting, putBack: scopeOf(ctx).putBack}, nil + acting: acting}, nil } // recordSent writes down what a machine was just sent, and returns the digest. @@ -1616,17 +1612,23 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body return "", err } digest := digestOf(body) - if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil { + // The digest and the generation it carried are written in one transaction (novox/hq issue 234), so the + // would-send is never stamped with a generation the machine was not sent; and the send itself, who sent + // it and from which generation, beside them. A record of the send that cannot be written does not make a + // send that is away look failed: it is said, loudly, and the machine's own record stands. + err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{ + Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, + Digest: digest, Modules: sent.modules}) + var unrecorded *inventory.SendNotRecordedError + switch { + case errors.As(err, &unrecorded): + fmt.Fprintf(os.Stderr, "mesh-controller: SEND NOT RECORDED: %s was sent declaration %s (sequence %d), and who "+ + "sent it and from which generation could not be written down, so a refusal of it will name no sender "+ + "(novox/hq issue 234): %v\n", node, short(digest), sent.sequence, unrecorded.Err) + fmt.Printf("%s: sent, and the record of who sent it could NOT be written: %v\n", node, unrecorded.Err) + case err != nil: return "", err } - // And the send itself, who sent it and from which generation (novox/hq issue 234): what the machine - // was last told, by whom, is read back from here — and what the would-send is stamped with. - if err := inv.RecordSend(kept, inventory.Send{Node: record.ID, Sequence: sent.sequence, - Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, PutBack: sent.putBack, - ToldGeneration: sent.toldGeneration, ToldPutBack: sent.toldPutBack, Digest: digest, - Modules: sent.modules}); err != nil { - return "", fmt.Errorf("%s was sent its declaration, and the send could not be recorded: %w", node, err) - } // And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217). // Never a reason to fail a send that is already away: a summary that cannot be kept means the // next comparison has nothing to hold against, which is how every machine starts. diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 31c1ef87..56b702c8 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -50,9 +50,6 @@ type sendScope struct { modules map[string]bool // person is a person's act: it carries what it carries. person bool - // putBack is a gate putting machines back after a failed send (novox/hq issue 234): its declarations - // say so, and a machine does not refuse them for the generation they carry. - putBack bool } type sendScopeKey struct{} diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go index cbc89943..0c1e2116 100644 --- a/cmd/mesh-controller/sendable.go +++ b/cmd/mesh-controller/sendable.go @@ -33,17 +33,11 @@ type sendable struct { // node-engine has not said it reads one is sent none, for the reason the epoch is not (an older // node-engine refuses a key it does not know, whole). Generation int64 - // PutBack marks a gate's put-back (novox/hq issue 234): it carries the generation of what it puts - // back and is not refused for it. Sent only beside a generation. - PutBack bool // composedFrom is the generation read before this declaration was composed, and actingEpoch the // lease epoch its sender acted under — whether or not the machine is sent either — for the record of // the send; never on the wire. composedFrom int64 actingEpoch uint64 - // putBackSend is whether a gate sent it to put the machine back, for the record; PutBack is the mark - // on the wire, only beside a generation. - putBackSend bool // modules are the modules this declaration names, for the record of the send; never on the wire. modules []string // Adoption is nil for a converged node, and then the body is byte for byte what it was before @@ -115,9 +109,6 @@ func (s sendable) Body() ([]byte, error) { } if s.Generation > 0 { envelope["generation"] = s.Generation - if s.PutBack { - envelope["put_back"] = true - } } // An empty declaration is deliberate here — the node owns nothing the mesh put there // (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness diff --git a/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql b/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql index 07a182d3..52dde45b 100644 --- a/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql +++ b/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql @@ -24,26 +24,33 @@ begin end $$; --- Per row, so a statement that changes nothing (an assignment repeated, `on conflict do nothing`) raises --- nothing; a statement that changes several raises once per row, which only ever moves it forward. +-- Per row, so a statement that changes nothing raises nothing: an assignment repeated (`on conflict do +-- nothing` inserts no row) and an update that leaves a row as it was (the WHEN below; a plain UPDATE fires a +-- row trigger whether or not it changed anything). A statement that changes several rows raises once per +-- row, which only ever moves it forward. Two triggers, because only an update has both OLD and NEW. create trigger assignment_generation_raised - after insert or update or delete on assignment + after insert or delete on assignment for each row execute function raise_assignment_generation(); +create trigger assignment_generation_raised_by_update + after update on assignment + for each row when (old is distinct from new) execute function raise_assignment_generation(); --- What a machine was last sent of it: the generation, and whether that send was a gate's put-back, so +-- The generation a machine was last sent, written in the same transaction as the digest of that send, so -- what the mesh WOULD send is stamped the same way and reads as byte for byte what it DID send when --- nothing else changed (as sent_epoch, migration 0068). Null and false for a send without. +-- nothing else changed (as sent_epoch, migration 0068). Null for a send without one. alter table node add column sent_generation bigint; -alter table node add column sent_put_back boolean not null default false; -- Whether the machine's node-engine said it reads a generation (its reports' `reads_generation`): until -- it has, it is sent none, because an older node-engine refuses a key it does not know, whole. alter table node add column reads_generation boolean not null default false; -- Every send: the machine, its sequence, who sent it — the lease epoch the sender acted under and the --- caller, process and build — the generation it was composed from, whether it was a put-back, its --- digest and the modules it named. Written after the declaration is away, as the node row's record is. --- A refusal by the machine of a send for its generation is kept on the send it refused, so the --- condition it raises names the sender from here. The newest 200 per machine are kept. +-- caller, process and build — the generation it was composed from, its digest and the modules it named. +-- Written after the declaration is away, with the node row's record of it. A refusal by the machine of a +-- send for its generation is kept on the send it refused, so the condition it raises names the sender +-- from here; a refusal of a sequence the mesh has no send for is kept as a row of its own, `recorded` +-- false, whose sender is said to be unknown. `counter_raised_to` is the generation the mesh's counter was +-- raised to when the refusal showed the counter behind the machine (a store put back from a backup). The +-- newest 200 per machine are kept. create table declaration_send ( id bigserial primary key, node uuid not null references node (id) on delete cascade, @@ -51,13 +58,14 @@ create table declaration_send ( epoch bigint, sender text not null, generation bigint, - put_back boolean not null default false, digest text not null, modules text[] not null default '{}', sent_at timestamptz not null default now(), refused_at timestamptz, -- refused_applied is the generation the machine said it had applied when it refused this send. - refused_applied bigint + refused_applied bigint, + counter_raised_to bigint, + recorded boolean not null default true ); create index declaration_send_node on declaration_send (node, id desc); create index declaration_send_sequence on declaration_send (node, sequence); diff --git a/internal/inventory/sends.go b/internal/inventory/sends.go index 085a2ac9..8a6efa36 100644 --- a/internal/inventory/sends.go +++ b/internal/inventory/sends.go @@ -2,6 +2,7 @@ package inventory import ( "context" + "encoding/json" "fmt" "time" @@ -47,20 +48,17 @@ func (i *Inventory) RaiseAssignmentGeneration(ctx context.Context, past int64) ( return g, nil } -// SentGeneration is the generation a machine was last sent and whether that send was a put-back, by its -// id; zero for one sent without. What the mesh WOULD send is stamped with these, so it reads as byte for -// byte what it DID send when nothing else changed. -func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, bool, error) { +// SentGeneration is the generation a machine was last sent, by its id; zero for one sent without. What the +// mesh WOULD send is stamped with it, so it reads as byte for byte what it DID send when nothing else changed. +func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, error) { var g *int64 - var putBack bool - if err := i.store.Pool().QueryRow(ctx, `select sent_generation, sent_put_back from node where id = $1`, id). - Scan(&g, &putBack); err != nil { - return 0, false, fmt.Errorf("reading the generation %s was last sent: %w", id, err) + if err := i.store.Pool().QueryRow(ctx, `select sent_generation from node where id = $1`, id).Scan(&g); err != nil { + return 0, fmt.Errorf("reading the generation %s was last sent: %w", id, err) } if g == nil { - return 0, putBack, nil + return 0, nil } - return *g, putBack, nil + return *g, nil } // ReadsGeneration says a machine's node-engine said it reads a generation in a declaration, by its id. @@ -88,66 +86,122 @@ type Send struct { Epoch int64 // Sender is who sent it, in words: the caller, and the process and build that composed it. Sender string - // Generation is the assignment generation it was composed from, zero when not known; PutBack whether a - // gate sent it to put the machine back. + // Generation is the assignment generation it was composed from, zero when not known. Generation int64 - PutBack bool - // ToldGeneration and ToldPutBack are what the machine was told of them on the wire: zero and false for a - // machine whose node-engine has not said it reads a generation. What the would-send is stamped with. - ToldGeneration int64 - ToldPutBack bool - Digest string + Digest string // Modules are the modules it named. Modules []string SentAt time.Time // RefusedAt is when the machine refused it for its generation, nil when it did not; RefusedApplied the - // generation the machine said it had applied then. - RefusedAt *time.Time - RefusedApplied int64 + // generation the machine said it had applied then; CounterRaisedTo what the mesh's counter was raised to + // because the refusal showed it behind the machine, zero when it was not. + RefusedAt *time.Time + RefusedApplied int64 + CounterRaisedTo int64 + // Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown. + Recorded bool } -// RecordSend writes one send down, and what the machine was last sent of its generation, in one -// transaction; the oldest beyond the newest 200 for the machine are let go. -func (i *Inventory) RecordSend(ctx context.Context, s Send) error { +// unknownSender is the sender of a refused sequence the mesh has no record of sending. +const unknownSender = "a sender the mesh has no record of (no send of this sequence was recorded: sent by " + + "hand, before sends were recorded, or by a controller whose record was not written)" + +// SendNotRecordedError is a send whose machine's record was written — its digest and the generation it +// carried — and whose record of who sent it was not. The send is away and the machine's record stands; the +// caller says this loudly and does not take the send for failed. +type SendNotRecordedError struct{ Err error } + +func (e *SendNotRecordedError) Error() string { + return "the record of the send was not written: " + e.Err.Error() +} +func (e *SendNotRecordedError) Unwrap() error { return e.Err } + +// RecordSentWith writes down what a machine was just sent — its digest, the builds it carried, the epoch and +// the generation on the wire — and the send itself, who sent it and from which generation (novox/hq issue +// 234), in one transaction. The digest and the generation are written together, so the would-send is never +// stamped with a generation the machine was not sent. The send's own record is written under a savepoint: if +// it cannot be, the machine's record is still committed and a *SendNotRecordedError says so. The oldest +// sends beyond the newest 200 for the machine are let go. +func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, builds map[string]string, epoch uint64, + generation int64, s Send) error { + var sentEpoch *int64 + if epoch > 0 { + e := int64(epoch) + sentEpoch = &e + } + var carried *string + if builds != nil { + raw, err := json.Marshal(builds) + if err != nil { + return err + } + text := string(raw) + carried = &text + } tx, err := i.store.Pool().Begin(ctx) if err != nil { return err } defer func() { _ = tx.Rollback(ctx) }() - if _, err := tx.Exec(ctx, `update node set sent_generation = $2, sent_put_back = $3 where id = $1`, - s.Node, nullIfZero(s.ToldGeneration), s.ToldPutBack); err != nil { + if _, err := tx.Exec(ctx, + `update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4, sent_generation = $5 + where id = $1`, node, digest, carried, sentEpoch, nullIfZero(generation)); err != nil { return err } + recordErr := recordSend(ctx, tx, node, s) + if err := tx.Commit(ctx); err != nil { + return err + } + if recordErr != nil { + return &SendNotRecordedError{Err: recordErr} + } + return nil +} + +// recordSend writes one send under a savepoint of tx, which it rolls back when the send cannot be written. +func recordSend(ctx context.Context, tx pgx.Tx, node string, s Send) error { + sp, err := tx.Begin(ctx) + if err != nil { + return err + } + defer func() { _ = sp.Rollback(ctx) }() modules := s.Modules if modules == nil { modules = []string{} } - if _, err := tx.Exec(ctx, - `insert into declaration_send (node, sequence, epoch, sender, generation, put_back, digest, modules) - values ($1, $2, $3, $4, $5, $6, $7, $8)`, - s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.PutBack, - s.Digest, modules); err != nil { + if _, err := sp.Exec(ctx, + `insert into declaration_send (node, sequence, epoch, sender, generation, digest, modules) + values ($1, $2, $3, $4, $5, $6, $7)`, + node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.Digest, + modules); err != nil { return err } - if _, err := tx.Exec(ctx, + if err := pruneSends(ctx, sp, node); err != nil { + return err + } + return sp.Commit(ctx) +} + +// pruneSends lets go of a machine's sends beyond the newest 200. +func pruneSends(ctx context.Context, q queries, node string) error { + _, err := q.Exec(ctx, `delete from declaration_send where node = $1 and id not in - (select id from declaration_send where node = $1 order by id desc limit $2)`, s.Node, sendsKept); err != nil { - return err - } - return tx.Commit(ctx) + (select id from declaration_send where node = $1 order by id desc limit $2)`, node, sendsKept) + return err } // sendColumns are a send's columns as scanSends reads them. const sendColumns = `s.node, n.name, coalesce(s.sequence, 0), coalesce(s.epoch, 0), s.sender, coalesce(s.generation, 0), - s.put_back, s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0)` + s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0), coalesce(s.counter_raised_to, 0), + s.recorded` func scanSends(rows pgx.Rows) ([]Send, error) { defer rows.Close() var out []Send for rows.Next() { var s Send - if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.PutBack, - &s.Digest, &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied); err != nil { + if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest, + &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded); err != nil { return nil, err } out = append(out, s) @@ -155,11 +209,12 @@ func scanSends(rows pgx.Rows) ([]Send, error) { return out, rows.Err() } -// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. +// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. A refusal +// of a sequence the mesh has no send for is not a send, and is not it. func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) { rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` from declaration_send s join node n on n.id = s.node - where n.name = $1 order by s.id desc limit 1`, name) + where n.name = $1 and s.recorded order by s.id desc limit 1`, name) if err != nil { return Send{}, false, err } @@ -171,15 +226,17 @@ func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, erro } // RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and -// answers that send, sender and all; false when no send of that sequence was recorded — sent before the -// record, or by a hand the mesh did not see. The newest of that sequence when there are several: a -// declaration a person sent by hand may repeat one. -func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied int64) (Send, bool, error) { +// answers that send, sender and all; false when no send of that sequence was recorded (RecordUnrecordedRefusal +// keeps that one). The newest of that sequence when there are several: a declaration a person sent by hand may +// repeat one. raisedTo is what the mesh's counter was raised to for it, zero for none. +func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied, raisedTo int64) (Send, bool, error) { rows, err := i.store.Pool().Query(ctx, `with refused as ( - update declaration_send set refused_at = now(), refused_applied = $3 - where id = (select id from declaration_send where node = $1 and sequence = $2 order by id desc limit 1) + update declaration_send set refused_at = now(), refused_applied = $3, counter_raised_to = $4 + where id = (select id from declaration_send where node = $1 and sequence = $2 and recorded + order by id desc limit 1) returning *) - select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied) + select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied, + nullIfZero(raisedTo)) if err != nil { return Send{}, false, err } @@ -190,6 +247,34 @@ func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, appl return sends[0], true, nil } +// RecordUnrecordedRefusal keeps a machine's refusal of a sequence the mesh has no send for, as a row of its +// own whose sender is unknown, and answers it: S20 raises it like any other, because a refusal nobody can +// attribute is no quieter for that. s carries the machine, the sequence, epoch, generation and digest the +// refused declaration carried, and the refusal's applied generation and counter raise. +func (i *Inventory) RecordUnrecordedRefusal(ctx context.Context, s Send) (Send, error) { + rows, err := i.store.Pool().Query(ctx, `with refused as ( + insert into declaration_send (node, sequence, epoch, sender, generation, digest, refused_at, refused_applied, + counter_raised_to, recorded) + values ($1, $2, $3, $4, $5, $6, now(), $7, $8, false) returning *) + select `+sendColumns+` from refused s join node n on n.id = s.node`, + s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), unknownSender, nullIfZero(s.Generation), s.Digest, + s.RefusedApplied, nullIfZero(s.CounterRaisedTo)) + if err != nil { + return Send{}, err + } + sends, err := scanSends(rows) + if err != nil { + return Send{}, err + } + if len(sends) == 0 { + return Send{}, fmt.Errorf("the refusal of sequence %d was not kept", s.Sequence) + } + if err := pruneSends(ctx, i.store.Pool(), s.Node); err != nil { + return Send{}, err + } + return sends[0], nil +} + // RefusedSendsSince is every send refused for its generation since a moment, newest first. func (i *Inventory) RefusedSendsSince(ctx context.Context, since time.Time) ([]Send, error) { rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` diff --git a/internal/inventory/sends_test.go b/internal/inventory/sends_test.go index 9f5dfe6b..96c98d15 100644 --- a/internal/inventory/sends_test.go +++ b/internal/inventory/sends_test.go @@ -1,6 +1,7 @@ package inventory import ( + "errors" "slices" "testing" "time" @@ -75,8 +76,8 @@ func TestTheGenerationIsRaisedPastWhatAMachineApplied(t *testing.T) { } // **Every send is recorded: its sequence, its sender, the generation it came from and the modules it -// named** — and the machine's last send is what `plan` reads, the would-send is stamped with its -// generation and put-back mark, and a refusal is kept on the send it refused, naming its sender. +// named** — and the machine's last send is what `plan` reads, the would-send is stamped with the generation +// written with the digest, and a refusal is kept on the send it refused, naming its sender. func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) { inv, node := aNodeWithModules(t) ctx := t.Context() @@ -84,42 +85,110 @@ func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) { if err != nil { t.Fatal(err) } - first := Send{Node: record.ID, Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)", + first := Send{Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: "d11", Modules: []string{"docker", "pacman", "sudo"}} - if err := inv.RecordSend(ctx, first); err != nil { + if err := inv.RecordSentWith(ctx, record.ID, "d11", nil, 57, 40, first); err != nil { t.Fatal(err) } - stale := Send{Node: record.ID, Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", - Generation: 38, ToldGeneration: 38, Digest: "d12", Modules: []string{"docker"}} - if err := inv.RecordSend(ctx, stale); err != nil { + stale := Send{Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", + Generation: 38, Digest: "d12", Modules: []string{"docker"}} + if err := inv.RecordSentWith(ctx, record.ID, "d12", nil, 57, 38, stale); err != nil { t.Fatal(err) } last, found, err := inv.LastSend(ctx, node) if err != nil || !found { t.Fatalf("the last send is not kept: %v", err) } - if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || + if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || !last.Recorded || !slices.Equal(last.Modules, []string{"docker"}) || last.SentAt.IsZero() { t.Fatalf("the last send reads %+v", last) } - generation, putBack, err := inv.SentGeneration(ctx, record.ID) - if err != nil || generation != 38 || putBack { - t.Fatalf("what the machine was last sent of it reads %d, %v (%v)", generation, putBack, err) + if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 38 { + t.Fatalf("what the machine was last sent of it reads %d (%v)", generation, err) } - refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40) - if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 { + refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40, 0) + if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 || + refused.CounterRaisedTo != 0 { t.Fatalf("the refusal is not kept on the send it refused: %+v, %v, %v", refused, found, err) } since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour)) if err != nil || len(since) != 1 || since[0].NodeName != node || since[0].Sequence != 12 { t.Fatalf("the refused sends within the hour read %+v (%v)", since, err) } - if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40); err != nil || found { + if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40, 0); err != nil || found { t.Errorf("a refusal of a send never recorded was found: %v, %v", found, err) } } +// **A refusal of a sequence the mesh has no send for is kept too, its sender unknown**, so S20 raises it; +// it is not the machine's last send, and it says what the counter was raised to. +func TestARefusalOfASendNeverRecordedIsKeptWithAnUnknownSender(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + kept, err := inv.RecordUnrecordedRefusal(ctx, Send{Node: record.ID, Sequence: 99, Epoch: 57, Generation: 41, + Digest: "d99", RefusedApplied: 44, CounterRaisedTo: 45}) + if err != nil { + t.Fatal(err) + } + if kept.Recorded || kept.Sender != unknownSender || kept.RefusedAt == nil || kept.CounterRaisedTo != 45 || + kept.NodeName != node { + t.Fatalf("the refusal reads %+v", kept) + } + since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour)) + if err != nil || len(since) != 1 || since[0].Sequence != 99 { + t.Fatalf("the refused sends within the hour read %+v (%v)", since, err) + } + if _, found, err := inv.LastSend(ctx, node); err != nil || found { + t.Errorf("a refusal of a send never recorded reads as the machine's last send: %v, %v", found, err) + } +} + +// **The machine's record and the send's are written together, and a send record that cannot be written +// leaves the machine's record standing and says so**: the digest and generation are the machine's, and a +// send that is away must not read as failed, nor the machine as behind. +func TestASendRecordThatCannotBeWrittenLeavesTheMachinesRecord(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + // A NUL byte is text PostgreSQL refuses: the send's own row cannot be written. + broken := Send{Sequence: 13, Sender: "a test", Generation: 42, Digest: "d\x0013"} + err = inv.RecordSentWith(ctx, record.ID, "d13", nil, 0, 42, broken) + var unrecorded *SendNotRecordedError + if !errors.As(err, &unrecorded) { + t.Fatalf("a send record that could not be written read as %v", err) + } + if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 42 { + t.Fatalf("the machine's generation was not written with its digest: %d (%v)", generation, err) + } + if sent, err := inv.Outstanding(ctx, node); err != nil || sent != "d13" { + t.Fatalf("the machine's digest was not written: %q (%v)", sent, err) + } +} + +// **An update that leaves an assignment as it was raises nothing** (the trigger's WHEN). +func TestAnUpdateThatChangesNoAssignmentRaisesNothing(t *testing.T) { + inv, node := aNodeWithModules(t, "web") + ctx := t.Context() + if _, err := inv.Assign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + before := generationNow(t, inv) + if _, err := inv.store.Pool().Exec(ctx, `update assignment set module = module`); err != nil { + t.Fatal(err) + } + if after := generationNow(t, inv); after != before { + t.Errorf("an update that changed nothing moved the generation from %d to %d", before, after) + } +} + // **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer. func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) { inv, node := aNodeWithModules(t) From 483c387b724954853f78b0433a227677cc9312a2 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 18:46:31 +0200 Subject: [PATCH 16/16] Judge the runtime's daemon.json against the docker module's own keys, not a frozen copy The test pinned daemon.json to live-restore alone, so every controller check failed once the docker module gained log rotation as a preference (mesh-catalog #205, #217). It still refuses dns (ADR 0196) and any key the module does not declare, and checks the log defaults where the module declares them (novox/hq issue 498). --- internal/catalogue/resolver_manifests_test.go | 50 ++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/internal/catalogue/resolver_manifests_test.go b/internal/catalogue/resolver_manifests_test.go index 8ea3e32b..7d149596 100644 --- a/internal/catalogue/resolver_manifests_test.go +++ b/internal/catalogue/resolver_manifests_test.go @@ -194,8 +194,31 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) { if err := json.Unmarshal([]byte(runtime["content"].(string)), &keys); err != nil { t.Fatalf("the runtime's keys are not JSON: %v", err) } - if len(keys) != 1 || keys["live-restore"] != true { - t.Errorf("the runtime is given %v; live-restore and nothing else", keys) + // What the runtime is given is what its own module declares, and nothing a layer or another module + // adds. This used to say "live-restore and nothing else", to hold the line ADR 0196 drew when `dns` + // left the file; since the docker module gained log rotation as a preference (mesh-catalog #205 and + // #217, novox/hq issue 452) that frozen copy failed every check (novox/hq issue 498). So the keys are + // judged against the module's own daemon.json instead: live-restore is given, `dns` never is, and no + // key the module does not declare appears. + if keys["live-restore"] != true { + t.Errorf("the runtime is given %v; a restart must keep every container running (live-restore)", keys) + } + if _, ok := keys["dns"]; ok { + t.Errorf("the runtime is given a resolver of its own, %v; a container copies its machine's (ADR 0196)", keys["dns"]) + } + declared := daemonKeysDeclared(t) + for key := range keys { + if !declared[key] { + t.Errorf("the runtime is given %q, which the docker module does not declare: %v", key, keys) + } + } + // The log rotation preference, at its defaults, wherever the module declares it. The copy captured in + // testdata/beside predates it, so outside a merge check it is not there to judge. + if declared["log-driver"] { + opts, _ := keys["log-opts"].(map[string]any) + if keys["log-driver"] != "json-file" || opts["max-size"] != "100m" || opts["max-file"] != "10" { + t.Errorf("the runtime is given %v; its containers' logs are cut at 100m and ten are kept (novox/hq issue 452)", keys) + } } // The runtime is reloaded when that file changes, and never restarted: a restart stops every // container on the machine (ADR 0102), and a reload is what turns live-restore on. @@ -275,3 +298,26 @@ func TestTheResolverOnAMachineOffTheNetworkIsRefused(t *testing.T) { t.Fatalf("a machine off the network was refused for another reason: %v", err) } } + +// daemonKeysDeclared is the top-level keys of the docker module's own daemon.json, read from its +// manifest: what the runtime may be given (novox/hq issue 498). A placeholder sits inside a JSON +// string there, so the content parses as it is. +func daemonKeysDeclared(t *testing.T) map[string]bool { + t.Helper() + for _, r := range catalogueManifest(t, "docker").Resources { + if r["id"] != "daemon" { + continue + } + var keys map[string]any + if err := json.Unmarshal([]byte(r["content"].(string)), &keys); err != nil { + t.Fatalf("the docker module's daemon.json is not JSON: %v", err) + } + declared := map[string]bool{} + for key := range keys { + declared[key] = true + } + return declared + } + t.Fatal("the docker module declares no daemon.json") + return nil +}