Compare commits

..
Author SHA1 Message Date
jschoubben 2fabbe556e Pin to the group's new mesh-sdk head bfc3883a, whose composer reads a fallback to a predefined key as the indexer does (ADR 0348 §7)
mesh/check failed: shanks: novox/mesh-delivery#35 does not me… | 2fabbe556e73 on 25176e1ccdb3 | judged by the candidate's mesh-build-go at 85f37b40efdd
mesh/delivery-group group feat/r4g-finish rejected: composed together: the checker's verdict at the members' heads (171a15384fa1): shanks
mesh/delivery ready: it delivers once merged
2026-10-24 14:57:35 +02:00
jschoubben 89b9212c4d Pin to the group's new mesh-sdk head 19c1d359, whose composer names the reader's pinned SDK instead of a bare (devel) (ADR 0482 clause 5)
mesh/delivery superseded: a newer head of the same pull request
mesh/check failed: shanks: novox/mesh-catalog#473: modules/mesh-build-go/go.mod names mesh-sdk 19c1d35939bc, which is neither on mesh-sdk's trunk nor …
2026-10-24 14:12:09 +02:00
jschoubben 865f3b6455 Say what pacman-tools reads outside: the root predefined.json (issue 1935)
mesh/delivery-group group feat/r4g-finish rejected: a member's own check failed
mesh/delivery superseded: a newer head of the same pull request
mesh/check failed: shanks: novox/mesh-catalog#473: modules/mesh-build-go/go.mod names mesh-sdk bc0c25e51052, which is neither on mesh-sdk's trunk nor …
2026-10-24 13:48:18 +02:00
jschoubben 03524ecd13 Mark pacman-tools as reading outside its directory: its upgrade-order test reads the root predefined.json, which its digest does not cover (issue 1935) 2026-10-24 13:48:09 +02:00
jschoubben f910b0ceb8 Pin mesh-build-go to the group's mesh-sdk head after main's merge, so it builds on the SDK the group delivers 2026-10-24 13:28:16 +02:00
jschoubben 91c3e954ae Merge main into feat/r4g-finish (#471) 2026-10-24 13:26:24 +02:00
jschoubben 16d7b1898d Mark the packages whose tests read files outside their directory, found by the review (issue 1935) 2026-10-24 04:38:49 +02:00
jschoubben f04465537b Say in realtree_test that a missing trunk fails, not skips (issue 1295) 2026-10-24 04:37:52 +02:00
jschoubben e870a7f65f Pin to the finish group's mesh-sdk head dd0d156b, so every member names one SDK (issue 1622) 2026-10-24 04:28:14 +02:00
jschoubben e8a59171c7 Merge remote-tracking branch 'origin/feat/r4g-finish-composer' into HEAD 2026-10-24 04:26:03 +02:00
jschoubben ebb25757a2 Merge remote-tracking branch 'origin/feat/r4g-finish-1935' into HEAD 2026-10-24 04:26:02 +02:00
jschoubben 6a70735489 Run a compose ask on the Go build seat: build mesh-sdk's composer at the asked commit once, fill its manifests from the indexer and run it shut off (issue 1622) 2026-10-24 04:14:21 +02:00
jschoubben a715c8821f mesh-build-go: a catalogue module's non-Go files enter its packages' digests; markers where tests read other modules (issue 1935) 2026-10-24 03:59:08 +02:00
jschoubben b6ed8599c8 mesh-build-go: a package marked reads-outside is never kept, so tests that read files outside their directory run on every check (issue 1935) 2026-10-24 03:46:45 +02:00
jschoubben ae6f7d5b54 Make a missing trunk copy a failure again, now the checker places it (issue 1295) 2026-10-24 03:43:49 +02:00
157 changed files with 11389 additions and 1671 deletions
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+2 -1
View File
@@ -251,7 +251,8 @@ func main() {
}
var grown []string
if baseDir == "" {
fmt.Println("NOT COMPARED: no base branch's lists were given, so additions to them were not judged")
fmt.Fprintln(os.Stderr, "NOT COMPARED: no base branch's lists were given (-base <dir>), so additions to them were not judged; that is a failure, not a pass (issue 1295)")
os.Exit(2)
} else {
baseShared, _, err1 := list(filepath.Join(baseDir, "shared"))
baseKnown, _, err2 := list(filepath.Join(baseDir, "known"))
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+2 -2
View File
@@ -32,8 +32,8 @@ func TestNoModuleNamesAnothersState(t *testing.T) {
}
}
// And neither list gains a line the trunk's copies at the merge base do not hold, unless the line says why after `# allow:`. Skipped,
// and said, where no trunk is beside.
// And neither list gains a line the trunk's copies at the merge base do not hold, unless the line says why after `# allow:`. Fails, saying
// NOT COMPARED, where no trunk is beside (issue 1295).
func TestTheListsOfPathsHoldNoLineTheTrunksDoNot(t *testing.T) {
sharedPath, okShared := trunk.Copy(t, "checks/paths/shared")
knownPath, okKnown := trunk.Copy(t, "checks/paths/known")
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+14 -18
View File
@@ -3,14 +3,14 @@
//
// The checker fetches the change's own repository at its base commit beside the change, as it fetches a sibling, and says where
// in MESH_CHECK_BESIDE; the copy is under the repository's name (the same convention the other repositories' tests use). With
// MESH_CHECK_BESIDE unset, or set without the catalogue's copy in it, the comparison is skipped and the test says NOT COMPARED
// WITH THE TRUNK, naming why: the checker that runs before R4g's delivery places siblings but not the change's own repository,
// so R4g's own check cannot hold the copy (issue 1295 notes that a missing copy becomes a failure once that checker is gone). A trunk that holds no such file had
// MESH_CHECK_BESIDE unset, or set without the catalogue's copy in it, the test fails saying NOT COMPARED WITH THE TRUNK and
// why: the checker places the change's own repository since R4g (issue 1295), so a missing copy is a fault, never a pass. A trunk that holds no such file had
// none: the list's first copy, which nothing was added to.
package trunk
import (
"errors"
"fmt"
"os"
"path/filepath"
"testing"
@@ -23,13 +23,11 @@ const Repository = "mesh-catalog"
const BesideEnv = "MESH_CHECK_BESIDE"
// Copy is the trunk's copy of the file at rel (a path from the catalogue's root), written to a file of the test's own: its path,
// and whether the trunk holds one (false: its first copy). It skips the test, saying so, when no copy of the catalogue is beside.
// and whether the trunk holds one (false: its first copy). It fails the test when no copy of the catalogue is beside.
func Copy(t testing.TB, rel string) (string, bool) {
t.Helper()
raw, found, skip, err := Read(rel)
raw, found, err := Read(rel)
switch {
case skip != "":
t.Skip(skip)
case err != nil:
t.Fatal(err)
case !found:
@@ -43,27 +41,25 @@ func Copy(t testing.TB, rel string) (string, bool) {
return path, true
}
// Read is the trunk's copy of rel and whether the trunk holds one. skip is the words to skip the comparison with, when no trunk is
// named; err is a trunk named and not there, or unreadable.
func Read(rel string) (content []byte, found bool, skip string, err error) {
// Read is the trunk's copy of rel and whether the trunk holds one. err is a trunk not beside the check (MESH_CHECK_BESIDE unset,
// or without the catalogue's directory), or a copy unreadable: a comparison that cannot be made is never a pass.
func Read(rel string) (content []byte, found bool, err error) {
beside := os.Getenv(BesideEnv)
if beside == "" {
return nil, false, "NOT COMPARED WITH THE TRUNK: " + BesideEnv + " is not set, so the trunk's copy of " + rel +
" is not beside this run; the checker places it", nil
return nil, false, fmt.Errorf("NOT COMPARED WITH THE TRUNK: %s is not set, so the trunk's copy of %s is not beside this run; the checker places the change's own repository there (issue 1295)", BesideEnv, rel)
}
dir := filepath.Join(beside, Repository)
if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) {
return nil, false, "NOT COMPARED WITH THE TRUNK: " + Repository + " is not beside this check in " + BesideEnv + "=" + beside +
", so " + rel + " is not judged against the trunk's copy (the checker before R4g places no copy of the change's own repository)", nil
return nil, false, fmt.Errorf("NOT COMPARED WITH THE TRUNK: %s is not beside this check in %s=%s, so %s is not judged against the trunk's copy; the checker places the change's own repository there at its trunk (issue 1295)", Repository, BesideEnv, beside, rel)
} else if err != nil {
return nil, false, "", err
return nil, false, err
}
raw, err := os.ReadFile(filepath.Join(dir, rel))
if errors.Is(err, os.ErrNotExist) {
return nil, false, "", nil
return nil, false, nil
}
if err != nil {
return nil, false, "", err
return nil, false, err
}
return raw, true, "", nil
return raw, true, nil
}
+14 -14
View File
@@ -17,27 +17,27 @@ func TestTheCloneBesideTheCheckIsTheTrunk(t *testing.T) {
t.Fatal(err)
}
t.Setenv(BesideEnv, beside)
raw, found, skip, err := Read("checks/x/known")
if err != nil || skip != "" || !found || string(raw) != "a\nb\n" {
t.Fatalf("read %q found %v skip %q err %v", raw, found, skip, err)
raw, found, err := Read("checks/x/known")
if err != nil || !found || string(raw) != "a\nb\n" {
t.Fatalf("read %q found %v err %v", raw, found, err)
}
if _, found, skip, err := Read("checks/x/absent"); err != nil || skip != "" || found {
t.Fatalf("a file the trunk does not hold: found %v skip %q err %v; wanted its first copy", found, skip, err)
if _, found, err := Read("checks/x/absent"); err != nil || found {
t.Fatalf("a file the trunk does not hold: found %v err %v; wanted its first copy", found, err)
}
}
func TestWithNothingBesideTheComparisonIsSkippedAndSaidNotMade(t *testing.T) {
func TestWithNothingBesideTheComparisonFailsAndSaysNotMade(t *testing.T) {
t.Setenv(BesideEnv, "")
_, found, skip, err := Read("checks/x/known")
if found || err != nil || len(skip) < 12 || skip[:12] != "NOT COMPARED" {
t.Fatalf("found %v, skip %q, err %v; wanted NOT COMPARED", found, skip, err)
_, found, err := Read("checks/x/known")
if found || err == nil || !strings.Contains(err.Error(), "NOT COMPARED") {
t.Fatalf("found %v, err %v; wanted a failure saying NOT COMPARED", found, err)
}
}
func TestACloneBesideWithoutTheCatalogueIsSkippedAndSaidNotMade(t *testing.T) {
t.Setenv(BesideEnv, t.TempDir()) // the siblings are there, the catalogue's own base is not (the checker before R4g)
_, found, skip, err := Read("checks/x/known")
if found || err != nil || len(skip) < 12 || skip[:12] != "NOT COMPARED" || !strings.Contains(skip, "not beside") {
t.Fatalf("found %v, skip %q, err %v; wanted NOT COMPARED naming the missing copy", found, skip, err)
func TestACloneBesideWithoutTheCatalogueFailsAndSaysNotMade(t *testing.T) {
t.Setenv(BesideEnv, t.TempDir()) // the siblings are there, the catalogue's own base is not
_, found, err := Read("checks/x/known")
if found || err == nil || !strings.Contains(err.Error(), "NOT COMPARED") || !strings.Contains(err.Error(), "not beside") {
t.Fatalf("found %v, err %v; wanted a failure naming the missing copy", found, err)
}
}
+1
View File
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
@@ -0,0 +1 @@
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
+5 -1
View File
@@ -102,7 +102,11 @@ integration level, the postgres major and the nats-server version; and, when the
workspace (`beside`), each one's repository and commit, which the answer echoes as `beside` (the checker keeps a verdict only
for the siblings it was judged with). The text's form is `DigestVersion` 3. The Go version is not in it: the toolchain enters a
fingerprint once, as the version the build nodes agree on (ADR 0477 §3), and the standard library's packages are that
version's. A package with no tests at the level has no digest and is named in `without_tests`. A directory that vendors
version's. A package with no tests at the level has no digest and is named in `without_tests`. A catalogue module's packages also hash the module's non-Go files (the nearest ancestor directory holding `module.json`: its manifest, README, `files/` and the rest, so `../../module.json` is covered; digest form 5). A package whose directory holds a file named `reads-outside` (any content; say what the tests read) is never kept (issue 1935): its tests read
files outside it, which a digest from `go list` cannot name, so its digest is a fresh random one on every call, the ask finds no kept
verdict for it and runs it, and the answer names it in `reads_outside` and the module's log says why. The checker still writes a
verdict under that digest, which nothing ever reads again. The marker is the plain way to say it until each verdict keeps the files
its tests opened (Go's test log). A directory that vendors
is listed and tested from its vendor directory (`-mod=vendor`).
## A change to this module is judged by its candidate
@@ -100,6 +100,9 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
if err := m.room(); err != nil {
return nil, err
}
if a.Compose != nil {
return m.compose(ctx, ask.Fetch, a)
}
return buildstep.RunAsk(ctx, ask, a)
}}
}
@@ -116,7 +116,7 @@ func TestTheIntegrationLevelDigestsUnderBothTags(t *testing.T) {
t.Errorf("the integration digest listed %q", s.command)
}
}
if DigestVersion != "4" {
if DigestVersion != "5" {
t.Errorf("the digest's form changed in kind and is still version %s", DigestVersion)
}
}
@@ -0,0 +1,210 @@
package main
// **A compose ask runs mesh-sdk's composer at a commit** (novox/hq ADR 0482 clause 4, issues 1621 and 1622): the
// assigner asks it, to compose a group with its mesh-sdk member's head, and to read the group's manifests with the
// mesh-sdk the running controller and node-engine pin (ADR 0482 clause 5, issue 1623). The module fetches the commit
// (its fetch step: the forge key and the network, none of the change's code), builds the composer package in the Go
// module the ask names, in a run step shut off with nothing fetched, and keeps the program under its key
// (delivery.ComposerKey), so each commit is built once on a node. It fills the request's manifests from the delivery
// indexer's manifests state, by digest, and runs the program shut off: no network, none of the account, the request a
// file of a workspace of its own. The answer is the composer's verdict whole (delivery.ComposeAnswer); the module
// judges nothing of it.
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ManifestsState is the delivery indexer's manifests by digest, which a compose ask's request names (the module reads
// it, module.json reads).
const ManifestsState = delivery.IndexerModule + ".manifests"
// composerLocks hold one mutex per composer key, so a composer two asks need at once is built once.
var composerLocks sync.Map
// composerPath is where the composer of key is kept: the cache directory, beside Go's own caches.
func (m *Module) composerPath(key string) string {
return filepath.Join(m.Cache, "composers", key, "mesh-compose")
}
// compose runs a compose ask: the composer of its commit, built once, over its request with the manifests filled.
func (m *Module) compose(ctx context.Context, fetch buildstep.StepFunc, a delivery.StepAsk) (any, error) {
job := *a.Compose
repository, _ := job.Fetch["repository"].(string)
commit, _ := job.Fetch["commit"].(string)
installed, err := m.Probe(ctx)
if err != nil {
return nil, err
}
key := delivery.ComposerKey(repository, commit, job.Dir, job.Package, delivery.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()})
exe, built, err := m.composerOf(ctx, fetch, job, key)
if err != nil {
return nil, err
}
req, err := composer.Decode(job.Request)
if err != nil {
return nil, buildstep.Refuse("the compose ask %s: %v", a.ID, err)
}
if err := composer.Fill(&req, m.manifestOf); err != nil {
return nil, err
}
verdict, err := m.runComposer(ctx, exe, req)
if err != nil {
return nil, err
}
return delivery.ComposeAnswer{Built: built, Key: key, Verdict: verdict}, nil
}
// manifestOf is the manifest the indexer keeps under digest, as its repository holds it.
func (m *Module) manifestOf(digest string) ([]byte, error) {
if m.ReadManifest == nil {
return nil, errors.New("this module reads no manifests state")
}
key, err := delivery.ManifestsKey(digest)
if err != nil {
return nil, err
}
raw, err := m.ReadManifest(key)
if err != nil {
return nil, err
}
if raw == nil {
return nil, fmt.Errorf("the indexer keeps no manifest of digest %s", digest)
}
man, err := delivery.DecodeManifest(raw)
if err != nil {
return nil, err
}
return []byte(man.JSON), nil
}
// composerOf is the composer kept under key, or, when none is, the one built now from the fetched commit (built true).
func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job delivery.ComposeJob, key string) (string, bool, error) {
exe := m.composerPath(key)
mu, _ := composerLocks.LoadOrStore(exe, &sync.Mutex{})
mu.(*sync.Mutex).Lock()
defer mu.(*sync.Mutex).Unlock()
if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() {
return exe, false, nil
}
if fetch == nil {
return "", false, buildstep.Refuse("this build module fetches nothing")
}
out, err := fetch(ctx, job.Fetch)
if err != nil {
return "", false, err
}
raw, err := json.Marshal(out)
if err != nil {
return "", false, err
}
var f struct {
Workspace string `json:"workspace"`
}
if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" {
return "", false, fmt.Errorf("the fetch answered no workspace: %s", raw)
}
defer m.releaseSaid(f.Workspace)
done := m.spaces().Hold(f.Workspace)
defer done()
dir := filepath.Join(m.Work, f.Workspace, "tree", job.Dir)
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil {
return "", false, fmt.Errorf("%s at %s holds no go.mod in %s, so no composer can be built there", job.Fetch["repository"], job.Fetch["commit"], job.Dir)
}
into := filepath.Join(m.Work, f.Workspace, "composer", "mesh-compose")
if err := os.MkdirAll(filepath.Dir(into), 0o700); err != nil {
return "", false, err
}
run, err := m.stepRunner(f.Workspace, false)
if err != nil {
return "", false, err
}
env := append(m.goEnvFor(dir), "GOPROXY=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0")
r := run(ctx, env, "go", "-C", dir, "build", "-trimpath", "-o", into, job.Package)
if r.Status != 0 || r.Err != "" {
// The words Go says when the module holds no such package (a mesh-sdk from before the composer) are kept whole,
// so the asker tells that apart from a fault.
return "", false, fmt.Errorf("the composer %s does not build at %s: %s", job.Package, job.Fetch["commit"], compileError(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
}
if err := copyExecutable(into, exe); err != nil {
return "", false, fmt.Errorf("the composer could not be kept: %w", err)
}
return exe, true, nil
}
// copyExecutable copies a built program into place through a file beside it, so a half-written program is never kept.
func copyExecutable(from, to string) error {
if err := os.MkdirAll(filepath.Dir(to), 0o700); err != nil {
return err
}
src, err := os.Open(from)
if err != nil {
return err
}
defer src.Close()
tmp := to + ".new"
dst, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o700)
if err != nil {
return err
}
if _, err := io.Copy(dst, src); err != nil {
dst.Close()
return err
}
if err := dst.Close(); err != nil {
return err
}
return os.Rename(tmp, to)
}
// releaseSaid removes a workspace once its steps ended, saying a failure: the sweep removes it later.
func (m *Module) releaseSaid(workspace string) {
if err := m.spaces().Release(workspace); err != nil && m.Say != nil {
m.Say("the compose ask's workspace %s: %v", workspace, err)
}
}
// runComposer runs the composer over req in a workspace of its own, shut off, and answers its verdict: the last line of
// its standard output, a composer.Verdict of the version this module reads.
func (m *Module) runComposer(ctx context.Context, exe string, req composer.Request) (json.RawMessage, error) {
id := buildstep.NewID()
done := m.spaces().Hold(id)
defer m.releaseSaid(id)
defer done()
ws := filepath.Join(m.Work, id)
if err := os.MkdirAll(ws, 0o700); err != nil {
return nil, err
}
raw, err := json.Marshal(req)
if err != nil {
return nil, err
}
file := filepath.Join(ws, "request.json")
if err := os.WriteFile(file, raw, 0o600); err != nil {
return nil, err
}
run := m.Run
if !m.Inside {
iso := m.isolation(id, false)
iso.ReadOnly = append(iso.ReadOnly, exe)
run = buildstep.Isolated(m.Run, iso)
}
r := run(ctx, nil, exe, "-request", file)
line := strings.TrimSpace(lastLine(r.Stdout))
var v composer.Verdict
if r.Status != 0 || r.Err != "" || json.Unmarshal([]byte(line), &v) != nil || v.V != composer.Version {
return nil, fmt.Errorf("the composer answered no verdict (status %d %s): %s", r.Status, r.Err, firstLine(r.Stderr+"\n"+r.Stdout))
}
return json.RawMessage(line), nil
}
@@ -0,0 +1,156 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/forgekey"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
// sdkPin is the mesh-sdk version this module's go.mod requires: the composer the test builds is that version's.
func sdkPin(t *testing.T) string {
t.Helper()
raw, err := os.ReadFile("../../go.mod")
if err != nil {
t.Fatal(err)
}
for _, l := range strings.Split(string(raw), "\n") {
if f := strings.Fields(l); len(f) >= 2 && f[0] == "git.novox.be/novox/mesh-sdk/go" {
return f[1]
}
}
t.Fatal("go.mod requires no mesh-sdk")
return ""
}
// goEnvOf is one value of this machine's go env.
func goEnvOf(t *testing.T, name string) string {
t.Helper()
out, err := exec.Command("go", "env", name).Output()
if err != nil {
t.Fatalf("go env %s: %v", name, err)
}
return strings.TrimSpace(string(out))
}
// 1622/AC1, AC3 against the real bus: a compose ask put on the Go build seat's check stream is pulled by the module
// started whole on that bus, which fetches the commit (here a tree of a module requiring mesh-sdk at this module's pin,
// as a reader repository is), builds mesh-sdk's composer there with nothing fetched, fills the request's manifests from
// the indexer's state and runs the composer; the step key <node>.<ask id> passes with the composer's verdict. A second
// ask of the same commit runs the kept composer: fetched and built once.
func TestAComposeAskIsBuiltOnceAndAnsweredOnTheBus(t *testing.T) {
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer nc.Close()
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
if err := buildstep.DeclareAskWorkers(ctx, js, delivery.AskGo, 2*time.Minute); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = js.DeleteStream(context.Background(), "SEAT_NODE_BUILD_GO") })
states := newStates()
m := moduleWith(states, "go1.26.0")
m.Work, m.Cache, m.Inside = t.TempDir(), t.TempDir(), true
m.Run = StepRunner(t.TempDir())
m.GoModCache = goEnvOf(t, "GOMODCACHE") // the module cache this module's own build filled, read and not fetched
run, gocache := m.Run, goEnvOf(t, "GOCACHE") // this machine's build cache, so the composer's packages compile once
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
return run(ctx, append(env, "GOCACHE="+gocache), name, args...)
}
read, digests := indexerHolds(t, `{"module":"demo","version":"1"}`)
m.ReadManifest = read
gosum, err := os.ReadFile("../../go.sum")
if err != nil {
t.Fatal(err)
}
fetches := 0
steps := fakeSteps(new([]string))
for i := range steps {
if steps[i].Name == "build_go_fetch" {
steps[i].Run = func(ctx context.Context, args map[string]any) (any, error) {
fetches++
ws := buildstep.NewID()
tree := filepath.Join(m.Work, ws, "tree")
if err := os.MkdirAll(tree, 0o700); err != nil {
return nil, err
}
mod := "module reader\n\ngo 1.26.0\n\nrequire git.novox.be/novox/mesh-sdk/go " + sdkPin(t) + "\n"
if err := os.WriteFile(filepath.Join(tree, "go.mod"), []byte(mod), 0o600); err != nil {
return nil, err
}
return map[string]any{"workspace": ws}, os.WriteFile(filepath.Join(tree, "go.sum"), gosum, 0o600)
}
}
}
p := m.Puller(steps, &buildstep.JetStreamAsks{JS: js, Language: delivery.AskGo})
p.PullWait = 200 * time.Millisecond
go p.Pull(ctx)
ask := func() delivery.Step {
a := composeAskOf(t, digests[0])
a.Compose.Fetch["repository"], a.Compose.Dir = "novox/mesh-controller", delivery.RootPath
raw, err := delivery.EncodeStepAsk(a)
if err != nil {
t.Fatal(err)
}
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck)
if _, err := js.Publish(ctx, subject, raw); err != nil {
t.Fatal(err)
}
key, _ := delivery.AskStepKey("ace", a.ID)
for deadline := time.Now().Add(90 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
states.mu.Lock()
v := states.steps[key]
states.mu.Unlock()
if len(v) > 0 && v[len(v)-1].State != delivery.StepRunning {
return v[len(v)-1]
}
}
t.Fatalf("the compose ask %s never ended", a.ID)
return delivery.Step{}
}
for n, wantBuilt := range []bool{true, false} {
step := ask()
if step.State != delivery.StepPassed {
t.Fatalf("ask %d: %s: %s", n+1, step.State, step.Reason)
}
var ans delivery.ComposeAnswer
var v composer.Verdict
if err := json.Unmarshal(step.Result, &ans); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(ans.Verdict, &v); err != nil {
t.Fatal(err)
}
if ans.Built != wantBuilt || !strings.HasPrefix(ans.Key, "composer-") || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted ||
!strings.Contains(v.SDK, sdkPin(t)) {
t.Fatalf("ask %d: answer built %v key %s, verdict %+v; want built %v and demo accepted by mesh-sdk %s", n+1, ans.Built, ans.Key, v, wantBuilt, sdkPin(t))
}
}
if fetches != 1 {
t.Errorf("two asks of one commit fetched %d times, want once", fetches)
}
}
@@ -0,0 +1,125 @@
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const composeCommit = "4444444444444444444444444444444444444444"
// indexerHolds is the delivery indexer's manifests state holding each body under its digest, as the indexer writes it.
func indexerHolds(t *testing.T, bodies ...string) (func(string) (json.RawMessage, error), []string) {
t.Helper()
kept, digests := map[string]json.RawMessage{}, []string{}
for _, b := range bodies {
d := delivery.ManifestDigest([]byte(b))
key, _ := delivery.ManifestsKey(d)
raw, err := delivery.EncodeManifest(delivery.Manifest{Module: "demo", JSON: b})
if err != nil {
t.Fatal(err)
}
kept[key], digests = raw, append(digests, d)
}
return func(key string) (json.RawMessage, error) { return kept[key], nil }, digests
}
func composeAskOf(t *testing.T, digest string) delivery.StepAsk {
t.Helper()
req, _ := json.Marshal(composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "change",
Manifests: map[string]composer.Manifest{"demo": {Digest: digest}}}}})
return delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-assigner",
At: time.Now(), Compose: &delivery.ComposeJob{Fetch: map[string]any{"repository": "novox/mesh-sdk", "commit": composeCommit},
Dir: "go", Package: composer.Package, Request: req}}
}
// composeModule is a module with work and cache directories of its own, running its commands as they are (Inside: a
// unit test has no bubblewrap to shut them off in).
func composeModule(t *testing.T) *Module {
t.Helper()
m := moduleWith(newStates(), "go1.26.0")
m.Work, m.Cache, m.Inside, m.Run = t.TempDir(), t.TempDir(), true, StepRunner(t.TempDir())
return m
}
// 1622/AC3: a composer kept under the ask's key is run again, nothing fetched or built: the answer says it was not
// built, names the key, and carries the composer's verdict whole; the request it read had its manifests filled from the
// indexer's state.
func TestAKeptComposerIsRunWithTheManifestsFilled(t *testing.T) {
m := composeModule(t)
read, digests := indexerHolds(t, `{"module":"demo","version":"1"}`)
m.ReadManifest = read
a := composeAskOf(t, digests[0])
key := delivery.ComposerKey("novox/mesh-sdk", composeCommit, "go", composer.Package, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()})
exe := m.composerPath(key)
if err := os.MkdirAll(filepath.Dir(exe), 0o700); err != nil {
t.Fatal(err)
}
// A composer that answers which manifests its request held, filled or not, as a verdict.
script := "#!/bin/sh\n" +
`if grep -q '"digest"' "$2"; then echo '{"v":1,"sdk":"stand-in","cases":[{"name":"unfilled","manifests":{}}]}'; ` +
`else echo '{"v":1,"sdk":"stand-in","cases":[{"name":"change","manifests":{"demo":{"accepted":true}}}]}'; fi` + "\n"
if err := os.WriteFile(exe, []byte(script), 0o700); err != nil {
t.Fatal(err)
}
fetched := 0
fetch := func(context.Context, map[string]any) (any, error) { fetched++; return nil, nil }
out, err := m.compose(context.Background(), fetch, a)
if err != nil {
t.Fatal(err)
}
ans := out.(delivery.ComposeAnswer)
var v composer.Verdict
if err := json.Unmarshal(ans.Verdict, &v); err != nil {
t.Fatal(err)
}
if ans.Built || ans.Key != key || fetched != 0 || len(v.Cases) != 1 || v.Cases[0].Name != "change" || !v.Cases[0].Manifests["demo"].Accepted {
t.Fatalf("answer = %+v (fetched %d), verdict %+v", ans, fetched, v)
}
if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
t.Errorf("the run's workspace was left: %v", entries)
}
}
// 1622: a manifest the indexer does not keep fails the ask, naming its digest; a fetched commit with no go.mod where
// the ask builds fails it naming the directory, and its workspace is released.
func TestAComposeAskFailsLoudly(t *testing.T) {
m := composeModule(t)
m.ReadManifest, _ = indexerHolds(t)
a := composeAskOf(t, "feedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeedfeed")
key := delivery.ComposerKey("novox/mesh-sdk", composeCommit, "go", composer.Package, delivery.ToolchainUsed{Version: "go1.26.0", Recipe: m.Recipe.Version()})
os.MkdirAll(filepath.Dir(m.composerPath(key)), 0o700)
os.WriteFile(m.composerPath(key), []byte("#!/bin/sh\nexit 0\n"), 0o700)
if _, err := m.compose(context.Background(), nil, a); err == nil || !strings.Contains(err.Error(), "feedfeed") {
t.Errorf("an unkept manifest: %v", err)
}
m2 := composeModule(t)
fetch := func(context.Context, map[string]any) (any, error) {
if err := os.MkdirAll(filepath.Join(m2.Work, "ws0000000000a", "tree", "go"), 0o700); err != nil {
return nil, err
}
return map[string]any{"workspace": "ws0000000000a"}, nil
}
if _, err := m2.compose(context.Background(), fetch, composeAskOf(t, "x")); err == nil || !strings.Contains(err.Error(), "no go.mod in go") {
t.Errorf("a commit with no go.mod: %v", err)
}
if entries, _ := os.ReadDir(m2.Work); len(entries) != 0 {
t.Errorf("the fetched workspace was left: %v", entries)
}
}
// 1622: the module reads the delivery indexer's manifests, which a compose ask names by digest.
func TestTheModuleReadsTheIndexersManifests(t *testing.T) {
if m := readManifest(t); !slices.Contains(m.Reads, ManifestsState) {
t.Errorf("reads = %v, lacks %s", m.Reads, ManifestsState)
}
}
@@ -7,6 +7,7 @@ package main
import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
@@ -24,12 +25,20 @@ import (
// 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3). 3: the siblings'
// commits (the lead's decision L5) and the integration level's tags integration,isolation (the checker's agreement).
// 4: the siblings' commits enter only the digest of a package whose tests reach a file naming MESH_CHECK_BESIDE (ADR 0536 §1).
const DigestVersion = "4"
// 5: a catalogue module's non-Go files enter the digest of its packages (issue 1935).
const DigestVersion = "5"
// besideVariable is the name a file bears when it reads where the siblings lie; a package whose tests reach such a file
// depends on the siblings' commits.
const besideVariable = "MESH_CHECK_BESIDE"
// ReadsOutsideMarker is the name of a file in a package's directory that says its tests read files outside it (a
// repository's documents, a sibling directory): such a package is never kept (issue 1935). The digest hashes only what
// the Go toolchain lists, so it cannot name what a test opens elsewhere; a marker file, beside the Go files where a
// reader of the package sees it, is the plain way to say so until the checker keeps the files a test opened (the Go
// test log) with each verdict.
const ReadsOutsideMarker = "reads-outside"
// integrationTags are the build tags of the integration level, its run and its digest alike: integration, and isolation
// for the bubblewrap proofs (mesh-sdk's and the build modules'), which a build node can run.
var integrationTags = []string{"integration", "isolation"}
@@ -89,6 +98,9 @@ type DigestResult struct {
// only when they are the siblings it gave the fetch.
Beside []string `json:"beside,omitempty"`
WithoutTests []string `json:"without_tests,omitempty"`
// ReadsOutside names the packages that carry the marker: each has a digest that no kept verdict can have (a fresh
// random one), so the check runs it every time.
ReadsOutside []string `json:"reads_outside,omitempty"`
// JudgedBy is the program that ran the digest (ADR 0535 §5): "installed", or "candidate <merge commit>" under the
// runner workspace.
JudgedBy string `json:"judged_by,omitempty"`
@@ -199,6 +211,12 @@ func (m *Module) digestHere(ctx context.Context, workspace, level string) (Diges
}
}
sort.Strings(res.WithoutTests)
sort.Strings(res.ReadsOutside)
for _, p := range res.ReadsOutside {
if m.Say != nil {
m.Say("the %s level of %s always runs: its directory carries %s, so its tests read files outside it that no digest names", level, p, ReadsOutsideMarker)
}
}
return res, nil
}
@@ -256,6 +274,17 @@ func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, he
if testMain == nil {
return fmt.Errorf("go list gave no test binary for %s", path)
}
if _, err := os.Stat(filepath.Join(root.Dir, ReadsOutsideMarker)); err == nil {
// The digest of a package that reads outside itself is one nothing can have kept: the key is a fresh random
// hex, so the ask finds no kept verdict and runs it, and what the checker writes under it is never read again.
var nonce [32]byte
if _, err := rand.Read(nonce[:]); err != nil {
return err
}
res.Packages[path] = hex.EncodeToString(nonce[:])
res.ReadsOutside = append(res.ReadsOutside, path)
continue
}
d, err := packageDigest(root, testMain, pkgs, tree, header, beside)
if err != nil {
return fmt.Errorf("%s: %w", path, err)
@@ -303,6 +332,15 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s
}
}
}
// A catalogue module's tests read its manifest, its README and its files from the package's directory (../../module.json):
// every non-Go file of the module's directory is an input (issue 1935). The module is the nearest ancestor with a module.json.
if modDir := enclosingModule(tree, root.Dir); modDir != "" {
more, err := moduleFileLines(tree, root, modDir)
if err != nil {
return "", err
}
lines = append(lines, more...)
}
data := filepath.Join(root.Dir, "testdata")
if _, err := os.Stat(data); err == nil {
err := filepath.WalkDir(data, func(path string, e fs.DirEntry, err error) error {
@@ -328,6 +366,41 @@ func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree s
return hex.EncodeToString(h.Sum(nil)), nil
}
// enclosingModule is the nearest ancestor of dir, inside tree, that holds a module.json, or "" when there is none.
func enclosingModule(tree, dir string) string {
tree = filepath.Clean(tree)
for d := filepath.Clean(dir); d != tree && strings.HasPrefix(d, tree+string(filepath.Separator)); d = filepath.Dir(d) {
if st, err := os.Stat(filepath.Join(d, "module.json")); err == nil && !st.IsDir() {
return d
}
}
return ""
}
// moduleFileLines are the content lines of every file of a module's directory that is not Go source (Go source is the
// packages' own, listed by go list), skipping .git and node_modules.
func moduleFileLines(tree string, p *goPackage, modDir string) ([]string, error) {
var lines []string
err := filepath.WalkDir(modDir, func(path string, e fs.DirEntry, err error) error {
if err != nil {
return err
}
if e.IsDir() {
if n := e.Name(); n == ".git" || n == "node_modules" {
return filepath.SkipDir
}
return nil
}
if strings.HasSuffix(path, ".go") {
return nil
}
l, err := contentLine(tree, p, path)
lines = append(lines, l)
return err
})
return lines, err
}
func fileLine(tree string, p *goPackage, name string) (string, error) {
return contentLine(tree, p, filepath.Join(p.Dir, name))
}
@@ -189,3 +189,71 @@ func TestAModuleOutsideTheTreeCountsByVersionAndChecksum(t *testing.T) {
t.Errorf("a new version left the digest as it was")
}
}
// Issue 1935: a package whose directory carries the reads-outside marker is never kept: its digest is one no kept verdict
// can have (it differs on every call, though nothing changed), and the answer names it. A package without the marker
// keeps its digest.
func TestAPackageWithTheReadsOutsideMarkerIsNeverKept(t *testing.T) {
t.Parallel()
m, tree := digestFixture(t)
plain := digests(t, m, "unit")
if len(plain.ReadsOutside) != 0 {
t.Fatalf("a package carries the marker in the fixture: %v", plain.ReadsOutside)
}
edit(t, tree, "c/"+ReadsOutsideMarker, "the tests read ../docs\n")
first, second := digests(t, m, "unit"), digests(t, m, "unit")
c := "fixture/digest/c"
if len(first.Packages[c]) != 64 || first.Packages[c] == second.Packages[c] || first.Packages[c] == plain.Packages[c] {
t.Errorf("c carries the marker and its digest is stable: %s, %s, before %s", first.Packages[c], second.Packages[c], plain.Packages[c])
}
if strings.Join(first.ReadsOutside, ",") != c {
t.Errorf("the answer does not name the package that reads outside: %v", first.ReadsOutside)
}
a := "fixture/digest/a"
if first.Packages[a] != plain.Packages[a] || first.Packages[a] != second.Packages[a] {
t.Errorf("a has no marker and its digest moved: %s, %s, before %s", first.Packages[a], second.Packages[a], plain.Packages[a])
}
}
// Issue 1935: a catalogue module's packages read ../../module.json and the module's files: a change to them changes the
// digest of the module's package, and a change to another module's files does not.
func TestACatalogueModulesFilesChangeTheDigestOfItsPackagesOnly(t *testing.T) {
t.Parallel()
m, tree := digestFixture(t)
for _, mod := range []string{"mod1", "mod2"} {
dir := filepath.Join(tree, mod, "cmd", "p")
if err := os.MkdirAll(filepath.Join(tree, mod, "files"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
edit(t, tree, mod+"/module.json", `{"module":"`+mod+`"}`)
edit(t, tree, mod+"/README.md", "# "+mod+"\n")
edit(t, tree, mod+"/files/f.env", "A=1\n")
edit(t, tree, mod+"/cmd/p/p.go", "package p\n")
edit(t, tree, mod+"/cmd/p/p_test.go", "package p\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestReads(t *testing.T) {\n\tif _, err := os.ReadFile(\"../../module.json\"); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n")
}
p1, p2 := "fixture/digest/mod1/cmd/p", "fixture/digest/mod2/cmd/p"
before := digests(t, m, "unit")
if len(before.Packages[p1]) != 64 || len(before.Packages[p2]) != 64 {
t.Fatalf("no digest for the module packages: %+v", before.Packages)
}
if before.Packages[p1] == before.Packages[p2] {
t.Errorf("two modules, one digest")
}
for _, file := range []string{"mod1/module.json", "mod1/README.md", "mod1/files/f.env"} {
edit(t, tree, file, "changed "+file+"\n")
after := digests(t, m, "unit")
if after.Packages[p1] == before.Packages[p1] {
t.Errorf("a change to %s did not change the module's digest", file)
}
if after.Packages[p2] != before.Packages[p2] {
t.Errorf("a change to %s changed the other module's digest", file)
}
before = after
}
if got := digests(t, m, "unit"); got.Packages["fixture/digest/c"] == "" {
t.Errorf("a package outside any module lost its digest")
}
}
@@ -288,10 +288,14 @@ func inTestdata(path string) bool {
// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory
// that vendors, see goEnvFor.
func (m *Module) goEnv() []string {
modcache := filepath.Join(m.Cache, "gomod")
if m.GoModCache != "" {
modcache = m.GoModCache
}
return []string{
"GOTOOLCHAIN=local",
"GOFLAGS=-mod=mod",
"GOMODCACHE=" + filepath.Join(m.Cache, "gomod"),
"GOMODCACHE=" + modcache,
"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
"GOPRIVATE=git.novox.be",
"GIT_TERMINAL_PROMPT=0",
@@ -54,6 +54,7 @@ func main() {
m.RegistryFile = m.Home + "/registry"
m.Settings = m.Home + "/settings.json"
m.ReadKept = readOf(stdio.State(KeptState)).Get
m.ReadManifest = readOf(stdio.State(ManifestsState)).Get
m.Probe = ProbeGo(m.Home, m.Cache)
m.Bubblewrap = ProbeBubblewrap
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
@@ -82,6 +82,10 @@ type Module struct {
Settings string
Capacity func() int
ReadKept func(key string) (json.RawMessage, error)
// ReadManifest reads a key of the delivery indexer's manifests state (ManifestsState), which a compose ask's request
// names its manifests by (compose.go); GoModCache, when set, is the Go module cache in place of Cache's (a test).
ReadManifest func(key string) (json.RawMessage, error)
GoModCache string
spacesOnce sync.Once
workspaces *buildstep.Workspaces
@@ -0,0 +1 @@
These tests read other catalogue modules' files, outside this module (issue 1935): never kept.
+1 -1
View File
@@ -3,7 +3,7 @@ module meshbuildgo
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980
github.com/distribution/distribution/v3 v3.1.2
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980 h1:ZEEeYibEPD1rngOAqbODzRY5rUStGC46OpyQwtNCYbs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
+2 -1
View File
@@ -20,7 +20,8 @@
}
],
"reads": [
"mesh-delivery-checker.kept"
"mesh-delivery-checker.kept",
"mesh-delivery-indexer.manifests"
],
"config": {
"capacity": {
@@ -0,0 +1 @@
These tests read other catalogue modules' files, outside this module (issue 1935): never kept.
@@ -0,0 +1 @@
These tests read other catalogue modules' files, outside this module (issue 1935): never kept.
+27 -12
View File
@@ -10,15 +10,15 @@ Each directory or archive this module declares says how it treats a file it did
- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
## What it keeps (ADR 0539 §17)
## A backup that failed before the send
The gate keeps the merge's states: proposed, ready, rejected, merged, held (a merge that reached the trunk without a passing
check) and closed. It asks the operator's Approve and merges once, moving `source.main`. It asks the controller nothing, follows
no plan event and derives nothing after the merge: whether a merged change reached the machines is the view's, from
`source.main`, the wants and the haves. The seat `mesh-delivery` keeps the verbs `deliveries`, `show`, `groups`, `checks`,
`table`, `stalled` (for a merge or a check past its bound), `recheck` and `close`; `times`, `what-if`, `release`, `stop` and
`retire-history` went with the plans, so the gate stays a bundle serving the seat. A delivery an earlier owner kept in a state that
is gone is read as the state it came to (merged, closed or proposed) when the gate starts.
The controller backs up the data of the modules a delivery moves before it asks or sends anything (ADR 0404). When its
walk's record, in `plan-moved`, names a backup that failed or stayed silent and nothing of the plan was sent, the
delivery goes from delivering to held (the observed row `backup failed`); the held reason names the machine, the
module and the backup's words, and the pull request's comment shows it. The hold does not lift by itself. `release`
of such a delivery asks the controller (`delivery-backups-again`) to take the backups again and the delivery goes on
delivering; a record from before the controller cleared the failure does not hold it again. A held delivery already
ends as covered or delivered when a later merge covers it.
## When a refused or rejected merge is said (ADR 0505 §4, §5)
@@ -33,8 +33,23 @@ names every refusal and rejection at once, and the pull request is told at once;
`needs: new-head` and names its author: the branch, and the session a `Pushed-by: <who@node>` line in the pull
request's description names.
## The seat's hand-over
## Taking the seat over from today's module
The gate took the seat `mesh-delivery` over from the module `mesh-delivery` (ADR 0449 §4, ADR 0450 §6, ADR 0454) and carried its
deliveries, groups and pull keys once; the carry, and the reading of the controller's assignments and the node-engine's have keys
that waited for it, went with release 6. `catalogue_test.go` keeps the catalogue to one claimant of the seat.
The gate claims the seat `mesh-delivery` and took it over from the module `mesh-delivery`, which the catalogue no longer holds.
The controller refuses a merge that removes a module that is assigned, so today's module stayed in the catalogue until it was
unassigned, and went by a handover of verbs after the merge, never by hand on a machine (novox/hq ADR 0449 §4, ADR 0450 §6, ADR 0454):
1. assign `mesh-delivery-gate` to the machine that holds the seat (`mesh-controller.assign`);
2. check that `mesh-controller.seats` still names `mesh-delivery`, held by the running module;
3. unassign `mesh-delivery` from that machine (`mesh-controller.unassign`) and push it (`mesh-controller.push`);
4. the seat passes: the gate reads the controller's assignments and the node-engine's have keys, finds the module on no machine,
carries its deliveries, groups and pull keys over once (the key `carried` in its `deliveries` says it did), reads the controller's
walks and the forge's open pull requests, and acts.
Until then the gate is silent: it takes no event, writes no key, asks nothing and answers no verb but `delivery_status`, which
says why (`holds-the-seat`, `seat-waits`).
The removal of `modules/mesh-delivery` followed the handover as a pull request of its own, merged by the gate (ADR 0454); the
catalogue has one claimant of the seat, and no manifest but the gate's and the issue tracker's names the old module but the names
the handover needed: the three buckets the gate carries over, and the event the issue tracker and the tool runner hear
(`catalogue_test.go`).
@@ -1,32 +1,50 @@
package main
import (
"strings"
"testing"
"time"
)
const (
cat = "novox/mesh-catalog"
headCC = "cc00000000000001"
headDu = "dd00000000000002"
)
// One build per change (novox/hq ADR 0350): a pull request closed unmerged has its check builds let go of, and a
// walk's record says per module whether its build reused a check's build or built, and why.
var atCC = time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
// A pull request closed unmerged is closed. Since issue 1318 (ADR 0503 §3) the controller is no longer asked to let go
// of what its checks built, and since ADR 0539 §17 the gate asks it nothing at all.
func TestAPullRequestClosedUnmergedIsClosedAndAsksNoOne(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeForge(), atCC)
// A pull request closed unmerged is stopped. Since issue 1318 (ADR 0503 §3) the controller is no longer asked to let go
// of what its checks built (§5's forget-checks goes with the merge gate's checks); a controller that is down changes
// nothing.
func TestAPullRequestClosedUnmergedIsStoppedAndAsksTheControllerNothing(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC)
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
w.h.PullUpdated(cc)
w.h.PullClosed(cc)
if s := w.state(IDOf(cat, headCC)); s != Closed {
if len(w.ctl.forgot) != 0 {
t.Fatalf("closed unmerged, the controller was asked to let go of %v", w.ctl.forgot)
}
if s := w.state(IDOf(cat, headCC)); s != Stopped {
t.Fatalf("closed unmerged it is %s", s)
}
w.ctl.down = true
du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json")
w.h.PullUpdated(du)
w.h.PullClosed(du)
if s := w.state(IDOf(cat, headDu)); s != Closed {
t.Fatalf("a closed delivery is %s", s)
if s := w.state(IDOf(cat, headDu)); s != Stopped {
t.Fatalf("with the controller down, a closed delivery is %s", s)
}
}
// The walk's record says per module what its build did, and the delivery keeps it beside its walk (§4).
func TestAWalksRecordSaysWhetherEachModuleWasReusedOrBuilt(t *testing.T) {
walk := catWalk("plan-1", mergeCC, atCC.Add(time.Minute), "claude-code", "dunst")
walk.Modules["claude-code"].Made = "reused from the check of novox/mesh-catalog#101 at head cccccccccccc " +
"(merged into trunk 553b71910000): tools sha256:aa"
walk.Modules["dunst"].Made = "built: no check of novox/mesh-catalog#101 built it"
seen := walk.Seen()
if !strings.HasPrefix(seen.Built["claude-code"], "reused from the check of novox/mesh-catalog#101") ||
seen.Built["dunst"] != "built: no check of novox/mesh-catalog#101 built it" {
t.Fatalf("the delivery keeps %v", seen.Built)
}
if catWalk("plan-2", mergeCC, atCC, "dunst").Seen().Built != nil {
t.Fatal("a walk from a controller that says nothing of its builds reads as if it said something")
}
}
@@ -1,11 +1,14 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The tests of novox/hq ADR 0505 "How it is checked" for the gate: a refusal the gate is already answering is no
@@ -73,7 +76,7 @@ func refusedForAMovedTrunk(t *testing.T) (*world, string) {
t.Helper()
w := announced(t)
id := IDOf("novox/app", readyHead)
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
w.forge.refuse = errAskedAgain
approve(w)
if len(w.forge.merges) != 1 || w.delivery(id).Ready.RefusedSince == nil {
@@ -122,7 +125,7 @@ func TestATrunkMovedRefusalIsStalledAtTwiceTheLastCheckTimeAtLeastTheFloor(t *te
func TestAnyOtherRefusalIsStalledAtOnce(t *testing.T) {
w := announced(t)
id := IDOf("novox/app", readyHead)
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
w.forge.refuse = refusedWords("novox/app#7")
approve(w)
if got := linesFor(w, id); len(got) != 1 {
@@ -143,7 +146,7 @@ func rejectedAfterItsApprove(t *testing.T, pushedBy string) (*world, string) {
w.h.PullUpdated(p)
w.settleAll()
id := IDOf("novox/app", readyHead)
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
// Judged again (the table's recheck, which the checker's verdict on a new trunk makes): proposed once more.
w.h.mu.Lock()
d := w.h.deliveries[id]
@@ -153,7 +156,7 @@ func rejectedAfterItsApprove(t *testing.T, pushedBy string) (*world, string) {
}
d.Check = nil
w.h.mu.Unlock()
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "fail", "t1t1t1t1t1t1"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "fail", "t1t1t1t1t1t1"))
approve(w)
w.h.React()
if w.state(id) != Rejected {
@@ -217,3 +220,48 @@ func TestANewHeadWithinTwoHoursKeepsTheRejectedHeadFromTheOperator(t *testing.T)
t.Fatalf("the superseded head reached the operator: %+v", got)
}
}
// ADR 0505 §2: a walk's end said again by the controller with its declared machines (`declared_read`, a touch marked
// declared) acts once: no transition, comment or ask of the controller the first saying did not make.
func TestAWalkSaidAgainWithItsDeclaredMachinesActsOnceInTheGate(t *testing.T) {
w, id, plan := aDeliveringWalk(t)
done := w.ctl.walk(plan)
sent := time.Date(2026, 10, 6, 12, 40, 0, 0, time.UTC)
done.Modules["app"] = &WalkModule{State: "built", First: []string{"anchor"}, FirstAt: &sent, SentAt: &sent, Build: "build-1",
Gate: &WalkGate{Machines: []string{"anchor"}, Since: &sent, Passes: 1, Verdict: "passed", JudgedAt: &sent}}
done.State = walkDone
done = w.ctl.put(done)
body := func(again bool) []byte {
raw, err := json.Marshal(done)
if err != nil {
t.Fatal(err)
}
var said map[string]any
_ = json.Unmarshal(raw, &said)
said["walk"], said["phase"], said["outcome"] = plan, "ended", "delivered"
said["touches"] = []map[string]any{{"machine": "anchor", "modules": []string{"app"}}}
if again {
said["touches"] = append(said["touches"].([]map[string]any),
map[string]any{"machine": "shanks", "modules": []string{}, "declared": true})
said["declared_read"] = true
}
out, _ := json.Marshal(said)
return out
}
if err := take(w.h, stdio.Envelope{Key: ControllerSeat + ".plan-moved", Body: body(false)}); err != nil {
t.Fatal(err)
}
w.settleAll()
if s := w.state(id); s != Delivered {
t.Fatalf("the walk's end left it %s", s)
}
transitions, comments, delivers := len(w.delivery(id).Transitions), len(w.forge.comments), len(w.ctl.delivers)
if err := take(w.h, stdio.Envelope{Key: ControllerSeat + ".plan-moved", Body: body(true)}); err != nil {
t.Fatal(err)
}
w.settleAll()
if len(w.delivery(id).Transitions) != transitions || len(w.forge.comments) != comments || len(w.ctl.delivers) != delivers {
t.Fatalf("said again it acted again: transitions %d→%d, comments %d→%d, asks of the controller %d→%d",
transitions, len(w.delivery(id).Transitions), comments, len(w.forge.comments), delivers, len(w.ctl.delivers))
}
}
@@ -0,0 +1,672 @@
package main
import (
"fmt"
"sort"
"strings"
"time"
)
// Which merges a walk answers (novox/hq issue 362, ADR 0276).
//
// On 2026-10-10 two merges of the catalogue 18 seconds apart were walked by one walk at the later merge's
// commit, which superseded the first. This owner found the walk a delivery follows by the walk's id or by the
// commit it was merged as, final or not: the surviving walk matched the dunst delivery, which the supersession
// had already ended, so nothing gave it its word, and the claude-code delivery, merged as another commit,
// matched no walk and was held for a person after ten minutes, for a reason that named neither.
//
// A walk answers every merge it carries: the merges it names (`delivery.merges`, said by a controller that
// batches merges), the merge of its own commit, a merge whose walk it took over, and a merge of its repository
// and branch the forge says its commit contains. A delivery follows the newest walk that answers it, and only a
// delivery not yet ended is ever tied to a walk by what it answers: an ended delivery is moved only by the
// record of its own walk, which no longer moves it.
// seenWalk is a walk's last record and when it was heard.
type seenWalk struct {
walk Walk
at time.Time
}
// keptWalkFor is how long a walk no longer open is kept for a merge heard after it ended; a batch is kept as
// long, past its window's latest close.
const keptWalkFor = 10 * time.Minute
// answer is a delivery a walk answers and the commit of its repository the walk carries for it.
type answer struct {
d *Delivery
carried string
}
// walkAnswers is whether a walk answers a delivery's merge, and the commit it carries for it.
func walkAnswers(w Walk, d *Delivery) (string, bool) {
if w.Release != nil || w.IsBatch() || d.MergedAs == "" {
return "", false
}
if c, ok := w.carriedFor(d.Repository, d.MergedAs); ok {
return c, true
}
if d.Walk != nil && d.Walk.TakenOverBy == w.ID && namesNoMerges(w) && strings.EqualFold(w.Repository, d.Repository) &&
sameBranch(w.Branch, d.Base) {
// The walk that took its walk over, from a controller that names no merges: it answers every merge the one
// it took over answered, by its commit of the same repository and branch. A walk that names its merges
// answers only those (carriedFor, above).
return w.Commit, true
}
return "", false
}
// containedAnswer is, for a controller that does not name the merges a walk answers, whether an open walk
// answers a ready merged delivery by containment: of its repository and branch, made for a merge after it, the
// forge says its commit contains this merge, and it moves every module this merge's plan moves. A later walk that
// moves only what its own merge moved does not deliver this one, however much of the trunk it contains.
func (h *Holder) containedAnswer(w Walk, d *Delivery) (string, bool) {
if !h.mayContain(w, d) || !d.ContainedIn[w.Commit] {
return "", false
}
return w.Commit, true
}
// mayContain is whether an open walk is one to ask the forge about for a ready merged delivery (containedAnswer).
func (h *Holder) mayContain(w Walk, d *Delivery) bool {
if w.Release != nil || w.IsBatch() || !w.Open() || !namesNoMerges(w) || d.State != Ready || d.MergedAs == "" || d.Walk != nil ||
d.Plan == nil || len(d.Plan.Moved) == 0 || !strings.EqualFold(w.Repository, d.Repository) ||
!sameBranch(w.Branch, d.Base) || sameCommit(w.Commit, d.MergedAs) {
return false
}
for _, m := range d.Plan.Moved {
if _, moves := w.Modules[m]; !moves {
return false
}
}
// Made for a merge after this one: the walk's own merge, where this owner holds it, or else the walk itself.
after := w.Created
for _, o := range h.deliveries {
if strings.EqualFold(o.Repository, w.Repository) && sameCommit(o.MergedAs, w.Commit) && !o.MergedAt.IsZero() {
after = o.MergedAt
}
}
return after.After(d.MergedAt)
}
// namesNoMerges is whether a walk is from a controller that does not name the merges it answers.
func namesNoMerges(w Walk) bool { return w.Delivery == nil || len(w.Delivery.Merges) == 0 }
func sameBranch(a, b string) bool { return a == "" || b == "" || strings.EqualFold(a, b) }
// follows is whether a delivery takes a walk that answers it as the one it follows: it has none, it is that walk,
// that walk took its own over, or that walk is newer than the one it follows. A walk record kept before its
// creation was read is followed past only once it is superseded or failed.
func follows(d *Delivery, w Walk) bool {
switch {
case d.Walk == nil || d.Walk.ID == w.ID || d.Walk.TakenOverBy == w.ID:
return true
case d.Walk.Created.IsZero():
return d.Walk.State == walkSuperseded || d.Walk.State == walkFailed
}
return w.Created.After(d.Walk.Created)
}
// deliveriesWalking are the deliveries a walk record moves: every delivery that already follows it — ended or
// not, its own record — and every delivery not ended that it answers and that follows it now. In id order.
func (h *Holder) deliveriesWalking(w Walk) []answer {
var out []answer
for _, d := range h.deliveries {
if d.Walk != nil && d.Walk.ID == w.ID {
carried := d.AnsweredBy
if c, ok := walkAnswers(w, d); ok {
carried = c
}
out = append(out, answer{d: d, carried: carried})
continue
}
if d.State.Final() || !follows(d, w) {
continue
}
if c, ok := walkAnswers(w, d); ok {
out = append(out, answer{d: d, carried: c})
}
}
sort.Slice(out, func(i, j int) bool { return out[i].d.ID < out[j].d.ID })
return out
}
// accounted is whether any delivery, ended or not, follows the walk or is answered by it: a walk known for
// what it answers is never made a delivery of its own.
func (h *Holder) accounted(w Walk) bool {
for _, d := range h.deliveries {
if d.Walk != nil && d.Walk.ID == w.ID {
return true
}
if _, ok := walkAnswers(w, d); ok {
return true
}
}
return false
}
// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. A walk whose
// word only stops that lapsed give (lapsedOn, novox/hq issue 753) is not held: it stays under the orphans' timer, so
// past the grace without its word it is listed by `stalled`, saying why it was not given.
func (h *Holder) heldWalk(w Walk) bool {
for _, d := range h.deliveries {
if d.State.Final() {
continue
}
if d.Walk != nil && d.Walk.ID == w.ID {
return true
}
if _, ok := walkAnswers(w, d); ok {
return true
}
}
return false
}
// pickWalk is the newest walk heard that answers a merged delivery with no walk yet; nil for none.
func (h *Holder) pickWalk(d *Delivery) (Walk, string, bool) {
var best Walk
var carried string
found := false
for _, s := range h.walks {
if !follows(d, s.walk) {
continue
}
c, ok := walkAnswers(s.walk, d)
if !ok {
c, ok = h.containedAnswer(s.walk, d)
}
if !ok {
continue
}
if !found || s.walk.Created.After(best.Created) || s.walk.Created.Equal(best.Created) && s.walk.ID > best.ID {
best, carried, found = s.walk, c, true
}
}
return best, carried, found
}
// linkAnswering ties a merged delivery with no walk to the newest walk heard that answers it; under the lock.
func (h *Holder) linkAnswering(d *Delivery) bool {
if d.State.Final() || d.MergedAs == "" || d.Walk != nil {
return false
}
w, carried, ok := h.pickWalk(d)
if !ok {
return false
}
h.linkWalk(d, w, carried)
delete(h.unmatched, w.ID)
return true
}
// containsAsks are the forge's asks whether an open walk of a merged delivery's repository and branch, at
// another commit, contains its merge: for a controller that does not name the merges a walk answers (issue 362),
// asked once per pair, the answer kept on the delivery.
func (h *Holder) containsAsks(d *Delivery, now time.Time) []func() {
var asks []func()
for _, s := range h.walks {
w := s.walk
if !h.mayContain(w, d) {
continue
}
if _, known := d.ContainedIn[w.Commit]; known {
continue
}
if a := h.askContains(d, w.Commit, now); a != nil {
asks = append(asks, a)
}
}
return asks
}
// batchOf is the batch a merged delivery's merge is in, when the controller said one; false for none.
func (h *Holder) batchOf(d *Delivery) (Walk, bool) {
if d.MergedAs == "" {
return Walk{}, false
}
for _, s := range h.walks {
if !s.walk.IsBatch() || s.walk.Delivery == nil {
continue
}
for _, m := range s.walk.Delivery.Merges {
if strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs) {
return s.walk, true
}
}
}
return Walk{}, false
}
// batchSaid is what a batch holding a delivery's merge waits for, in words.
func batchSaid(b Walk, now time.Time) string {
said := "its batch " + b.ID + ": " + b.State
bt := b.Delivery.Batch
switch {
case b.State == walkQueued && bt != nil && bt.Behind != "":
said += " behind " + bt.Behind
case b.State == walkAssembling && bt != nil && bt.ClosesAt != nil:
left := bt.ClosesAt.Sub(now).Round(time.Second)
if left < 0 {
left = 0
}
said += fmt.Sprintf(", %s left", left)
if bt.AtMost != nil {
said += " (at the latest " + bt.AtMost.Local().Format("15:04:05") + ")"
}
}
return said + "; plan not yet calculated"
}
// unwalkedWhy is why a merged delivery no walk answers is held: the open walks of its repository and that none
// answers its merge, or that the controller holds none — what happened, not what might have.
func (h *Holder) unwalkedWhy(d *Delivery) string {
var open []string
for _, s := range h.walks {
w := s.walk
if w.Release == nil && !w.IsBatch() && w.Open() && strings.EqualFold(w.Repository, d.Repository) {
open = append(open, fmt.Sprintf("%s at %s", w.ID, shortOf(w.Commit, 8)))
}
}
sort.Strings(open)
head := fmt.Sprintf("merged as %s, and no walk answers its merge within %s: ", shortOf(d.MergedAs, 8), unwalked)
if len(open) == 0 {
return head + "the controller holds no open walk of " + d.Repository + " — its merge was not heard, or " +
"nothing the mesh holds follows " + orNone(d.Base) + ": a person decides"
}
return head + "the open walk(s) of " + d.Repository + " — " + strings.Join(open, ", ") + " — name no merge of " +
"it and the forge did not say they contain it: a person decides"
}
func orNone(s string) string {
if s == "" {
return "its branch"
}
return s
}
// orphanLines are the waiting walks no live delivery holds, past the grace a merge is given to be heard, as
// stalled lines in the state Unheld: nothing would ever give such a walk its word (issue 362). Under the lock.
func (h *Holder) orphanLines(now time.Time) []StalledLine {
var out []StalledLine
for id, first := range h.orphans {
s, known := h.walks[id]
if !known || now.Sub(first) < matchGrace {
continue
}
w := s.walk
if len(h.lapsedOn(w)) > 0 {
// Owed its word by stops that lapsed, and not given it past the grace (review of issue 753): said, why.
says := fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, which the stopped deliveries whose "+
"merge it answers owe it (novox/hq issue 753), and it has not started", w.ID, w.Repository, shortOf(w.Commit, 8))
if why := h.lapsedSaid(w); why != "" {
says += ": " + why
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: w.ID,
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
H2: "none: its word is given at its turn — the operator's past the bound", Says: says})
continue
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID,
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's",
Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+
"follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+
"starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)})
out[len(out)-1].Says += h.endedSaid(w)
if hold := h.coverHeld(id); hold != "" {
out[len(out)-1].Says += "; it is held back meanwhile: " + hold
}
}
return out
}
// Unheld is a waiting walk no live delivery holds: not a state of the table — only what a stalled line says.
const Unheld State = "unheld"
// seeOrphans keeps when each waiting walk no live delivery holds was first seen so, and forgets the rest; a
// walk still unmatched waits for its merge or its adoption instead. Under the lock.
func (h *Holder) seeOrphans(now time.Time) {
for id, s := range h.walks {
w := s.walk
_, unmatched := h.unmatched[id]
if unmatched || w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits || h.heldWalk(w) {
delete(h.orphans, id)
continue
}
if _, was := h.orphans[id]; !was {
h.orphans[id] = now
h.Logf("[mesh-delivery] the walk %s waits for this owner's word and no live delivery holds it: said once "+
"it is %s so", id, matchGrace)
}
}
for id := range h.orphans {
if _, known := h.walks[id]; !known {
delete(h.orphans, id)
}
}
}
// forgetWalks drops a walk record no longer open after keptWalkFor, and a batch heard of that long past its
// window's latest close. Under the lock.
func (h *Holder) forgetWalks(now time.Time) {
for id, s := range h.walks {
w := s.walk
switch {
case w.IsBatch():
end := s.at
if w.Delivery != nil && w.Delivery.Batch != nil && w.Delivery.Batch.AtMost != nil && w.Delivery.Batch.AtMost.After(end) {
end = *w.Delivery.Batch.AtMost
}
if now.Sub(end) > 3*time.Hour {
delete(h.walks, id)
}
case !w.Open() && now.Sub(s.at) > keptWalkFor:
delete(h.walks, id)
delete(h.deliverErr, id)
}
}
}
// --- the word, once per walk ------------------------------------------------------------------------------
// turnWant is a walk whose turn came for some of the deliveries it carries, and why.
type turnWant struct {
ids []string
whys []string
}
// endedSaid names the ended deliveries a waiting walk answers or that follow it, with their states, so a walk no live
// delivery holds says what it carries instead of only that nothing holds it (novox/hq issue 753); empty for none.
func (h *Holder) endedSaid(w Walk) string {
var ended []string
for _, d := range h.deliveries {
if !d.State.Final() {
continue
}
_, answers := walkAnswers(w, d)
if answers || d.Walk != nil && d.Walk.ID == w.ID {
ended = append(ended, fmt.Sprintf("%s (%s)", d.ID, d.State))
}
}
if len(ended) == 0 {
return ""
}
sort.Strings(ended)
return "; every delivery it carries ended: " + strings.Join(ended, ", ")
}
// lapsedOn are the deliveries, by id, whose word an open walk waiting for this owner's word takes from a stop that
// lapsed (novox/hq issue 753): each was stopped by its group for a member delivered since, so its own change waits for
// a later walk of its repository that contains its merge (issue 747), and this walk answers that merge — it names it,
// or is made at it — and has not left it out. Such a walk is the one that delivery waits for: it gives it its word, so
// a walk whose every other delivery ended is never left without one. A walk at the merge of a delivery stopped for
// itself, or by a person, gets nothing from it. Under the lock.
//
// No member is ordered before another (novox/hq ADR 0418 §2): each such delivery gives its word at once.
func (h *Holder) lapsedOn(w Walk) []string {
if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits {
return nil
}
var out []string
for _, d := range h.deliveries {
if h.lapsedAnswered(w, d) {
out = append(out, d.ID)
}
}
sort.Strings(out)
return out
}
// lapsedAnswered is whether a delivery's stop lapsed and the walk answers its merge, not left out.
func (h *Holder) lapsedAnswered(w Walk, d *Delivery) bool {
if d.MergedAs == "" || !h.stopLapsed(d) {
return false
}
if _, ok := walkAnswers(w, d); !ok {
return false
}
_, left := w.leftOut(d.Repository, d.MergedAs)
return !left
}
// lapsedSaid is why a walk whose word stops that lapsed give still waits: the members of their groups not yet
// delivered before them, what the last look held it for, or the controller's refusal of the word. Under the lock.
func (h *Holder) lapsedSaid(w Walk) string {
var parts []string
if why := h.waitsWhy[w.ID]; why != "" {
parts = append(parts, "not let go yet: "+why)
}
if err := h.deliverErr[w.ID]; err != "" {
parts = append(parts, "the controller refused its word: "+err)
}
for _, id := range h.lapsedOn(w) {
if d := h.deliveries[id]; d != nil && strings.Contains(d.LetGoWhy, lapsedWord(w.ID)) {
parts = append(parts, fmt.Sprintf("its word was given for %s at %s, and the controller has not started it",
id, d.LetGo.Local().Format("15:04:05")))
}
}
return strings.Join(parts, "; ")
}
// lapsedOwed is whether a walk's word is owed by stops that lapsed.
func (h *Holder) lapsedOwed(w Walk) bool {
return len(h.lapsedOn(w)) > 0
}
// lapsedTurns registers, for every open walk waiting for this owner's word, the turn of each delivery whose stop
// lapsed and whose merge it answers (lapsedOn, novox/hq issue 753). The word is then given as for any walk: once,
// when every live delivery it carries is at its turn too and it carries no stopped change still held back (words).
// Given once per walk: the walk's id is in the delivery's LetGoWhy once its word was given. Under the lock.
func (h *Holder) lapsedTurns() {
for id, s := range h.walks {
for _, did := range h.lapsedOn(s.walk) {
d := h.deliveries[did]
if strings.Contains(d.LetGoWhy, lapsedWord(id)) {
continue
}
if h.turning == nil {
h.turning = map[string]*turnWant{}
}
want := h.turning[id]
if want == nil {
want = &turnWant{}
h.turning[id] = want
}
if !containsString(want.ids, d.ID) {
want.ids = append(want.ids, d.ID)
want.whys = append(want.whys, fmt.Sprintf("%s: %s its merge %s, stopped by its group for %s, whose change "+
"was delivered since; it waits for a later walk of %s that contains its merge, and this is that "+
"walk (novox/hq issue 753)", d.ID, lapsedWord(id), shortOf(d.MergedAs, 8), d.stoppedFor(), d.Repository))
}
}
}
}
// lapsedWord is how the word a lapsed stop gives a walk names that walk.
func lapsedWord(walk string) string { return "the walk " + walk + " answers" }
// words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per
// walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5).
// Under the lock.
func (h *Holder) words() []func() {
walks := make([]string, 0, len(h.turning))
for walk := range h.turning {
walks = append(walks, walk)
}
sort.Strings(walks)
var asks []func()
h.waitsWhy = map[string]string{}
for _, walk := range walks {
want := h.turning[walk]
var waiting []string
for _, d := range h.deliveries {
if d.State.Final() || d.Walk == nil || d.Walk.ID != walk || containsString(want.ids, d.ID) {
continue
}
waiting = append(waiting, fmt.Sprintf("%s (%s)", d.ID, d.State))
}
if len(waiting) > 0 {
sort.Strings(waiting)
h.waitsWhy[walk] = strings.Join(waiting, ", ") + " carried by it not at its turn"
h.Logf("[mesh-delivery] the walk %s is not let go yet: %s", walk, h.waitsWhy[walk])
continue
}
ids, why := append([]string(nil), want.ids...), strings.Join(want.whys, "; ")
// **Never the word for a walk that carries a stopped change** (ADR 0299, review of #221): it is asked to
// leave the merge out first (leaveStopped).
if stopped, _ := h.carriesStopped(walk); len(stopped) > 0 {
h.waitsWhy[walk] = "it carries " + strings.Join(stopped, ", ") + ", whose delivery was stopped: it is " +
"asked to leave that merge out first"
if hold := h.coverHeld(walk); hold != "" {
h.waitsWhy[walk] = hold
}
h.Logf("[mesh-delivery] the walk %s is not let go: it carries %s, whose delivery was stopped", walk,
strings.Join(stopped, ", "))
continue
}
if h.asking[walk] {
continue // its word is being asked
}
h.asking[walk] = true
walk := walk
asks = append(asks, func() {
err := h.Controller.Deliver(walk, why)
h.mu.Lock()
defer h.mu.Unlock()
delete(h.asking, walk)
if err != nil {
if h.deliverErr == nil {
h.deliverErr = map[string]string{}
}
h.deliverErr[walk] = err.Error() // said by `stalled` past the grace (review of novox/hq issue 753)
h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err)
h.again("a walk's word the controller refused or did not answer")
return
}
delete(h.deliverErr, walk)
for _, id := range ids {
d := h.deliveries[id]
if d == nil {
continue
}
d.LetGo, d.LetGoWhy = h.Now(), why
if d.Walk != nil && d.Walk.ID == walk {
d.Walk.Waits, d.Walk.GoBy = false, byOwner
}
h.settleAndKeep(d)
}
})
}
h.turning = nil
return asks
}
// endedMovingNothing is whether a delivery ended delivered with nothing for a walk to move: merged, its plan
// moving no module the mesh holds, and no walk followed.
func endedMovingNothing(d *Delivery) bool {
return d.State == Delivered && d.MergedAs != "" && d.Walk == nil && d.MovesNothing()
}
// endedAsNothingToMove are the deliveries a walk answers that ended delivered as nothing for a walk to move (novox/hq
// issues 362, 529: the nzbgeek case); empty when it answers any other. Such a walk is held by nothing: said, never
// let go by this owner. Under the lock.
func (h *Holder) endedAsNothingToMove(w Walk) []string {
var out []string
for _, d := range h.deliveries {
_, answers := walkAnswers(w, d)
if !answers && (d.Walk == nil || d.Walk.ID != w.ID) {
continue
}
if !endedMovingNothing(d) {
return nil
}
out = append(out, d.ID)
}
sort.Strings(out)
return out
}
// WalksWaiting is `stalled` asked with `walks` (novox/hq issue 529): for each walk waiting for this owner's word,
// what holds it, in the shape of a stalled line naming the walk — each held delivery it carries (State held, its
// reason and the verb that releases it), else why its word was not given at the last look (State published), else
// that no live delivery holds it (State unheld). A walk whose word is being given is not listed. The controller's
// `plans` says these instead of pointing at a hand start; the self-check never raises a condition for them.
func (h *Holder) WalksWaiting() []StalledLine {
h.mu.Lock()
defer h.mu.Unlock()
now := h.Now()
var out []StalledLine
for id, s := range h.walks {
w := s.walk
if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits || h.asking[id] {
continue
}
waited := now.Sub(w.Created).Round(time.Second).String()
var held []StalledLine
live := false
for _, d := range h.deliveries {
if d.State.Final() || d.Walk == nil || d.Walk.ID != id {
continue
}
live = true
if d.State == Held {
held = append(held, StalledLine{ID: d.ID, Number: d.Number, State: Held, Walk: id, For: waited,
Bound: "0s", H2: "none: held for a person — the operator's", Says: HeldLine(d)})
}
}
switch {
case len(held) > 0:
out = append(out, held...)
case h.coverHeld(id) != "":
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
Bound: coverAskBound.String(), H2: "none: it is asked to leave the merge out past the bound",
Says: "held back: " + h.coverHeld(id)})
case h.waitsWhy[id] != "":
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]})
case !live && h.lapsedOwed(w):
// Its word comes from the stopped deliveries whose merge it answers (novox/hq issue 753): said, never
// unheld, while the controller has not taken it, with what holds it.
lapsed := h.lapsedOn(w)
says := "its word is owed by the stopped deliveries whose merge it answers, each stopped for a member " +
"delivered since (novox/hq issue 753)"
if len(lapsed) > 0 {
says = "its word is given for " + strings.Join(lapsed, ", ") + ", stopped for a member delivered since, " +
"whose merge it answers (novox/hq issue 753)"
}
if why := h.lapsedSaid(w); why != "" {
says += "; " + why
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
Bound: "0s", H2: "none: it goes at its turn", Says: says})
case !live:
says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " +
"`mesh-delivery.show` for the deliveries of " + w.Repository
if ended := h.endedAsNothingToMove(w); len(ended) > 0 {
// The nzbgeek case (novox/hq issues 362, 529): its merges' deliveries ended as nothing for a walk to
// move, yet the walk moves modules — nothing judged those, so nothing here lets it go.
says = "held by nothing: its merges' deliveries ended as nothing-to-move (" + strings.Join(ended, ", ") +
"), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word"
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited,
Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says + h.endedSaid(w)})
}
}
sort.Slice(out, func(i, j int) bool {
if out[i].Walk != out[j].Walk {
return out[i].Walk < out[j].Walk
}
return out[i].ID < out[j].ID
})
return out
}
// walkModules are the modules a walk's record names, sorted.
func walkModules(w Walk) []string {
out := make([]string, 0, len(w.Modules))
for m := range w.Modules {
out = append(out, m)
}
sort.Strings(out)
return out
}
@@ -140,14 +140,6 @@ func repoName(repository string) string {
func pullSaid(d *Delivery) string { return fmt.Sprintf("%s#%d", repoName(d.Repository), d.Number) }
// touchesSaid is the modules the delivery's check says its change touches, for the ask; the gate plans no delivery.
func touchesSaid(d *Delivery) string {
if d.Check == nil || len(d.Check.Modules) == 0 {
return "no module the check named"
}
return strings.Join(d.Check.Modules, ", ")
}
// composeMergeAsk is the ask for a delivery, or for a group's members together.
func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery, heads map[string]string, now time.Time) asks.Ask {
var a asks.Ask
@@ -163,10 +155,13 @@ func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery,
unreviewed = pullSaid(m) + " is " + words
}
reviewers = append(reviewers, fmt.Sprintf("%s: %s", pullSaid(m), words))
touches := touchesSaid(m)
reaches = append(reaches, touches)
details = append(details, fmt.Sprintf("%s %q at %s: issues: %s; review: %s; touches: %s",
pullSaid(m), m.Title, shortOf(m.Commit, 12), issuesSaid([]*Delivery{m}), words, touches))
plan := "no delivery plan yet"
if m.Plan != nil && m.Plan.Summary != "" {
plan = m.Plan.Summary
}
reaches = append(reaches, plan)
details = append(details, fmt.Sprintf("%s %q at %s: issues: %s; review: %s; reaches: %s",
pullSaid(m), m.Title, shortOf(m.Commit, 12), issuesSaid([]*Delivery{m}), words, plan))
}
if group == "" {
a.Headline = phone("Merge into main: "+pullSaid(d)+" "+d.Title, asks.HeadlineLength)
@@ -179,7 +174,7 @@ func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery,
phone(summary, asks.SummaryLength),
phone("Issues: "+issuesSaid(members), asks.PointLength),
phone("Review: "+reviewers[0], asks.PointLength),
phone("Touches: "+reaches[0], asks.PointLength),
phone("Reaches: "+reaches[0], asks.PointLength),
}, "\n")
} else {
var names []string
@@ -196,7 +191,7 @@ func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery,
phone(summary, asks.SummaryLength),
phone("Issues: "+issuesSaid(members), asks.PointLength),
phone("Review: "+strings.Join(reviewers, "; "), asks.PointLength),
phone("Touches: "+strings.Join(reaches, "; "), asks.PointLength),
phone("Reaches: "+strings.Join(reaches, "; "), asks.PointLength),
}, "\n")
}
a.Details = phone(strings.Join(details, "\n"), 1500)
@@ -43,7 +43,7 @@ func TestAMarkedGreenDeliveryIntoMainIsAskedAndNotMergedUntilApproved(t *testing
len(a.Options) != 2 || a.Options[0].ID != OptionApprove || a.Options[1].ID != OptionDecline {
t.Fatalf("the ask is not the merge ask: %+v", a)
}
for _, want := range []string{"issue 948", "issue 12", "reviewed by reviewer on shanks", "Touches: "} {
for _, want := range []string{"issue 948", "issue 12", "reviewed by reviewer on shanks", "builds app"} {
if !strings.Contains(a.Explanation, want) {
t.Fatalf("the ask does not say %q: %s", want, a.Explanation)
}
@@ -72,6 +72,7 @@ func TestAMarkedGreenDeliveryIntoMainIsAskedAndNotMergedUntilApproved(t *testing
t.Fatalf("the warrant is not kept: %+v", d.Ready)
}
w.h.PullMerged(merged("novox/app", 7, readyHead, "m7m7m7m7m7m7"))
w.h.WalkMoved(w.ctl.put(aWalk("plan-7", "novox/app", "m7m7m7m7m7m7", true)))
w.settleAll()
var last Transition
for _, tr := range w.delivery(id).Transitions {
@@ -372,7 +373,7 @@ func TestIssue1025AnOwedMergedIsSaidAfterARestartAndOnce(t *testing.T) {
t.Fatal(err)
}
w.h.mu.Unlock()
again := newWorldOver(t, w.store, w.forge, w.now.Add(time.Minute))
again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute))
again.settleAll()
again.later(time.Hour)
again.settleAll()
@@ -0,0 +1,154 @@
package main
import (
"strings"
"testing"
)
// A failed backup before the send holds the delivery, and a release asks the backups again (novox/hq ADR 0404
// clause 4, issue 1068). The walk's backups field is the controller's contract.
func failedBackup() *WalkBackups {
return &WalkBackups{State: "failed", Asked: []WalkBackupAsked{
{Machine: "ace", Module: "mail", State: "done"},
{Machine: "ace", Module: "pg", State: "failed", Words: "no space left on device"}}}
}
// aDeliveringWalk is a delivery whose walk was let go and runs, nothing sent yet.
func aDeliveringWalk(t *testing.T) (*world, string, string) {
t.Helper()
w := newWorld(t)
id := IDOf("novox/app", head)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "pass"))
w.h.PullMerged(merged("novox/app", 7, head, merge))
w.h.WalkMoved(w.ctl.put(aWalk("plan-1", "novox/app", merge, true)))
w.settleAll()
if s := w.state(id); s != Delivering {
t.Fatalf("setup: %s, want delivering", s)
}
return w, id, "plan-1"
}
func TestABackupFailedTakesADeliveringDeliveryToHeldOnlyWhileNothingWasSent(t *testing.T) {
w, id, plan := aDeliveringWalk(t)
sent := w.ctl.walk(plan)
sent.Backups = failedBackup()
sent.Modules["app"].SentAt = &w.now
w.h.WalkMoved(w.ctl.put(sent))
w.settleAll()
if s := w.state(id); s == Held {
t.Fatal("held for a failed backup after a module of the plan was sent")
}
w, id, plan = aDeliveringWalk(t)
held := w.ctl.walk(plan)
held.Backups = failedBackup()
w.h.WalkMoved(w.ctl.put(held))
w.settleAll()
d := w.delivery(id)
if d.State != Held {
t.Fatalf("with a failed backup and nothing sent it is %s", d.State)
}
for _, want := range []string{"ace", "pg", "no space left on device"} {
if !strings.Contains(d.HeldWhy, want) {
t.Errorf("the held reason %q does not name %q", d.HeldWhy, want)
}
}
// It does not lift by itself: the same facts read again leave it held.
w.h.WalkMoved(w.ctl.put(w.ctl.walk(plan)))
w.settleAll()
if s := w.state(id); s != Held {
t.Fatalf("read again it is %s, want it still held", s)
}
}
func TestAReleaseOfADeliveryHeldForAFailedBackupAsksTheControllerAgainAndGoesToDelivering(t *testing.T) {
w, id, plan := aDeliveringWalk(t)
held := w.ctl.walk(plan)
held.Backups = failedBackup()
w.h.WalkMoved(w.ctl.put(held))
w.settleAll()
if s := w.state(id); s != Held {
t.Fatalf("setup: %s, want held", s)
}
lettingGo := len(w.ctl.delivers)
if _, err := w.h.Release(id, "freed some space", "jochen"); err != nil {
t.Fatal(err)
}
if got := w.ctl.backupsAgain; len(got) != 1 || got[0] != plan {
t.Fatalf("the controller was asked to take the backups again %v, want once for %s", got, plan)
}
if len(w.ctl.delivers) != lettingGo {
t.Errorf("a started walk was let go again: %v", w.ctl.delivers)
}
if s := w.state(id); s != Delivering {
t.Fatalf("released, it is %s", s)
}
// The record the controller cleared is read: it stays delivering.
w.h.WalkMoved(w.ctl.walk(plan))
w.settleAll()
if s := w.state(id); s != Delivering {
t.Fatalf("with the backups cleared it is %s", s)
}
// Another failure, in a later record, holds it again.
again := w.ctl.walk(plan)
again.Backups = failedBackup()
w.h.WalkMoved(w.ctl.put(again))
w.settleAll()
if s := w.state(id); s != Held {
t.Fatalf("a failed backup in a later record left it %s", s)
}
}
func TestARecordThatStillHoldsTheFailureAfterAReleaseDoesNotHoldItAgain(t *testing.T) {
w, id, plan := aDeliveringWalk(t)
held := w.ctl.walk(plan)
held.Backups = failedBackup()
stale := w.ctl.put(held)
w.h.WalkMoved(stale)
w.settleAll()
if _, err := w.h.Release(id, "freed some space", "jochen"); err != nil {
t.Fatal(err)
}
w.h.WalkMoved(stale) // the record as it was before the controller cleared it
w.settleAll()
if s := w.state(id); s != Delivering {
t.Fatalf("a record from before the release held it again: %s", s)
}
}
func TestADeliveryHeldForAFailedBackupIsNotLetGoByItsWalkAlone(t *testing.T) {
// The controller's own path starts a walk with nobody's word: the held-to-delivering row for that must not
// undo the backup hold.
d := &Delivery{ID: "novox/app@c0", State: Held, MergedAs: "aaaa", Walk: &WalkSeen{ID: "plan-1", State: walkBuilding, Revision: 3,
BackupsFailed: []WalkBackupFailure{{Machine: "ace", Module: "pg", Words: "no space left on device"}}}}
if _, err := Apply(d, EvGo, false, Facts{}); err == nil {
t.Fatal("a walk started on the controller's own path lifted the backup hold")
}
}
func TestAReleaseWhoseBackupsTheControllerRefusedToAskAgainLeavesTheDeliveryHeld(t *testing.T) {
w, id, plan := aDeliveringWalk(t)
held := w.ctl.walk(plan)
held.Backups = failedBackup()
w.h.WalkMoved(w.ctl.put(held))
w.settleAll()
w.ctl.refuseBackupsAgain = true
_, err := w.h.Release(id, "freed some space", "jochen")
if err == nil || !strings.Contains(err.Error(), "held again") {
t.Fatalf("the release of a refused word answered %v, want an error saying it is held again", err)
}
w.settleAll()
d := w.delivery(id)
if d.State != Held || d.BackupsAgainAfter != 0 || !strings.Contains(d.HeldWhy, "no space left on device") {
t.Fatalf("after the refusal it is %s (again-after %d, why %q), want held for the backup", d.State, d.BackupsAgainAfter, d.HeldWhy)
}
// The controller takes the word the next time: the delivery goes on.
w.ctl.refuseBackupsAgain = false
if _, err := w.h.Release(id, "freed some space", "jochen"); err != nil {
t.Fatal(err)
}
if s := w.state(id); s != Delivering {
t.Fatalf("released again, it is %s", s)
}
}
@@ -11,6 +11,8 @@ import (
"io"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
@@ -83,8 +85,8 @@ func newTestBus(t *testing.T) *testBus {
}
b.kvs[name] = kv
}
// The buckets of other modules the manifest reads (`<module>.<name>` is the bucket `<module>_<name>`): the teller's answers
// and the checker's verdicts.
// The buckets of other modules the manifest reads (`<module>.<name>` is the bucket `<module>_<name>`): the gate reads
// who has the seat there, and today's state to carry (seat.go).
for _, name := range m.Reads {
kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: strings.Replace(name, ".", "_", 1), History: 10})
if err != nil {
@@ -92,9 +94,13 @@ func newTestBus(t *testing.T) *testBus {
}
b.kvs[name] = kv
}
// The node-engine has said today's module removed on the one machine: the gate may hold the seat (ADR 0450 §6, issue 1369).
if _, err := b.kvs["node-engine.have"].Put(context.Background(), "test.mesh-delivery", []byte(`{"v":1,"phase":"removed","health":"unknown"}`)); err != nil {
t.Fatal(err)
}
// The streams the events are on (mesh.mod.<module>.event.<name>, mesh.seat.<seat>.event.<name>).
if _, err := js.CreateStream(context.Background(), jetstream.StreamConfig{Name: "EVENTS",
Subjects: []string{"mesh.mod.gitea.event.>", "mesh.seat." + asks.Seat + ".event.>",
Subjects: []string{"mesh.mod.gitea.event.>", "mesh.seat." + ControllerSeat + ".event.>", "mesh.seat." + asks.Seat + ".event.>",
"mesh.mod." + sdk.ForgeBridgeModule + ".event.>"}}); err != nil {
t.Fatal(err)
}
@@ -202,29 +208,15 @@ func (r *runtime) read(out io.Reader) {
}
}
// untilListening waits for the gate to have read its kept state back and subscribed to every event it follows: it takes no
// event before (loadKept), so a subscription is its word that it is ready.
func (r *runtime) untilListening() {
// untilHolding waits for the gate to hold the seat: it holds it when it reads the controller's walks once, the first thing it
// does after (readOnHolding), and it takes no event before.
func (r *runtime) untilHolding() {
r.t.Helper()
r.until("the gate to listen to every event it follows", 20*time.Second, func() bool {
r.mu.Lock()
defer r.mu.Unlock()
return len(r.subs) >= len(followed)
r.until("the gate to hold the seat", 20*time.Second, func() bool {
return len(r.asksOf("seat:"+ControllerSeat+".delivery-walks")) > 0
})
}
// noControllerAsk fails the test when the gate asked the controller anything: it asks it nothing (ADR 0539 §17, R6-C3).
func (r *runtime) noControllerAsk() {
r.t.Helper()
r.mu.Lock()
defer r.mu.Unlock()
for _, a := range r.asked {
if strings.HasPrefix(a.Key, "seat:mesh-controller.") {
r.t.Fatalf("the gate asked the controller %s: it asks it nothing", a.Key)
}
}
}
func (r *runtime) ping() {
select {
case r.notify <- struct{}{}:
@@ -342,8 +334,11 @@ func (r *runtime) answer(m rpcMessage) {
pulls := append([]OpenPull{}, r.openPulls...)
r.mu.Unlock()
switch a.Key {
case "":
return nil, errors.New("an ask with no key")
case "seat:" + ControllerSeat + ".delivery-walks":
return toolAnswer(map[string]any{"held": true, "walks": []any{}}), nil
case "seat:" + ControllerSeat + ".delivery-plan":
return toolAnswer(map[string]any{"plan": DeliveryPlan{Repository: "novox/app", Moved: []string{"app"},
Summary: "builds app, sent to the node"}}), nil
case "gitea.gitea_open_pulls":
return toolAnswer(map[string]any{"pulls": pulls}), nil
case "gitea.gitea_commit_statuses":
@@ -360,10 +355,6 @@ func (r *runtime) answer(m rpcMessage) {
"gitea.gitea_add_comment":
return toolAnswer(map[string]any{}), nil
}
if strings.HasPrefix(a.Key, "seat:mesh-controller.") {
// The fake controller of this test fails on any call: the gate has no business with it.
return nil, fmt.Errorf("the controller is not to be called by the gate: %s", a.Key)
}
return nil, fmt.Errorf("the runtime answers no ask %s in this test", a.Key)
}
kv := r.b.kvs[p.State]
@@ -504,7 +495,12 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
defer cancel()
rt.deliverEvents(evctx)
// The module has started when it hears what it follows.
rt.untilListening()
rt.until("the module to subscribe to the events it follows", 20*time.Second, func() bool {
rt.mu.Lock()
defer rt.mu.Unlock()
return len(rt.subs) >= len(followed)
})
rt.untilHolding()
for _, k := range followed {
if strings.Contains(k, "labeled") || strings.Contains(k, "commented") {
t.Fatalf("the module follows %s: a label or a comment starts nothing here (ADR 0421 §4)", k)
@@ -584,8 +580,6 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
// The bridge says it merged; the mesh's source then holds the merge, in one write.
b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: "novox/app", Number: 7, Head: head, Merge: strings.Repeat("b", 40),
Root: "r", Trees: map[string]string{"app": "t"}})
// The forge's own word that the pull request merged follows the bridge's (its webhook): the delivery ends merged.
b.publish(t, "mesh.mod.gitea.event.pull.merged", merged("novox/app", 7, head, strings.Repeat("b", 40)))
rt.until("source.main to hold the merge", eventWithin, func() bool {
e, err := b.kvs["source"].Get(ctx, sdk.SourceMainKey)
if err != nil {
@@ -594,13 +588,6 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
src, err := sdk.DecodeSource(e.Value())
return err == nil && src.Group == "fix/x" && src.Heads["novox/app"] == strings.Repeat("b", 40)
})
// The delivery ends merged (ADR 0539 §17), and the gate asked the controller nothing (R6-C3).
rt.until("the delivery to be merged", eventWithin, func() bool {
e, err := b.kvs["deliveries"].Get(ctx, kvKey(IDOf("novox/app", head)))
var d Delivery
return err == nil && json.Unmarshal(e.Value(), &d) == nil && d.State == Merged
})
rt.noControllerAsk()
// Quiet follows: nothing asks the forge or the controller again with no event to say so.
quiet := len(rt.asked)
time.Sleep(1500 * time.Millisecond)
@@ -703,7 +690,12 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe
evctx, cancel := context.WithCancel(ctx)
defer cancel()
rt.deliverEvents(evctx)
rt.untilListening()
rt.until("the module to subscribe to the events it follows", 20*time.Second, func() bool {
rt.mu.Lock()
defer rt.mu.Unlock()
return len(rt.subs) >= len(followed)
})
rt.untilHolding()
const ctlHead, catHead = "c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2"
for _, p := range []PullEvent{pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go"),
pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")} {
@@ -714,6 +706,9 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe
// Both members pass at the checker, whose one verdict covers both heads: the group is asked once, no composed check is asked
// of the controller (issue 1318).
b.checkPasses(t, rt, "feat/x", "a1a1a1a1a1a1", judgedPull{"novox/mesh-controller", 1, ctlHead}, judgedPull{"novox/mesh-catalog", 2, catHead})
if n := len(rt.asksOf("seat:" + ControllerSeat + ".delivery-order")); n != 0 {
t.Fatalf("the controller's delivery-order was asked %d times: no order is asked for", n)
}
var key string
rt.until("the operator to be asked once for the group", 10*time.Second, func() bool { key = onlyAsk(b); return key != "" })
time.Sleep(500 * time.Millisecond)
@@ -763,25 +758,163 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe
if keys, _ := b.kvs["asks"].Keys(ctx); len(keys) != 1 {
t.Fatalf("a second ask was made: %v", keys)
}
// The forge's own word that each pull request merged follows the bridge's (its webhook), after the second was handed (the
// checker judges the members still open by the trunk the first merge moved, which this test does not play): both members
// end merged, the group with them, and the controller was never called (R6-C3).
b.publish(t, "mesh.mod.gitea.event.pull.merged", merged(one.Repository, one.Number, one.Head.Head, strings.Repeat("a", 40)))
b.publish(t, "mesh.mod.gitea.event.pull.merged", merged(two.Repository, two.Number, two.Head.Head, strings.Repeat("b", 40)))
for _, id := range []string{IDOf("novox/mesh-controller", ctlHead), IDOf("novox/mesh-catalog", catHead)} {
id := id
rt.until(id+" to be merged", eventWithin, func() bool {
e, err := b.kvs["deliveries"].Get(ctx, kvKey(id))
var d Delivery
return err == nil && json.Unmarshal(e.Value(), &d) == nil && d.State == Merged
})
}
rt.until("the group to read merged", eventWithin, func() bool {
e, err := b.kvs["groups"].Get(ctx, kvKey("feat/x"))
var g Group
return err == nil && json.Unmarshal(e.Value(), &g) == nil && g.Said == "merged"
}
// **The acceptance of a failed backup before the send, on the mesh-delivery side** (novox/hq ADR 0404 clause 4, design 47
// milestone 4, issue 1068): a merged delivery whose walk runs is held when the walk's record, as the controller's
// `plan-moved` carries it, names a backup that failed and nothing was sent; the hold names the machine, the module and the
// backup's words, and nothing asks the controller to let it go. One test function, one bus. The walk's backups field is the controller's
// contract (issue 1065).
func TestAPlanMovedCarryingAFailedBackupHoldsTheDeliveryWithTheBackupsWordsOverARealBus(t *testing.T) {
b := newTestBus(t)
rt := b.start(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
rt.deliverEvents(ctx)
rt.until("the module to subscribe to the events it follows", 20*time.Second, func() bool {
rt.mu.Lock()
defer rt.mu.Unlock()
return len(rt.subs) >= len(followed)
})
rt.noControllerAsk()
rt.untilHolding()
const head, merge = "3333333333333333333333333333333333333333", "4444444444444444444444444444444444444444"
announced := pr("novox/app", 9, head, "fix/y", "src/a.go")
announced.BaseSHA = "a1a1a1a1a1a1"
deliveryOf := func() Delivery {
e, err := b.kvs["deliveries"].Get(context.Background(), kvKey(IDOf("novox/app", head)))
var d Delivery
if err == nil {
_ = json.Unmarshal(e.Value(), &d)
}
return d
}
// Each event is published after the module has taken the one before it: they are on two subjects, and the runner
// may hand over a later one first, which the table would rightly judge on its own (merged before checked).
b.publish(t, "mesh.mod.gitea.event.pull.updated", announced)
rt.until("the pull request to be taken", 10*time.Second, func() bool { return deliveryOf().State != "" })
b.checkPasses(t, rt, "fix/y", "a1a1a1a1a1a1", judgedPull{"novox/app", 9, head})
rt.until("the verdict to be taken", 10*time.Second, func() bool { d := deliveryOf(); return d.State != "" && d.State != Proposed })
b.publish(t, "mesh.mod.gitea.event.pull.merged", merged("novox/app", 9, head, merge))
rt.until("the merge to be taken", 10*time.Second, func() bool { return deliveryOf().MergedAs != "" })
// Its walk runs, let go: the record names no failed backup yet.
now := time.Now()
walk := aWalk("plan-1", "novox/app", merge, true)
walk.Created, walk.State, walk.Revision = now, walkRolling, 1
walk.Delivery.Go, walk.Delivery.By = &now, "mesh-delivery"
b.publish(t, "mesh.seat."+ControllerSeat+".event.plan-moved", walk)
rt.until("the delivery to be delivering", 10*time.Second, func() bool { return deliveryOf().State == Delivering })
// A backup failed before the send: the delivery is held with the backup's words.
walk.Revision, walk.Backups = 2, &WalkBackups{State: "failed", Asked: []WalkBackupAsked{{Machine: "ace", Module: "pg", State: "failed", Words: "no space left on device"}}}
b.publish(t, "mesh.seat."+ControllerSeat+".event.plan-moved", walk)
rt.until("the delivery to be held for the failed backup", 10*time.Second, func() bool { return deliveryOf().State == Held })
d := deliveryOf()
for _, want := range []string{"ace", "pg", "no space left on device"} {
if !strings.Contains(d.HeldWhy, want) {
t.Fatalf("the held reason %q does not name %q", d.HeldWhy, want)
}
}
// It stays held, and the controller is asked nothing to let it go.
time.Sleep(500 * time.Millisecond)
if s := deliveryOf().State; s != Held {
t.Fatalf("the hold lifted by itself: %s", s)
}
if n := len(rt.asksOf("seat:"+ControllerSeat+".deliver")) + len(rt.asksOf("seat:"+ControllerSeat+".delivery-backups-again")); n != 0 {
t.Fatalf("the controller was asked %d times to let the held walk go or take its backups again", n)
}
}
// **AC3 of issue 1347 (ADR 0437 §7's check): the gate carries a live mesh's open deliveries and groups, once.** The buckets of
// today's module are filled from testdata/live-capture-2026-10-20.json (its source is said in the file), the gate starts with
// today's module on no machine, and every delivery and group arrives in its own buckets unchanged but for the ask an Approve
// waited on; a second start copies nothing.
func TestTheGateCarriesALiveCapturesDeliveriesAndGroupsOnceOverARealBus(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("testdata", "live-capture-2026-10-20.json"))
if err != nil {
t.Fatal(err)
}
var capture struct {
Deliveries []json.RawMessage `json:"deliveries"`
Groups []json.RawMessage `json:"groups"`
}
if err := json.Unmarshal(raw, &capture); err != nil {
t.Fatal(err)
}
b := newTestBus(t)
ctx := context.Background()
want := map[string]Delivery{}
for _, one := range capture.Deliveries {
var d Delivery
if err := json.Unmarshal(one, &d); err != nil {
t.Fatal(err)
}
if _, err := b.kvs["mesh-delivery.deliveries"].Put(ctx, kvKey(d.ID), one); err != nil {
t.Fatal(err)
}
if d.Ready != nil { // what the carry clears: the ask nobody answered
d.Ready.AskID, d.Ready.Ask, d.Ready.Heads = "", nil, nil
}
want[kvKey(d.ID)] = d
}
for _, one := range capture.Groups {
var g Group
if err := json.Unmarshal(one, &g); err != nil {
t.Fatal(err)
}
if _, err := b.kvs["mesh-delivery.groups"].Put(ctx, kvKey(g.ID), one); err != nil {
t.Fatal(err)
}
}
rt := b.start(t)
rt.untilHolding()
for key, d := range want {
e, err := b.kvs["deliveries"].Get(ctx, key)
if err != nil {
t.Fatalf("the delivery %s did not arrive: %v", key, err)
}
var got Delivery
if err := json.Unmarshal(e.Value(), &got); err != nil {
t.Fatal(err)
}
// What is compared is what the carry copies and the gate never rewrites: who the delivery is. By the time this reads, the
// gate holds the seat and acts on the delivery (it asks a check, owes a note), which changes its check ask, its owed
// effects and its transitions, so a whole-value comparison depends on timing. The carry's own exactness is shown without a
// bus by TestTheCarryCopiesWhatItDoesNotClearByteForByte and TestTheGateCarriesTodaysStateOnceWhenItFirstHoldsTheSeat.
if got.ID != d.ID || got.Repository != d.Repository || got.Commit != d.Commit || got.Number != d.Number || got.Base != d.Base ||
got.Group != d.Group || !got.Created.Equal(d.Created) {
t.Fatalf("the delivery %s changed in the carry:\n got %s\nwant %s", key, mustJSON(got), mustJSON(d))
}
}
for _, one := range capture.Groups {
var g Group
_ = json.Unmarshal(one, &g)
e, err := b.kvs["groups"].Get(ctx, kvKey(g.ID))
if err != nil {
t.Fatalf("the group %s did not arrive: %v", g.ID, err)
}
// The carry copies the group byte for byte (TestTheCarryCopiesWhatItDoesNotClearByteForByte); by the time this reads, the gate
// holds the seat and acts, and writes a group's said state as it does, so what is compared is what only the carry could change.
got := mustDecode[Group](t, e.Value())
if got.ID != g.ID || !slices.Equal(got.Members, g.Members) || !got.Created.Equal(g.Created) {
t.Fatalf("the group %s changed in the carry: %+v", g.ID, got)
}
}
if _, err := b.kvs["deliveries"].Get(ctx, carriedKey); err != nil {
t.Fatalf("the mark carried was not written: %v", err)
}
// A second start copies nothing: a delivery today's module gains meanwhile stays where it is.
_ = rt.in.Close()
_ = rt.cmd.Wait()
late := mustJSON(&Delivery{ID: "novox/app@999999999999", Repository: "novox/app", Commit: "999999999999", Number: 99, State: Proposed})
if _, err := b.kvs["mesh-delivery.deliveries"].Put(ctx, kvKey("novox/app@999999999999"), late); err != nil {
t.Fatal(err)
}
again := b.start(t)
again.untilHolding()
if _, err := b.kvs["deliveries"].Get(ctx, kvKey("novox/app@999999999999")); !errors.Is(err, jetstream.ErrKeyNotFound) {
t.Fatalf("a second start copied a key (%v)", err)
}
}
func mustDecode[T any](t *testing.T, raw []byte) T {
@@ -12,10 +12,11 @@ import (
// **The gate is the only claimant of the seat `mesh-delivery` in the catalogue** (novox/hq ADR 0437 §1, ADR 0449 §4, ADR 0454,
// issue 1351; how it is checked). Today's module `modules/mesh-delivery` was removed by a pull request of its own after the
// handover, so the claimants are exactly one. And no manifest but the gate's and mesh-issues' names the old module in what it
// reads or hears, but through the seat's name (`mesh-delivery` as a seat or a repository, `seat:mesh-delivery.<verb>`). The
// three buckets the gate once carried over (ADR 0450 §6) went with the carry (ADR 0539 §17), and the event
// `mesh-delivery.transition` is heard by no one (issue 1406).
var oldModuleNames = map[string][]string{}
// reads or hears, but through the seat's name (`mesh-delivery` as a seat or a repository, `seat:mesh-delivery.<verb>`) and the
// names below: the three buckets the gate carries over (ADR 0450 §6). The event `mesh-delivery.transition` is no longer heard by anyone (issue 1406).
var oldModuleNames = map[string][]string{
"mesh-delivery-gate": {"mesh-delivery.deliveries", "mesh-delivery.groups", "mesh-delivery.pulls"},
}
// seatClaimants are the modules that claim the seat: the gate alone.
var seatClaimants = []string{"mesh-delivery-gate"}
@@ -15,9 +15,6 @@ const (
ctxRepo = "mesh/repo-check"
)
// shimCheckedKey is the key a test hears a verdict under: said as the checker's (see Checked).
const shimCheckedKey = "mesh-controller.checked"
// CheckedEvent is a verdict as these tests state it. The gate follows no verdict of the controller any more (issue 1318,
// ADR 0524 §5): the shim below says it the way the mesh says it now, as the head's mesh/check and the checker's verdict at
// the heads, so the table, the Approve and the merges are exercised on the one path they have.
@@ -41,6 +38,7 @@ type CheckedEvent struct {
Verdict string `json:"verdict"`
Summary string `json:"summary"`
}
Plan *DeliveryPlan
Group string
Members []struct {
Owner string `json:"owner"`
@@ -87,6 +85,9 @@ func (h *Holder) Checked(c CheckedEvent) {
if d == nil {
return
}
if c.Plan != nil {
d.Plan = c.Plan
}
words := map[string]string{"success": "pass", "warning": "pass", "failure": "failed: " + c.Summary, "error": "error"}[state]
line := words + " " + shortOf(c.Commit, 12)
if c.Trunk != "" {
@@ -69,11 +69,9 @@ type ChecksOf struct {
Since string `json:"since"`
Waits string `json:"waits,omitempty"`
Group string `json:"group,omitempty"`
// MergedAs is the commit its merge landed as on the trunk, once merged: what the delivery view's `delivered` event is
// matched to a fix by (ADR 0539 clause 17).
MergedAs string `json:"merged_as,omitempty"`
// Opened and MergedAt are what the issue tracker measures a fix's time by (novox/hq ADR 0313): when its pull request
// was first heard, and when it merged.
// Class, Opened and MergedAt are what the issue tracker measures a fix's time by (novox/hq ADR 0313): the
// class of what it moves, when its pull request was first heard, and when it merged.
Class string `json:"class,omitempty"`
Opened string `json:"opened,omitempty"`
MergedAt string `json:"merged_at,omitempty"`
}
@@ -115,7 +113,7 @@ func (h *Holder) Checks(repository string, number int, commit string) (*ChecksAn
}
if d != nil {
a.Delivery = &ChecksOf{ID: d.ID, State: d.State, Since: d.Since.UTC().Format(time.RFC3339), Waits: d.waits,
Group: d.Group, MergedAs: d.MergedAs}
Group: d.Group, Class: fixClass(&d.Delivery)}
if !d.opened.IsZero() {
a.Delivery.Opened = d.opened.UTC().Format(time.RFC3339)
}
@@ -200,8 +198,8 @@ func (h *Holder) deliveryOf(repository string, number int, commit string) *check
// newerHead is whether d is a pull request's head to read before e: one not superseded first, then the newest.
func newerHead(d, e *Delivery) bool {
if (d.State == Closed) != (e.State == Closed) {
return e.State == Closed
if (d.State == Superseded) != (e.State == Superseded) {
return e.State == Superseded
}
return d.Created.After(e.Created)
}
@@ -0,0 +1,587 @@
package main
import (
"fmt"
"strings"
"testing"
"time"
)
// novox/hq issues 319 and 316: a held delivery whose change already runs, or that builds nothing, is not a
// question for a person. A merge that moves nothing is delivered at once; a merge whose every module a later
// merge on the same trunk delivered is covered — never stopped, never an error on its pull request; and a
// group member held for a composed check that had not answered goes on once it passes.
var t0 = time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
// mergedAt is the forge's `pull.merged` at a moment.
func mergedAt(p PullEvent, merge string, at time.Time) PullEvent {
p.MergeCommit = merge
p.MergedAt = at.UTC().Format(time.RFC3339)
return p
}
// verdictMoving is a passing verdict whose plan moves these modules — or builds nothing, given none.
func verdictMoving(repo string, number int, sha string, modules ...string) CheckedEvent {
c := verdict(repo, number, sha, "pass")
c.Plan = planMoving(repo, modules...)
return c
}
func planMoving(repo string, modules ...string) *DeliveryPlan {
if len(modules) == 0 {
return &DeliveryPlan{Repository: repo, Base: "main", Summary: "builds nothing: the change touches no module of the mesh's graph"}
}
return &DeliveryPlan{Repository: repo, Base: "main", Moved: modules, Summary: "builds " + strings.Join(modules, ", ") + " → anchor"}
}
// deliverLater is a later merge into a trunk, walked: its walk opened for its merge commit, let go, and left
// in the state given, every module of it sent.
func deliverLater(w *world, repo string, number int, base, headSHA, merge string, at time.Time, state string,
modules ...string) string {
w.t.Helper()
p := pr(repo, number, headSHA, fmt.Sprintf("feat/later-%d", number))
p.Base = base
w.h.PullUpdated(p)
w.h.Checked(verdictMoving(repo, number, headSHA, modules...))
w.h.PullMerged(mergedAt(p, merge, at))
walk := aWalk("plan-"+merge[:6], repo, merge, true)
walk.Branch = base
walk.Modules = map[string]*WalkModule{}
for _, m := range modules {
walk.Modules[m] = &WalkModule{}
}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
walk = w.ctl.walk(walk.ID)
sent := at.Add(5 * time.Minute)
for _, m := range modules {
walk.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
}
walk.State = state
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
return IDOf(repo, headSHA)
}
func statusOn(w *world, repo, commit string) string {
w.forge.mu.Lock()
defer w.forge.mu.Unlock()
return w.forge.statuses[repo+"@"+commit+" mesh/delivery"]
}
// **A merge that moves nothing is delivered at once** (issue 319): there is nothing for a walk to move and
// nothing for a person to decide — whether its check passed, failed or never answered, and whether its plan
// builds nothing at all or only a new module the controller builds and sends nowhere (issue 300).
func TestAMergeThatMovesNothingIsDeliveredAtOnce(t *testing.T) {
for _, c := range []struct {
name string
verdict string // "" is a merge before any verdict
plan *DeliveryPlan
}{
{"builds nothing, merged before its check answered", "", planMoving("novox/hq")},
{"builds nothing, merged on a failed check", "fail", planMoving("novox/hq")},
{"builds nothing, merged on a passing check", "pass", planMoving("novox/hq")},
{"a new module, sent nowhere", "pass", &DeliveryPlan{Repository: "novox/hq", New: []string{"modules/zfs"},
Summary: "builds new: modules/zfs, sent nowhere; no bus step"}},
{"a new module, sent nowhere, merged before its check answered", "", &DeliveryPlan{Repository: "novox/hq",
New: []string{"modules/zfs"}, Summary: "builds new: modules/zfs, sent nowhere; no bus step"}},
} {
t.Run(c.name, func(t *testing.T) {
w := newWorld(t)
w.ctl.planned = c.plan
p := pr("novox/hq", 7, head, "docs/x", "02-DECISIONS/x.md")
w.h.PullUpdated(p)
if c.verdict != "" {
v := verdict("novox/hq", 7, head, c.verdict)
v.Plan = c.plan
w.h.Checked(v)
}
w.later(31 * time.Minute)
w.h.PullMerged(mergedAt(p, merge, t0.Add(30*time.Minute)))
w.settleAll()
// Past the bound a merge waits for its walk: nothing holds it then either.
w.later(15 * time.Minute)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
d := w.delivery(IDOf("novox/hq", head))
if d.State != Delivered {
t.Fatalf("a merge whose plan is %q is %s (%s), not delivered", c.plan.Summary, d.State, d.HeldWhy)
}
if s := statusOn(w, "novox/hq", head); !strings.HasPrefix(s, "success") {
t.Fatalf("its pull request's mesh/delivery says %q", s)
}
})
}
}
// **A held delivery whose every module a later merge on its trunk delivered is covered** (issue 319): the
// later merge commit contains it, its walk is done, and it sent each module this one moves. Anything short
// of that — a module not carried, an earlier merge, another trunk, a walk that did not finish — leaves it held
// for a person, as before.
func TestAMergeALaterDeliveredWalkCarriedIsCovered(t *testing.T) {
const ctl = "novox/mesh-controller"
for _, c := range []struct {
name string
// The later merge: its trunk, minutes after t0 it merged (this one merged at 30), its walk's end, and
// what it sent.
base string
at int
state string
modules []string
// Whether this one was held for a person before the later merge was delivered.
heldFirst bool
// What the forge says of the later merge containing this one: yes, no, or away until asked again.
forge string
want State
}{
{"carried every module, delivered after this one was held", "main", 35, walkDone,
[]string{"build-agent", "mesh-controller"}, true, "yes", Covered},
{"carried every module, delivered before this one was due to be held", "main", 35, walkDone,
[]string{"build-agent", "mesh-controller", "route-proxy"}, false, "yes", Covered},
{"carried one of its two modules", "main", 35, walkDone, []string{"mesh-controller"}, true, "yes", Held},
{"merged before this one", "main", 25, walkDone, []string{"build-agent", "mesh-controller"}, true, "yes", Held},
{"merged into another branch", "release", 35, walkDone, []string{"build-agent", "mesh-controller"}, true, "yes", Held},
{"its walk failed", "main", 35, walkFailed, []string{"build-agent", "mesh-controller"}, true, "yes", Held},
{"the forge says the later merge does not contain it", "main", 35, walkDone,
[]string{"build-agent", "mesh-controller"}, true, "no", Held},
{"the forge could not say at first, then said it does", "main", 35, walkDone,
[]string{"build-agent", "mesh-controller"}, true, "away", Covered},
} {
t.Run(c.name, func(t *testing.T) {
w := newWorld(t)
pair := merge + " in nnnnnnnnnnnn8888"
switch c.forge {
case "yes", "away":
w.forge.contains[pair] = true
case "no":
w.forge.contains[pair] = false
}
w.forge.containsDown = c.forge == "away"
p := pr(ctl, 1, head, "feat/one", "cmd/a.go")
w.h.PullUpdated(p)
w.h.Checked(verdictMoving(ctl, 1, head, "mesh-controller", "build-agent"))
w.later(31 * time.Minute)
w.h.PullMerged(mergedAt(p, merge, t0.Add(30*time.Minute)))
w.settleAll()
if c.heldFirst {
w.later(15 * time.Minute)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
if s := w.state(IDOf(ctl, head)); s != Held {
t.Fatalf("a merge no walk opened for is %s before the later one", s)
}
}
later := deliverLater(w, ctl, 2, c.base, "bbbbbbbbbbbb2222", "nnnnnnnnnnnn8888", t0.Add(time.Duration(c.at)*time.Minute),
c.state, c.modules...)
if c.state == walkDone && w.state(later) != Delivered {
t.Fatalf("the later merge is %s", w.state(later))
}
w.later(20 * time.Minute)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
if c.forge == "away" {
// A forge that cannot say is "not yet covered": held, and asked again a minute on.
if s := w.state(IDOf(ctl, head)); s != Held {
t.Fatalf("with the forge away it is %s", s)
}
w.forge.mu.Lock()
w.forge.containsDown = false
w.forge.mu.Unlock()
w.later(2 * time.Minute)
w.settleAll()
}
d := w.delivery(IDOf(ctl, head))
if d.State != c.want {
t.Fatalf("it is %s (%s), want %s", d.State, d.HeldWhy, c.want)
}
s := statusOn(w, ctl, head)
if c.want == Covered {
if !strings.HasPrefix(s, "success covered: a later merge that contains it was delivered") ||
!strings.Contains(s, "nnnnnnnn") {
t.Fatalf("its pull request's mesh/delivery says %q", s)
}
last := d.Transitions[len(d.Transitions)-1]
if !strings.Contains(last.Why, IDOf(ctl, "bbbbbbbbbbbb2222")) {
t.Fatalf("its transition does not name what covered it: %q", last.Why)
}
}
if strings.HasPrefix(s, "error") {
t.Fatalf("its pull request's mesh/delivery is an error: %q", s)
}
// Asked once for the pair, however many ticks read it — once more only after the forge was away.
asked := 0
for _, a := range w.forge.containsAsks {
if a == pair {
asked++
}
}
candidate := c.base == "main" && c.at > 30 && c.state == walkDone && len(c.modules) >= 2
switch {
case !candidate && asked != 0:
t.Fatalf("the forge was asked whether %s for a merge order and walk that cannot cover it", pair)
case candidate && c.forge != "away" && asked != 1:
t.Fatalf("the forge was asked %d time(s) whether %s, want once", asked, pair)
case candidate && c.forge == "away" && asked < 2:
t.Fatalf("the forge, away, was asked %d time(s) whether %s: never again", asked, pair)
}
})
}
}
// A covered member of a group counts as delivered: the member after it is let go, not stopped.
func TestACoveredGroupMemberLetsTheNextOneGo(t *testing.T) {
w := newWorld(t)
const ctl, lab = "novox/mesh-controller", "novox/mesh-lab"
const ctlHead, labHead = "c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2"
ctlID, labID := IDOf(ctl, ctlHead), IDOf(lab, labHead)
pc, pl := pr(ctl, 1, ctlHead, "feat/g", "cmd/a.go"), pr(lab, 2, labHead, "feat/g", "replays/a.go")
w.h.PullUpdated(pc)
w.h.PullUpdated(pl)
w.h.Checked(verdictMoving(ctl, 1, ctlHead, "mesh-controller"))
w.h.Checked(verdictMoving(lab, 2, labHead, "mesh-lab"))
w.settleAll()
w.h.Checked(CheckedEvent{Group: "feat/g", ID: "check-1", Verdict: "pass", Summary: "both compose"})
w.later(31 * time.Minute)
w.h.PullMerged(mergedAt(pc, "m1m1m1m1m1m1", t0.Add(30*time.Minute)))
w.h.PullMerged(mergedAt(pl, "m2m2m2m2m2m2", t0.Add(31*time.Minute)))
w.h.WalkMoved(w.ctl.put(aWalk("plan-lab", lab, "m2m2m2m2m2m2", true)))
w.settleAll()
w.forge.contains["m1m1m1m1m1m1 in m3m3m3m3m3m3"] = true
deliverLater(w, ctl, 3, "main", "e3e3e3e3e3e3e3e3", "m3m3m3m3m3m3", t0.Add(35*time.Minute), walkDone, "mesh-controller")
w.later(20 * time.Minute)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
if w.state(ctlID) != Covered {
t.Fatalf("the controller's member is %s", w.state(ctlID))
}
if w.state(labID) != Delivering || !containsString(w.ctl.delivers, "plan-lab") {
t.Fatalf("the member after it is %s, let go %v", w.state(labID), w.ctl.delivers)
}
lw := w.ctl.walk("plan-lab")
lw.State = walkDone
w.h.WalkMoved(w.ctl.put(lw))
w.settleAll()
if state, why := GroupState(w.h.groups["feat/g"], w.h.membersOf(w.h.groups["feat/g"])); state != "delivered" {
t.Fatalf("the group is %s: %s", state, why)
}
}
// **On rollout, the deliveries already held close by themselves** (issue 319's held ones), and a restart
// after that moves nothing again: what was read back is where it stays.
func TestHeldDeliveriesReadBackCloseByThemselvesOnce(t *testing.T) {
const ctl, cat = "novox/mesh-controller", "novox/mesh-catalog"
merged1, merged2 := t0, t0.Add(2*time.Hour)
coverer := func(repo, module string) *Delivery {
return &Delivery{ID: IDOf(repo, "c0c0c0c0c0c0c0c0"), Repository: repo, Commit: "c0c0c0c0c0c0c0c0", Number: 90,
Base: "main", State: Delivered, Since: merged2.Add(time.Hour), Created: merged2.Add(-time.Hour),
MergedAs: "f0f0f0f0f0f0f0f0", MergedAt: merged2, Plan: planMoving(repo, module), Check: &Verdict{Gate: "pass"},
Walk: &WalkSeen{ID: "plan-90", Commit: "f0f0f0f0f0f0f0f0", State: walkDone},
Steps: []Step{{Machine: "anchor", Module: module, State: StepPassed}, {Machine: "the rest running it", Module: module, State: StepSent}}}
}
held := func(repo string, plan *DeliveryPlan, why string) *Delivery {
return &Delivery{ID: IDOf(repo, head), Repository: repo, Commit: head, Number: 7, Base: "main", State: Held,
Since: merged1.Add(10 * time.Minute), Created: merged1.Add(-time.Hour), MergedAs: merge, MergedAt: merged1,
HeldWhy: why, Plan: plan, Transitions: []Transition{{At: merged1, From: Ready, To: Held, Event: EvHold, Why: why}}}
}
const unwalkedWhy = "merged, and the controller opened no walk for it within 10m0s — nothing it holds follows that branch, " +
"or the merge was not heard: a person decides"
withCoverer := func(d *Delivery, change func(c *Delivery)) []*Delivery {
c := coverer(d.Repository, d.Plan.Moved[0])
change(c)
return []*Delivery{d, c}
}
const uncheckedWhy = "merged without a passing check: only a person decides that it goes on"
for _, c := range []struct {
name string
kept []*Delivery
group *Group
want State
}{
{"merged without a passing check, builds nothing",
[]*Delivery{held("novox/hq", planMoving("novox/hq"), "merged without a passing check: only a person decides that it goes on")},
nil, Delivered},
{"a new module sent nowhere, no walk opened",
[]*Delivery{held(cat, &DeliveryPlan{New: []string{"modules/zfs"}, Summary: "builds new: modules/zfs, sent nowhere"}, unwalkedWhy)},
nil, Delivered},
{"no walk opened, a later delivered walk carried its module",
[]*Delivery{held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), coverer(ctl, "mesh-controller")}, nil, Covered},
{"merged without a passing check, a later delivered walk whose own check passed carried its module",
withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), uncheckedWhy), func(*Delivery) {}), nil, Covered},
{"merged without a passing check, the later delivery's own check did not pass",
withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), uncheckedWhy), func(c *Delivery) {
c.Check = &Verdict{Gate: "fail"}
}), nil, Held},
{"no walk opened, the later delivery's walk was of another commit",
withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), func(c *Delivery) {
c.Walk.Commit = "e0e0e0e0e0e0e0e0"
}), nil, Held},
{"no walk opened, the later walk sent its module to its first machine only",
withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), func(c *Delivery) {
c.Steps = c.Steps[:1]
}), nil, Held},
{"no walk opened, the later walk's branch is another",
withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), func(c *Delivery) {
c.Walk.Branch = "release"
}), nil, Held},
{"no walk opened, a later delivered walk carried another module",
[]*Delivery{held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), coverer(ctl, "route-proxy")}, nil, Held},
{"held for its group, its own walk superseded, a later delivered walk carried its module",
func() []*Delivery {
d := held(cat, planMoving(cat, "systemd"), "its group feat/j's composed check did not pass for the heads that "+
"merged; a person decides that it goes on")
d.Group = "feat/j"
d.Walk = &WalkSeen{ID: "plan-7", Commit: merge, State: walkSuperseded, Waits: true, WaitedFor: "mesh-delivery"}
other := &Delivery{ID: IDOf("novox/mesh-host", "abababababababab"), Repository: "novox/mesh-host",
Commit: "abababababababab", Number: 3, Base: "main", Group: "feat/j", State: Delivered,
MergedAs: "acacacacacacacac", MergedAt: merged1}
return []*Delivery{d, other, coverer(cat, "systemd")}
}(),
&Group{ID: "feat/j", Members: []string{IDOf(cat, head), IDOf("novox/mesh-host", "abababababababab")}, Closed: true,
Check: &GroupCheck{Asked: "check-1", Verdict: "pass", Commits: map[string]string{IDOf(cat, head): head,
IDOf("novox/mesh-host", "abababababababab"): "abababababababab"}}},
Covered},
} {
t.Run(c.name, func(t *testing.T) {
store := newMemStore()
for _, d := range c.kept {
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
}
if c.group != nil {
if err := store.PutGroup(c.group); err != nil {
t.Fatal(err)
}
}
now := merged2.Add(30 * time.Hour)
forge := newFakeForge()
forge.contains[merge+" in f0f0f0f0f0f0f0f0"] = true
w := newWorldOver(t, store, newFakeController(), forge, now)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
id := c.kept[0].ID
d := w.delivery(id)
if d.State != c.want {
t.Fatalf("read back held, it is %s, want %s", d.State, c.want)
}
if s := statusOn(w, d.Repository, d.Commit); c.want != Held && !strings.HasPrefix(s, "success") {
t.Fatalf("its pull request's mesh/delivery says %q", s)
}
// A restart over what it kept: nothing moves again.
again := newWorldOver(t, store, newFakeController(), forge, now.Add(time.Hour))
if err := again.h.Reconcile(); err != nil {
t.Fatal(err)
}
again.settleAll()
if d2 := again.delivery(id); d2.State != d.State || len(d2.Transitions) != len(d.Transitions) {
t.Fatalf("a restart moved it again: %s with %d transitions, was %s with %d", d2.State, len(d2.Transitions),
d.State, len(d.Transitions))
}
})
}
}
// **A module a release carried to every machine the held delivery sends it to covers it** (novox/hq issue
// 351). A build the controller held back for a person and released on their word reaches each machine by a
// backlog walk, which credits the delivery that published the build one step per machine — never the walk's
// own "every machine running it" or "the rest running it". The held delivery's plan says which machines
// receive the module: a passed step on each of them is the module sent everywhere. A machine still judging,
// one that failed, or a plan that names no machines leaves it held, as before.
func TestAModuleAReleaseCarriedToEveryMachineCovers(t *testing.T) {
const cat = "novox/mesh-catalog"
merged1, merged2 := t0, t0.Add(2*time.Hour)
laptop, desktop := "laptop", "desktop"
plan := func(machines ...string) *DeliveryPlan {
p := planMoving(cat, "openrazer")
for _, m := range machines {
p.Machines = append(p.Machines, MachinePlan{Machine: m, Receives: []string{"openrazer"}})
}
return p
}
held := func(p *DeliveryPlan) *Delivery {
why := "its group feat/j's composed check did not pass for the heads that merged; a person decides that it goes on"
return &Delivery{ID: IDOf(cat, head), Repository: cat, Commit: head, Number: 7, Base: "main", State: Held,
Since: merged1.Add(10 * time.Minute), Created: merged1.Add(-time.Hour), MergedAs: merge, MergedAt: merged1,
HeldWhy: why, Plan: p, Walk: &WalkSeen{ID: "plan-7", Commit: merge, State: walkSuperseded, Waits: true, WaitedFor: "mesh-delivery"},
Transitions: []Transition{{At: merged1, From: Published, To: Held, Event: EvHold, Why: why}}}
}
step := func(machine string, state StepState) Step {
return Step{Machine: machine, Module: "openrazer", Build: "build-1", State: state, Why: "carried by release-1"}
}
coverer := func(steps ...Step) *Delivery {
return &Delivery{ID: IDOf(cat, "c0c0c0c0c0c0c0c0"), Repository: cat, Commit: "c0c0c0c0c0c0c0c0", Number: 90,
Base: "main", State: Delivered, Since: merged2.Add(time.Hour), Created: merged2.Add(-time.Hour),
MergedAs: "f0f0f0f0f0f0f0f0", MergedAt: merged2, Plan: planMoving(cat, "power"), Check: &Verdict{Gate: "pass"},
Walk: &WalkSeen{ID: "plan-90", Commit: "f0f0f0f0f0f0f0f0", State: walkDone},
Steps: append([]Step{{Machine: "anchor", Module: "power", State: StepPassed},
{Machine: "the rest running it", Module: "power", State: StepSent}}, steps...)}
}
for _, c := range []struct {
name string
held *Delivery
by *Delivery
want State
}{
{"passed on every machine its plan sends it to", held(plan(laptop, desktop)),
coverer(step(laptop, StepPassed), step(desktop, StepPassed)), Covered},
{"still judging on one of them", held(plan(laptop, desktop)),
coverer(step(laptop, StepPassed), step(desktop, StepJudging)), Held},
{"failed on one of them", held(plan(laptop, desktop)),
coverer(step(laptop, StepPassed), step(desktop, StepFailed)), Held},
{"never sent to one of them", held(plan(laptop, desktop)), coverer(step(laptop, StepPassed)), Held},
{"its plan names no machines", held(plan()), coverer(step(laptop, StepPassed), step(desktop, StepPassed)), Held},
} {
t.Run(c.name, func(t *testing.T) {
store := newMemStore()
for _, d := range []*Delivery{c.held, c.by} {
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
}
forge := newFakeForge()
forge.contains[merge+" in f0f0f0f0f0f0f0f0"] = true
w := newWorldOver(t, store, newFakeController(), forge, merged2.Add(30*time.Hour))
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
d := w.delivery(c.held.ID)
if d.State != c.want {
t.Fatalf("it is %s (%s), want %s", d.State, d.HeldWhy, c.want)
}
if c.want == Covered {
if s := statusOn(w, cat, head); !strings.HasPrefix(s, "success covered: a later merge that contains it was delivered") {
t.Fatalf("its pull request's mesh/delivery says %q", s)
}
if last := d.Transitions[len(d.Transitions)-1]; !strings.Contains(last.Why, "openrazer") {
t.Fatalf("its transition does not name the module: %q", last.Why)
}
}
})
}
}
// **A finished release moves the steps it carried to passed** (novox/hq issue 351): a backlog walk's gate
// credits a step per machine while it judges, and its last record keeps no gate — only which machines it is
// done on. A step credited to it, on a machine it is done on, is passed; a step it failed on, one on a machine
// it is not done on, and a step another walk carried are left as they are.
func TestAFinishedReleaseMovesTheStepsItCarriedToPassed(t *testing.T) {
const cat = "novox/mesh-catalog"
carried := func(machine string, state StepState, by string) Step {
return Step{Machine: machine, Module: "openrazer", Build: "build-1", State: state, Why: "carried by " + by}
}
for _, c := range []struct {
name string
steps []Step
done []string
want map[string]StepState
}{
{"done on both", []Step{carried("laptop", StepJudging, "release-1"), carried("desktop", StepJudging, "release-1")},
[]string{"laptop", "desktop"}, map[string]StepState{"laptop": StepPassed, "desktop": StepPassed}},
{"done on one, the other still to come", []Step{carried("laptop", StepJudging, "release-1"), carried("desktop", StepJudging, "release-1")},
[]string{"laptop"}, map[string]StepState{"laptop": StepPassed, "desktop": StepJudging}},
{"a step it failed on stays failed", []Step{carried("laptop", StepFailed, "release-1")},
[]string{"laptop"}, map[string]StepState{"laptop": StepFailed}},
{"a step another walk carried is not its", []Step{carried("laptop", StepJudging, "release-2")},
[]string{"laptop"}, map[string]StepState{"laptop": StepJudging}},
} {
t.Run(c.name, func(t *testing.T) {
d := &Delivery{ID: IDOf(cat, "c0c0c0c0c0c0c0c0"), Repository: cat, Commit: "c0c0c0c0c0c0c0c0", Number: 90,
Base: "main", State: Delivered, Since: t0.Add(time.Hour), Created: t0, MergedAs: "f0f0f0f0f0f0f0f0", MergedAt: t0,
Plan: planMoving(cat, "power"), Check: &Verdict{Gate: "pass"},
Walk: &WalkSeen{ID: "plan-90", Commit: "f0f0f0f0f0f0f0f0", State: walkDone}, Steps: c.steps}
store := newMemStore()
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
w := newWorldOver(t, store, newFakeController(), newFakeForge(), t0.Add(2*time.Hour))
w.h.WalkMoved(Walk{ID: "release-1", Repository: "the builds waiting for a gate", State: walkDone,
Tiers: [][]string{{"openrazer"}}, Modules: map[string]*WalkModule{},
Release: &WalkRelease{Order: []string{"laptop", "desktop"}, Next: len(c.done), Done: c.done}})
got := map[string]StepState{}
for _, s := range w.delivery(d.ID).Steps {
got[s.Machine] = s.State
}
for machine, want := range c.want {
if got[machine] != want {
t.Fatalf("openrazer on %s is %s, want %s (steps %v)", machine, got[machine], want, got)
}
}
})
}
}
// The day's case whole (novox/hq issue 351): a member held for its group's check, its own walk taken over by
// later merges; the one later walk that carried its module failed its gate; a later merge's walk was delivered
// without the module; the controller then held the module's build back for a person, who released it, and the
// release reached both machines. The held delivery is covered by that later merge once the release is done —
// read by its id, since a walk that ended before the holder's restart may be older than the ended walks the
// controller lists back.
func TestAHeldDeliveryWhoseModuleAReleaseCarriedIsCoveredOnceTheReleaseIsDone(t *testing.T) {
const cat = "novox/mesh-catalog"
merged1, merged2 := t0, t0.Add(2*time.Hour)
plan := planMoving(cat, "openrazer")
plan.Machines = []MachinePlan{{Machine: "laptop", Receives: []string{"openrazer"}}, {Machine: "desktop", Receives: []string{"openrazer"}}}
why := "its group feat/j's composed check did not pass for the heads that merged; a person decides that it goes on"
held := &Delivery{ID: IDOf(cat, head), Repository: cat, Commit: head, Number: 7, Base: "main", State: Held,
Since: merged1.Add(10 * time.Minute), Created: merged1.Add(-time.Hour), MergedAs: merge, MergedAt: merged1,
HeldWhy: why, Plan: plan, Walk: &WalkSeen{ID: "plan-7", Commit: merge, State: walkSuperseded, Waits: true, WaitedFor: "mesh-delivery"},
Transitions: []Transition{{At: merged1, From: Published, To: Held, Event: EvHold, Why: why}}}
later := &Delivery{ID: IDOf(cat, "c0c0c0c0c0c0c0c0"), Repository: cat, Commit: "c0c0c0c0c0c0c0c0", Number: 90,
Base: "main", State: Delivered, Since: merged2.Add(time.Hour), Created: merged2.Add(-time.Hour),
MergedAs: "f0f0f0f0f0f0f0f0", MergedAt: merged2, Plan: planMoving(cat, "power"), Check: &Verdict{Gate: "pass"},
Walk: &WalkSeen{ID: "plan-90", Commit: "f0f0f0f0f0f0f0f0", State: walkDone},
Steps: []Step{{Machine: "anchor", Module: "power", State: StepPassed}, {Machine: "the rest running it", Module: "power", State: StepSent}}}
store := newMemStore()
for _, d := range []*Delivery{held, later} {
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
}
forge := newFakeForge()
forge.contains[merge+" in f0f0f0f0f0f0f0f0"] = true
w := newWorldOver(t, store, newFakeController(), forge, merged2.Add(3*time.Hour))
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
if s := w.state(held.ID); s != Held {
t.Fatalf("before any release it is %s", s)
}
// The release judges the build on each machine in turn: its gate names what it carried.
release := Walk{ID: "release-1", Repository: "the builds waiting for a gate", State: walkRolling,
Tiers: [][]string{{"openrazer"}}, Modules: map[string]*WalkModule{},
Release: &WalkRelease{Order: []string{"laptop", "desktop"}, Gate: &WalkGate{Machines: []string{"laptop"},
Carried: []WalkCarry{{Module: "openrazer", Node: "laptop", To: "f0f0f0f0f0f0f0f0", Build: "build-1"},
{Module: "openrazer", Node: "desktop", To: "f0f0f0f0f0f0f0f0", Build: "build-1"}}}}}
w.h.WalkMoved(release)
w.settleAll()
if s := w.state(held.ID); s != Held {
t.Fatalf("while the release judges it is %s", s)
}
// Done on both: its last record keeps no gate. It ended while this holder was down, and is older than the
// ended walks the controller lists back: the holder asks for it by its id, since a step it carried still judges.
release.State, release.Release.Gate, release.Release.Done, release.Release.Next = walkDone, nil, []string{"laptop", "desktop"}, 2
w.ctl.put(release)
w.later(time.Hour) // a walk a step still judges is asked for once an hour; at a start, at once
w.settleAll()
d := w.delivery(held.ID)
if d.State != Covered {
t.Fatalf("once the release is done it is %s (%s), want covered", d.State, d.HeldWhy)
}
if last := d.Transitions[len(d.Transitions)-1]; !strings.Contains(last.Why, IDOf(cat, "c0c0c0c0c0c0c0c0")) ||
!strings.Contains(last.Why, "openrazer") {
t.Fatalf("its transition does not name what covered it: %q", last.Why)
}
}
@@ -31,6 +31,15 @@ func transitionEvent(d *Delivery, t Transition, opened time.Time) map[string]any
if d.MergedAs != "" {
e["merged_as"] = d.MergedAs
}
if d.Walk != nil {
e["walk"] = d.Walk.ID
}
if d.Plan != nil {
e["plan"] = d.Plan.Summary
}
if c := fixClass(d); c != "" {
e["class"] = c
}
if !opened.IsZero() {
e["opened"] = opened.UTC()
}
@@ -40,6 +49,24 @@ func transitionEvent(d *Delivery, t Transition, opened time.Time) map[string]any
return e
}
// fixClass is the class of what a delivery moves (novox/hq ADR 0313 §2): its plan's, as the planner said it;
// none for a plan that moves no module, from a planner that did not say a class; else its walk's class once its
// delivery time is known; empty when nothing says it.
func fixClass(d *Delivery) string {
switch {
case d.Plan != nil && d.Plan.Class != "":
return d.Plan.Class
case d.Plan != nil && len(d.Plan.Moved) == 0 && len(d.Plan.Dependents) == 0 && len(d.Plan.New) == 0:
return classNone
case d.Time != nil:
return d.Time.Class
}
return ""
}
// classNone is the class of a delivery that moves no module: nothing for a walk to move, and no fix budget.
const classNone = "none"
// openedOf is when a delivery's pull request was first heard: the earliest head of it this owner holds, never
// one made from the forge's word at its merge; zero when none is held or it has no pull request. Under the lock.
func (h *Holder) openedOf(d *Delivery) time.Time {
@@ -69,50 +96,88 @@ func noteLine(d *Delivery, t Transition) string {
}
line := fmt.Sprintf("%s delivery %s: %s -> %s (%s%s): %s", t.At.UTC().Format(time.RFC3339), d.ID, orNothing(t.From),
t.To, t.Event, by, t.Why)
if d.Group != "" && (t.To == Proposed || t.To == Merged) {
if t.To == Proposed && d.Plan != nil {
line += "; predicted: " + d.Plan.Summary
}
if d.Group != "" && (t.To == Proposed || t.To == Published) {
line += "; group " + d.Group
}
return oneLine(line)
}
// executedLine is what a delivery did, written on the commit it landed as when it ends.
func executedLine(d *Delivery) string {
var parts []string
for _, s := range d.Steps {
parts = append(parts, fmt.Sprintf("%s on %s %s", s.Module, s.Machine, s.State))
}
walk := ""
if d.Walk != nil {
walk = " (walk " + d.Walk.ID + ")"
}
return oneLine(fmt.Sprintf("%s delivery %s executed%s: %s", time.Now().UTC().Format(time.RFC3339), d.ID, walk,
strings.Join(parts, "; ")))
}
func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }
// statusOf is the commit status `mesh/delivery`: where the delivery stands, in the forge's words. It says the merge's
// states only (ADR 0539 §17): whether a merged change reached the machines is the view's.
// statusOf is the commit status `mesh/delivery`: where the delivery stands, in the forge's words.
func statusOf(d *Delivery) (string, string) {
why := ""
if len(d.Transitions) > 0 {
why = d.Transitions[len(d.Transitions)-1].Why
}
switch d.State {
case Proposed:
case Proposed, Checked:
return "pending", "checking"
case Ready:
if d.holdsForReview() {
return "pending", clip("ready, "+d.Ready.Waits, 140) // novox/hq ADR 0496: no Approve is asked before a review
}
return "success", "ready: it is merged on the Approve"
return "success", "ready: it delivers once merged"
case Rejected:
return "failure", clip("rejected: "+why, 140)
case Merged:
return "success", clip("merged: "+d.sourceSaid(), 140)
case Published:
return "pending", "published: its walk waits for its turn"
case Held:
return "pending", clip("held for a person: "+d.HeldWhy, 140)
case Closed:
return "warning", clip("closed: "+why, 140)
case Delivering:
passed := 0
for _, s := range d.Steps {
if s.State == StepPassed {
passed++
}
}
return "pending", fmt.Sprintf("delivering: %d machine step(s) passed", passed)
case Delivered:
return "success", "delivered"
case Failed:
if d.coveredStop() {
return "success", clip(d.coveredStopSaid(), 140) // its change reached the machines (novox/hq issue 747)
}
return "failure", clip("failed: "+why, 140)
case Superseded:
if d.coveredStop() {
return "success", clip(d.coveredStopSaid(), 140) // its walk's taker delivered it (issue 1786)
}
return "warning", clip("superseded: "+why, 140)
case Stopped:
if d.coveredStop() {
return "success", clip(d.coveredStopSaid(), 140) // its change reached the machines (novox/hq issue 480)
}
return "error", clip("stopped: "+why, 140)
case Retired:
return "success", "history: merged before this owner delivered anything, never walked"
case Covered:
by := d.CoveredBy
if by == "" {
by = d.Carrier
}
return "success", clip("covered: a later merge that contains it was delivered: "+by, 140)
}
return "pending", string(d.State)
}
// sourceSaid names the source revision a merge moved: the trunk commit of its repository that source.main now holds
// (ADR 0539 §17: the gate's last word on a change; what the machines run is the view's).
func (d *Delivery) sourceSaid() string {
if d.MergedAs == "" {
return "not on the trunk"
}
return "source.main moved to " + d.Repository + "@" + shortOf(d.MergedAs, 12)
}
// viewMarker is how the forge's holder finds the one view comment of a pull request.
const viewMarker = "<!-- mesh-delivery:view -->"
@@ -120,12 +185,31 @@ const viewMarker = "<!-- mesh-delivery:view -->"
func ViewBody(d *Delivery, g *Group) string {
var b strings.Builder
fmt.Fprintf(&b, "%s\n**Delivery** `%s` — **%s** since %s\n\n", viewMarker, d.ID, d.State, d.Since.UTC().Format(time.RFC3339))
if d.HeldWhy != "" && d.State == Held {
fmt.Fprintf(&b, "Held for a person: %s — the merge reached the trunk without a passing check.\n\n", d.HeldWhy)
if d.coveredStop() {
fmt.Fprintf(&b, "**Covered**: %s\n\n", d.coveredStopSaid())
}
if d.MergedAs != "" {
fmt.Fprintf(&b, "**Source** — %s. Whether it reached the machines is `mesh-delivery-view`'s, derived from "+
"`source.main`, the wants and the haves.\n\n", d.sourceSaid())
if d.HeldWhy != "" && d.State == Held {
fmt.Fprintf(&b, "Held for a person: %s — `mesh-delivery.release` with why lets it go on.\n\n", d.HeldWhy)
}
if d.Plan != nil {
fmt.Fprintf(&b, "**Delivery plan** — %s\n", d.Plan.Summary)
for i, tier := range d.Plan.Tiers {
fmt.Fprintf(&b, "- tier %d: %s\n", i, strings.Join(tier, ", "))
}
for _, m := range d.Plan.Machines {
var parts []string
if len(m.Receives) > 0 {
parts = append(parts, "receives "+strings.Join(m.Receives, ", "))
}
if len(m.Waits) > 0 {
parts = append(parts, "waits for a person: "+strings.Join(m.Waits, ", "))
}
fmt.Fprintf(&b, "- %s: %s\n", m.Machine, strings.Join(parts, "; "))
}
for _, s := range d.Plan.Steps {
fmt.Fprintf(&b, "- %s\n", s)
}
b.WriteString("\n")
}
if g != nil {
fmt.Fprintf(&b, "**Group** `%s`: %s\n", g.ID, strings.Join(g.Members, ", "))
@@ -134,6 +218,17 @@ func ViewBody(d *Delivery, g *Group) string {
}
b.WriteString("\n")
}
if len(d.Steps) > 0 {
b.WriteString("**Machines**\n")
for _, s := range d.Steps {
fmt.Fprintf(&b, "- %s on %s: %s", s.Module, s.Machine, s.State)
if s.Why != "" {
fmt.Fprintf(&b, " — %s", s.Why)
}
b.WriteString("\n")
}
b.WriteString("\n")
}
b.WriteString("**Transitions**\n")
from := 0
if len(d.Transitions) > 12 {
@@ -142,7 +237,7 @@ func ViewBody(d *Delivery, g *Group) string {
for _, t := range d.Transitions[from:] {
fmt.Fprintf(&b, "- %s %s → %s (%s): %s\n", t.At.UTC().Format("2006-01-02 15:04"), orNothing(t.From), t.To, t.Event, t.Why)
}
b.WriteString("\nThe commit's note under `refs/notes/mesh-plan` keeps every transition of this change up to its merge: `git log --notes=mesh-plan`.\n")
b.WriteString("\nThe commit's note under `refs/notes/mesh-plan` keeps every transition: `git log --notes=mesh-plan`.\n")
return b.String()
}
@@ -1,62 +0,0 @@
package main
import (
"strings"
"testing"
"time"
)
// R6-C4 (ADR 0539 §17): the delivery's status and view say the merge's six states and nothing after the merge: no delivery plan,
// no machine step, no covered or walk line; a merged delivery's view names the source revision its merge moved.
func TestStatusAndViewSayEachOfTheSixStatesWithNoPlanAndNoMachineStep(t *testing.T) {
at := time.Date(2026, 10, 24, 12, 0, 0, 0, time.UTC)
base := Delivery{ID: "novox/app@c0ffee000000", Repository: "novox/app", Commit: "c0ffee000000", Number: 7, Since: at,
Transitions: []Transition{{At: at, From: Proposed, To: Rejected, Event: EvRefused, Why: "the gate said fail"}}}
for _, c := range []struct {
state State
mutate func(d *Delivery)
status string
desc string
inView []string
keepOut []string
}{
{Proposed, nil, "pending", "checking", nil, nil},
{Ready, nil, "success", "ready: it is merged on the Approve", nil, nil},
{Rejected, nil, "failure", "rejected: the gate said fail", nil, nil},
{Merged, func(d *Delivery) { d.MergedAs = "bbbbbbbbbbbb1111" }, "success",
"merged: source.main moved to novox/app@bbbbbbbbbbbb", []string{"**Source**", "source.main moved to novox/app@bbbbbbbbbbbb", "mesh-delivery-view"}, nil},
{Held, func(d *Delivery) { d.MergedAs, d.HeldWhy = "bbbbbbbbbbbb1111", "merged without a passing check" }, "pending",
"held for a person: merged without a passing check", []string{"Held for a person: merged without a passing check"}, nil},
{Closed, nil, "warning", "closed: the gate said fail", nil, nil},
} {
d := base
d.State = c.state
if c.mutate != nil {
c.mutate(&d)
}
state, desc := statusOf(&d)
if state != c.status || desc != c.desc {
t.Errorf("%s: the status is %s %q, wanted %s %q", c.state, state, desc, c.status, c.desc)
}
view := ViewBody(&d, nil)
if !strings.Contains(view, "**"+string(c.state)+"**") {
t.Errorf("%s: the view does not say the state: %s", c.state, view)
}
for _, want := range c.inView {
if !strings.Contains(view, want) {
t.Errorf("%s: the view lacks %q: %s", c.state, want, view)
}
}
for _, never := range []string{"**Delivery plan**", "**Machines**", "tier ", "receives", "walk", "Covered", "plans go", "mesh-delivery.release"} {
if strings.Contains(view, never) {
t.Errorf("%s: the view still says %q: %s", c.state, never, view)
}
}
}
// A merge not on the trunk names no source revision.
d := base
d.State = Merged
if s := d.sourceSaid(); s != "not on the trunk" {
t.Errorf("a merged delivery with no merge commit says %q", s)
}
}
@@ -42,7 +42,9 @@ func (h *Holder) takeForgeWanted() bool {
// AnswersChanged takes a change of the teller's answers: the answers to the asks pending are read at the look it asks for.
func (h *Holder) AnswersChanged() {
h.Poke()
if h.Holding() {
h.Poke()
}
}
// CommitStatus takes a commit status set on the forge: one of the merge checks (mesh/check and, on a
@@ -19,7 +19,7 @@ func hear(t *testing.T, w *world, key string, body any) {
if err != nil {
t.Fatal(err)
}
if key == shimCheckedKey {
if key == ControllerSeat+".checked" {
// The gate follows no verdict of the controller any more (issue 1318): a test says it as the checker's.
var c CheckedEvent
if err := json.Unmarshal(raw, &c); err != nil {
@@ -47,7 +47,7 @@ func TestAPassingVerdictAsksTheOperatorAtOnceWithNoMark(t *testing.T) {
if w.op.count() != 0 {
t.Fatalf("asked before the checks passed: %d", w.op.count())
}
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
if w.op.count() != 1 {
t.Fatalf("a pull request whose checks passed asked the operator %d times, wanted once, with no mark", w.op.count())
}
@@ -63,7 +63,7 @@ func TestAPassingVerdictAsksTheOperatorAtOnceWithNoMark(t *testing.T) {
func TestAFailedVerdictAsksNothing(t *testing.T) {
w := announced(t)
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "fail", "t1t1t1t1t1t1"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "fail", "t1t1t1t1t1t1"))
if w.op.count() != 0 {
t.Fatalf("a pull request whose checks failed asked the operator %d times", w.op.count())
}
@@ -78,7 +78,7 @@ func approve(w *world) {
func TestAnApproveHeardAsAnEventMergesAtOnceAtTheApprovedHead(t *testing.T) {
w := announced(t)
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
if len(w.op.open()) != 1 || len(w.forge.merges) != 0 {
t.Fatalf("asks %v, merges %v", w.op.open(), w.forge.merges)
}
@@ -95,7 +95,7 @@ func TestAnApproveHeardAsAnEventMergesAtOnceAtTheApprovedHead(t *testing.T) {
func TestAPassingVerdictAsksTheMergeAtOnceEvenJustAfterARefusal(t *testing.T) {
w := announced(t)
id := IDOf("novox/app", readyHead)
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
w.forge.refuse = errAskedAgain
approve(w)
if len(w.forge.merges) != 1 {
@@ -104,7 +104,7 @@ func TestAPassingVerdictAsksTheMergeAtOnceEvenJustAfterARefusal(t *testing.T) {
w.forge.refuse = nil
w.h.PullMerged(merged("novox/app", 9, "9999999999999999", "t2t2t2t2t2t2"))
w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go"))
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t2t2t2t2t2t2"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t2t2t2t2t2t2"))
if len(w.forge.merges) != 2 || w.delivery(id).Ready.Merged == nil {
t.Fatalf("the passing verdict did not ask the merge at once: %v %+v", w.forge.merges, w.delivery(id).Ready)
}
@@ -130,7 +130,7 @@ func TestAHeadNotReviewedAtThisHeadIsNotAsked(t *testing.T) {
p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go")
p.Body = c.body
w.h.PullUpdated(p)
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
if w.op.count() != 0 {
t.Fatalf("an unreviewed pull request was asked: %d", w.op.count())
}
@@ -144,7 +144,7 @@ func TestTheAskNamesTheReviewerOfTheHead(t *testing.T) {
p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go")
p.Body = "Fixes issue 7.\n\nReviewed-by: lead@shanks at " + readyHead + "\n"
w.h.PullUpdated(p)
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
a := w.op.ask(w.op.open()[0])
if strings.Contains(a.Explanation, "not reviewed") || !strings.Contains(a.Explanation, "reviewed by lead") {
t.Fatalf("a reviewed head is not said reviewed:\n%s", a.Explanation)
@@ -167,7 +167,7 @@ func TestTheReviewLineCorrectedInTheDescriptionIsReadWithoutAPush(t *testing.T)
// Issue 1022 (ADR 0421 §6): a push ends the Approve and says so once on the pull request.
func TestAPushAfterTheAskEndsItAndSaysSoOnceOnThePullRequest(t *testing.T) {
w := announced(t)
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
old := w.op.open()[0]
w.h.PullUpdated(pr("novox/app", 7, pushed, "feat/x", "src/a.go"))
w.h.React()
@@ -180,7 +180,7 @@ func TestAPushAfterTheAskEndsItAndSaysSoOnceOnThePullRequest(t *testing.T) {
t.Fatalf("the end of the Approve is not said once on the pull request: %v", w.forge.comments)
}
// The new head is asked on its own when its checks pass.
hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, pushed, "pass", "t1t1t1t1t1t1"))
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, pushed, "pass", "t1t1t1t1t1t1"))
if open := w.op.open(); len(open) != 1 || open[0] == old {
t.Fatalf("the new head was not asked afresh: %v", open)
}
@@ -63,7 +63,7 @@ func (s *memStore) Deliveries() ([]*Delivery, error) {
defer s.mu.Unlock()
var out []*Delivery
for key, raw := range s.deliveries {
if key == carriedKey { // the mark an earlier owner's carry-over wrote, as the real store skips it
if key == carriedKey { // the gate's mark that today's state was carried (seat.go), as the real store skips it
continue
}
var d Delivery
@@ -88,6 +88,299 @@ func (s *memStore) Groups() ([]*Group, error) {
return out, nil
}
type fakeController struct {
mu sync.Mutex
walks map[string]Walk
held bool
checks []string
asked int
delivers []string
// backupsAgain are the walks whose failed backups were asked again.
backupsAgain []string
// refuseBackupsAgain refuses the word to take a walk's backups again.
refuseBackupsAgain bool
stops []string
// withouts are the stops that left merges out (novox/hq ADR 0299): "<walk> without <merge>,…".
withouts []string
// refuseWithout refuses every stop with merges to leave out, as a controller before ADR 0299 does; mergedAt are
// the merges' times, by commit, for what a later merge contains.
refuseWithout bool
mergedAt map[string]time.Time
// refuseStop refuses every whole stop too; moves are the modules each merge moves, by commit, for what a left-out
// merge shares (ADR 0299).
refuseStop bool
moves map[string][]string
plans int
down bool
// emptyCheck makes Check answer with no id and no error, as the port does when the answer holds none.
emptyCheck bool
// walkReads counts the reads of the controller's walks (Walks).
walkReads int
// checkReady is the ready flag of each member of each check asked.
checkReady []bool
// baseSHAs is the base head each member of each check asked was sent with.
baseSHAs []string
// refuseDeliver refuses every word, as a controller that will not take it does (review of novox/hq issue 753).
refuseDeliver bool
// onCheck runs as a check is asked, before it is answered: what the forge held at that moment.
onCheck func()
// planned is what Plan answers instead of "builds app", when set.
planned *DeliveryPlan
// forgot are the pull requests whose check builds were let go of (novox/hq ADR 0350 §5), "owner/repo#n".
forgot []string
}
// BackupsAgain records the word, and clears the walk's failed backups as the controller does.
func (c *fakeController) BackupsAgain(walk, why string) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.down {
return errors.New("down")
}
if c.refuseBackupsAgain {
return errors.New("the controller refuses the backups")
}
w, known := c.walks[walk]
if !known || w.Backups == nil || w.Backups.State != "failed" {
return fmt.Errorf("%s holds no failed backup", walk)
}
w.Backups = nil
w.Revision++
c.walks[walk] = w
c.backupsAgain = append(c.backupsAgain, walk)
return nil
}
func (c *fakeController) ForgetChecks(owner, repo string, number int) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.down {
return errors.New("down")
}
c.forgot = append(c.forgot, fmt.Sprintf("%s/%s#%d", owner, repo, number))
return nil
}
func newFakeController() *fakeController {
return &fakeController{walks: map[string]Walk{}, held: true, mergedAt: map[string]time.Time{},
moves: map[string][]string{}}
}
func (c *fakeController) Plan(repository, base, head string, paths, dirs, removed []string) (*DeliveryPlan, error) {
c.mu.Lock()
defer c.mu.Unlock()
c.plans++
if c.down {
return nil, errors.New("down")
}
if c.planned != nil {
p := *c.planned
return &p, nil
}
return &DeliveryPlan{Repository: repository, Base: base, Head: head, Moved: []string{"app"}, Summary: "builds app"}, nil
}
func (c *fakeController) Deliver(walk, why string) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.down {
return errors.New("down")
}
if c.refuseDeliver {
return errors.New("the controller refuses the word")
}
w, known := c.walks[walk]
if !known || w.Delivery == nil || w.Delivery.Go != nil {
return fmt.Errorf("%s waits for nobody", walk)
}
now := time.Now()
w.Delivery.Go, w.Delivery.By, w.Delivery.Why = &now, byOwner, why
w.Revision++
c.walks[walk] = w
c.delivers = append(c.delivers, walk)
return nil
}
func (c *fakeController) Stop(walk, why, by string) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.down {
return errors.New("down")
}
c.stops = append(c.stops, walk)
if c.refuseStop {
return errors.New("the controller refuses")
}
if w, known := c.walks[walk]; known && !w.Open() {
// As the controller: only a walk that is building or rolling is stopped (`delivery-stop`).
return fmt.Errorf("%s is already %s", walk, w.State)
} else if known {
w.State = walkFailed
if w.Delivery == nil {
w.Delivery = &WalkDelivery{}
}
w.Delivery.Stopped, w.Delivery.StoppedWhy = by, why
w.Revision++
c.walks[walk] = w
}
return nil
}
// StopWithout ends a walk without the merges named, as the controller does (novox/hq ADR 0299): each merge named,
// and each later merge of its repository by merge time (mergedAt), is left out; the others are put in a batch,
// "<walk>-again", assembling until a test cuts it. Given a batch, the merges left out go to a stopped record,
// "<walk>-left", and the batch goes on with the others. refuseWithout refuses every such stop, as an older
// controller does.
func (c *fakeController) StopWithout(walk, why, by string, without []string) (StopAnswer, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.down {
return StopAnswer{}, errors.New("down")
}
c.withouts = append(c.withouts, walk+" without "+strings.Join(without, ","))
if c.refuseWithout {
return StopAnswer{}, errors.New(`delivery-stop takes no argument "without"`)
}
w, known := c.walks[walk]
if !known || !(w.Open() || w.IsBatch()) {
return StopAnswer{}, fmt.Errorf("%s is no open walk or batch", walk)
}
wd := WalkDelivery{}
if w.Delivery != nil {
wd = *w.Delivery
}
named := func(m WalkMerge) bool { return containsString(without, m.Repository+"@"+m.Commit) }
contains := func(m WalkMerge) string {
for _, s := range wd.Merges {
if named(s) && s.Repository == m.Repository && c.mergedAt[m.Commit].After(c.mergedAt[s.Commit]) {
return s.Commit
}
}
return ""
}
shares := func(m WalkMerge) (string, []string) {
for _, s := range wd.Merges {
if named(s) {
var both []string
for _, x := range c.moves[m.Commit] {
if containsString(c.moves[s.Commit], x) {
both = append(both, x)
}
}
if len(both) > 0 {
return s.Commit, both
}
}
}
return "", nil
}
a := StopAnswer{Stopped: walk}
var again, left []WalkMerge
for _, m := range wd.Merges {
s, both := shares(m)
switch {
case named(m):
a.LeftOut = append(a.LeftOut, WalkLeftOut{Repository: m.Repository, Commit: m.Commit})
left = append(left, m)
case len(both) > 0:
a.LeftOut = append(a.LeftOut, WalkLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: s, Shares: both})
left = append(left, m)
case contains(m) != "":
a.LeftOut = append(a.LeftOut, WalkLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains(m)})
left = append(left, m)
default:
m.Carried = ""
again = append(again, m)
a.WalkedAgain = append(a.WalkedAgain, m.Repository+"@"+m.Commit)
}
}
if len(a.LeftOut) == 0 {
return StopAnswer{}, fmt.Errorf("%s answers no merge %s", walk, strings.Join(without, ", "))
}
if w.IsBatch() && len(again) == 0 {
// Every merge left out: the batch itself is stopped, as the controller does.
wd.Stopped, wd.StoppedWhy, wd.LeftOut, wd.Batch = by, why, a.LeftOut, nil
w.Delivery, w.State = &wd, walkFailed
w.Revision++
c.walks[walk] = w
return a, nil
}
if w.IsBatch() {
rid := walk + "-left"
c.walks[rid] = Walk{ID: rid, Repository: left[0].Repository, Branch: "main", Commit: left[0].Commit,
Created: w.Created, State: walkFailed, Revision: 1, Modules: map[string]*WalkModule{},
Delivery: &WalkDelivery{Stopped: by, StoppedWhy: why, Merges: left, LeftOut: a.LeftOut, WalkedAgainBy: walk}}
wd.Merges = again
w.Delivery = &wd
w.Repository, w.Commit = again[0].Repository, again[0].Commit
w.Revision++
c.walks[walk] = w
a.Stopped, a.WalkedAgainBy = rid, walk
return a, nil
}
if len(again) > 0 {
id := walk + "-again"
closes := w.Created.Add(time.Minute)
c.walks[id] = Walk{ID: id, Repository: again[0].Repository, Branch: "main", Commit: again[0].Commit,
Created: w.Created.Add(time.Minute), State: walkAssembling, Revision: 1,
Modules: map[string]*WalkModule{}, Delivery: &WalkDelivery{Merges: again, Batch: &WalkBatch{ClosesAt: &closes}}}
wd.WalkedAgainBy, a.WalkedAgainBy = id, id
}
wd.Stopped, wd.StoppedWhy, wd.LeftOut = by, why, a.LeftOut
w.Delivery = &wd
w.State = walkFailed
w.Note = "stopped by " + by + ": " + why
w.Revision++
c.walks[walk] = w
return a, nil
}
// cut makes a batch the walk under the same id, waiting for its word, as the controller cuts it.
func (c *fakeController) cut(id string) Walk {
c.mu.Lock()
w := c.walks[id]
c.mu.Unlock()
wd := *w.Delivery
wd.Batch, wd.Awaits = nil, "mesh-delivery"
w.Delivery = &wd
w.State, w.Tiers, w.Modules = walkBuilding, [][]string{{"app"}}, map[string]*WalkModule{"app": {}}
return c.put(w)
}
func (c *fakeController) Walks(walk string) (bool, []Walk, error) {
c.mu.Lock()
defer c.mu.Unlock()
c.walkReads++
if c.down {
return false, nil, errors.New("down")
}
var out []Walk
for id, w := range c.walks {
if walk == "" || walk == id {
out = append(out, w)
}
}
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
return c.held, out, nil
}
// put is the controller keeping a walk; it answers the walk as `plan-moved` would say it.
func (c *fakeController) put(w Walk) Walk {
c.mu.Lock()
defer c.mu.Unlock()
if old, known := c.walks[w.ID]; known {
w.Revision = old.Revision + 1
} else if w.Revision == 0 {
w.Revision = 1
}
c.walks[w.ID] = w
return w
}
func (c *fakeController) walk(id string) Walk {
c.mu.Lock()
defer c.mu.Unlock()
return c.walks[id]
}
type fakeForge struct {
mu sync.Mutex
notes map[string][]string
@@ -276,6 +569,7 @@ type world struct {
t *testing.T
h *Holder
store *memStore
ctl *fakeController
forge *fakeForge
op *fakeOperator
merges *fakeMerges
@@ -290,12 +584,12 @@ type world struct {
}
func newWorld(t *testing.T) *world {
return newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC))
return newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC))
}
func newWorldOver(t *testing.T, store *memStore, forge *fakeForge, now time.Time) *world {
w := &world{t: t, store: store, forge: forge, now: now}
w.h = &Holder{Store: store, Forge: forge,
func newWorldOver(t *testing.T, store *memStore, ctl *fakeController, forge *fakeForge, now time.Time) *world {
w := &world{t: t, store: store, ctl: ctl, forge: forge, now: now}
w.h = &Holder{Store: store, Controller: ctl, Forge: forge,
Now: func() time.Time { w.mu.Lock(); defer w.mu.Unlock(); return w.now },
Logf: func(format string, a ...any) { t.Logf(format, a...); fmt.Fprintf(&w.logbuf, format+"\n", a...) },
Emit: func(event string, body any) error {
@@ -373,7 +667,7 @@ func pr(repo string, number int, sha, branch string, paths ...string) PullEvent
func verdict(repo string, number int, sha, gate string) CheckedEvent {
owner, name, _ := strings.Cut(repo, "/")
c := CheckedEvent{Owner: owner, Repo: name, Number: number, Commit: sha, Verdict: gate, Summary: "the gate said " + gate,
ID: "check-of-" + sha}
ID: "check-of-" + sha, Plan: &DeliveryPlan{Repository: repo, Moved: []string{"app"}, Summary: "builds app → anchor"}}
return c
}
@@ -385,6 +679,16 @@ func merged(repo string, number int, head, merge string) PullEvent {
return p
}
// aWalk is the controller's walk of a merge commit, waiting for its delivery's word.
func aWalk(id, repo, commit string, waits bool) Walk {
w := Walk{ID: id, Repository: repo, Branch: "main", Commit: commit, Created: time.Date(2026, 10, 6, 12, 31, 0, 0, time.UTC),
State: walkBuilding, Tiers: [][]string{{"app"}}, Modules: map[string]*WalkModule{"app": {}}}
if waits {
w.Delivery = &WalkDelivery{Awaits: "mesh-delivery"}
}
return w
}
// Comment records what was asked (novox/hq issue 722).
func (f *fakeForge) Comment(owner, repo string, number int, body string) error {
f.mu.Lock()
@@ -107,3 +107,15 @@ func TestARefusalLoopInAClosedGroupIsSaidAtOnce(t *testing.T) {
// check set its gate (issue 1049: the forge's "not a success: mesh/repo-check (pending)").
var ownPending = errors.New("the forge refused gitea_merge_pull_request: novox/mesh-controller#1 is not merged: main requires " +
"mesh/merge-gate, mesh/repo-check; not a success: mesh/repo-check (pending). A pull request merges once its checks succeed")
// ownChecks are the checks asked from index on that were a head's own (no group), not a group's composed check.
func ownChecks(w *world, from int) (own, composed []string) {
for _, c := range w.ctl.checks[from:] {
if strings.HasPrefix(c, ":") {
own = append(own, c)
} else {
composed = append(composed, c)
}
}
return
}
@@ -1,23 +0,0 @@
package main
import "time"
var t0 = time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
// mergedAt is the forge's `pull.merged` at a moment.
func mergedAt(p PullEvent, merge string, at time.Time) PullEvent {
p.MergeCommit = merge
p.MergedAt = at.UTC().Format(time.RFC3339)
return p
}
// verdictMoving is a passing verdict. The gate plans no delivery any more (ADR 0539 §17), so it names no module.
func verdictMoving(repo string, number int, sha string, _ ...string) CheckedEvent {
return verdict(repo, number, sha, "pass")
}
func statusOn(w *world, repo, commit string) string {
w.forge.mu.Lock()
defer w.forge.mu.Unlock()
return w.forge.statuses[repo+"@"+commit+" mesh/delivery"]
}
@@ -0,0 +1,227 @@
package main
import (
"errors"
sdk "git.novox.be/novox/mesh-sdk/go/delivery"
"strings"
"testing"
"time"
)
// History (novox/hq issue 313): merges long past, heard only from the forge's word when this owner first held
// the seat, reached the machines before it existed. They are retired — never held for a person, released or
// delivered — and what is owed for a pull request the forge no longer holds is given up, not tried for ever.
// aPastMerge is the forge's `pull.merged` replayed from the bus's history: merged days before it is heard.
func aPastMerge(repo string, number int, head, merge string, at time.Time) PullEvent {
p := merged(repo, number, head, merge)
p.MergedAt = at.UTC().Format(time.RFC3339)
return p
}
func TestAMergeLongPastHeardFromTheForgesWordIsRetiredAtAdoption(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
w.h.PullMerged(aPastMerge("novox/app", 274, head, merge, w.now.Add(-4*24*time.Hour)))
w.settleAll()
d := w.delivery(id)
if d.State != Retired || !strings.Contains(d.History, "history") {
t.Fatalf("a merge four days past heard from the forge's word is %s (%q)", d.State, d.History)
}
if got := statesOf([]State{d.Transitions[0].To, d.Transitions[1].To}); strings.Join(got, ",") != "proposed,retired" ||
d.Transitions[1].Event != EvHistory {
t.Fatalf("transitions %+v", d.Transitions)
}
// Nothing is said on its pull request — the forge may hold another under its number — and nothing stalls.
if w.forge.viewCalls != 0 || len(w.forge.statuses) != 0 {
t.Fatalf("history was shown on a pull request: %d view(s), statuses %v", w.forge.viewCalls, w.forge.statuses)
}
if len(d.Owed) != 0 {
t.Fatalf("still owed: %+v", d.Owed)
}
if s := w.h.Stalled(); len(s) != 0 {
t.Fatalf("history stalls: %+v", s)
}
// Its record is still kept, on the commit's note.
if len(w.forge.notesOn(head)) == 0 {
t.Fatal("no note of it on its commit")
}
// A merge heard from the forge's word within moments of happening is a live one: held for a person.
live := IDOf("novox/app", "bbbbbbbbbbbb2222")
w.h.PullMerged(aPastMerge("novox/app", 9, "bbbbbbbbbbbb2222", "cccccccccccc3333", w.now.Add(-time.Minute)))
if s := w.state(live); s != Held {
t.Fatalf("a live merge from the forge's word is %s", s)
}
// A merge long past whose walk the controller still holds open is delivered by it, never history.
walked := IDOf("novox/app", "dddddddddddd4444")
w.h.WalkMoved(aWalk("plan-9", "novox/app", "eeeeeeeeeeee5555", true))
w.h.PullMerged(aPastMerge("novox/app", 10, "dddddddddddd4444", "eeeeeeeeeeee5555", w.now.Add(-48*time.Hour)))
if s := w.state(walked); s == Retired {
t.Fatal("a merge with an open walk was retired as history")
}
// A merge the forge announces under a number an older delivery already merged with is not that delivery.
w.h.PullMerged(aPastMerge("novox/app", 274, "", "ffffffffffff6666", w.now.Add(-time.Minute)))
if d := w.delivery(id); d.MergedAs != merge {
t.Fatalf("a reused number merged the old delivery again as %s", d.MergedAs)
}
}
// oldHistory is a held delivery as the first adoption left it: made from the forge's word, held, owing a view
// on a pull request the forge no longer holds.
func oldHistory(id, repo string, number int, created, mergedAt time.Time) *Delivery {
commit := strings.SplitN(id, "@", 2)[1] + "0000"
return &Delivery{ID: id, Repository: repo, Commit: commit, Number: number, Base: "main", MergedAs: commit,
MergedAt: mergedAt, Created: created, Since: created, State: Held,
HeldWhy: "merged without a passing check: only a person decides that it goes on",
Transitions: []Transition{
{At: created, To: Proposed, Event: EvAnnounced, Why: repo + "#274 merged; its head was not heard by this " +
"owner, so it is made from the forge's word"},
{At: created, From: Proposed, To: Held, Event: EvMergedUnchecked, Why: "merged without a passing check"}},
Owed: []Effect{{Kind: EffectView, Since: created, Tries: 11051,
Last: "Gitea API /repos/" + repo + "/issues/274/comments: 500 issue does not exist"}}}
}
func TestRetireHistoryRetiresTheAdoptedHistoryInBulkAndLeavesEveryOtherHeldOne(t *testing.T) {
store := newMemStore()
start := time.Date(2026, 10, 6, 23, 5, 27, 0, time.UTC)
for _, d := range []*Delivery{
oldHistory("novox/hq@018ee359ae52", "novox/hq", 274, start, start.Add(-4*24*time.Hour)),
oldHistory("novox/lab@a4f6bb8ce054", "novox/lab", 53, start, start.Add(-2*time.Hour)),
// Made from the forge's word, but heard a minute after its merge: a live merge, a person's.
oldHistory("novox/hq@7777777777aa", "novox/hq", 3, start, start.Add(-time.Minute)),
} {
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
}
w := newWorldOver(t, store, newFakeController(), newFakeForge(), start.Add(36*time.Hour))
// And one heard live: rejected, then merged — a person's to decide.
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "fail"))
w.h.PullMerged(merged("novox/app", 7, head, merge))
if len(w.h.Stalled()) != 3 { // the three adopted at the switch; the one held just now is within its bound
t.Fatalf("stalled before: %+v", w.h.Stalled())
}
dry, err := w.h.RetireHistory("", "", "history adopted at the switch", aPerson, true)
if err != nil || dry.Retired != 2 || dry.LeftHeld != 2 || len(dry.Left) != 2 {
t.Fatalf("a dry retire: %+v %v", dry, err)
}
if s := w.state("novox/hq@018ee359ae52"); s != Held {
t.Fatalf("a dry retire moved a delivery to %s", s)
}
if _, err := w.h.RetireHistory("", "", "", aPerson, false); err == nil {
t.Fatal("a retire with no why was taken")
}
got, err := w.h.RetireHistory("", "", "history adopted at the switch", aPerson, false)
if err != nil || got.Retired != 2 || got.ByRepository["novox/hq"] != 1 || got.ByRepository["novox/lab"] != 1 {
t.Fatalf("retire: %+v %v", got, err)
}
for _, id := range []string{"novox/hq@018ee359ae52", "novox/lab@a4f6bb8ce054"} {
d := w.delivery(id)
if d.State != Retired || d.Transitions[len(d.Transitions)-1].By != aPerson {
t.Fatalf("%s is %s", id, d.State)
}
for _, e := range d.Owed {
if e.Kind == EffectView || e.Kind == EffectStatus {
t.Fatalf("%s still owes its pull request: %+v", id, e)
}
}
}
for _, id := range []string{"novox/hq@7777777777aa", IDOf("novox/app", head)} {
if s := w.state(id); s != Held {
t.Fatalf("%s, not history, is %s", id, s)
}
}
// Nothing was released, delivered or let go.
if len(w.ctl.delivers) != 0 || len(w.ctl.stops) != 0 {
t.Fatalf("a retire asked the controller: deliver %v stop %v", w.ctl.delivers, w.ctl.stops)
}
if s := w.h.Stalled(); len(s) != 1 || s[0].ID != "novox/hq@7777777777aa" {
t.Fatalf("stalled after: %+v", s)
}
// Retired is final: a second retire takes nothing, and nothing leaves it.
again, _ := w.h.RetireHistory("", "", "again", aPerson, false)
if again.Retired != 0 {
t.Fatalf("retired twice: %+v", again)
}
if _, err := w.h.Release("novox/hq@018ee359ae52", "go on", aPerson); err == nil {
t.Fatal("a retired delivery was released")
}
}
func TestRetireHistoryIsRefusedForARealHeldDelivery(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "fail"))
w.h.PullMerged(merged("novox/app", 7, head, merge))
w.later(48 * time.Hour)
_, err := w.h.RetireHistory(id, "", "looks old", aPerson, false)
var refused ErrRefused
if !errors.As(err, &refused) || !strings.Contains(err.Error(), "heard its head") {
t.Fatalf("a held delivery whose head was heard was retired: %v", err)
}
if s := w.state(id); s != Held {
t.Fatalf("a refused retire left it %s", s)
}
// A delivery that is not held is refused too, by the table.
other := IDOf("novox/app", "bbbbbbbbbbbb2222")
w.h.PullUpdated(pr("novox/app", 8, "bbbbbbbbbbbb2222", "feat/y", "src/b.go"))
if _, err := w.h.RetireHistory(other, "", "old", aPerson, false); err == nil || w.state(other) != Proposed {
t.Fatalf("a proposed delivery was retired: %v", err)
}
if _, err := w.h.RetireHistory("novox/app@nothing", "", "old", aPerson, false); err == nil {
t.Fatal("retired a delivery that does not exist")
}
}
// What the bus cannot take is owed and tried again later each time, and given up after retryLimit tries, saying so. What the
// forge refuses is the bridge's to say (issue 1349): the gate has no retry for it.
func TestOwedWorkTheBusCannotTakeIsTriedAgainLaterAndGivenUpAfterRetryLimit(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
w.owed.down = true
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
tries := func() int {
for _, e := range w.delivery(id).Owed {
if e.Kind == EffectView {
return e.Tries
}
}
return -1
}
w.h.Flush()
w.h.Flush()
w.h.Flush()
if tries() != 1 {
t.Fatalf("a view the bus could not take was tried %d time(s) at once", tries())
}
w.later(retryFirst + time.Second)
w.h.Flush()
if tries() != 2 {
t.Fatalf("a failed view was not tried again after its wait: %d", tries())
}
if retryAfter(1) != retryFirst || retryAfter(2) != 2*retryFirst || retryAfter(100) != retryCap {
t.Fatalf("the waits: %s %s %s", retryAfter(1), retryAfter(2), retryAfter(100))
}
for range retryLimit {
w.later(retryCap)
w.h.Flush()
}
if tries() != -1 {
t.Fatalf("a view failing %d times is still owed", tries())
}
if st := w.h.Status("listening"); !strings.Contains(strings.Join(st["refused"].([]string), "\n"), "given up") {
t.Fatalf("its drop was not said: %v", st["refused"])
}
// The bus back: what is owed anew is written.
w.owed.down = false
w.h.Checked(verdict("novox/app", 7, head, "pass"))
w.h.Flush()
if len(w.owed.acts(sdk.OwedView)) == 0 {
t.Fatal("nothing was written once the bus was back")
}
}
File diff suppressed because it is too large Load Diff
@@ -7,16 +7,17 @@ import (
"time"
)
// The holder, end to end over fakes (novox/hq ADR 0239 *How it is checked*, cut by ADR 0539 §17): a pull request's head
// proposed and ready; merged; a newer head closing the older; a restart resuming; the trunk rule; and every transition
// kept before it is said, noted on the commit and shown on the pull request.
// The holder, end to end over fakes (novox/hq ADR 0239 *How it is checked*): a pull request's head proposed,
// checked and ready; merged, published, let go, delivering and delivered; a gate that fails, put back and
// failed; a newer head superseding; a restart resuming; the trunk rule; and every transition kept before
// it is said, noted on the commit and shown on the pull request.
const (
head = "aaaaaaaaaaaa1111"
merge = "mmmmmmmmmmmm9999"
)
func TestAPullRequestsHeadIsProposedAndReady(t *testing.T) {
func TestAPullRequestsHeadIsProposedCheckedAndReady(t *testing.T) {
w := newWorld(t)
p := pr("novox/app", 7, head, "feat/x", "src/a.go")
p.Body = "" // an empty description is never read again, so the review set below stays
@@ -29,19 +30,19 @@ func TestAPullRequestsHeadIsProposedAndReady(t *testing.T) {
}
w.h.Checked(verdict("novox/app", 7, head, "pass"))
d := w.delivery(id)
if d.State != Ready {
t.Fatalf("a passing verdict left it %s", d.State)
if d.State != Ready || d.Plan == nil || d.Plan.Summary != "builds app → anchor" {
t.Fatalf("a passing verdict left it %s with plan %+v", d.State, d.Plan)
}
var path []State
for _, tr := range d.Transitions {
path = append(path, tr.To)
}
if strings.Join(statesOf(path), ",") != "proposed,ready" {
if strings.Join(statesOf(path), ",") != "proposed,checked,ready" {
t.Fatalf("its transitions are %v", path)
}
w.settleAll()
notes := w.forge.notesOn(head)
if len(notes) != 2 || !strings.Contains(notes[1], "proposed -> ready") {
if len(notes) != 3 || !strings.Contains(notes[2], "checked -> ready") {
t.Fatalf("the head's note is %q", notes)
}
if !strings.Contains(w.forge.statuses["novox/app@"+head+" mesh/delivery"], "success ready") {
@@ -50,9 +51,9 @@ func TestAPullRequestsHeadIsProposedAndReady(t *testing.T) {
if v := w.forge.views["novox/app#7"]; !strings.Contains(v, viewMarker) || !strings.Contains(v, "**ready**") {
t.Fatalf("the pull request's view is %q", v)
}
// A failing verdict for another head rejects it, and a newer head closes the older.
// A failing verdict for another head rejects it, and a newer head supersedes the older.
w.h.PullUpdated(pr("novox/app", 7, "bbbbbbbbbbbb2222", "feat/x"))
if w.state(id) != Closed {
if w.state(id) != Superseded {
t.Fatalf("a newer head left the older %s", w.state(id))
}
w.h.Checked(verdict("novox/app", 7, "bbbbbbbbbbbb2222", "fail"))
@@ -69,32 +70,58 @@ func statesOf(ss []State) []string {
return out
}
// A merge puts the delivery on the trunk and ends it there: merged, with the merge commit's note and the pull request's
// status naming the source revision it moved. Nothing after the merge is derived (ADR 0539 §17).
func TestAMergeIsMergedAndTheGateDerivesNothingAfterIt(t *testing.T) {
// A merge publishes; the walk waits for this owner's word; the word is given; the walk's record moves the
// delivery through its machines to delivered — and the note on the merge commit says what was executed.
func TestAMergeIsPublishedLetGoDeliveringAndDelivered(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "pass"))
// The walk is heard before the merge: it waits for its delivery.
walk := w.ctl.put(aWalk("plan-1", "novox/app", merge, true))
w.h.WalkMoved(walk)
if w.state(id) != Ready {
t.Fatalf("a walk for a commit no merge was heard of moved the delivery to %s", w.state(id))
}
w.h.PullMerged(merged("novox/app", 7, head, merge))
if w.state(id) != Published {
t.Fatalf("a merge with its walk open left it %s", w.state(id))
}
w.settleAll()
if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-1" {
t.Fatalf("its walk was let go %v", w.ctl.delivers)
}
if w.state(id) != Delivering {
t.Fatalf("let go, it is %s", w.state(id))
}
// The first machine sent, judged, passed; the rest sent; the walk done.
sent := time.Date(2026, 10, 6, 12, 40, 0, 0, time.UTC)
walk = w.ctl.walk("plan-1")
walk.State = walkRolling
walk.Modules["app"] = &WalkModule{State: "built", First: []string{"anchor"}, FirstAt: &sent, Build: "build-1",
Gate: &WalkGate{Machines: []string{"anchor"}, Since: &sent, Passes: 1}}
w.h.WalkMoved(w.ctl.put(walk))
if d := w.delivery(id); len(d.Steps) != 1 || d.Steps[0].State != StepJudging {
t.Fatalf("judging on its first machine reads %+v", d.Steps)
}
judged := sent.Add(3 * time.Minute)
walk.Modules["app"].Gate.Verdict, walk.Modules["app"].Gate.JudgedAt = "passed", &judged
walk.Modules["app"].SentAt = &judged
walk.State = walkDone
w.h.WalkMoved(w.ctl.put(walk))
d := w.delivery(id)
if d.State != Merged || d.MergedAs != merge {
t.Fatalf("a merge left it %s as %q", d.State, d.MergedAs)
if d.State != Delivered {
t.Fatalf("a walk done left it %s", d.State)
}
if len(d.Steps) != 2 || d.Steps[0].State != StepPassed || d.Steps[1].Machine != "the rest running it" {
t.Fatalf("its machine steps are %+v", d.Steps)
}
w.settleAll()
w.later(72 * time.Hour)
w.settleAll()
if s := w.state(id); s != Merged {
t.Fatalf("time moved a merged delivery to %s", s)
}
if st := w.forge.statuses["novox/app@"+head+" mesh/delivery"]; !strings.HasPrefix(st, "success merged: source.main moved to novox/app@"+merge[:12]) {
t.Fatalf("the pull request's mesh/delivery status is %q", st)
}
view := w.forge.views["novox/app#7"]
if !strings.Contains(view, "**Source**") || !strings.Contains(view, "source.main moved to novox/app@"+merge[:12]) ||
!strings.Contains(view, "mesh-delivery-view") || strings.Contains(view, "**Delivery plan**") || strings.Contains(view, "**Machines**") {
t.Fatalf("the pull request's view is %q", view)
onMerge := w.forge.notesOn(merge)
if len(onMerge) == 0 || !strings.Contains(strings.Join(onMerge, "\n"), "executed (walk plan-1): app on anchor passed") {
t.Fatalf("the merge commit's note is %q", onMerge)
}
// Said on the bus, kept first: every transition emitted once.
n := 0
for _, s := range w.said {
if s["id"] == id {
@@ -106,52 +133,110 @@ func TestAMergeIsMergedAndTheGateDerivesNothingAfterIt(t *testing.T) {
}
}
// The trunk rule: a ready delivery whose commit never reached the trunk is never merged.
func TestACommitOffTheTrunkIsNeverMerged(t *testing.T) {
// A first machine's gate fails: what it carried is put back (the controller's), the step is rolled back,
// and the delivery failed — never delivered.
func TestAFailedGateIsRolledBackAndTheDeliveryFails(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "pass"))
w.h.PullMerged(merged("novox/app", 7, head, merge))
w.h.WalkMoved(w.ctl.put(aWalk("plan-1", "novox/app", merge, true)))
w.settleAll()
sent := time.Date(2026, 10, 6, 12, 40, 0, 0, time.UTC)
walk := w.ctl.walk("plan-1")
walk.State = walkRolling
walk.Modules["app"] = &WalkModule{State: "built", First: []string{"anchor"}, FirstAt: &sent,
Gate: &WalkGate{Machines: []string{"anchor"}, Since: &sent}}
w.h.WalkMoved(w.ctl.put(walk))
at := sent.Add(10 * time.Minute)
walk.Modules["app"].Gate.Verdict, walk.Modules["app"].Gate.Why, walk.Modules["app"].Gate.JudgedAt =
"failed", "its tools were not served", &at
walk.Modules["app"].Gate.Rollback = "rolled-back"
walk.State = walkFailed
walk.Note = "app failed its gate on anchor"
w.h.WalkMoved(w.ctl.put(walk))
d := w.delivery(id)
if d.State != Failed || len(d.Steps) != 1 || d.Steps[0].State != StepRolledBack {
t.Fatalf("a failed gate left it %s, steps %+v", d.State, d.Steps)
}
// An older record arriving late changes nothing; a step the table does not hold is refused.
w.h.WalkMoved(aWalk("plan-1", "novox/app", merge, false))
if w.state(id) != Failed {
t.Fatal("an older record of the walk moved a final delivery")
}
}
// A walk superseded that names no walk taking it over (a record from before ADR 0276): superseded, from
// published or delivering. One that names its successor is followed (TestADeliveryFollowsTheWalkThatTookItsOver).
func TestANewerWalkSupersedesAnOlderDelivery(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "pass"))
w.h.PullMerged(merged("novox/app", 7, head, merge))
walk := aWalk("plan-1", "novox/app", merge, true)
w.h.WalkMoved(w.ctl.put(walk))
walk.State = walkSuperseded
walk.Note = "superseded"
w.ctl.down = true // its word could not be given meanwhile
w.settleAll()
w.h.WalkMoved(w.ctl.put(walk))
if s := w.state(id); s != Superseded {
t.Fatalf("its walk superseded left it %s", s)
}
}
// The trunk rule: a ready delivery whose commit never reached the trunk is never published, whatever walks
// are heard of; and a walk for a commit no delivery landed as is not taken as its.
func TestACommitOffTheTrunkIsNeverPublished(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "pass"))
w.h.WalkMoved(w.ctl.put(aWalk("plan-9", "novox/app", head, true))) // a walk of the head itself: no merge
w.settleAll()
if s := w.state(id); s != Ready {
t.Fatalf("an off-trunk head is %s", s)
}
if len(w.ctl.delivers) != 0 {
t.Fatalf("a walk was let go for a commit off the trunk: %v", w.ctl.delivers)
}
}
// A restart resumes: a new holder over the same state stands every delivery where it was.
// A restart resumes: a new holder over the same state stands every delivery where it was, reads the walks it
// missed back from the controller, and goes on.
func TestARestartResumesFromTheState(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "pass"))
w.h.PullMerged(merged("novox/app", 7, head, merge))
w.ctl.put(aWalk("plan-1", "novox/app", merge, true))
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
if s := w.state(id); s != Merged {
if s := w.state(id); s != Delivering {
t.Fatalf("before the restart it is %s", s)
}
again := newWorldOver(t, w.store, w.forge, w.now.Add(time.Hour))
if s := again.state(id); s != Merged {
// While it is down, the walk finishes and nobody says so.
walk := w.ctl.walk("plan-1")
walk.State = walkDone
w.ctl.put(walk)
again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Hour))
if s := again.state(id); s != Delivering {
t.Fatalf("read back, it is %s", s)
}
}
// A delivery kept in a state an earlier owner had and this one does not is read as the state it came to and kept so.
func TestADeliveryKeptInAGoneStateIsReadAsTheMergeStateItCameTo(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "pass"))
w.h.PullMerged(merged("novox/app", 7, head, merge))
w.settleAll()
raw := w.store.deliveries[id]
w.store.deliveries[id] = []byte(strings.Replace(string(raw), `"state":"merged"`, `"state":"delivering"`, 1))
again := newWorldOver(t, w.store, w.forge, w.now.Add(time.Hour))
if s := again.state(id); s != Merged {
t.Fatalf("a delivering delivery was read as %s", s)
if err := again.h.Reconcile(); err != nil {
t.Fatal(err)
}
again.settleAll()
if !strings.Contains(string(w.store.deliveries[id]), `"state":"merged"`) {
t.Fatalf("the read state was not kept: %s", w.store.deliveries[id])
if s := again.state(id); s != Delivered {
t.Fatalf("the walk it missed was not read back: %s", s)
}
if len(w.ctl.delivers) != 1 {
t.Fatalf("a restarted holder let the walk go again: %v", w.ctl.delivers)
}
}
@@ -180,7 +265,7 @@ func TestATransitionIsKeptBeforeItIsSaid(t *testing.T) {
w.later(time.Hour) // each failure put its next try later: past every one of them
w.settleAll()
w.settleAll()
if notes := w.forge.notesOn(head); len(notes) != 2 {
if notes := w.forge.notesOn(head); len(notes) != 3 {
t.Fatalf("after the bus came back the note has %d line(s): %q", len(notes), notes)
}
if d := w.delivery(IDOf("novox/app", head)); len(d.Owed) != 0 {
@@ -188,8 +273,66 @@ func TestATransitionIsKeptBeforeItIsSaid(t *testing.T) {
}
}
// At the switch: a walk already running is adopted as a delivery that is delivering; a waiting walk whose pull
// request this owner never heard of waits for a person, and a person's release lets it go.
func TestTheSwitchAdoptsRunningWalksAndHoldsUnknownWaitingOnes(t *testing.T) {
w := newWorld(t)
running := aWalk("plan-old", "novox/mesh-catalog", "cccccccccccc3333", false)
running.Created = w.now.Add(-time.Hour)
running.State = walkRolling
w.ctl.put(running)
waiting := aWalk("plan-new", "novox/app", "dddddddddddd4444", true)
waiting.Created = w.now.Add(time.Minute)
w.ctl.put(waiting)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.h.React()
if s := w.state(IDOf("novox/mesh-catalog", "cccccccccccc3333")); s != Delivering {
t.Fatalf("a running walk at the switch was adopted as %s", s)
}
if _, made := w.h.deliveries[IDOf("novox/app", "dddddddddddd4444")]; made {
t.Fatal("a new walk was made a delivery before its merge had time to be heard")
}
w.later(3 * time.Minute)
w.h.React()
held := IDOf("novox/app", "dddddddddddd4444")
if s := w.state(held); s != Held {
t.Fatalf("a waiting walk with no pull request is %s", s)
}
if _, err := w.h.Release(held, "", "jochen"); err == nil {
t.Fatal("released without why")
}
if _, err := w.h.Release(held, "checked by hand", "jochen"); err != nil {
t.Fatal(err)
}
if s := w.state(held); s != Delivering || len(w.ctl.delivers) != 1 {
t.Fatalf("released, it is %s and let go %v", s, w.ctl.delivers)
}
}
// mesh-delivery's controller down: nothing is lost — its word is asked again on the next tick.
func TestAControllerThatDoesNotAnswerIsAskedAgain(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "pass"))
w.h.PullMerged(merged("novox/app", 7, head, merge))
w.h.WalkMoved(w.ctl.put(aWalk("plan-1", "novox/app", merge, true)))
w.ctl.down = true
w.settleAll()
if s := w.state(id); s != Published {
t.Fatalf("with the controller down it is %s", s)
}
w.ctl.down = false
w.settleAll()
if s := w.state(id); s != Delivering {
t.Fatalf("with the controller back it is %s", s)
}
}
// A merge before the check passed is held for a person; a merge this owner never heard the head of is made
// from the forge's word and merged when the forge says its gate passed.
// from the forge's word and published when the forge says its gate passed.
func TestAMergeWithoutAPassingCheckIsHeld(t *testing.T) {
w := newWorld(t)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
@@ -205,38 +348,50 @@ func TestAMergeWithoutAPassingCheckIsHeld(t *testing.T) {
w.h.PullMerged(m)
w.settleAll()
d := w.delivery(IDOf("novox/lab", "eeeeeeeeeeee5555"))
if d.State != Merged {
// The fake planner says it moves app; with no walk yet it waits published.
if d.State != Ready && d.State != Published {
t.Fatalf("a merge whose head passed by the forge's word is %s", d.State)
}
}
// A group merged whole ends merged, member by member and as a group, and source.main holds its heads (ADR 0539 §17, R6-C2).
func TestAMergedGroupEndsMergedWithSourceMainMoved(t *testing.T) {
// A member that fails stops no other (ADR 0418 §5): the plan that fails fails as every plan does, and what was delivered
// stays. Nothing is stopped member by member.
func TestAFailedMemberStopsNoOtherMember(t *testing.T) {
w := newWorld(t)
heads := map[string]string{"novox/a": "a1a1a1a1a1a1a1", "novox/b": "b2b2b2b2b2b2b2"}
heads := map[string]string{"novox/a": "a1a1a1a1a1a1a1", "novox/b": "b2b2b2b2b2b2b2", "novox/c": "c3c3c3c3c3c3c3"}
ids := map[string]string{}
for i, repo := range []string{"novox/a", "novox/b"} {
for i, repo := range []string{"novox/a", "novox/b", "novox/c"} {
ids[repo] = IDOf(repo, heads[repo])
w.h.PullUpdated(pr(repo, i+1, heads[repo], "feat/y", "x"))
w.h.Checked(verdict(repo, i+1, heads[repo], "pass"))
}
w.settleAll()
w.h.Checked(CheckedEvent{Group: "feat/y", ID: "check-1", Verdict: "pass", Summary: "compose"})
for i, repo := range []string{"novox/a", "novox/b"} {
w.h.PullMerged(merged(repo, i+1, heads[repo], strings.Repeat(string(rune('p'+i)), 12)))
for i, repo := range []string{"novox/a", "novox/b", "novox/c"} {
mergeCommit := strings.Repeat(string(rune('p'+i)), 12)
w.h.PullMerged(merged(repo, i+1, heads[repo], mergeCommit))
w.h.WalkMoved(w.ctl.put(aWalk("plan-"+repo[6:], repo, mergeCommit, true)))
}
w.settleAll()
for repo, id := range ids {
if s := w.state(id); s != Merged {
t.Errorf("%s is %s, wanted merged", repo, s)
}
if len(w.ctl.delivers) != 3 {
t.Fatalf("every member merged, yet the walks were let go %v", w.ctl.delivers)
}
g := w.h.groups["feat/y"]
if state, why := GroupState(g, w.h.membersOf(g)); state != "merged" {
t.Fatalf("the group is %s (%s), wanted merged", state, why)
a := w.ctl.walk("plan-a")
a.State = walkDone
w.h.WalkMoved(w.ctl.put(a))
b := w.ctl.walk("plan-b")
b.State = walkFailed
b.Note = "b failed its gate on anchor"
w.h.WalkMoved(w.ctl.put(b))
w.settleAll()
if w.state(ids["novox/a"]) != Delivered || w.state(ids["novox/b"]) != Failed || w.state(ids["novox/c"]) == Stopped {
t.Fatalf("a %s, b %s, c %s", w.state(ids["novox/a"]), w.state(ids["novox/b"]), w.state(ids["novox/c"]))
}
if w.source.main == nil || w.source.main.Heads["novox/a"] != strings.Repeat("p", 12) || w.source.main.Heads["novox/b"] != strings.Repeat("q", 12) {
t.Fatalf("source.main is %+v, wanted both merges", w.source.main)
if len(w.ctl.stops) != 0 || len(w.ctl.withouts) != 0 {
t.Fatalf("a walk was stopped because another failed: %v %v", w.ctl.stops, w.ctl.withouts)
}
if state, _ := GroupState(w.h.groups["feat/y"], w.h.membersOf(w.h.groups["feat/y"])); state != "failed" {
t.Fatalf("the group is %s", state)
}
}
@@ -279,38 +434,44 @@ func TestAnAfterLineIsNotReadAndSaidOnceAndNothingWaitsForAnOrder(t *testing.T)
}
}
// Stalled: a delivery past its state's bound is listed with what H2 may do; close takes only that. Only a proposed
// delivery has a bound now: its check was asked and not answered.
// Stalled: a delivery past its state's bound is listed with what H2 may do; close takes only that.
func TestStalledAndCloseWorkFromTheTable(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "pass"))
w.h.PullMerged(merged("novox/app", 7, head, merge))
walk := w.ctl.put(aWalk("plan-1", "novox/app", merge, false))
w.h.WalkMoved(walk)
if w.state(id) != Delivering {
t.Fatalf("it is %s", w.state(id))
}
if _, err := w.h.Close(id, "too early"); err == nil {
t.Fatal("closed within its bound")
}
w.later(3 * time.Hour)
stalled := w.h.Stalled()
if len(stalled) != 1 || stalled[0].ID != id || stalled[0].State != Proposed {
if len(stalled) != 1 || stalled[0].ID != id || !strings.Contains(stalled[0].H2, "close") {
t.Fatalf("stalled %+v", stalled)
}
// A merge is the gate's last word: it is never stalled, however old.
w.h.Checked(verdict("novox/app", 7, head, "pass"))
w.h.PullMerged(merged("novox/app", 7, head, merge))
w.later(72 * time.Hour)
for _, s := range w.h.Stalled() {
if s.ID == id {
t.Fatalf("a merged delivery is stalled: %+v", s)
}
if _, err := w.h.Close(id, "delivery stalled"); err == nil {
t.Fatal("closed with nothing in the walk's record saying it moved")
}
if _, err := w.h.Close(id, "x"); err == nil {
t.Fatal("H2 closed a merged delivery")
walk.State = walkDone
w.ctl.put(walk) // finished, and the event was missed
said, err := w.h.Close(id, "delivery stalled")
if err != nil || w.state(id) != Delivered {
t.Fatalf("H2's close left it %s: %v", w.state(id), err)
}
// A merge without a passing check is held: a person's, with no transition for H2.
if !strings.Contains(said, "delivered") {
t.Fatalf("close says %q", said)
}
// Held is the operator's: H2 has no transition there.
w.h.PullUpdated(pr("novox/app", 8, "ffffffffffff0000", "feat/q"))
w.h.PullMerged(merged("novox/app", 8, "ffffffffffff0000", "eeeeeeeeeeee0000"))
w.later(48 * time.Hour)
if _, err := w.h.Close(IDOf("novox/app", "ffffffffffff0000"), "x"); err == nil {
t.Fatal("H2 closed a held delivery")
if _, err := w.h.Close(IDOf("novox/app", "ffffffffffff0000"), "x"); err == nil || !strings.Contains(err.Error(), "operator's") {
t.Fatalf("H2 closed a held delivery: %v", err)
}
}
@@ -359,3 +520,14 @@ func TestGroupsNamesEachMembersPullRequestHeadBaseAndOpen(t *testing.T) {
}
}
}
// composed are the asks of a group's composed check among the checks asked: a member's own check has no group before its colon.
func composed(checks []string) []string {
var out []string
for _, c := range checks {
if !strings.HasPrefix(c, ":") {
out = append(out, c)
}
}
return out
}
@@ -1,6 +1,7 @@
package main
import (
"strings"
"testing"
)
@@ -15,7 +16,40 @@ const (
branch1083 = "feat/backup-before-send"
)
// A closed record that was merged is not taken over, and the same pull request announced again stays as it was.
func TestAPullRequestAtACommitWhoseDeliveryAPersonStoppedTakesItOver(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head1083)
w.h.PullUpdated(pr(catalog783, 375, head1083, branch1083, "src/a.go"))
w.h.Checked(verdict(catalog783, 375, head1083, "pass"))
if w.state(id) != Ready {
t.Fatalf("its passing verdict left it %s", w.state(id))
}
if _, err := w.h.Stop(id, "to break the loop", "jochen"); err != nil {
t.Fatal(err)
}
// The person stopped it first; the pull request closed afterwards: a final state is not moved by the close.
w.h.PullClosed(pr(catalog783, 375, head1083, branch1083))
if d := w.delivery(id); d.State != Stopped || d.ClosedUnmerged {
t.Fatalf("setup: %s, closed unmerged %v; wanted stopped by the person alone", d.State, d.ClosedUnmerged)
}
w.h.PullUpdated(pr(catalog783, 376, head1083, branch1083+"-2", "src/a.go"))
d := w.delivery(id)
if d.State != Proposed || d.Number != 376 {
t.Fatalf("#376 at the same commit left it %s, Number %d; wanted proposed under #376", d.State, d.Number)
}
why := d.Transitions[len(d.Transitions)-1].Why
if !strings.Contains(why, "#375 with the same head was stopped by jochen: to break the loop") {
t.Fatalf("the take-over is not said, with who stopped it and why: %q", why)
}
w.h.Checked(verdict(catalog783, 376, head1083, "pass"))
if w.state(id) != Ready {
t.Fatalf("a passing verdict for #376 left it %s", w.state(id))
}
}
// A stopped record that was merged is not taken over, and the same pull request announced again stays stopped.
func TestAStoppedRecordThatWasMergedIsNotTakenOverByAnotherPullRequest(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783c+"1")
@@ -33,6 +67,9 @@ func TestATakenOverRecordLeavesTheGroupItWasStoppedIn(t *testing.T) {
w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go"))
w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json"))
id := IDOf(gCtl, gHc)
if _, err := w.h.Stop(id, "no", "jochen"); err != nil {
t.Fatal(err)
}
w.h.PullClosed(pr(gCtl, 1, gHc, "feat/g"))
w.h.PullUpdated(pr(gCtl, 9, gHc, "feat/other", "internal/a.go"))
w.h.mu.Lock()
@@ -1,11 +1,15 @@
package main
import (
"encoding/json"
"go/ast"
"go/parser"
"go/token"
"path/filepath"
"slices"
"strconv"
"strings"
"sync"
"testing"
sdk "git.novox.be/novox/mesh-sdk/go/delivery"
@@ -53,6 +57,7 @@ func TestANoteOnTheMergeCommitIsOwedWithNumberZero(t *testing.T) {
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "pass"))
w.h.PullMerged(merged("novox/app", 7, head, merge))
w.h.WalkMoved(w.ctl.put(aWalk("plan-1", "novox/app", merge, true)))
w.settleAll()
key, _ := sdk.OwedKey("novox/app", 0, merge, "note")
if n, there := w.owed.keys[key]; !there || n.Act != sdk.OwedNote {
@@ -81,15 +86,19 @@ func TestTwoCommentsOnOneHeadAreTwoOwedKeys(t *testing.T) {
}
}
// AC2 (R6-C1): the gate's code calls no gitea write tool and no controller verb at all (ADR 0539 §17: the gate asks the
// controller nothing, and holds no port to it). Tests are left out: they name the words to prove them gone.
func TestTheGatesCodeCallsNoForgeWriteToolAndNoControllerVerb(t *testing.T) {
// AC2: the gate's code calls no gitea write tool and no `delivery-plan`, and no controller verb beyond the ones its manifest
// names (ADR 0450 §7). Tests are left out: they name the words to prove them gone.
func TestTheGatesCodeCallsNoForgeWriteToolAndNoDeliveryPlan(t *testing.T) {
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
forbidden := []string{"delivery-plan", "gitea_commit_status\"", "gitea_delivery_view", "gitea_note_append", "gitea_add_comment",
"gitea_merge_pull_request\"", "gitea_open_pulls_write", "seat:mesh-controller", "plan-moved"}
"gitea_merge_pull_request\"", "gitea_open_pulls_write"}
// The controller's verbs the gate calls: ADR 0450 §7, less the checks and the queue, which the controller no longer
// keeps (issue 1318, ADR 0524 §5).
allowed := []string{"deliver", "delivery-stop", "delivery-backups-again", "delivery-walks", "plans"}
called := map[string]bool{}
fset := token.NewFileSet()
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
@@ -109,8 +118,79 @@ func TestTheGatesCodeCallsNoForgeWriteToolAndNoControllerVerb(t *testing.T) {
}
}
}
case *ast.CallExpr:
if sel, ok := x.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "verb" && len(x.Args) > 0 {
if lit, ok := x.Args[0].(*ast.BasicLit); ok {
verb, _ := strconv.Unquote(lit.Value)
called[verb] = true
if !slices.Contains(allowed, verb) {
t.Errorf("%s calls the controller's verb %q", f, verb)
}
}
}
}
return true
})
}
for _, verb := range []string{"plans"} {
if !called[verb] {
t.Errorf("the gate calls no %s: the test reads nothing", verb)
}
}
}
// stubController is a controller that answers every verb with one document, and keeps what it was asked.
type stubController struct {
mu sync.Mutex
keys []string
bodies []map[string]any
answer string
}
func (s *stubController) ask(key string, body any) (json.RawMessage, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.keys = append(s.keys, key)
b, _ := body.(map[string]any)
s.bodies = append(s.bodies, b)
return json.RawMessage(s.answer), nil
}
func (s *stubController) called(key string) []map[string]any {
s.mu.Lock()
defer s.mu.Unlock()
var out []map[string]any
for i, k := range s.keys {
if k == key {
out = append(out, s.bodies[i])
}
}
return out
}
// AC3: a delivery with no plan gets it from `plans`, asked as data, with the change's diffset.
func TestADeliveryWithNoPlanGetsItFromPlansAsData(t *testing.T) {
w := newWorld(t)
stub := &stubController{answer: `{"ok":true,"output":"","answer":{"asked":"check-1","held":true,"walks":[],
"plan":{"repository":"novox/app","base":"main","head":"` + head + `","moved":["app"],"summary":"builds app, sent to the node"}}}`}
w.h.Controller = seatController{ask: stub.ask}
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
c := verdict("novox/app", 7, head, "pass")
c.Plan = nil
w.h.Checked(c)
w.settleAll()
asked := stub.called("seat:mesh-controller.plans")
if len(asked) == 0 {
t.Fatalf("`plans` was not asked; the controller was asked %v", stub.keys)
}
if len(stub.called("seat:mesh-controller.delivery-plan")) != 0 {
t.Fatal("delivery-plan was asked")
}
got := asked[0]
if got["repository"] != "novox/app" || got["paths"] != "src/a.go" || got["data"] != "true" || got["head"] != head || got["base"] != "main" {
t.Fatalf("plans was asked %v; wanted the repository, the paths, the head, the base and data", got)
}
if d := w.delivery(IDOf("novox/app", head)); d.Plan == nil || d.Plan.Summary != "builds app, sent to the node" {
t.Fatalf("the delivery's plan is %+v", d.Plan)
}
}
@@ -0,0 +1,72 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 1581: a delivery stopped because its walk ended without building its modules (issue 474) waited for
// "a later delivered merge to carry" them, listed by `stalled` for good. When no later merge can: the pull request
// removed the module from the catalogue (mesh-catalog #384, scan), or the module is built for no machine (#391, "sent
// nowhere"), the stop reads ended, as covered reads covered (issue 480) — a reading of the stop, no transition — says
// why, and leaves `stalled`, so the condition that listed it ends by itself.
func strandedWith(t *testing.T, change func(d *Delivery)) (*world, string) {
w, id := strandedNight(t)
if len(w.h.Stalled()) != 1 {
t.Fatalf("before, stalled lists %+v", w.h.Stalled())
}
w.h.mu.Lock()
d := w.h.deliveries[id]
change(d)
if err := w.store.PutDelivery(d); err != nil {
t.Fatal(err)
}
w.h.mu.Unlock()
w.later(time.Minute)
w.settleAll()
return w, id
}
func TestAStrandedStopWhoseModuleIsSentNowhereEndsWithoutALaterMerge(t *testing.T) {
w, id := strandedWith(t, func(d *Delivery) {
d.Plan.Summary = "builds build-agent, sent nowhere; no bus step"
})
if s := w.h.Stalled(); len(s) != 0 {
t.Fatalf("sent nowhere, stalled still lists %+v", s)
}
shown, err := w.h.Show(id)
if err != nil {
t.Fatal(err)
}
reads, _ := shown.(map[string]any)["reads"].(string)
if !strings.HasPrefix(reads, "ended") || !strings.Contains(reads, "build-agent") {
t.Fatalf("show reads %q", reads)
}
if waits, _ := shown.(map[string]any)["waits"].(string); strings.Contains(waits, "later delivered merge") {
t.Fatalf("it still waits for %q", waits)
}
if d := w.delivery(id); d.State != Stopped {
t.Fatalf("a reading, not a transition: it is %s", d.State)
}
}
func TestAStrandedStopWhoseModuleThePullRequestRemovedEndsWithoutALaterMerge(t *testing.T) {
w, _ := strandedWith(t, func(d *Delivery) {
d.Removed = []string{"modules/build-agent/module.json", "modules/build-agent/cmd/main.go"}
})
if s := w.h.Stalled(); len(s) != 0 {
t.Fatalf("removed, stalled still lists %+v", s)
}
}
// A stop whose module still runs somewhere keeps waiting for a later merge, and stays listed.
func TestAStrandedStopWhoseModuleCanStillBeCarriedStaysListed(t *testing.T) {
w, _ := strandedWith(t, func(d *Delivery) {
d.Removed = []string{"modules/other/module.json"}
})
if s := w.h.Stalled(); len(s) != 1 {
t.Fatalf("stalled lists %+v", s)
}
}
@@ -98,8 +98,8 @@ func TestAnAnnouncementFillingAVerdictFirstHeadSupersedesTheOlderHeadAndItsAsk(t
w.settleAll()
w.h.PullUpdated(pr("novox/app", 419, pushed, "fix/x", "src/a.go"))
w.settleAll()
if got := w.delivery(a); got.NewerHead != pushed || got.State != Closed {
t.Fatalf("head A is %s with newer head %q; wanted closed by %s", got.State, got.NewerHead, pushed)
if got := w.delivery(a); got.NewerHead != pushed || got.State != Superseded {
t.Fatalf("head A is %s with newer head %q; wanted superseded by %s", got.State, got.NewerHead, pushed)
}
if got := w.delivery(b); got.Base != "main" || got.Branch != "fix/x" {
t.Fatalf("head B not filled: %+v", got)
@@ -1,75 +0,0 @@
package main
import (
"testing"
"time"
sdk "git.novox.be/novox/mesh-sdk/go/delivery"
)
// Issue 1726 (2026-10-24, mesh-catalog #471 and #472), kept in release 6 where no seat holder arrives to run a catch-up: the
// checker takes a repository's trunk head only from source.main, and the gate moves it only on a merge it hears. A merge marked
// in the source and never written (as a gate before the fix left #471) reaches source.main when the gate starts, once; one older
// than the head source.main holds never moves it back.
const (
i1726Repo = "novox/mesh-catalog"
i1726Head1 = "aaaa000000000000000000000000000000000001"
i1726Merge1 = "dddd000000000000000000000000000000000002"
i1726Merge2 = "bbbb000000000000000000000000000000000003"
)
func TestStartingWritesTheNewestMergeOfEachRepositoryNotYetInSourceMain(t *testing.T) {
w := newWorld(t)
w.h.PullUpdated(pr(i1726Repo, 470, i1726Head1, "fix/a", "modules/app/a.go"))
w.h.PullMerged(i888MergedAt(i1726Repo, 470, i1726Head1, i1726Merge1, w.now.Add(-time.Hour)))
const head2 = "cccc000000000000000000000000000000000001"
w.h.PullUpdated(pr(i1726Repo, 471, head2, "fix/b", "modules/app/b.go"))
w.h.PullMerged(i888MergedAt(i1726Repo, 471, head2, i1726Merge2, w.now.Add(-time.Minute)))
w.settleAll()
if got := w.source.main.Heads[i1726Repo]; got != i1726Merge2 {
t.Fatalf("the heard merges left source.main at %s", got)
}
// What a gate before the fix left: the merge marked in the source, source.main still at the merge before.
w.source.main = &sdk.Source{V: sdk.Version, Group: "before", Heads: map[string]string{i1726Repo: i1726Merge1, "novox/other": "b0"}}
w.settleAll()
if got := w.source.main.Heads[i1726Repo]; got != i1726Merge1 {
t.Fatalf("source.main moved to %s with no catch-up asked", got)
}
w.h.SourceCatchUp()
w.settleAll()
if got := w.source.main.Heads[i1726Repo]; got != i1726Merge2 || w.source.main.Heads["novox/other"] != "b0" {
t.Fatalf("source.main is %+v after the catch-up; wanted %s at %s and the other kept", w.source.main.Heads, i1726Repo, i1726Merge2)
}
n := len(w.source.moves)
w.h.SourceCatchUp()
w.settleAll()
if len(w.source.moves) != n {
t.Fatalf("a second catch-up moved source.main again: %d moves, then %d", n, len(w.source.moves))
}
o := newWorld(t)
o.source.main = &sdk.Source{V: sdk.Version, Group: "newer", Heads: map[string]string{i1726Repo: i1726Merge2}}
o.h.PullUpdated(pr(i1726Repo, 471, head2, "fix/b", "modules/app/b.go"))
o.h.PullMerged(i888MergedAt(i1726Repo, 471, head2, i1726Merge2, o.now.Add(-time.Minute)))
o.h.PullMerged(i888MergedAt(i1726Repo, 470, i1726Head1, i1726Merge1, o.now.Add(-time.Hour)))
o.settleAll()
o.h.SourceCatchUp()
o.settleAll()
if got := o.source.main.Heads[i1726Repo]; got != i1726Merge2 {
t.Fatalf("the catch-up moved source.main back to %s", got)
}
}
// A delivery kept with no merge time (adopted from a walk by an earlier owner) is dated by when the gate made it, so it does not
// make every older merge look newer, and the catch-up at the start weighs it.
func TestAMergeKeptWithNoMergeTimeIsDatedByWhenTheGateMadeIt(t *testing.T) {
d := &Delivery{MergedAs: i1726Merge2, Created: time.Date(2026, 10, 24, 12, 0, 0, 0, time.UTC)}
if got := mergedWhen(d); !got.Equal(d.Created) {
t.Fatalf("dated %s", got)
}
d.MergedAt = d.Created.Add(-time.Hour)
if got := mergedWhen(d); !got.Equal(d.MergedAt) {
t.Fatalf("dated %s with a merge time", got)
}
}
@@ -0,0 +1,126 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 1786: two merges whose walks the controller folded, before they were cut, into a batch that was
// still queued (not yet cut into a walk) ended superseded: the batch is no walk that answers a merge, so the follow
// loop took the walk's taken_over_by off ("the walk that took its own over does not answer it") and the table's
// supersession then held ("names no walk that took it over"). Delivered by the batch's walk, they showed
// superseded for good. A batch that took a walk over is waited for; and one already ended so follows its taker.
const (
head1786 = "1786178617861786178617861786178617861786"
merge1786 = "a1786a1786a1786a1786a1786a1786a1786a17860"
)
// folded1786 is a delivery whose walk was folded into a batch that is still queued.
func folded1786(t *testing.T) (*world, string) {
t.Helper()
w := newWorld(t)
id := IDOf("novox/app", head1786)
w.h.PullUpdated(pr("novox/app", 7, head1786, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head1786, "pass"))
w.h.PullMerged(merged("novox/app", 7, head1786, merge1786))
w.h.WalkMoved(w.ctl.put(aWalk("plan-own", "novox/app", merge1786, true)))
w.settleAll()
if s := w.state(id); s != Delivering {
t.Fatalf("setup: %s, want delivering", s)
}
closes := w.now.Add(time.Minute)
batch := Walk{ID: "plan-batch", Repository: "novox/other", Branch: "main", Commit: "ffff", Created: w.now, State: walkQueued,
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Batch: &WalkBatch{ClosesAt: &closes},
Merges: []WalkMerge{{Repository: "novox/app", Commit: merge1786}}}}
w.h.WalkMoved(w.ctl.put(batch))
folded := w.ctl.walk("plan-own")
folded.State = walkSuperseded
folded.Note = "folded into plan-batch before it was cut"
folded.Delivery.TakenOverBy = "plan-batch"
w.h.WalkMoved(w.ctl.put(folded))
w.settleAll()
return w, id
}
func cut1786(w *world, state string) {
b := w.ctl.walk("plan-batch")
b.State = state
b.Delivery.Batch = nil
b.Tiers, b.Modules = [][]string{{"app"}}, map[string]*WalkModule{"app": {}}
w.h.WalkMoved(w.ctl.put(b))
w.settleAll()
}
func TestAWalkFoldedIntoAQueuedBatchIsNotSupersededAndFollowsTheBatchWhenItIsCut(t *testing.T) {
w, id := folded1786(t)
if d := w.delivery(id); d.State.Final() || d.Walk.TakenOverBy != "plan-batch" {
t.Fatalf("folded into a queued batch it is %s, taken over by %q; wanted it still following plan-batch", d.State, d.Walk.TakenOverBy)
}
cut1786(w, walkBuilding)
cut1786(w, walkDone)
if s := w.state(id); s != Delivered {
t.Fatalf("the batch's walk is done and it is %s; wanted delivered", s)
}
}
func TestADeliverySupersededWhileItsTakerWasABatchIsHealedWhenTheTakerEndsDone(t *testing.T) {
w, id := folded1786(t)
d := w.h.deliveries[id]
// The state kept by the fault: superseded, its walk record naming the walk that took it over.
d.State = Superseded
d.Transitions = append(d.Transitions, Transition{At: w.now, From: Delivering, To: Superseded, Event: EvSuperseded})
if err := w.h.Store.PutDelivery(d); err != nil {
t.Fatal(err)
}
cut1786(w, walkBuilding)
cut1786(w, walkDone)
w.later(time30s)
w.settleAll()
d = w.delivery(id)
// A final state stays final (ADR 0239 §5): the taker's end is kept as DeliveredBy, as issue 442 and 747 do.
if d.State != Superseded || d.DeliveredBy == "" || !memberReached(d) || !d.coveredStop() {
t.Fatalf("superseded by the fault and its taker done, it is %s, delivered by %q, reached %v; wanted superseded, delivered by the taker, reached",
d.State, d.DeliveredBy, memberReached(d))
}
if st, _ := statusOf(d); st != "success" {
t.Fatalf("its status is %s; wanted success", st)
}
}
func TestAFinalDeliveryWhoseTakerDoesNotAnswerItIsNeitherHealedNorLoggedEveryPass(t *testing.T) {
w, id := folded1786(t)
d := w.h.deliveries[id]
d.State = Superseded
d.Transitions = append(d.Transitions, Transition{At: w.now, From: Delivering, To: Superseded, Event: EvSuperseded})
other := aWalk("plan-batch", "novox/other", "ffff", false)
other.State = walkDone
w.h.WalkMoved(w.ctl.put(other))
w.settleAll()
before := w.logs()
w.later(time30s)
w.settleAll()
if d := w.delivery(id); d.DeliveredBy != "" || d.State != Superseded {
t.Fatalf("healed by a walk that does not answer it: %s %q", d.State, d.DeliveredBy)
}
if n := strings.Count(w.logs(), "does not answer its merge") - strings.Count(before, "does not answer its merge"); n != 0 {
t.Fatalf("logged %d times on a pass over a final record", n)
}
}
func TestASupersededDeliveryWithNoWalkThatTookItOverStaysSuperseded(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head1786)
w.h.PullUpdated(pr("novox/app", 7, head1786, "feat/x", "src/a.go"))
w.h.PullUpdated(pr("novox/app", 7, "b"+head1786[1:], "feat/x", "src/a.go"))
w.settleAll()
if s := w.state(id); s != Superseded {
t.Fatalf("a head superseded by a newer one is %s", s)
}
w.later(time30s)
w.settleAll()
if s := w.state(id); s != Superseded {
t.Fatalf("it moved: %s", s)
}
}
@@ -0,0 +1,39 @@
package main
import "testing"
const gHk2 = "f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4"
func composedPassAt(w *world, hc, hk string) {
g := w.h.groups["feat/g"]
w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed",
Members: []struct {
Owner string `json:"owner"`
Repo string `json:"repo"`
Commit string `json:"commit"`
Trunk string `json:"trunk,omitempty"`
}{{Owner: "novox", Repo: "mesh-controller", Commit: hc, Trunk: gTrunk0[:12]},
{Owner: "novox", Repo: "mesh-catalog", Commit: hk, Trunk: gTrunk0[:12]}}})
}
// heldOnOwnPending is the group of issue 1792 after its first own ask: the member's own check asked, once.
func heldOnOwnPending(t *testing.T) *world {
w := newWorld(t)
w.op.auto = false
w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go"))
w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json"))
w.h.Checked(judged(gCtl, 1, gHc, "pass", gTrunk0))
w.h.Checked(judged(gCat, 2, gHk, "pass", gTrunk0))
w.settleAll()
setChecks(w, gCat, gHk, "success", "success")
setChecks(w, gCtl, gHc, "success", "pending")
asked := len(w.ctl.checks)
composedPassAt(w, gHc, gHk)
w.settleAll()
w.later(time30s)
w.settleAll()
if own, _ := ownChecks(w, asked); len(own) != 1 {
t.Fatalf("the first own ask: %v", own)
}
return w
}
@@ -5,36 +5,55 @@ import (
"time"
)
// novox/hq issue 326: a delivery whose state did not change is not written again — a pass over deliveries that did not
// move writes nothing — and one that changed is written, and said.
// novox/hq issue 326: a delivery whose state did not change is not written again — a reconcile pass over
// walks that did not move writes nothing — and one that changed is written, and said.
func TestADeliveryIsWrittenOnlyWhenItChanged(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head, "pass"))
w.ctl.put(aWalk("plan-1", "novox/app", merge, true))
w.h.PullMerged(merged("novox/app", 7, head, merge))
w.settleAll()
if w.state(id) != Ready {
t.Fatalf("it is %s", w.state(id))
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
// The word given moves the walk's record once more: read back, it is a change, and written.
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
if w.state(id) != Delivering {
t.Fatalf("let go, it is %s", w.state(id))
}
// Passes over the same deliveries, as every event asks for one: nothing changed, nothing written.
// Passes over the same walks, as main.go's tickers run them: nothing changed, nothing written.
before, said := w.store.puts, len(w.said)
for range 5 {
w.later(30 * time.Second)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
}
if w.store.puts != before {
t.Fatalf("five passes over deliveries that did not move wrote %d delivery(ies)", w.store.puts-before)
t.Fatalf("five passes over walks that did not move wrote %d delivery(ies)", w.store.puts-before)
}
if len(w.said) != said {
t.Fatalf("five passes over deliveries that did not move said %d event(s)", len(w.said)-said)
t.Fatalf("five passes over walks that did not move said %d event(s)", len(w.said)-said)
}
// The merge is heard: the delivery changes, is written, and is said.
w.h.PullMerged(merged("novox/app", 7, head, merge))
// The walk moves: the delivery changes, is written, and is said.
walk := w.ctl.walk("plan-1")
walk.State = walkDone
w.ctl.put(walk)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
if w.state(id) != Merged {
t.Fatalf("a merge left it %s", w.state(id))
if w.state(id) != Delivered {
t.Fatalf("a walk done left it %s", w.state(id))
}
if w.store.puts == before {
t.Fatal("a delivery that changed was not written")
@@ -44,8 +63,8 @@ func TestADeliveryIsWrittenOnlyWhenItChanged(t *testing.T) {
t.Fatal(err)
}
for _, d := range back {
if d.ID == id && d.State != Merged {
t.Fatalf("the store holds it %s, the holder %s", d.State, Merged)
if d.ID == id && d.State != Delivered {
t.Fatalf("the store holds it %s, the holder %s", d.State, Delivered)
}
}
if len(w.said) == said {
@@ -24,6 +24,9 @@ func judged(repo string, number int, sha, gate, trunk string) CheckedEvent {
c.ID = "check-of-" + sha + "-at-" + shortOf(trunk, 8)
c.Took = "4m12s"
c.Beside = map[string]string{"mesh-catalog": "0123456789ab"}
plan := *c.Plan
plan.Head, plan.Trunk = sha, c.Trunk
c.Plan = &plan
return c
}
@@ -41,6 +44,7 @@ func TestATrunkMoveSendsAReadyDeliveryBackToProposed(t *testing.T) {
if s := w.state(id66); s != Ready {
t.Fatalf("#66 checked green against the trunk is %s", s)
}
asked := len(w.ctl.checks)
// #64 merges: the trunk moves to its merge commit.
w.h.PullMerged(merged(host, 64, h64, merge1))
@@ -59,6 +63,9 @@ func TestATrunkMoveSendsAReadyDeliveryBackToProposed(t *testing.T) {
if s := w.state(id67); s != Rejected {
t.Fatalf("a rejected delivery went to %s on the trunk's move", s)
}
if len(w.ctl.checks) != asked {
t.Fatalf("the trunk's move asked a check: %v", w.ctl.checks[asked:])
}
// Out of date and not asked: no bound, so stalled does not list it, however long it waits.
w.later(5 * time.Hour)
@@ -68,6 +75,9 @@ func TestATrunkMoveSendsAReadyDeliveryBackToProposed(t *testing.T) {
t.Fatalf("stalled lists a proposed delivery whose check is out of date and not asked: %+v", l)
}
}
if len(w.ctl.checks) != asked {
t.Fatalf("an out-of-date check was asked by this owner: %v", w.ctl.checks[asked:])
}
// A late verdict for the old trunk is kept, does not count, and moves nothing: the table's fixed point.
before := len(w.delivery(id66).Transitions)
@@ -143,6 +153,9 @@ func TestChecksAndShowNameTheHeadAndTheTrunkCommit(t *testing.T) {
if d.Check.Head != head[:12] || d.Check.Trunk != trunk0[:12] {
t.Fatalf("show gives the verdict %+v", d.Check)
}
if d.Plan == nil || d.Plan.Trunk != trunk0[:12] || d.Plan.Head != head {
t.Fatalf("the kept delivery plan records %+v", d.Plan)
}
w.h.PullMerged(merged(repo, 8, "8888888888888888", merge1))
if a, _ = w.h.Checks(repo, 0, head); !strings.Contains(a.OutOfDate, merge1[:12]) {
@@ -224,3 +237,26 @@ func TestAnOutOfDateDeliveryAskedAgainIsBoundFromTheAsk(t *testing.T) {
t.Fatalf("judged again it is %s, asked again %v", d.State, d.AskedAgain)
}
}
// A check this owner asks — a recheck — sends the base head the forge last announced the head with, so the
// controller's verdict for a head no build judges names its trunk commit (ADR 0323 §3, review of mesh-controller#271).
func TestARecheckSendsTheAnnouncedBaseHead(t *testing.T) {
const repo = "novox/mesh-tools-go"
const head = "1a2b3c4d5e6f7a8b"
w := newWorld(t)
p := pr(repo, 3, head, "x")
p.BaseSHA = trunk0
w.h.PullUpdated(p)
id := IDOf(repo, head)
if m := MemberOf(w.delivery(id)); m.BaseSHA != trunk0 {
t.Fatalf("the member sends base head %q", m.BaseSHA)
}
p.BaseSHA = merge1 // announced again, by the merge tool
w.h.PullUpdated(p)
if m := MemberOf(w.delivery(id)); m.BaseSHA != merge1 {
t.Fatalf("announced again, the member sends base head %q", m.BaseSHA)
}
if raw := string(mustJSON(MemberOf(w.delivery(id)))); !strings.Contains(raw, `"base_sha":"`+merge1+`"`) {
t.Fatalf("the member as the controller reads it: %s", raw)
}
}
@@ -0,0 +1,434 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 362, ADR 0276: two merges of one repository 18 seconds apart, walked by one walk at the later
// merge's commit, which superseded the first. Every delivery the walk answers follows it and ends with it; no
// walk is tied to a delivery that ended; and what cannot move is said at once.
// A controller that names the merges a walk answers (ADR 0276): the forge is not asked, and the earlier merge is
// tied at once.
func TestAWalkThatNamesItsMergesIsFollowedByEachOfThem(t *testing.T) {
w := twoMerges(t)
walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst")
walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}}
w.h.WalkMoved(w.ctl.put(walk))
for _, id := range []string{IDOf(cat, headCC), IDOf(cat, headDu)} {
if d := w.delivery(id); d.State != Published || d.Walk == nil || d.Walk.ID != "plan-9" {
t.Fatalf("%s is %s on %v", id, d.State, d.Walk)
}
}
w.settleAll()
if len(w.forge.containsAsks) != 0 {
t.Fatalf("the forge was asked %v for a walk that names its merges", w.forge.containsAsks)
}
if len(w.ctl.delivers) != 1 {
t.Fatalf("the word was asked %v", w.ctl.delivers)
}
}
// A walk that failed marks no merge delivered: the merge it carried its own commit for fails, and the earlier
// merge it carried a later commit for waits for a walk of its own.
func TestAFailedWalkMarksNoMergeItCarriedDelivered(t *testing.T) {
w := twoMerges(t)
walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst")
walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
walk = w.ctl.walk("plan-9")
walk.State, walk.Note = walkFailed, "dunst failed its gate on anchor"
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
if s := w.state(IDOf(cat, headDu)); s != Failed {
t.Fatalf("dunst is %s", s)
}
cc := w.delivery(IDOf(cat, headCC))
if cc.State != Published || cc.State.Reached() {
t.Fatalf("claude-code is %s", cc.State)
}
// Its own walk, when the controller opens one, is followed.
own := catWalk("plan-10", mergeCC, atDu.Add(time.Hour), "claude-code")
w.h.WalkMoved(w.ctl.put(own))
w.settleAll()
w.h.WalkMoved(w.ctl.put(done(w.ctl.walk("plan-10"), w.now)))
w.settleAll()
if s := w.state(IDOf(cat, headCC)); s != Delivered {
t.Fatalf("claude-code on its own walk is %s", s)
}
}
// Never revived: a walk at the commit of a delivery that ended is not tied to it, not adopted, and not let go —
// it is listed by `stalled` as a walk no live delivery holds.
func TestAWalkIsNeverTiedToADeliveryThatEnded(t *testing.T) {
w := twoMerges(t)
first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst"))
w.h.WalkMoved(first)
w.h.WalkMoved(w.ctl.put(supersededBy(first, "")))
// A record from before ADR 0276: superseded, naming no walk — the delivery ends superseded.
sup := w.ctl.walk("plan-1")
sup.Note = "superseded"
w.h.WalkMoved(w.ctl.put(sup))
idDu := IDOf(cat, headDu)
if s := w.state(idDu); s != Superseded {
t.Fatalf("dunst is %s", s)
}
orphan := catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "dunst")
w.h.WalkMoved(w.ctl.put(orphan))
w.settleAll()
w.later(3 * time.Minute)
w.settleAll()
if d := w.delivery(idDu); d.State != Superseded || d.Walk.ID != "plan-1" {
t.Fatalf("the ended delivery is %s on %s", d.State, d.Walk.ID)
}
for _, asked := range w.ctl.delivers {
if asked == "plan-2" {
t.Fatal("a walk no live delivery holds was let go")
}
}
var found bool
for _, l := range w.h.Stalled() {
if l.State == Unheld && l.Walk == "plan-2" {
found = true
}
}
if !found {
t.Fatalf("stalled does not list the walk no delivery holds: %+v", w.h.Stalled())
}
}
// A merge no walk answers is held after ten minutes with the true reason, and listed by `stalled` at once; a walk
// that answers it later lifts it.
func TestAMergeNoWalkAnswersIsHeldTrulyAndSaidAtOnce(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC)
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
w.h.PullUpdated(cc)
w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code"))
w.h.PullMerged(mergedAt(cc, mergeCC, atCC))
// An open walk of the repository that moves something else: it does not answer this merge.
w.h.WalkMoved(w.ctl.put(catWalk("plan-5", "eeeeeeeeeeee5555", atCC.Add(time.Minute), "dunst")))
w.later(11 * time.Minute)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
id := IDOf(cat, headCC)
d := w.delivery(id)
if d.State != Held || !strings.Contains(d.HeldWhy, "plan-5") || !strings.Contains(d.HeldWhy, "name no merge") {
t.Fatalf("it is %s: %q", d.State, d.HeldWhy)
}
lines := w.h.Stalled()
if len(lines) != 1 || lines[0].ID != id || lines[0].Bound != "0s" {
t.Fatalf("stalled lists %+v", lines)
}
// The controller opens a walk that names it.
walk := catWalk("plan-6", mergeCC, w.now, "claude-code")
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
if s := w.state(id); s != Delivering || len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-6" {
t.Fatalf("with its walk it is %s, let go %v", s, w.ctl.delivers)
}
}
// One word per walk, and none while a delivery it carries is not at its turn.
func TestAWalkIsLetGoOnlyWhenEveryDeliveryItCarriesIsAtItsTurn(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second))
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json")
w.h.PullUpdated(cc)
w.h.PullUpdated(du)
w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code"))
// dunst merges unchecked: held for a person.
w.h.PullMerged(mergedAt(cc, mergeCC, atCC))
w.h.PullMerged(mergedAt(du, mergeDu, atDu))
walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst")
walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
if s := w.state(IDOf(cat, headDu)); s != Held {
t.Fatalf("dunst is %s", s)
}
if len(w.ctl.delivers) != 0 {
t.Fatalf("the walk was let go %v while it carries a held delivery", w.ctl.delivers)
}
if _, err := w.h.Release(IDOf(cat, headDu), "checked by hand", "jochen"); err != nil {
t.Fatal(err)
}
w.settleAll()
if len(w.ctl.delivers) != 1 {
t.Fatalf("released, the word was asked %v", w.ctl.delivers)
}
}
// A merge in a batch still assembling is said so, is not held for a person, and nothing is let go.
func TestAMergeInAnAssemblingBatchIsShownAndNotHeld(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC)
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
w.h.PullUpdated(cc)
w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code"))
w.h.PullMerged(mergedAt(cc, mergeCC, atCC))
closes, atMost := atCC.Add(90*time.Second), atCC.Add(10*time.Minute)
batch := Walk{ID: "plan-b1", Repository: cat, Branch: "main", Commit: mergeCC, Created: atCC, State: walkAssembling,
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: cat, Commit: mergeCC}},
Batch: &WalkBatch{ClosesAt: &closes, AtMost: &atMost}}}
w.h.WalkMoved(w.ctl.put(batch))
w.later(48 * time.Second)
w.settleAll()
id := IDOf(cat, headCC)
var line Line
for _, l := range w.h.Deliveries("", "", "", true) {
if l.ID == id {
line = l
}
}
if !strings.Contains(line.Waits, "plan-b1: assembling, 42s left") || !strings.Contains(line.Waits, "plan not yet calculated") {
t.Fatalf("it waits for %q", line.Waits)
}
// Queued behind a walk for longer than a walk opens within: still not held.
batch = w.ctl.walk("plan-b1")
batch.State, batch.Delivery.Batch.Behind = walkQueued, "plan-7"
w.h.WalkMoved(w.ctl.put(batch))
w.later(20 * time.Minute)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
if d := w.delivery(id); d.State != Ready || d.Walk != nil {
t.Fatalf("a merge in a queued batch is %s on %v", d.State, d.Walk)
}
if len(w.ctl.delivers) != 0 {
t.Fatalf("a batch was let go: %v", w.ctl.delivers)
}
// Cut: the batch is a walk now, under its id.
batch.State, batch.Delivery.Batch = walkBuilding, nil
batch.Tiers, batch.Modules = [][]string{{"claude-code"}}, map[string]*WalkModule{"claude-code": {}}
w.h.WalkMoved(w.ctl.put(batch))
w.settleAll()
if d := w.delivery(id); d.State != Delivering || d.Walk.ID != "plan-b1" {
t.Fatalf("cut, it is %s on %v", d.State, d.Walk)
}
}
// An older record of the walk a delivery no longer follows, arriving late, does not take it back.
func TestALateRecordOfAnOlderWalkDoesNotTakeADeliveryBack(t *testing.T) {
w := twoMerges(t)
w.forge.contains[mergeCC+" in "+mergeDu] = true
first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst"))
w.h.WalkMoved(first)
w.h.WalkMoved(w.ctl.put(catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst")))
w.h.WalkMoved(first) // the older record, again
if d := w.delivery(IDOf(cat, headDu)); d.Walk.ID != "plan-2" {
t.Fatalf("dunst went back to %s", d.Walk.ID)
}
}
// A walk no merge was heard for, at the commit of a delivery that ended, is not adopted by that delivery: it is
// listed by `stalled` as a walk no live delivery holds.
func TestAnUnheardWalkIsNotAdoptedByADeliveryThatEnded(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC)
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
w.h.PullUpdated(cc)
w.h.PullClosed(cc)
id := IDOf(cat, headCC)
if s := w.state(id); s != Stopped {
t.Fatalf("closed unmerged it is %s", s)
}
w.h.WalkMoved(w.ctl.put(catWalk("plan-3", headCC, atCC.Add(time.Minute), "claude-code")))
w.later(3 * time.Minute)
w.settleAll()
w.later(3 * time.Minute)
w.settleAll()
if d := w.delivery(id); d.State != Stopped || d.Walk != nil {
t.Fatalf("the stopped delivery is %s on %v", d.State, d.Walk)
}
var found bool
for _, l := range w.h.Stalled() {
found = found || l.State == Unheld && l.Walk == "plan-3"
}
if !found || len(w.ctl.delivers) != 0 {
t.Fatalf("stalled lists %+v; let go %v", w.h.Stalled(), w.ctl.delivers)
}
}
// Two members of one delivery group carried by one walk: let go once, for both (review of #171: the group's turn
// stopped at its first member, and the walk waited for the second for ever).
func TestAWalkCarryingTwoMembersOfAGroupIsLetGoOnce(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second))
a := pr("novox/a", 1, "aaaaaaaaaaaa0001", "feat/y", "x.go")
b := pr("novox/b", 2, "bbbbbbbbbbbb0002", "feat/y", "y.go")
w.h.PullUpdated(a)
w.h.PullUpdated(b)
w.h.Checked(verdictMoving("novox/a", 1, "aaaaaaaaaaaa0001", "app-a"))
w.h.Checked(verdictMoving("novox/b", 2, "bbbbbbbbbbbb0002", "app-b"))
w.settleAll()
g := w.h.groups["feat/y"]
if g == nil || g.Check == nil {
t.Fatalf("the group is %+v", g)
}
w.h.Checked(CheckedEvent{Group: "feat/y", ID: g.Check.Asked, Verdict: "pass", Summary: "composed"})
w.h.PullMerged(mergedAt(a, "aaaaaaaaaaaa1111", atCC))
w.h.PullMerged(mergedAt(b, "bbbbbbbbbbbb2222", atDu))
batch := Walk{ID: "plan-batch", Repository: "novox/a", Branch: "main", Commit: "aaaaaaaaaaaa1111",
Created: atDu.Add(time.Minute), State: walkBuilding, Tiers: [][]string{{"app-a"}, {"app-b"}},
Modules: map[string]*WalkModule{"app-a": {}, "app-b": {}},
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
{Repository: "novox/a", Commit: "aaaaaaaaaaaa1111"}, {Repository: "novox/b", Commit: "bbbbbbbbbbbb2222"}}}}
w.h.WalkMoved(w.ctl.put(batch))
w.settleAll()
if got := w.ctl.delivers; len(got) != 1 || got[0] != "plan-batch" {
t.Fatalf("the word was asked %v, want once for plan-batch", got)
}
for _, id := range []string{IDOf("novox/a", "aaaaaaaaaaaa0001"), IDOf("novox/b", "bbbbbbbbbbbb0002")} {
if s := w.state(id); s != Delivering {
t.Fatalf("%s is %s", id, s)
}
}
}
// A walk that names its merges answers only those: a delivery whose walk it took over and which it does not name
// is not tied to it, and ends superseded once that is known.
func TestAWalkThatNamesItsMergesAnswersOnlyThose(t *testing.T) {
w := twoMerges(t)
first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst"))
w.h.WalkMoved(first)
other := catWalk("plan-2", "ffffffffffff9999", atDu.Add(6*time.Second), "dunst")
other.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: "ffffffffffff9999"}}
w.h.WalkMoved(w.ctl.put(other))
w.h.WalkMoved(w.ctl.put(supersededBy(first, "plan-2")))
w.settleAll()
d := w.delivery(IDOf(cat, headDu))
if d.State != Superseded || d.Walk.ID != "plan-1" {
t.Fatalf("dunst is %s on %s", d.State, d.Walk.ID)
}
for _, asked := range w.ctl.delivers {
if asked == "plan-2" {
t.Fatal("a walk that does not name dunst was let go for it")
}
}
}
// The forge's containment ties a merge to a walk only when the walk moves everything the merge moves and was made
// for a later merge.
func TestContainmentAloneDoesNotTieAMerge(t *testing.T) {
for _, c := range []struct {
name string
modules []string
created time.Time
names bool
}{
{"the later walk moves only its own module", []string{"dunst"}, atDu.Add(5 * time.Second), false},
{"the walk was made for an earlier merge", []string{"claude-code", "dunst"}, atCC.Add(-time.Minute), false},
{"the walk names its merges, and not this one", []string{"claude-code", "dunst"}, atDu.Add(5 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second))
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
w.h.PullUpdated(cc)
w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code"))
w.h.PullMerged(mergedAt(cc, mergeCC, atCC))
w.forge.contains[mergeCC+" in "+mergeDu] = true
walk := catWalk("plan-2", mergeDu, c.created, c.modules...)
if c.names {
walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeDu}}
}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
if d := w.delivery(IDOf(cat, headCC)); d.Walk != nil || len(w.forge.containsAsks) != 0 {
t.Fatalf("it follows %v; the forge was asked %v", d.Walk, w.forge.containsAsks)
}
})
}
}
// After the walk that carried a later commit for it failed, a merge stays published — once — and is said within
// the ten minutes a walk opens within, naming what happened.
func TestAMergeWhoseCarryingWalkFailedWaitsOnceAndIsSaid(t *testing.T) {
w := twoMerges(t)
walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst")
walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
walk = w.ctl.walk("plan-9")
walk.State = walkFailed
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
id := IDOf(cat, headCC)
d := w.delivery(id)
rewalks := 0
for _, tr := range d.Transitions {
if tr.Event == EvRewalk {
rewalks++
}
}
if d.State != Published || rewalks != 1 {
t.Fatalf("it is %s after %d rewalk(s)", d.State, rewalks)
}
w.later(11 * time.Minute)
var line *StalledLine
for _, l := range w.h.Stalled() {
if l.ID == id {
l := l
line = &l
}
}
if line == nil || !strings.Contains(line.Says, "no walk of its own commit") {
t.Fatalf("stalled says %+v", line)
}
}
// A delivery that ended is never tied to a walk after: not one that answers its merge, not the walk that took its
// own over.
func TestADeliveryThatEndedFollowsNoWalk(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC)
w.ctl.planned = planMoving(cat)
p := pr(cat, 103, "eeeeeeeeeeee0003", "docs/x", "README.md")
w.h.PullUpdated(p)
w.h.PullMerged(mergedAt(p, "eeeeeeeeeeee3333", atCC))
w.settleAll()
nothing := IDOf(cat, "eeeeeeeeeeee0003")
if s := w.state(nothing); s != Delivered {
t.Fatalf("a merge moving nothing is %s", s)
}
w.h.WalkMoved(w.ctl.put(catWalk("plan-4", "eeeeeeeeeeee3333", atCC.Add(time.Minute), "app")))
w.settleAll()
if d := w.delivery(nothing); d.Walk != nil {
t.Fatalf("a delivered merge was tied to %s", d.Walk.ID)
}
w2 := twoMerges(t)
first := w2.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst"))
w2.h.WalkMoved(first)
idDu := IDOf(cat, headDu)
if _, err := w2.h.Stop(idDu, "not now", "jochen"); err != nil {
t.Fatal(err)
}
w2.ctl.put(catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst"))
w2.h.WalkMoved(w2.ctl.put(supersededBy(w2.ctl.walk("plan-1"), "plan-2")))
w2.settleAll()
if d := w2.delivery(idDu); d.State != Stopped || d.Walk.ID != "plan-1" {
t.Fatalf("the stopped delivery is %s on %s", d.State, d.Walk.ID)
}
}
// Today's controller, merges A then B minutes apart: B's walk took A's over before it started. A follows it and
// ends covered by B, rather than superseded (ADR 0276 decision 4).
func TestAnEarlierMergeFollowsTheWalkThatTookItsOver(t *testing.T) {
w := twoMerges(t)
first := w.ctl.put(catWalk("plan-1", mergeCC, atCC.Add(5*time.Second), "claude-code"))
w.h.WalkMoved(first)
w.h.WalkMoved(w.ctl.put(catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst")))
w.h.WalkMoved(w.ctl.put(supersededBy(w.ctl.walk("plan-1"), "plan-2")))
w.settleAll()
idCC := IDOf(cat, headCC)
if d := w.delivery(idCC); d.Walk.ID != "plan-2" || len(w.ctl.delivers) != 1 {
t.Fatalf("claude-code follows %s; let go %v", d.Walk.ID, w.ctl.delivers)
}
w.h.WalkMoved(w.ctl.put(done(w.ctl.walk("plan-2"), w.now)))
w.settleAll()
if s := w.state(idCC); s != Covered {
t.Fatalf("claude-code is %s", s)
}
}
@@ -21,7 +21,7 @@ func transitionsTo(w *world, id string, to State) []map[string]any {
return out
}
func TestATransitionSaysWhenItsPullRequestOpenedAndMerged(t *testing.T) {
func TestATransitionSaysWhenItsPullRequestOpenedMergedAndItsClass(t *testing.T) {
w := newWorld(t)
repo := "novox/mesh-catalog"
opened := w.now
@@ -30,19 +30,29 @@ func TestATransitionSaysWhenItsPullRequestOpenedAndMerged(t *testing.T) {
w.later(3 * time.Minute)
second := pr(repo, 9, "head9second", "fix/403-x", "modules/app/a.go")
w.h.PullUpdated(second)
w.h.Checked(verdictMoving(repo, 9, "head9second", "app"))
c := verdictMoving(repo, 9, "head9second", "app")
c.Plan.Class = "core"
w.h.Checked(c)
w.later(2 * time.Minute)
mergedWhen := w.now
w.h.PullMerged(mergedAt(second, "merge9aaaaaa", mergedWhen))
walk := aWalk("plan-merge9", repo, "merge9aaaaaa", false)
walk.Delivery = &WalkDelivery{Merges: []WalkMerge{{Repository: repo, Commit: "merge9aaaaaa"}}}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
walk = w.ctl.walk(walk.ID)
walk.State, walk.Revision = walkDone, walk.Revision+1
walk.Modules = map[string]*WalkModule{"app": {State: "built"}}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
id := IDOf(repo, "head9second")
if s := w.state(id); s != Merged {
t.Fatalf("the merge is %s, not merged", s)
if s := w.state(id); s != Delivered {
t.Fatalf("the merge is %s, not delivered", s)
}
said := transitionsTo(w, id, Merged)
said := transitionsTo(w, id, Delivered)
if len(said) == 0 {
t.Fatalf("no transition to merged was said: %v", w.said)
t.Fatalf("no transition to delivered was said: %v", w.said)
}
e := said[len(said)-1]
if e["opened"] != opened.UTC().Format(time.RFC3339) {
@@ -51,19 +61,35 @@ func TestATransitionSaysWhenItsPullRequestOpenedAndMerged(t *testing.T) {
if e["merged_at"] != mergedWhen.UTC().Format(time.RFC3339) {
t.Errorf("merged_at %v, want %s", e["merged_at"], mergedWhen.Format(time.RFC3339))
}
if _, has := e["class"]; has {
t.Errorf("the transition names a class %v: the gate plans nothing", e["class"])
}
if _, has := e["plan"]; has {
t.Errorf("the transition names a plan %v", e["plan"])
if e["class"] != "core" {
t.Errorf("class %v, want the plan's core", e["class"])
}
a, err := w.h.Checks(repo, 9, "")
if err != nil {
t.Fatal(err)
}
if a.Delivery == nil || a.Delivery.Opened != opened.UTC().Format(time.RFC3339) ||
a.Delivery.MergedAt != mergedWhen.UTC().Format(time.RFC3339) || a.Delivery.MergedAs != "merge9aaaaaa" {
if a.Delivery == nil || a.Delivery.Opened != opened.UTC().Format(time.RFC3339) || a.Delivery.Class != "core" ||
a.Delivery.MergedAt != mergedWhen.UTC().Format(time.RFC3339) {
t.Errorf("checks says %+v", a.Delivery)
}
}
// A plan that moves no module is class none, so the tracker gives its fix no budget; a plan from a controller that
// says no class falls back to its walk's class once known, and says nothing before.
func TestAFixClassIsNoneWhenItMovesNothing(t *testing.T) {
for _, c := range []struct {
why string
d Delivery
want string
}{
{"the planner's class", Delivery{Plan: &DeliveryPlan{Moved: []string{"app"}, Class: "leaf"}}, "leaf"},
{"a plan that moves nothing, no class said", Delivery{Plan: planMoving("novox/x")}, "none"},
{"no class said, its walk's known", Delivery{Plan: planMoving("novox/x", "app"), Time: &DeliveryTime{Class: "core"}}, "core"},
{"no class said, nothing known", Delivery{Plan: planMoving("novox/x", "app")}, ""},
} {
if got := fixClass(&c.d); got != c.want {
t.Errorf("%s: %q, want %q", c.why, got, c.want)
}
}
}
@@ -97,7 +97,7 @@ func TestOwedTriedCountsSinceTheStart(t *testing.T) {
if countOf(t, triedSaid(t, w), "tried") == 0 {
t.Fatal("nothing counted before the restart")
}
again := newWorldOver(t, w.store, w.forge, w.now.Add(time.Minute))
again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute))
tried := triedSaid(t, again)
if countOf(t, tried, "tried") != 0 || tried["since"] != again.now.Format(time.RFC3339) {
t.Fatalf("a restarted holder says %v", tried)
@@ -42,7 +42,7 @@ const startedNeverAnswered = `{"pulls": [{
// this owner first saw it so, it is a stalled line naming each check that did not answer and since when. Before,
// any merge-check status counted as checked, pending included, and such a head was never said.
func TestAHeadWhoseCheckStartedAndNeverAnsweredIsSaid(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC))
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC))
w.forge.open = openPullsOf(t, startedNeverAnswered)
w.h.SeeOpenPulls()
w.later(Bounds[Proposed].For - time.Minute)
@@ -88,7 +88,7 @@ func TestAHeadWhoseCheckStartedAndNeverAnsweredIsSaid(t *testing.T) {
// **A check that answers does not start the bound again**: merge-gate answers at fifty minutes, repo-check is never
// set, and the head is said on the bound from this owner's first sight of it, not fifty minutes later.
func TestACheckThatAnswersDoesNotStartTheBoundAgain(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC))
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC))
w.forge.open = openPullsOf(t, startedNeverAnswered)
w.h.SeeOpenPulls()
w.later(50 * time.Minute)
@@ -106,7 +106,7 @@ func TestACheckThatAnswersDoesNotStartTheBoundAgain(t *testing.T) {
// A forge time with an offset is said in UTC, as the module's other lines say their times.
func TestAPendingTimeIsSaidInUTC(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC))
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC))
w.forge.open = openPullsOf(t, strings.Replace(startedNeverAnswered, "2026-10-11T00:11:39Z", "2026-10-11T02:11:39+02:00", 1))
w.h.SeeOpenPulls()
w.later(Bounds[Proposed].For + time.Minute)
@@ -118,7 +118,7 @@ func TestAPendingTimeIsSaidInUTC(t *testing.T) {
// A check asked again — its pending status set anew, as a recheck or a new announcement does — starts the bound again:
// the head is said only once that ask too has gone unanswered past it.
func TestACheckAskedAgainStartsItsBoundAgain(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC))
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC))
w.forge.open = openPullsOf(t, startedNeverAnswered)
w.h.SeeOpenPulls()
w.later(50 * time.Minute)
@@ -140,7 +140,7 @@ func TestACheckAskedAgainStartsItsBoundAgain(t *testing.T) {
// pull request says it itself. Nor is one whose only pending check is one its base does not require: hq requires
// mesh/repo-check alone.
func TestAHeadWhoseRequiredChecksAnsweredIsNotSaid(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC))
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC))
w.forge.open = openPullsOf(t, `{"pulls": [{
"base": "main", "head_sha": "99df6b999bad77c51dfdd13470cf57b1f38e8c24", "number": 194, "owner": "novox", "repo": "mesh-catalog",
"required": ["mesh/merge-gate", "mesh/repo-check"],
@@ -69,7 +69,7 @@ func heldIn(t *testing.T, w *world, gate string, want State) string {
// owner first saw it so, as an unanswered line naming the check and since when. Before, any head with a delivery was
// skipped, and ready has no bound of its own, so it waited for ever — unable to merge — with nothing said.
func TestADeliveryInReadyWhoseCheckIsPendingIsSaid(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC))
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC))
id := heldIn(t, w, "pass", Ready)
w.forge.open = openPullsOf(t, readyRepoCheckPending)
w.h.SeeOpenPulls()
@@ -111,7 +111,7 @@ func TestADeliveryInReadyWhoseCheckIsPendingIsSaid(t *testing.T) {
// request with no verdict to show, and nothing here asks again. It was announced — a delivery exists — so it is
// unanswered, never unannounced, though its head has no merge check at all.
func TestADeliveryInRejectedWhoseCheckIsMissingIsSaid(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC))
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC))
id := heldIn(t, w, "fail", Rejected)
w.forge.open = openPullsOf(t, strings.Replace(readyRepoCheckMissing,
`"mesh/merge-gate": {"state": "success", "description": "pass", "updated_at": "2026-10-11T00:42:01Z"}`,
@@ -126,7 +126,7 @@ func TestADeliveryInRejectedWhoseCheckIsMissingIsSaid(t *testing.T) {
}
// No merge check on its head at all — both writes lost — is still a delivery's: unanswered, never unannounced.
w2 := newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC))
w2 := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC))
heldIn(t, w2, "fail", Rejected)
w2.forge.open = openPullsOf(t, `{"pulls": [{"base": "main", "head_sha": "`+readyHeadSHA+`", "number": 215,
"owner": "novox", "repo": "mesh-controller", "required": ["mesh/merge-gate", "mesh/repo-check"],
@@ -147,7 +147,7 @@ func TestADeliveryInRejectedWhoseCheckIsMissingIsSaid(t *testing.T) {
// **A delivery in a state with a bound of its own is not said here**: a proposed one waits on its check under
// Bounds[Proposed] and H2's re-ask, and a second line for it would say the same thing twice.
func TestAProposedDeliveryIsSaidByItsOwnBoundOnly(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC))
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC))
w.h.PullUpdated(pr("novox/mesh-controller", 215, readyHeadSHA, "fix/215"))
id := IDOf("novox/mesh-controller", readyHeadSHA)
w.forge.open = openPullsOf(t, readyRepoCheckPending)
@@ -163,7 +163,7 @@ func TestAProposedDeliveryIsSaidByItsOwnBoundOnly(t *testing.T) {
// **A ready delivery whose required checks answered is not said**, as the forge said every one on 2026-10-11.
func TestAReadyDeliveryWhoseChecksAnsweredIsNotSaid(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC))
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC))
heldIn(t, w, "pass", Ready)
w.forge.open = openPullsOf(t, readyAnswered)
w.h.SeeOpenPulls()
@@ -177,7 +177,7 @@ func TestAReadyDeliveryWhoseChecksAnsweredIsNotSaid(t *testing.T) {
// saidReady is a ready delivery of #215's head whose mesh/repo-check reads pending on the forge, said past the bound.
func saidReady(t *testing.T) (*world, string) {
t.Helper()
w := newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC))
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC))
id := heldIn(t, w, "pass", Ready)
w.forge.open = openPullsOf(t, readyRepoCheckPending)
w.h.SeeOpenPulls()
@@ -199,22 +199,23 @@ func unansweredOf(w *world) []StalledLine {
return out
}
// **A said line clears when its delivery's state changes** (novox/hq issue 441): a merge ends it, past the pull
// **A said line clears when its delivery's state changes** (novox/hq issue 441): a merge publishes it, past the pull
// request. It clears the line at once —
// before the forge is read again, which may lag — and stay cleared once it is read.
func TestASaidLineClearsWhenItsDeliveryMovesOn(t *testing.T) {
t.Run("a merge ends it", func(t *testing.T) {
t.Run("a merge publishes it", func(t *testing.T) {
w, id := saidReady(t)
w.h.WalkMoved(w.ctl.put(aWalk("plan-1", "novox/mesh-controller", merge, true)))
w.h.PullMerged(merged("novox/mesh-controller", 215, readyHeadSHA, merge))
if s := w.state(id); s != Merged {
if s := w.state(id); s != Published {
t.Fatalf("a merge left it %s", s)
}
if lines := unansweredOf(w); len(lines) != 0 {
t.Fatalf("a merged delivery is still said as unanswered: %+v", lines)
t.Fatalf("a published delivery is still said as unanswered: %+v", lines)
}
w.h.SeeOpenPulls() // the forge's open list lags the merge: the head is still on it
if lines := unansweredOf(w); len(lines) != 0 {
t.Fatalf("a merged delivery still on a lagging open list is said: %+v", lines)
t.Fatalf("a published delivery still on a lagging open list is said: %+v", lines)
}
})
}
@@ -0,0 +1,200 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 442: five closed groups read "delivering, 1 of 2 delivered" for days, each beside a member
// whose walk a later walk of the same trunk superseded. Nothing in them was still in progress. A superseded
// member counts as reached when a later delivered walk on the same trunk carried every module it moves (ADR
// 0270's covered test); a closed group whose members all ended reads delivered when every one reached the
// machines, and stopped otherwise — never delivering.
//
// The shapes are the live groups' as mesh-delivery showed them on 2026-10-11: feat/delivery-checks-verb (the
// superseded member last in the order) and fix/assign-says-why-a-module-is-not-there (the superseded member
// first).
func TestAGroupWhoseMembersEndedEnds(t *testing.T) {
const cat, ctl, lab = "novox/mesh-catalog", "novox/mesh-controller", "novox/mesh-lab"
at := time.Date(2026, 10, 7, 19, 4, 10, 0, time.UTC)
// superseded is a member merged and walked, its walk superseded by a later walk of the same trunk.
superseded := func(repo, sha, merged, group string, modules ...string) *Delivery {
return &Delivery{ID: IDOf(repo, sha), Repository: repo, Commit: sha, Number: 109, Base: "main",
Branch: group, Group: group, State: Superseded, Since: at.Add(time.Minute), Created: at.Add(-time.Hour),
MergedAs: merged, MergedAt: at, Plan: planMoving(repo, modules...), Check: &Verdict{Gate: "pass"},
Walk: &WalkSeen{ID: "plan-" + merged[:6], Commit: merged, State: walkSuperseded,
Note: "superseded at tier 0 by plan-later (" + repo + " at 68aeee2c); planned there again"},
Transitions: []Transition{{At: at.Add(time.Minute), From: Delivering, To: Superseded, Event: EvSuperseded,
Why: "a newer delivery to the same trunk took over its walk"}}}
}
// delivered is the other member, delivered by its own walk.
delivered := func(repo, sha, merged, group string) *Delivery {
return &Delivery{ID: IDOf(repo, sha), Repository: repo, Commit: sha, Number: 117, Base: "main", Branch: group,
Group: group, State: Delivered, Since: at.Add(-10 * time.Minute), Created: at.Add(-2 * time.Hour),
MergedAs: merged, MergedAt: at.Add(-time.Hour), Plan: planMoving(repo, "mesh-controller"),
Check: &Verdict{Gate: "pass"}, Walk: &WalkSeen{ID: "plan-" + merged[:6], Commit: merged, State: walkDone}}
}
// later is the later merge of the same repository whose walk took the member's over and finished, sending
// each module given to every machine running it.
later := func(repo, merged string, modules ...string) *Delivery {
d := &Delivery{ID: IDOf(repo, "c0c0c0c0c0c0c0c0"), Repository: repo, Commit: "c0c0c0c0c0c0c0c0", Number: 110,
Base: "main", State: Delivered, Since: at.Add(30 * time.Minute), Created: at.Add(-time.Hour),
MergedAs: merged, MergedAt: at.Add(time.Minute), Plan: planMoving(repo, modules...), Check: &Verdict{Gate: "pass"},
Walk: &WalkSeen{ID: "plan-later", Commit: merged, State: walkDone}}
for _, m := range modules {
d.Steps = append(d.Steps, Step{Machine: "novox", Module: m, State: StepPassed},
Step{Machine: "the rest running it", Module: m, State: StepSent})
}
return d
}
group := func(id string, order ...string) *Group {
return &Group{ID: id, Members: append([]string(nil), order...), Closed: true, Said: "delivering"}
}
const checks, assign = "feat/delivery-checks-verb", "fix/assign-says-why-a-module-is-not-there"
catID, ctlID := IDOf(cat, "13894744a701fe59"), IDOf(ctl, "1fdff00794e95470")
asCtlID, asLabID := IDOf(ctl, "8adb7f1a051941e9"), IDOf(lab, "33e63ee5b47a7552")
for _, c := range []struct {
name string
kept []*Delivery
group *Group
contains bool // what the forge says of the later merge containing the superseded member's
want string
whySays string
}{
{"superseded last, a later delivered walk carried every module it moves",
[]*Delivery{superseded(cat, "13894744a701fe59", "9d4bbbd2bd271049", checks, "gitea", "mesh-delivery"),
delivered(ctl, "1fdff00794e95470", "85d664438f87f885", checks),
later(cat, "68aeee2c00000000", "gitea", "mesh-delivery")},
group(checks, ctlID, catID), true, "delivered", "every member"},
{"superseded last, the later walk carried one of its two modules",
[]*Delivery{superseded(cat, "13894744a701fe59", "9d4bbbd2bd271049", checks, "gitea", "mesh-delivery"),
delivered(ctl, "1fdff00794e95470", "85d664438f87f885", checks),
later(cat, "68aeee2c00000000", "mesh-delivery")},
group(checks, ctlID, catID), true, "stopped", "not carried by a later delivery: "},
{"superseded last, the forge says the later merge does not contain it",
[]*Delivery{superseded(cat, "13894744a701fe59", "9d4bbbd2bd271049", checks, "gitea", "mesh-delivery"),
delivered(ctl, "1fdff00794e95470", "85d664438f87f885", checks),
later(cat, "68aeee2c00000000", "gitea", "mesh-delivery")},
group(checks, ctlID, catID), false, "stopped", "not carried by a later delivery: "},
{"superseded first, a later delivered walk carried every module it moves",
[]*Delivery{superseded(ctl, "8adb7f1a051941e9", "a5f53ef9ebdf3650", assign, "build-agent", "mesh-controller", "route-proxy"),
delivered(lab, "33e63ee5b47a7552", "77777777aaaaaaaa", assign),
later(ctl, "3f68a49500000000", "build-agent", "mesh-controller", "route-proxy")},
group(assign, asCtlID, asLabID), true, "delivered", "every member"},
{"superseded first, nothing later delivered",
[]*Delivery{superseded(ctl, "8adb7f1a051941e9", "a5f53ef9ebdf3650", assign, "build-agent", "mesh-controller", "route-proxy"),
delivered(lab, "33e63ee5b47a7552", "77777777aaaaaaaa", assign)},
group(assign, asCtlID, asLabID), true, "stopped", "not carried by a later delivery: "},
{"still in progress: a member delivering beside a superseded one",
func() []*Delivery {
d := delivered(ctl, "1fdff00794e95470", "85d664438f87f885", checks)
d.State, d.Walk.State = Delivering, walkRolling
return []*Delivery{superseded(cat, "13894744a701fe59", "9d4bbbd2bd271049", checks, "gitea", "mesh-delivery"), d}
}(),
group(checks, ctlID, catID), true, "delivering", "0 of 2"},
} {
t.Run(c.name, func(t *testing.T) {
store := newMemStore()
for _, d := range c.kept {
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
}
if err := store.PutGroup(c.group); err != nil {
t.Fatal(err)
}
forge := newFakeForge()
forge.contains[c.kept[0].MergedAs+" in "+"68aeee2c00000000"] = c.contains
forge.contains[c.kept[0].MergedAs+" in "+"3f68a49500000000"] = c.contains
w := newWorldOver(t, store, newFakeController(), forge, at.Add(4*24*time.Hour))
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
g := w.h.groups[c.group.ID]
state, why := GroupState(g, w.h.membersOf(g))
if state != c.want || !strings.Contains(why, c.whySays) {
t.Fatalf("the group reads %s (%s), want %s saying %q", state, why, c.want, c.whySays)
}
// The forge is asked once per pair, however many ticks follow.
for range 3 {
w.later(2 * time.Minute)
w.settleAll()
}
w.forge.mu.Lock()
asked := map[string]int{}
for _, a := range w.forge.containsAsks {
asked[a]++
}
w.forge.mu.Unlock()
for pair, n := range asked {
if n != 1 {
t.Fatalf("the forge was asked %d times whether %s", n, pair)
}
}
// What was learned survives a read-back, and the group reads the same with no new ask.
sup := w.delivery(c.kept[0].ID)
again := newWorldOver(t, store, newFakeController(), forge, w.now.Add(time.Hour))
if err := again.h.Reconcile(); err != nil {
t.Fatal(err)
}
again.settleAll()
back := again.delivery(c.kept[0].ID)
if back.CoveredBy != sup.CoveredBy || len(back.ContainedIn) != len(sup.ContainedIn) {
t.Fatalf("read back, covered by %q and contained in %v; was %q and %v", back.CoveredBy, back.ContainedIn,
sup.CoveredBy, sup.ContainedIn)
}
for k, v := range sup.ContainedIn {
if back.ContainedIn[k] != v {
t.Fatalf("read back, contained in %s is %v, was %v", k, back.ContainedIn[k], v)
}
}
if s2, _ := GroupState(again.h.groups[c.group.ID], again.h.membersOf(again.h.groups[c.group.ID])); s2 != state {
t.Fatalf("read back, the group reads %s, was %s", s2, state)
}
forge.mu.Lock()
n := len(forge.containsAsks)
forge.mu.Unlock()
if want := len(asked); c.want != "delivering" && n != want {
t.Fatalf("read back, the forge was asked again: %d asks, was %d", n, want)
}
// Each member keeps its own end: the group's label changes, no delivery is moved by hand.
for _, d := range c.kept {
if got := w.state(d.ID); got != d.State {
t.Fatalf("%s moved from %s to %s", d.ID, d.State, got)
}
}
// What the group said reached every member's head, once it ended.
if c.want != "delivering" {
w.forge.mu.Lock()
s := w.forge.statuses[c.kept[0].Repository+"@"+c.kept[0].Commit+" mesh/delivery-group"]
w.forge.mu.Unlock()
if !strings.Contains(s, c.want) {
t.Fatalf("mesh/delivery-group on the superseded member's head says %q", s)
}
}
})
}
}
// The forge's mesh/delivery-group line, "group <id> stopped: <why>", is clipped at 140 characters: for each of
// the five live groups of novox/hq issue 442, the superseded member's id fits in it whole.
func TestAStoppedGroupsLineNamesItsSupersededMemberWhole(t *testing.T) {
for group, member := range map[string]string{
"feat/delivery-checks-verb": "novox/mesh-catalog@13894744a701",
"feat/machine-network-health": "novox/mesh-controller@b8bbf9c79fd8",
"fix/373-one-message-per-condition": "novox/mesh-catalog@8dfc19c9f7c8",
"fix/assign-says-why-a-module-is-not-there": "novox/mesh-controller@8adb7f1a0519",
"fix/needs-you-from-the-console": "novox/mesh-controller@2e6a9b1efc05",
} {
g := &Group{ID: group, Members: []string{member, "novox/hq@39585e3bb2ea"}, Closed: true}
state, why := GroupState(g, []*Delivery{{ID: member, State: Superseded}, {ID: "novox/hq@39585e3bb2ea", State: Delivered}})
line := clip("group "+group+" "+state+": "+why, 140)
if state != "stopped" || !strings.HasSuffix(line, member) {
t.Fatalf("group %s: the line %q (%d characters) does not end in %s", group, line, len(line), member)
}
}
}
@@ -13,7 +13,7 @@ import (
// inside a finished sentence cannot be said again in another zone, and the operator read "since
// 2026-10-11T00:11:39Z". The finished words stay beside the data, for a controller that does not read it yet.
func TestAnUnansweredLineGivesItsChecksAsData(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC))
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC))
w.forge.open = openPullsOf(t, strings.Replace(startedNeverAnswered, "2026-10-11T00:11:39Z", "2026-10-11T02:11:39+02:00", 1))
w.h.SeeOpenPulls()
w.later(Bounds[Proposed].For + time.Minute)
@@ -16,7 +16,7 @@ func TestAnUnansweredLineNamesTheStateOfItsDelivery(t *testing.T) {
gate string
want State
}{{"pass", Ready}, {"fail", Rejected}} {
w := newWorldOver(t, newMemStore(), newFakeForge(), at)
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at)
id := heldIn(t, w, c.gate, c.want)
w.forge.open = openPullsOf(t, readyRepoCheckPending)
w.h.SeeOpenPulls()
@@ -33,7 +33,7 @@ func TestAnUnansweredLineNamesTheStateOfItsDelivery(t *testing.T) {
}
// The same head with no delivery held here: unanswered, and no state to say.
w := newWorldOver(t, newMemStore(), newFakeForge(), at)
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at)
w.forge.open = openPullsOf(t, readyRepoCheckPending)
w.h.SeeOpenPulls()
w.later(Bounds[Proposed].For + time.Minute)
@@ -0,0 +1,338 @@
package main
import (
"fmt"
"strings"
"testing"
"time"
)
// novox/hq issue 459, ADR 0299: since the merge window (ADR 0276) one walk carries every merge of its batch. A group
// stop asked the controller to end the stopped member's walk, and with it every other delivery on it, which no one
// stopped. Now a group stop asks `delivery-stop` with the stopped members' merges to leave out: the walk ends, and
// the controller walks its other merges again in a fresh batch; a later merge of a stopped merge's repository
// contains it and ends with it, named. A person's stop of a delivery on a shared walk answers first with what
// ending the walk would take down, and acts only on a way given: `again` or `walk`.
// shape459 is a world of a delivery group feat/g (its members in the repositories group names, in that order) and
// lone deliveries (the repositories alone names), each checked and merged. The group's first member is on a walk of
// its own, plan-a; every other merge is carried by one walk, plan-2, waiting for its word.
type shape459 struct {
*world
ids, heads, merges map[string]string // by repository
}
func open459(t *testing.T, group, alone []string) *shape459 {
t.Helper()
w := newWorld(t)
s := &shape459{world: w, ids: map[string]string{}, heads: map[string]string{}, merges: map[string]string{}}
n := 0
add := func(repo, branch string) {
n++
head := fmt.Sprintf("%02d%s", n, strings.Repeat("h", 14))
s.ids[repo], s.heads[repo] = IDOf(repo, head), head
s.merges[repo] = fmt.Sprintf("%02d%s", n, strings.Repeat("m", 10))
w.h.PullUpdated(pr(repo, n, head, branch, "x"))
w.h.Checked(verdict(repo, n, head, "pass"))
}
for _, r := range group {
add(r, "feat/g")
}
for _, r := range alone {
add(r, "fix/"+r[strings.LastIndex(r, "/")+1:])
}
w.settleAll()
w.h.Checked(CheckedEvent{Group: "feat/g", ID: "check-1", Verdict: "pass", Summary: "compose"})
w.settleAll()
for i, r := range append(append([]string{}, group...), alone...) {
w.h.PullMerged(merged(r, i+1, s.heads[r], s.merges[r]))
}
first := aWalk("plan-a", group[0], s.merges[group[0]], true)
first.Delivery.Merges = []WalkMerge{{Repository: group[0], Commit: s.merges[group[0]]}}
w.h.WalkMoved(w.ctl.put(first))
two := aWalk("plan-2", alone[0], s.merges[alone[0]], true)
two.Created = two.Created.Add(time.Minute)
for _, r := range append(append([]string{}, group[1:]...), alone...) {
two.Delivery.Merges = append(two.Delivery.Merges, WalkMerge{Repository: r, Commit: s.merges[r]})
}
w.h.WalkMoved(w.ctl.put(two))
w.settleAll()
return s
}
// failFirst fails the group's first member's walk: the members after it are stopped by their group.
func (s *shape459) failFirst() {
a := s.ctl.walk("plan-a")
a.State, a.Note = walkFailed, "the first member failed its gate on anchor"
s.h.WalkMoved(s.ctl.put(a))
s.settleAll()
}
// heard is the controller's records of the stopped walk and the walk that walks its merges again, as plan-moved says
// them, and then a walk done.
func (s *shape459) heard(walks ...string) {
for _, id := range walks {
s.h.WalkMoved(s.ctl.walk(id))
}
s.settleAll()
}
// cutAgain is the controller cutting the batch that walks a stopped walk's merges again, as plan-moved says it.
func (s *shape459) cutAgain(id string) {
s.h.WalkMoved(s.ctl.cut(id))
s.settleAll()
}
func (s *shape459) done(walk string) {
r := s.ctl.walk(walk)
sent := s.now
for m := range r.Modules {
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
}
r.State = walkDone
s.h.WalkMoved(s.ctl.put(r))
s.settleAll()
}
func lastWhy(d *Delivery) string {
if len(d.Transitions) == 0 {
return ""
}
return d.Transitions[len(d.Transitions)-1].Why
}
// retarget makes the delivery of repo `later` a later merge of repository `of`: same repository, carried by plan-2
// for the merge of `of`.
func (s *shape459) retarget(later, of string) {
s.t.Helper()
s.h.mu.Lock()
d := s.h.deliveries[s.ids[later]]
d.Repository, d.MergedAt = of, d.MergedAt.Add(time.Minute)
s.h.mu.Unlock()
s.ctl.mu.Lock()
s.ctl.mergedAt[s.merges[of]], s.ctl.mergedAt[s.merges[later]] = d.MergedAt.Add(-time.Minute), d.MergedAt
s.ctl.mu.Unlock()
two := s.ctl.walk("plan-2")
merges := []WalkMerge{}
for _, m := range two.Delivery.Merges {
switch m.Repository {
case of:
m.Carried = s.merges[later]
case later:
m.Repository = of
}
merges = append(merges, m)
}
wd := *two.Delivery
wd.Merges = merges
two.Delivery = &wd
s.h.WalkMoved(s.ctl.put(two))
s.settleAll()
}
// shared is two lone deliveries, y and z, let go together on one walk, plan-2.
func shared459(t *testing.T) *shape459 {
t.Helper()
s := open459(t, []string{"novox/a"}, []string{"novox/y", "novox/z"})
for _, r := range []string{"novox/y", "novox/z"} {
if st := s.state(s.ids[r]); st != Delivering {
t.Fatalf("%s is %s, let go %v; want both delivering on plan-2", r, st, s.ctl.delivers)
}
}
return s
}
// A person's stop of y, which shares its walk with z, does nothing at first: it names z and the two ways.
func TestAPersonsStopOfASharedWalkAsksFirst(t *testing.T) {
s := shared459(t)
said, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "")
if err == nil {
t.Fatalf("the stop acted at once: %q", said)
}
for _, want := range []string{s.ids["novox/z"], "plan-2", "again", "walk", "not stopped"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("the stop said %q; want it to name %q", err, want)
}
}
if st := s.state(s.ids["novox/y"]); st != Delivering || len(s.ctl.stops)+len(s.ctl.withouts) != 0 {
t.Fatalf("y is %s, the controller was asked %v %v; want nothing done", st, s.ctl.stops, s.ctl.withouts)
}
if _, err := s.h.StopWay(s.ids["novox/y"], "x", "jochen", "sideways"); err == nil {
t.Fatal("a way that is neither again nor walk was taken")
}
}
// With `again`, y is stopped, the walk is stopped without y's merge, and z goes back to published and follows the
// walk that walks it again.
func TestAPersonsStopAgainWalksTheOthersAgain(t *testing.T) {
s := shared459(t)
said, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "again")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(said, "plan-2-again") || !strings.Contains(said, "novox/z@") {
t.Fatalf("the stop said %q; want it to name the walk that walks z again", said)
}
if want := "plan-2 without novox/y@" + s.merges["novox/y"]; !containsString(s.ctl.withouts, want) ||
containsString(s.ctl.stops, "plan-2") {
t.Fatalf("the controller was asked %v, ended whole %v; want %q", s.ctl.withouts, s.ctl.stops, want)
}
s.heard("plan-2")
if z := s.delivery(s.ids["novox/z"]); z.State != Published {
t.Fatalf("z is %s once plan-2 was stopped and walked again; want published, waiting for that walk", z.State)
}
s.heard("plan-2-again")
s.cutAgain("plan-2-again")
if z := s.delivery(s.ids["novox/z"]); z.Walk == nil || z.Walk.ID != "plan-2-again" || z.State != Delivering {
t.Fatalf("z is %s on %+v; want it delivering on plan-2-again", z.State, z.Walk)
}
if st := s.state(s.ids["novox/y"]); st != Stopped {
t.Fatalf("y is %s", st)
}
}
// With `walk`, the whole walk is stopped, and z ends stopped, naming y's stop.
func TestAPersonsStopOfTheWalkStopsEveryDelivery(t *testing.T) {
s := shared459(t)
if _, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "walk"); err != nil {
t.Fatal(err)
}
if !containsString(s.ctl.stops, "plan-2") || len(s.ctl.withouts) != 0 {
t.Fatalf("the controller was asked %v %v; want plan-2 ended whole", s.ctl.stops, s.ctl.withouts)
}
s.heard("plan-2")
z := s.delivery(s.ids["novox/z"])
if z.State != Stopped || !strings.Contains(lastWhy(z), s.ids["novox/y"]) {
t.Fatalf("z is %s (%q); want it stopped, naming y's stop", z.State, lastWhy(z))
}
}
// Whatever brought it there, a walk that carries a stopped delivery's merge is asked to leave it out, and is never
// given its word while it carries it.
func TestAWalkCarryingAStoppedMergeIsAskedToLeaveItOutAndNeverLetGo(t *testing.T) {
s := shared459(t)
if _, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "again"); err != nil {
t.Fatal(err)
}
// A walk the controller made that carries y's merge anyway, waiting for its word, with z on it.
x := aWalk("plan-x", "novox/z", s.merges["novox/z"], true)
x.Created = x.Created.Add(10 * time.Minute)
x.Delivery.Merges = []WalkMerge{{Repository: "novox/y", Commit: s.merges["novox/y"]},
{Repository: "novox/z", Commit: s.merges["novox/z"]}}
s.ctl.refuseWithout = true // it cannot be left out: the walk is stopped whole
s.h.WalkMoved(s.ctl.put(x))
s.settleAll()
if want := "plan-x without novox/y@" + s.merges["novox/y"]; !containsString(s.ctl.withouts, want) {
t.Fatalf("the controller was asked %v; want %q", s.ctl.withouts, want)
}
if containsString(s.ctl.delivers, "plan-x") {
t.Fatalf("a walk carrying a stopped merge was let go: %v", s.ctl.delivers)
}
if !containsString(s.ctl.stops, "plan-x") {
t.Fatalf("refused the leave-out, the walk was not stopped whole: %v", s.ctl.stops)
}
}
// A person's stop the controller cannot make leaves the delivery as it was, and says how to go on.
func TestAPersonsStopThatCannotBeMadeStopsNothing(t *testing.T) {
s := shared459(t)
s.ctl.refuseWithout = true
_, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "again")
if err == nil || !strings.Contains(err.Error(), "is not stopped") || !strings.Contains(err.Error(), "plans stop") {
t.Fatalf("the refused stop said %v", err)
}
if st := s.state(s.ids["novox/y"]); st != Delivering {
t.Fatalf("y is %s; want it delivering, its stop not made", st)
}
}
// What a stop that left merges out says: the walk that walks the others, what it left out and what each contains,
// as the controller says it, what stays on which machines, and what will still deliver the stopped change.
func TestAStopSaysWhatStaysAndWhatStillCarriesIt(t *testing.T) {
said := againWords(StopAnswer{Stopped: "plan-2", WalkedAgainBy: "plan-3", WalkedAgain: []string{"novox/z@zz"},
LeftOut: []WalkLeftOut{{Repository: "novox/b", Commit: "b1b1b1b1b1b1"},
{Repository: "novox/b", Commit: "b2b2b2b2b2b2", Contains: "b1b1b1b1b1b1"}},
Kept: []WalkKept{{Module: "lab", Machines: []string{"anchor"}}},
StillCarriedBy: []string{"novox/b@b3b3 (in plan-4)"}})
for _, want := range []string{"plan-3", "novox/z@zz", "b2b2b2b2b2b2 (contains b1b1b1b1b1b1)", "lab on anchor",
"stays on the machines", "novox/b@b3b3 (in plan-4)", "revert"} {
if !strings.Contains(said, want) {
t.Fatalf("the stop said %q; want %q in it", said, want)
}
}
}
// A person's first answer names the deliveries that would be left out for moving a module this merge moves, and says
// the controller may leave out more.
func TestAPersonsFirstAnswerNamesWhatSharesItsModules(t *testing.T) {
s := shared459(t)
_, err := s.h.StopWay(s.ids["novox/y"], "x", "jochen", "")
if err == nil || !strings.Contains(err.Error(), "for moving a module this merge moves: "+s.ids["novox/z"]+" (moves app)") ||
!strings.Contains(err.Error(), "may leave out more") {
t.Fatalf("the first answer said %v", err)
}
}
// The safety net holds back only what a person or a group stopped: a delivery ended by what its walk did (the
// observed rows, which name nobody) is not asked out of a walk that carries its merge.
func TestTheSafetyNetLeavesAStrandedStopAlone(t *testing.T) {
s := shared459(t)
s.h.mu.Lock()
y := s.h.deliveries[s.ids["novox/y"]]
y.State = Stopped
y.Transitions = append(y.Transitions, Transition{At: s.now, From: Published, To: Stopped, Event: EvStop,
Why: "its walk ended done before its word"})
s.h.mu.Unlock()
s.settleAll()
if len(s.ctl.withouts) != 0 {
t.Fatalf("a stop nobody asked for was held back: %v", s.ctl.withouts)
}
}
// A walk the safety net holds — its leave-out and its whole stop both refused — says why on the deliveries waiting on
// it, in `waits`, and in `stalled` past their bound.
func TestAHeldWalkIsSaidOnItsDeliveriesAndStalls(t *testing.T) {
s := shared459(t)
if _, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "again"); err != nil {
t.Fatal(err)
}
s.ctl.refuseWithout, s.ctl.refuseStop = true, true
x := aWalk("plan-x", "novox/z", s.merges["novox/z"], true)
x.Created = x.Created.Add(10 * time.Minute)
x.Delivery.Merges = []WalkMerge{{Repository: "novox/y", Commit: s.merges["novox/y"]},
{Repository: "novox/z", Commit: s.merges["novox/z"]}}
s.h.WalkMoved(s.ctl.put(x))
s.settleAll()
z := s.delivery(s.ids["novox/z"])
if !strings.Contains(z.HeldBack, "plan-x") || !strings.Contains(z.HeldBack, "refused") {
t.Fatalf("z says it is held back by %q; want plan-x and the refusal", z.HeldBack)
}
if w := s.h.waitsFor(z); w != z.HeldBack {
t.Fatalf("z waits for %q", w)
}
s.later(Bounds[Published].For + time.Minute)
s.settleAll()
for _, l := range s.h.Stalled() {
if l.ID == z.ID && strings.Contains(l.Says, "plan-x") {
return
}
}
t.Fatalf("z is not listed stalled naming plan-x: %+v", s.h.Stalled())
}
// A person's stop of a delivery whose merge waits in a batch asks that batch now to leave it out, and says so.
func TestAPersonsStopOfAMergeInABatchAsksTheBatch(t *testing.T) {
s := shared459(t)
if _, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "again"); err != nil {
t.Fatal(err)
}
s.heard("plan-2", "plan-2-again")
said, err := s.h.StopWay(s.ids["novox/z"], "this one too", "jochen", "")
if err != nil {
t.Fatal(err)
}
if want := "plan-2-again without novox/z@" + s.merges["novox/z"]; !containsString(s.ctl.withouts, want) ||
!strings.Contains(said, "batch plan-2-again") {
t.Fatalf("said %q, asked %v; want the batch asked now, and said", said, s.ctl.withouts)
}
}
@@ -0,0 +1,211 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 474: a merge batched behind a later one of its branch (ADR 0276) whose walk ended done before
// any word was asked — the controller judged the batch's files moved nothing it holds, "nothing to walk" — was
// left published, waiting for "its walk's start", which a done walk never has. Its walk's record settles it: it
// is delivered when that walk moved its modules, covered when a later delivered merge carried them (ADR 0270),
// and otherwise stopped, saying why — never left waiting.
//
// The shape is the night's own (2026-10-11): mesh-controller #215 (merged as fbc7bc97, its check's plan moving
// build-agent) and #217 (merged as 8305e4e3 a minute later, moving nothing) in one batch, walked as
// plan-1791681624611167730 at 8305e4e3: done, no tier, still saying it awaits mesh-delivery.
const (
ctl474 = "novox/mesh-controller"
head215 = "d2e9dc61590f950b"
merge215 = "fbc7bc976b3c115b"
head217 = "68fbd99e894443ce"
merge217 = "8305e4e3d126594d"
head219 = "ef551fdfb6b29d4a"
merge219 = "0a2e58c0704ccc09"
batchWalk = "plan-1791681624611167730"
walk219 = "plan-1791686195314774150"
nothingNote = "moved nothing the mesh holds: nothing to walk"
)
var at215 = time.Date(2026, 10, 11, 1, 20, 19, 0, time.UTC)
// theNight is #215 and #217 checked and merged, and their batch's walk done with no tier, as the controller said it.
func theNight(t *testing.T) *world {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at215.Add(-time.Minute))
p215 := pr(ctl474, 215, head215, "fix/431-make-check-runs-what-the-gate-runs", "Makefile")
p217 := pr(ctl474, 217, head217, "fix/432-a-test-reads-what-it-carries", "cmd/mesh-controller/check_test.go")
w.h.PullUpdated(p215)
w.h.PullUpdated(p217)
w.h.Checked(verdictMoving(ctl474, 215, head215, "build-agent"))
w.h.Checked(verdictMoving(ctl474, 217, head217))
w.later(time.Minute)
w.h.PullMerged(mergedAt(p215, merge215, at215))
w.later(time.Minute)
w.h.PullMerged(mergedAt(p217, merge217, at215.Add(56*time.Second)))
w.later(90 * time.Second)
cut := w.now
w.h.WalkMoved(w.ctl.put(Walk{ID: batchWalk, Repository: ctl474, Branch: "main", Commit: merge217, Created: cut,
Updated: cut, State: walkDone, Tiers: [][]string{}, Modules: map[string]*WalkModule{}, Note: nothingNote,
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
{Repository: ctl474, Commit: merge215, Carried: merge217}, {Repository: ctl474, Commit: merge217}}}}))
w.settleAll()
return w
}
// The live delivery: its walk ended without moving build-agent, and a later merge (#219) delivered build-agent
// everywhere from main holding its merge — covered, from the walk's record and the forge's word, never waiting.
func TestAMergeWhoseBatchWalkEndedIsCoveredByALaterDeliveredMerge(t *testing.T) {
w := theNight(t)
id := IDOf(ctl474, head215)
if s := w.state(IDOf(ctl474, head217)); s != Delivered {
t.Fatalf("#217, which moves nothing, is %s", s)
}
w.forge.contains[merge215+" in "+merge219] = true
deliverLater(w, ctl474, 219, "main", head219, merge219, at215.Add(76*time.Minute), walkDone, "build-agent")
w.later(2 * time.Minute)
w.settleAll()
d := w.delivery(id)
if d.State != Covered {
t.Fatalf("#215, its walk %s done with nothing moved and build-agent delivered later by #219, is %s, waiting "+
"for %q", batchWalk, d.State, w.h.waitsFor(d))
}
if why := d.Transitions[len(d.Transitions)-1].Why; !strings.Contains(why, IDOf(ctl474, head219)) {
t.Fatalf("covered, but not by #219: %q", why)
}
if len(w.h.Stalled()) != 0 {
t.Fatalf("stalled still lists %+v", w.h.Stalled())
}
}
// With nothing later to carry its modules, it is stopped once the cover test says so and the walk ended past the
// grace — saying which walk ended without moving what — and not before. Stopped, it stays loud: `stalled` lists it
// until a later delivered merge carries its modules.
func TestAMergeWhoseBatchWalkEndedAndNothingCoversItIsStoppedSayingWhy(t *testing.T) {
w := theNight(t)
id := IDOf(ctl474, head215)
w.later(5 * time.Minute)
w.settleAll()
if s := w.state(id); s != Published {
t.Fatalf("within the grace it is %s", s)
}
w.later(strandedAfter)
w.settleAll()
d := w.delivery(id)
if d.State != Stopped {
t.Fatalf("its walk done without build-agent and nothing later carrying it, it is %s, waiting for %q", d.State,
w.h.waitsFor(d))
}
why := d.Transitions[len(d.Transitions)-1].Why
for _, want := range []string{"never reached the machines", batchWalk, "build-agent", nothingNote} {
if !strings.Contains(why, want) {
t.Fatalf("its stop says %q, not %q", why, want)
}
}
lines := w.h.Stalled()
if len(lines) != 1 || lines[0].ID != id || !strings.Contains(lines[0].Says, "never reached the machines") {
t.Fatalf("stopped, it is not listed loudly: %+v", lines)
}
// A later merge that delivers build-agent ends the listing; the stop itself stays, final.
w.forge.contains[merge215+" in "+merge219] = true
deliverLater(w, ctl474, 219, "main", head219, merge219, w.now, walkDone, "build-agent")
w.later(2 * time.Minute)
w.settleAll()
if d := w.delivery(id); d.State != Stopped || d.CoveredBy == "" {
t.Fatalf("after #219 it is %s, covered by %q", d.State, d.CoveredBy)
}
if lines := w.h.Stalled(); len(lines) != 0 {
t.Fatalf("stalled still lists %+v", lines)
}
}
// A later merge whose walk is still queued, and whose plan moves what this one's walk left unbuilt, holds the stop
// off: a stop is final, and that merge may yet cover it (review of issue 474).
func TestAMergeWhoseBatchWalkEndedWaitsForALaterMergeOnItsWay(t *testing.T) {
w := theNight(t)
id := IDOf(ctl474, head215)
p := pr(ctl474, 219, head219, "fix/334", "internal/builder/builder.go")
w.h.PullUpdated(p)
w.h.Checked(verdictMoving(ctl474, 219, head219, "build-agent", "mesh-controller"))
w.later(3 * time.Minute)
w.h.PullMerged(mergedAt(p, merge219, w.now))
w.settleAll()
w.later(strandedAfter)
w.settleAll()
if d := w.delivery(id); d.State != Published {
t.Fatalf("with #219 on its way it is %s (%s)", d.State, d.Stranded)
}
// #219's walk delivers build-agent: covered.
w.forge.contains[merge215+" in "+merge219] = true
walk := aWalk(walk219, ctl474, merge219, true)
walk.Modules = map[string]*WalkModule{"build-agent": {}, "mesh-controller": {}}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
walk = w.ctl.walk(walk219)
sent := w.now
for m := range walk.Modules {
walk.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
}
walk.State = walkDone
w.h.WalkMoved(w.ctl.put(walk))
w.later(2 * time.Minute)
w.settleAll()
if s := w.state(IDOf(ctl474, head219)); s != Delivered {
t.Fatalf("#219 is %s", s)
}
if d := w.delivery(id); d.State != Covered {
t.Fatalf("after #219 delivered build-agent it is %s, waiting for %q", d.State, w.h.waitsFor(d))
}
}
// A walk that ended before its word built nothing: a module its record names with why it was not sent is not read
// as delivered. A merge whose plan moves nothing, answered by such a walk, is delivered.
func TestAWalkThatEndedBeforeItsWordBuiltNothing(t *testing.T) {
for _, moves := range [][]string{{"build-agent"}, nil} {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at215.Add(-time.Minute))
p := pr(ctl474, 215, head215, "fix/x", "cmd/mesh-builder/main.go")
w.h.PullUpdated(p)
w.h.Checked(verdictMoving(ctl474, 215, head215, moves...))
w.later(time.Minute)
w.h.PullMerged(mergedAt(p, merge215, at215))
w.later(2 * time.Minute)
w.h.WalkMoved(w.ctl.put(Walk{ID: batchWalk, Repository: ctl474, Branch: "main", Commit: merge215, Created: w.now,
Updated: w.now, State: walkDone, Tiers: [][]string{{"build-agent"}},
Modules: map[string]*WalkModule{"build-agent": {Why: "not sent: its walk never started"}},
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: ctl474, Commit: merge215}}}}))
w.settleAll()
d := w.delivery(IDOf(ctl474, head215))
want := Published
if len(moves) == 0 {
want = Delivered
}
if d.State != want {
t.Fatalf("moving %v: it is %s, waiting for %q, not %s", moves, d.State, w.h.waitsFor(d), want)
}
}
}
// A walk record kept before it named the modules it built, with tiers, is not read as building nothing: the walk is
// asked for again, and what it answers settles the delivery.
func TestAnOldWalkRecordIsAskedForAgain(t *testing.T) {
w := theNight(t)
id := IDOf(ctl474, head215)
w.h.mu.Lock()
d := w.h.deliveries[id]
d.Walk.Moved, d.Walk.MovedSaid, d.Walk.Tiers = nil, false, 1
w.h.mu.Unlock()
if got := w.h.waitsFor(w.delivery(id)); !strings.Contains(got, "walk's record") {
t.Fatalf("an old record waits for %q", got)
}
w.later(time.Minute)
w.settleAll()
if d := w.delivery(id); !d.Walk.MovedSaid || d.Walk.Tiers != 0 {
t.Fatalf("the walk was not read again: %+v", d.Walk)
}
w.later(strandedAfter)
w.settleAll()
if s := w.state(id); s != Stopped {
t.Fatalf("read again, it is %s", s)
}
}
@@ -0,0 +1,134 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 480: a delivery stopped because its walk ended done before its word without building its modules
// (issue 474) is still read for a later merge that delivers them, and kept CoveredBy once one did. Its state stays
// stopped — a final state never moves, as for a covered superseded member (issue 442) — but the forge's status, its
// view, its note and `show` went on saying "stopped" for a change that reached every machine. Once covered they say
// covered, and by which merge. And while `stalled` lists such a stop, `deliveries` lists it too, past a day.
// strandedNight is theNight with #215 stopped, its walk past the grace with nothing later carrying build-agent.
func strandedNight(t *testing.T) (*world, string) {
w := theNight(t)
id := IDOf(ctl474, head215)
w.later(strandedAfter + 5*time.Minute)
w.settleAll()
if d := w.delivery(id); d.State != Stopped || d.Stranded == "" {
t.Fatalf("#215 is %s, stranded %q", d.State, d.Stranded)
}
return w, id
}
func TestAStopALaterMergeCoversReadsCoveredByThatMerge(t *testing.T) {
w, id := strandedNight(t)
if got := statusOn(w, ctl474, head215); !strings.HasPrefix(got, "error stopped: ") {
t.Fatalf("stopped, its status says %q", got)
}
w.forge.contains[merge215+" in "+merge219] = true
by := deliverLater(w, ctl474, 219, "main", head219, merge219, w.now, walkDone, "build-agent")
w.later(2 * time.Minute)
w.settleAll()
d := w.delivery(id)
if d.State != Stopped || !strings.Contains(d.CoveredBy, by) {
t.Fatalf("after #219 it is %s, covered by %q", d.State, d.CoveredBy)
}
got := statusOn(w, ctl474, head215)
if !strings.HasPrefix(got, "success covered") || !strings.Contains(got, by) {
t.Fatalf("covered by %s, its status still says %q", by, got)
}
shown, err := w.h.Show(id)
if err != nil {
t.Fatal(err)
}
if reads, _ := shown.(map[string]any)["reads"].(string); !strings.HasPrefix(reads, "covered") || !strings.Contains(reads, by) {
t.Fatalf("show reads %q", reads)
}
view := w.forge.views[ctl474+"#215"]
if !strings.Contains(view, "Covered") || !strings.Contains(view, by) {
t.Fatalf("its view does not say it was covered by %s:\n%s", by, view)
}
noted := false
for _, l := range w.forge.notesOn(merge215) {
noted = noted || strings.Contains(l, "covered") && strings.Contains(l, by)
}
if !noted {
t.Fatalf("its merge's note does not say it was covered: %v", w.forge.notesOn(merge215))
}
// Told once: further ticks owe nothing more.
w.later(time.Hour)
w.settleAll()
if d := w.delivery(id); len(d.Owed) != 0 || !d.CoverTold {
t.Fatalf("still owes %+v, told %v", d.Owed, d.CoverTold)
}
}
// A stop covered before this was built (CoveredBy kept, nothing told) is told after a restart, from what was kept.
func TestACoveredStopKeptBeforeIsToldAfterARestart(t *testing.T) {
w, id := strandedNight(t)
w.h.mu.Lock()
d := w.h.deliveries[id]
d.CoveredBy = IDOf(ctl474, head219) + " (merged as " + merge219[:8] + ")"
if err := w.store.PutDelivery(d); err != nil {
t.Fatal(err)
}
w.h.mu.Unlock()
again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Hour))
again.settleAll()
if got := statusOn(again, ctl474, head215); !strings.HasPrefix(got, "success covered") ||
!strings.Contains(got, IDOf(ctl474, head219)) {
t.Fatalf("after a restart its status says %q", got)
}
if d := again.delivery(id); d.State != Stopped || len(d.Transitions) != len(w.delivery(id).Transitions) {
t.Fatalf("a restart moved it: %s", d.State)
}
}
// Only a stop for a walk that never built its modules reads covered: a person's stop, or one the cover test has
// not answered, keeps its error.
func TestOnlyACoveredStrandedStopReadsCovered(t *testing.T) {
for _, d := range []*Delivery{
{State: Stopped, CoveredBy: "x"}, // a person's stop, with a cover found while it was held
{State: Stopped, Stranded: "never reached"}, // stranded, not covered
} {
if state, _ := statusOf(d); state != "error" {
t.Fatalf("%+v reads %s", d, state)
}
if d.coveredStop() {
t.Fatalf("%+v reads as a covered stop", d)
}
}
}
func TestAStopStalledListsStaysInDeliveries(t *testing.T) {
w, id := strandedNight(t)
w.later(30 * time.Hour)
w.settleAll()
listed := func() *Line {
for _, l := range w.h.Deliveries("", "", "", false) {
if l.ID == id {
return &l
}
}
return nil
}
if len(w.h.Stalled()) != 1 {
t.Fatalf("stalled lists %+v", w.h.Stalled())
}
l := listed()
if l == nil || !l.Stalled || !strings.Contains(l.Waits, "build-agent") {
t.Fatalf("listed by stalled, deliveries shows %+v", l)
}
// Covered, it ends like any final delivery: a day after its stop.
w.forge.contains[merge215+" in "+merge219] = true
deliverLater(w, ctl474, 219, "main", head219, merge219, w.now, walkDone, "build-agent")
w.later(2 * time.Minute)
w.settleAll()
if len(w.h.Stalled()) != 0 || listed() != nil {
t.Fatalf("covered, stalled lists %+v and deliveries %+v", w.h.Stalled(), listed())
}
}
@@ -0,0 +1,142 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 362's note of 2026-10-11 (the nzbgeek case) and issue 529: mesh-media-catalog #18 added a module
// whose delivery plan said "sent nowhere". This owner ended its delivery at once, "nothing for a walk to move", but
// the controller opened a walk for the merge that moved modules to a machine and waited for this owner's word.
// With its delivery ended, nothing gave the walk its word: it held 39 minutes and two walks queued behind it.
const (
media = "novox/mesh-media-catalog"
headNzb = "abababab00001818"
mergeNzb = "cdcdcdcd00001818"
nzbWalkID = "plan-1791748193836841649"
)
var atNzb = time.Date(2026, 10, 11, 19, 43, 0, 0, time.UTC)
// nzbgeekMerged is #18 checked on a plan that builds a new module and sends it nowhere, and merged: its delivery
// ends at once, nothing for a walk to move.
func nzbgeekMerged(t *testing.T) *world {
t.Helper()
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atNzb.Add(-time.Minute))
p := pr(media, 18, headNzb, "feat/nzbgeek", "modules/nzbgeek/module.json")
w.h.PullUpdated(p)
v := verdict(media, 18, headNzb, "pass")
v.Plan = &DeliveryPlan{Repository: media, Base: "main", New: []string{"modules/nzbgeek"},
Summary: "builds new: modules/nzbgeek, sent nowhere; no bus step"}
w.h.Checked(v)
w.later(time.Minute)
w.h.PullMerged(mergedAt(p, mergeNzb, atNzb))
w.settleAll()
if d := w.delivery(IDOf(media, headNzb)); d.State != Delivered {
t.Fatalf("#18 is %s, not delivered as nothing for a walk to move", d.State)
}
return w
}
// nzbWalk is the walk the controller cut for the merge's batch: it names the merge and moves nzbgeek and nzbget.
func nzbWalk(created time.Time, merges ...WalkMerge) Walk {
w := Walk{ID: nzbWalkID, Repository: media, Branch: "main", Commit: mergeNzb, Created: created, State: walkBuilding,
Tiers: [][]string{{"nzbgeek", "nzbget"}}, Modules: map[string]*WalkModule{"nzbgeek": {}, "nzbget": {}},
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: merges}}
return w
}
// **A walk answering only merges that ended as nothing for a walk to move is named as held by nothing** (novox/hq
// issues 362 and 529): never let go by this owner — nothing judged what it moves — and said, by `stalled` with
// `walks`, with why, so the controller's `plans` says so instead of pointing nowhere.
func TestAWalkOfNothingToMoveMergesIsNamedAsHeldByNothing(t *testing.T) {
w := nzbgeekMerged(t)
w.later(90 * time.Second)
w.h.WalkMoved(w.ctl.put(nzbWalk(w.now, WalkMerge{Repository: media, Commit: mergeNzb})))
w.settleAll()
if len(w.ctl.delivers) != 0 {
t.Fatalf("a walk no live delivery holds was let go: %v", w.ctl.delivers)
}
lines := w.h.WalksWaiting()
if len(lines) != 1 || lines[0].Walk != nzbWalkID || lines[0].State != Unheld ||
!strings.Contains(lines[0].Says, "ended as nothing-to-move") || !strings.Contains(lines[0].Says, IDOf(media, headNzb)) ||
!strings.Contains(lines[0].Says, "nzbget") {
t.Fatalf("the walk is said as %+v", lines)
}
}
// **The status names each held delivery, why, and the verb that releases it** (novox/hq issue 529): "held: 1"
// alone pointed a reader nowhere.
func TestTheStatusNamesAHeldDeliveryAndWhy(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atNzb)
p := pr(media, 22, headNzb, "adr-0307-state-history", "modules/nzbgeek/module.json")
w.h.PullUpdated(p)
w.h.PullMerged(mergedAt(p, mergeNzb, atNzb)) // merged before any verdict: held for a person
w.settleAll()
id := IDOf(media, headNzb)
d := w.delivery(id)
if d.State != Held {
t.Fatalf("#22 is %s", d.State)
}
held, _ := w.h.Status("listening")["held"].([]string)
if len(held) != 1 || !strings.Contains(held[0], id) || !strings.Contains(held[0], d.HeldWhy) ||
!strings.Contains(held[0], "mesh-delivery.release") {
t.Fatalf("the status says held %q", held)
}
}
// **A walk waiting for this owner's word is told why** (novox/hq issue 529): asked with `walks`, `stalled` says of
// each walk waiting for its word what holds it — a held delivery, naming it, its reason and `mesh-delivery.release`,
// or the deliveries not at their turn — so the controller's `plans` points at the person's decision, not at a hand
// start.
func TestAWaitingWalkIsToldWhatHoldsIt(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second))
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json")
w.h.PullUpdated(cc)
w.h.PullUpdated(du)
w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code"))
w.h.PullMerged(mergedAt(cc, mergeCC, atCC))
w.h.PullMerged(mergedAt(du, mergeDu, atDu)) // unchecked: held for a person
walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst")
walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
idDu := IDOf(cat, headDu)
lines := w.h.WalksWaiting()
if len(lines) != 1 || lines[0].Walk != "plan-9" || lines[0].State != Held || lines[0].ID != idDu ||
!strings.Contains(lines[0].Says, w.delivery(idDu).HeldWhy) || !strings.Contains(lines[0].Says, "mesh-delivery.release") {
t.Fatalf("the waiting walk is said as %+v", lines)
}
// Not by default: `stalled` stays what the self-check raises conditions for.
for _, l := range w.h.Stalled() {
if l.Walk == "plan-9" {
t.Fatalf("stalled lists the waiting walk without being asked: %+v", l)
}
}
// Released, its word is asked: nothing holds it any more.
if _, err := w.h.Release(idDu, "checked by hand", "jochen"); err != nil {
t.Fatal(err)
}
w.h.WalkMoved(w.ctl.walk("plan-9"))
w.settleAll()
if lines := w.h.WalksWaiting(); len(lines) != 0 {
t.Fatalf("a walk let go is still said waiting: %+v", lines)
}
}
// A walk waiting while a delivery it carries is not at its turn says so, naming the delivery — what was said only
// in this owner's journal (novox/hq issue 529, note 2).
func TestAWaitingWalkSaysWhichDeliveryIsNotAtItsTurn(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atNzb)
w.h.mu.Lock()
w.h.waitsWhy = map[string]string{"plan-7": "novox/x@1 (published) carried by it not at its turn"}
w.h.walks["plan-7"] = seenWalk{walk: aWalk("plan-7", "novox/x", "1111", true), at: atNzb}
w.h.mu.Unlock()
lines := w.h.WalksWaiting()
if len(lines) != 1 || lines[0].Walk != "plan-7" || !strings.Contains(lines[0].Says, "not at its turn") {
t.Fatalf("the waiting walk is said as %+v", lines)
}
}
@@ -0,0 +1,150 @@
package main
import (
"sort"
"strings"
"testing"
"time"
)
// novox/hq issue 611: mesh-controller #293 changed one test file. Its check's plan, made while another walk built
// build-agent and route-proxy, read them whole and named them; at the cut their recorded build sources held none of
// its files, so its walk was done with no tier, "moved nothing the mesh holds: nothing to walk". The stranded rule
// (issue 474) read that as "ended without building build-agent, route-proxy", stopped it, and the operator was asked
// about a change already on every machine. The controller now says per module that the build matched what runs
// (the walk's delivery.unchanged), and such a delivery is delivered: its merge moves nothing (ADR 0270 §1). A walk
// that says neither — not built, not unchanged — still stops it.
const (
head293 = "94c928b4c7e1d2aa"
merge293 = "8f1aec0f2b9d6e11"
walk293 = "plan-1791867648768608552"
)
var at293 = time.Date(2026, 10, 13, 5, 10, 0, 0, time.UTC)
// matched is the why the controller gives a module it says unchanged.
const matched = "its build source in novox/mesh-controller holds none of the 1 changed file(s) " +
"(e.g. internal/builder/replay363_test.go): the build the mesh holds is its build"
// replay293 is #293 checked moving build-agent and route-proxy, merged, and its walk done with no tier, saying the
// modules named unchanged.
func replay293(t *testing.T, unchanged map[string]string) *world {
t.Helper()
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at293.Add(-time.Minute))
p := pr(ctl474, 293, head293, "fix/363-replay", "internal/builder/replay363_test.go")
w.h.PullUpdated(p)
w.h.Checked(verdictMoving(ctl474, 293, head293, "build-agent", "route-proxy"))
w.later(time.Minute)
w.h.PullMerged(mergedAt(p, merge293, at293))
w.later(90 * time.Second)
note := nothingNote
if len(unchanged) > 0 {
note += "; unchanged, the build the mesh holds is theirs: " + strings.Join(sortedKeys(unchanged), ", ")
}
w.h.WalkMoved(w.ctl.put(Walk{ID: walk293, Repository: ctl474, Branch: "main", Commit: merge293, Created: w.now,
Updated: w.now, State: walkDone, Tiers: [][]string{}, Modules: map[string]*WalkModule{}, Note: note,
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: ctl474, Commit: merge293}},
Unchanged: unchanged}}))
w.settleAll()
return w
}
// The live case: every module its plan named, the walk said matched what runs. Delivered at once, naming them, a
// success on the pull request, and nothing listed by stalled or asked of the operator, then or past the grace.
func TestAMergeWhoseModulesMatchWhatRunsIsDelivered(t *testing.T) {
w := replay293(t, map[string]string{"build-agent": matched, "route-proxy": matched})
id := IDOf(ctl474, head293)
d := w.delivery(id)
if d.State != Delivered {
t.Fatalf("#293, its modules said unchanged by its walk, is %s, waiting for %q", d.State, w.h.waitsFor(d))
}
why := d.Transitions[len(d.Transitions)-1].Why
for _, want := range []string{walk293, "build-agent, route-proxy", "matched what runs"} {
if !strings.Contains(why, want) {
t.Fatalf("delivered as %q, not saying %q", why, want)
}
}
if d.Stranded != "" {
t.Fatalf("it was read as stranded: %q", d.Stranded)
}
status := w.forge.statuses["novox/mesh-controller@"+head293+" mesh/delivery"]
if !strings.HasPrefix(status, "success ") {
t.Fatalf("mesh/delivery says %q", status)
}
w.later(strandedAfter + time.Minute)
w.settleAll()
if s := w.state(id); s != Delivered {
t.Fatalf("past the grace it is %s", s)
}
if lines := w.h.Stalled(); len(lines) != 0 {
t.Fatalf("stalled lists %+v", lines)
}
}
// A walk that said only one of its modules unchanged left the other unbuilt: it is still stopped past the grace,
// naming only the module nothing delivered.
func TestAMergeWithAModuleNeitherBuiltNorUnchangedIsStillStopped(t *testing.T) {
w := replay293(t, map[string]string{"build-agent": matched})
id := IDOf(ctl474, head293)
if s := w.state(id); s != Published {
t.Fatalf("within the grace it is %s", s)
}
w.later(strandedAfter + time.Minute)
w.settleAll()
d := w.delivery(id)
if d.State != Stopped {
t.Fatalf("route-proxy neither built nor unchanged, it is %s", d.State)
}
why := d.Transitions[len(d.Transitions)-1].Why
if !strings.Contains(why, "without building route-proxy (") {
t.Fatalf("its stop says %q", why)
}
}
// A walk that says nothing unchanged — a controller from before, or one that built nothing at all — stops it as
// before (issue 474).
func TestAMergeWhoseWalkSaysNothingUnchangedIsStillStopped(t *testing.T) {
w := replay293(t, nil)
w.later(strandedAfter + time.Minute)
w.settleAll()
if d := w.delivery(IDOf(ctl474, head293)); d.State != Stopped ||
!strings.Contains(d.Stranded, "without building build-agent, route-proxy") {
t.Fatalf("it is %s: %q", d.State, d.Stranded)
}
}
// Batched behind a later merge of its branch (issue 474's shape), a merge whose module the walk said unchanged moves
// nothing at either commit: delivered, not covered by a later merge that delivered nothing for it.
func TestABatchedMergeWhoseModuleMatchesWhatRunsIsDelivered(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at215.Add(-time.Minute))
p215 := pr(ctl474, 215, head215, "fix/431-make-check-runs-what-the-gate-runs", "Makefile")
p217 := pr(ctl474, 217, head217, "fix/432-a-test-reads-what-it-carries", "cmd/mesh-controller/check_test.go")
w.h.PullUpdated(p215)
w.h.PullUpdated(p217)
w.h.Checked(verdictMoving(ctl474, 215, head215, "build-agent"))
w.h.Checked(verdictMoving(ctl474, 217, head217))
w.later(time.Minute)
w.h.PullMerged(mergedAt(p215, merge215, at215))
w.later(time.Minute)
w.h.PullMerged(mergedAt(p217, merge217, at215.Add(56*time.Second)))
w.later(90 * time.Second)
w.h.WalkMoved(w.ctl.put(Walk{ID: batchWalk, Repository: ctl474, Branch: "main", Commit: merge217, Created: w.now,
Updated: w.now, State: walkDone, Tiers: [][]string{}, Modules: map[string]*WalkModule{}, Note: nothingNote,
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
{Repository: ctl474, Commit: merge215, Carried: merge217}, {Repository: ctl474, Commit: merge217}},
Unchanged: map[string]string{"build-agent": "its build source holds none of the 2 changed file(s)"}}}))
w.settleAll()
if d := w.delivery(IDOf(ctl474, head215)); d.State != Delivered {
t.Fatalf("#215, build-agent said unchanged by its batch's walk, is %s, waiting for %q", d.State, w.h.waitsFor(d))
}
}
func sortedKeys(m map[string]string) []string {
var out []string
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
@@ -0,0 +1,258 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 747: in the group fix/a-module-moves-only-when-its-program-changes, the controller's member #336
// (merged as 5d22d7df) failed at 22:32 during an outage of the forge's tools. The walk of the controller's next merge,
// #339 at 60cc9ec3, which contains 5d22d7df, failed with it and was resumed to done by a rebuild. mesh-delivery still
// held #336 failed, so its group stopped every later walk of the catalogue that carried the catalogue member's merge
// 2edef5a0: the catalogue could deliver nothing. A failed or stopped delivery is covered once a later finished walk of
// its repository that contains its merge built its modules, and a group stops a walk only for a member whose change is
// still undelivered.
//
// The shape is the live state of 2026-10-14 23:07, as mesh-delivery.show gave it, read back after a restart: the
// controller holds none of the walks of the night any more, only the walk of the catalogue's next merge.
const (
g747 = "fix/a-module-moves-only-when-its-program-changes"
ctl747, cat747 = "novox/mesh-controller", "novox/mesh-catalog"
head336, merge336 = "b34055f580d4e2a4", "5d22d7df86c73452"
head339, merge339 = "e4acab32d6a20ade", "60cc9ec363dfa7c6"
head301, merge301 = "d94764f00168471f", "2edef5a00ae05605"
merge307 = "0f76b6e2aaaaaaaa"
)
var (
at747 = time.Date(2026, 10, 14, 20, 21, 6, 0, time.UTC)
id336, id339, id301 = IDOf(ctl747, head336), IDOf(ctl747, head339), IDOf(cat747, head301)
unchangedRouteProxy747 = map[string]string{"route-proxy": "its build source holds none of the changed files"}
)
// night747 is the stuck state, with the forge saying, or not, that 60cc9ec3 contains 5d22d7df.
func night747(t *testing.T, contained bool) *world { return night747With(t, contained, nil) }
// night747With is night747 with #339's walk having built only the modules given, saying the others unchanged.
func night747With(t *testing.T, contained bool, built []string) *world {
t.Helper()
return night747Over(t, func(f *fakeForge) { f.contains[merge336+" in "+merge339] = contained }, built)
}
// night747Over is night747With with the forge set as given: one that is not told the pair cannot say.
func night747Over(t *testing.T, set func(*fakeForge), built []string) *world {
t.Helper()
at := at747
failedAt := at.Add(11 * time.Minute)
store := newMemStore()
moved339, unchanged339 := []string{"build-agent", "mesh-controller"}, unchangedRouteProxy747
if built != nil {
moved339, unchanged339 = built, map[string]string{}
for _, m := range []string{"build-agent", "mesh-controller", "route-proxy"} {
if !containsString(built, m) {
unchanged339[m] = "its build source holds none of the files #339 changed"
}
}
}
m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
State: Failed, Since: failedAt, Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
AnsweredBy: merge336, Check: &Verdict{Gate: "pass"},
Plan: planMoving(ctl747, "build-agent", "mesh-controller", "route-proxy"),
Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkFailed, Branch: "main", MovedSaid: true,
Note: "mesh-controller failed to build in tier 0", Created: at.Add(7 * time.Minute), Tiers: 2,
Unchanged: unchangedRouteProxy747},
Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
Why: "its walk failed, carrying its own commit"}}}
m339 := &Delivery{ID: id339, Repository: ctl747, Commit: head339, Number: 339, Base: "main", Branch: "fix/726",
State: Failed, Since: failedAt, Created: at.Add(-10 * time.Minute), MergedAs: merge339, MergedAt: at.Add(7 * time.Minute),
AnsweredBy: merge339, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "mesh-controller"),
// Its walk failed with #336's, and a rebuild resumed it to done: the record it keeps says so.
Walk: &WalkSeen{ID: "plan-339", Commit: merge339, State: walkDone, Branch: "main", MovedSaid: true,
Moved: moved339, Unchanged: unchanged339,
Created: at.Add(11 * time.Minute), Tiers: 2, Tier: 2},
Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
Why: "its walk failed, carrying its own commit"}}}
m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
State: Stopped, Since: failedAt, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"),
Transitions: []Transition{{At: failedAt, From: Ready, To: Stopped, Event: EvStop, By: byOwner,
Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
for _, d := range []*Delivery{m336, m339, m301} {
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
}
g := &Group{ID: g747, Members: []string{id301, id336}, Closed: true, Said: "failed",
Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
Verdict: "pass", Summary: "both compose", Commits: commitsOf([]*Delivery{m301, m336})}}
if err := store.PutGroup(g); err != nil {
t.Fatal(err)
}
ctlr := newFakeController()
// The catalogue's next merge: its walk carries 2edef5a0, merged before it and never walked, and waits for its word.
ctlr.put(Walk{ID: "plan-cat", Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
State: walkBuilding, Tiers: [][]string{{"gitea"}}, Modules: map[string]*WalkModule{"gitea": {}},
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge307}}}})
ctlr.mergedAt[merge301], ctlr.mergedAt[merge307] = m301.MergedAt, at.Add(46*time.Minute)
forge := newFakeForge()
set(forge)
w := newWorldOver(t, store, ctlr, forge, at.Add(47*time.Minute))
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
return w
}
func groupState747(w *world) (string, string) {
g := w.h.groups[g747]
return GroupState(g, w.h.membersOf(g))
}
// **A failed member is covered by a later finished walk that contains it**: #339's walk, done at 60cc9ec3, which the
// forge says contains 5d22d7df, built build-agent and mesh-controller, and #336's own walk said route-proxy unchanged.
// #336 stays failed, reads covered by that walk on its pull request, and its group no longer reads failed. #339, whose
// own walk was resumed to done, is covered by it too.
func TestAFailedMemberIsCoveredByALaterDoneWalkThatContainsIt(t *testing.T) {
w := night747(t, true)
d := w.delivery(id336)
if state, desc := statusOf(d); d.State != Failed || state != "success" || !strings.Contains(desc, "covered after its failure") ||
!strings.Contains(desc, "plan-339") {
t.Fatalf("#336's status reads %s %q", state, desc)
}
if s := statusOn(w, ctl747, head336); !strings.HasPrefix(s, "success") {
t.Fatalf("the forge was told %q of #336", s)
}
if state, desc := statusOf(w.delivery(id339)); state != "success" || !strings.Contains(desc, "plan-339") {
t.Fatalf("#339 reads %s %q; want it covered by its own walk, resumed to done", state, desc)
}
if state, why := groupState747(w); state == "failed" || state == "stopped" {
t.Fatalf("the group reads %s: %s", state, why)
}
// Told once, and kept: a restart tells nothing again.
notes := len(w.forge.notesOn(merge336))
again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute))
again.settleAll()
if state, _ := statusOf(again.delivery(id336)); len(w.forge.notesOn(merge336)) != notes || state != "success" {
t.Fatalf("a restart told it again or forgot it: %d notes, was %d", len(w.forge.notesOn(merge336)), notes)
}
}
// **The group then no longer stops the other repository's walk**: the catalogue member was stopped for #336 alone,
// so the walk of the catalogue's next merge, which carries its merge, is neither stopped nor held back, and the
// group waits for that walk to deliver the catalogue member's change.
func TestACoveredMemberNoLongerStopsTheOtherRepositorysWalk(t *testing.T) {
w := night747(t, true)
if len(w.ctl.withouts) > 0 || containsString(w.ctl.stops, "plan-cat") {
t.Fatalf("the catalogue's walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
if stopped, _ := w.h.carriesStopped("plan-cat"); len(stopped) > 0 {
t.Fatalf("the catalogue's walk is held back for %v", stopped)
}
if state, why := groupState747(w); state != "delivering" || !strings.Contains(why, id301) {
t.Fatalf("the group reads %s: %s; want delivering, the catalogue member waiting for a later walk", state, why)
}
// Once a later walk of the catalogue that contains its merge built its modules, it is covered, and the group is
// delivered.
w.forge.contains[merge301+" in "+merge307] = true
r := w.ctl.walk("plan-cat")
sent := w.now
r.Modules = map[string]*WalkModule{}
for _, m := range []string{"build-agent", "route-proxy", "gitea"} {
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
}
r.State = walkDone
w.h.WalkMoved(w.ctl.put(r))
w.later(time.Minute)
w.settleAll()
if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, "plan-cat") {
t.Fatalf("the catalogue member reads %s %q", state, desc)
}
if state, why := groupState747(w); state != "delivered" {
t.Fatalf("the group reads %s: %s", state, why)
}
}
// **A failed member not contained still stops it**: the forge says 60cc9ec3 does not contain 5d22d7df, so #336's change
// never reached the machines; it stays failed and uncovered, its group failed, and the catalogue's walk that carries
// the stopped member's merge is asked to leave it out.
func TestAFailedMemberNotContainedStillStopsTheOtherRepositorysWalk(t *testing.T) {
w := night747(t, false)
d := w.delivery(id336)
if state, desc := statusOf(d); d.State != Failed || state != "failure" {
t.Fatalf("#336 is %s and reads %s %q", d.State, state, desc)
}
if state, _ := groupState747(w); state != "failed" {
t.Fatalf("the group reads %s", state)
}
if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
t.Fatalf("the catalogue's walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
}
}
// **A later walk that only said a module unchanged covers nothing of an earlier merge that moves it**: #339's walk
// built mesh-controller alone and said build-agent unchanged for #339's own files, which says nothing of #336's; only
// #336's own walk says that of its change (route-proxy). #336 stays failed and its group still stops the catalogue.
func TestALaterWalksUnchangedCoversNoModuleOfAnEarlierMerge(t *testing.T) {
w := night747With(t, true, []string{"mesh-controller"})
if state, desc := statusOf(w.delivery(id336)); state != "failure" {
t.Fatalf("#336 reads %s %q, though no walk built build-agent for it", state, desc)
}
if state, _ := groupState747(w); state != "failed" {
t.Fatalf("the group reads %s", state)
}
if len(w.ctl.withouts) == 0 {
t.Fatal("the catalogue's walk was not asked to leave the stopped merge out")
}
}
// **A forge that cannot say holds the walk only within the bound** (review of issue 747): while the forge errors on
// whether 60cc9ec3 contains 5d22d7df, the catalogue's walk is held back, loudly, neither stopped nor asked to leave the
// merge out, and #336 still reads failed. Past the bound the forge's silence holds nothing back any more: the walk
// is asked to leave the stopped merge out, as with no later walk at all.
func TestAForgeThatCannotSayHoldsTheWalkOnlyWithinTheBound(t *testing.T) {
for _, c := range []struct {
name string
set func(*fakeForge)
}{
{"erroring", func(f *fakeForge) { f.containsDown = true }},
{"not answering for the pair", func(*fakeForge) {}},
} {
t.Run(c.name, func(t *testing.T) {
w := night747Over(t, c.set, nil)
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
if state, desc := statusOf(w.delivery(id336)); state != "failure" {
t.Fatalf("#336 reads %s %q while the forge cannot say", state, desc)
}
said := ""
for _, l := range w.h.WalksWaiting() {
if l.Walk == "plan-cat" {
said += l.Says
}
}
w.later(3 * time.Minute) // past the grace an unheld walk is given, so `stalled` lists it
w.settleAll()
for _, l := range w.h.Stalled() {
if l.Walk == "plan-cat" {
said += " | " + l.Says
}
}
if strings.Count(said, "the forge is asked since") != 2 {
t.Fatalf("the hold is not said by `stalled walks` and `stalled`: %q", said)
}
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
w.later(coverAskBound + time.Minute)
w.settleAll()
if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
t.Fatalf("past the bound the walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
}
})
}
}
@@ -0,0 +1,272 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 753: once issue 747's fix was live (2026-10-14 23:35), the catalogue's walk plan-1792012015539062661 at
// 0f76b6e2 (#307's merge; build-agent, gitea, route-proxy to all four machines) was resumed and said "published; its
// walk waits for mesh-delivery's word: no delivery that has not ended follows it or is answered by it". Every delivery
// it carried had ended: #301 stopped by its group for the controller's #336, delivered since, and waiting for "a later
// walk of its repository that contains its merge"; #304 stopped; #307 delivered. The only walk that contained #301's
// merge waited for #301's word: a deadlock, released by a hand `plans go`. A delivery whose stop lapsed gives its word
// to the walk that answers its merge, and a walk with no living delivery behind it says what it carries.
//
// The shape is the live state as `mesh-delivery.show` and `mesh-controller.plans` gave it, read back after a restart.
const (
merge304, head304, head307 = "4c0ffee0bbbbbbbb", "a304a304a304a304", "b307b307b307b307"
plan753 = "plan-1792012015539062661"
)
var id304, id307 = IDOf(cat747, head304), IDOf(cat747, head307)
// night753 is the stuck walk: #336 delivered (its change reached the machines), #301 stopped by the given act, #304
// stopped by its group, #307 delivered, and the catalogue's walk at #307's merge naming all three merges, waiting.
func night753(t *testing.T, stop Transition) *world { return night753With(t, stop) }
// night753With is night753 with more deliveries kept: one in #301's group is ordered after #336 and before #301.
func night753With(t *testing.T, stop Transition, more ...*Delivery) *world {
t.Helper()
return night753Over(t, stop, nil, nil, more...)
}
// night753Over is night753With with more merges named by the walk and the controller set as given before the first
// tick.
func night753Over(t *testing.T, stop Transition, merges []WalkMerge, set func(*fakeController), more ...*Delivery) *world {
t.Helper()
at := at747
store := newMemStore()
m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
State: Delivered, Since: at.Add(70 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "build-agent", "mesh-controller"),
Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkDone, Branch: "main", MovedSaid: true,
Moved: []string{"build-agent", "mesh-controller"}, Created: at.Add(7 * time.Minute), Tiers: 2, Tier: 2}}
m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
State: Stopped, Since: stop.At, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), Transitions: []Transition{stop}}
if stop.By == byOwner {
m301.StoppedFor = id336
}
m304 := &Delivery{ID: id304, Repository: cat747, Commit: head304, Number: 304, Base: "main", Branch: "fix/forge-events",
State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge304,
MergedAt: at.Add(30 * time.Minute), Check: &Verdict{Gate: "pass"},
Plan: planMoving(cat747, "build-agent", "gitea", "route-proxy"),
// Stopped with its walk, which the walk of #307 took over, and covered since by the walk of #306 (issue 747).
DeliveredBy: "plan-1792011830935225963 (novox/mesh-catalog at 0361c44a into main, done, contains its merge 4c0ffee0)",
CoverTold: true,
Walk: &WalkSeen{ID: "plan-1792009301770286799", Commit: merge304, State: walkSuperseded, Branch: "main",
TakenOverBy: plan753, Waits: true, WaitedFor: "mesh-delivery", Created: at.Add(31 * time.Minute)},
Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Published, To: Stopped, Event: EvStop,
Why: "its walk plan-1792009301770286799 was stopped by mesh-delivery"}}}
m307 := &Delivery{ID: id307, Repository: cat747, Commit: head307, Number: 307, Base: "main", Branch: "fix/ts-check",
State: Delivered, Since: at.Add(90 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge307,
MergedAt: at.Add(46 * time.Minute), Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent")}
for _, d := range append([]*Delivery{m336, m301, m304, m307}, more...) {
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
}
inGroup, order := []*Delivery{m301, m336}, []string{id336}
for _, d := range more {
if d.Group == g747 {
inGroup, order = append(inGroup, d), append(order, d.ID)
}
}
order = append(order, id301)
var ids []string
for _, d := range inGroup {
ids = append(ids, d.ID)
}
g := &Group{ID: g747, Members: ids, Closed: true, Said: "delivering",
Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
Verdict: "pass", Summary: "all compose", Commits: commitsOf(inGroup)}}
if err := store.PutGroup(g); err != nil {
t.Fatal(err)
}
ctlr := newFakeController()
modules := map[string]*WalkModule{"build-agent": {}, "gitea": {}, "route-proxy": {}}
ctlr.put(Walk{ID: plan753, Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
State: walkBuilding, Tiers: [][]string{{"build-agent", "gitea", "route-proxy"}}, Modules: modules,
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge304, Carried: merge307},
{Repository: cat747, Commit: merge307}}}})
if len(merges) > 0 {
r := ctlr.walk(plan753)
r.Delivery.Merges = append(merges, r.Delivery.Merges...)
ctlr.put(r)
}
if set != nil {
set(ctlr)
}
ctlr.mergedAt[merge301], ctlr.mergedAt[merge304], ctlr.mergedAt[merge307] = m301.MergedAt, m304.MergedAt, m307.MergedAt
w := newWorldOver(t, store, ctlr, newFakeForge(), at.Add(100*time.Minute))
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
return w
}
// groupStop753 is #301's stop as its group made it: for #336, whose change was delivered since.
func groupStop753() Transition {
return Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner,
Why: "its group " + g747 + " stopped: " + id336 + " failed"}
}
func saidOf(lines []StalledLine, walk string) (string, State) {
said, state := "", State("")
for _, l := range lines {
if l.Walk == walk {
said, state = said+l.Says, l.State
}
}
return said, state
}
// **A stopped delivery gives its word to the later walk it waits for**: #301, stopped for #336, delivered since, waits for
// a later walk of the catalogue that contains its merge; plan-1792012015539062661 names that merge and waits for this
// owner's word. #301 gives it, once, saying why, and `stalled walks` never calls the walk unheld.
func TestAStoppedDeliveryGivesItsWordToTheWalkItWaitsFor(t *testing.T) {
w := night753(t, groupStop753())
if n := strings.Count(strings.Join(w.ctl.delivers, " "), plan753); n != 1 {
t.Fatalf("the walk was let go %d times (%v); want once", n, w.ctl.delivers)
}
d := w.delivery(id301)
if d.State != Stopped || !strings.Contains(d.LetGoWhy, lapsedWord(plan753)) || !strings.Contains(d.LetGoWhy, "issue 753") {
t.Fatalf("#301 is %s, its word given for %q", d.State, d.LetGoWhy)
}
if said, state := saidOf(w.h.WalksWaiting(), plan753); state == Unheld {
t.Fatalf("the walk reads unheld: %q", said)
}
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
t.Fatalf("the walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
// Once that walk built build-agent and route-proxy and ended done, #301 is covered by it (issue 747).
w.forge.contains[merge301+" in "+merge307] = true
r := w.ctl.walk(plan753)
sent := w.now
for _, m := range []string{"build-agent", "gitea", "route-proxy"} {
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
}
r.State = walkDone
w.h.WalkMoved(w.ctl.put(r))
w.later(time.Minute)
w.settleAll()
if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, plan753) {
t.Fatalf("#301 reads %s %q", state, desc)
}
}
// **A walk still holding a live delivery not at its turn waits for it**: the stopped delivery's word is one turn among
// the walk's, never a way past a live delivery it carries.
func TestALapsedStopDoesNotLetGoPastALiveDelivery(t *testing.T) {
at := at747
held := &Delivery{ID: IDOf(cat747, "c308c308c308c308"), Repository: cat747, Commit: "c308c308c308c308", Number: 308,
Base: "main", Branch: "fix/held", State: Held, HeldWhy: "a person's", Since: at, Created: at.Add(-time.Hour),
MergedAs: "c308aaaaaaaaaaaa", MergedAt: at.Add(40 * time.Minute), Check: &Verdict{Gate: "pass"},
Plan: planMoving(cat747, "gitea"), Walk: &WalkSeen{ID: plan753, Commit: merge307, State: walkBuilding, Branch: "main",
Waits: true, Created: at.Add(46 * time.Minute)}}
w := night753With(t, groupStop753(), held)
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go past a held delivery it carries: %v", w.ctl.delivers)
}
if said, state := saidOf(w.h.WalksWaiting(), plan753); state != Held {
t.Fatalf("`stalled walks` reads %s %q; want the held delivery", state, said)
}
}
// **A stop by a person gives no word**: #301 stopped by hand holds its change back; the walk is asked to leave it out,
// never let go for it.
func TestAPersonsStopGivesNoWord(t *testing.T) {
w := night753(t, Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: "jochen",
Why: "stopped by hand"})
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go for a person's stop: %v", w.ctl.delivers)
}
if len(w.ctl.withouts) == 0 && len(w.ctl.stops) == 0 {
t.Fatal("the walk carrying a person's stop was neither left without it nor stopped")
}
}
// **A walk no living delivery holds says what it carries** (the issue's third expectation): with nothing to give it
// its word, `stalled walks` and, past the grace, `stalled` name every ended delivery it carries and its state, never
// only that nothing holds it.
func TestAWalkWithNoLivingDeliverySaysWhatItCarries(t *testing.T) {
w := night753(t, groupStop753())
// The walk read again as waiting, naming only #307's merge, which ended delivered: no delivery whose stop lapsed
// gives it its word, and no live one holds it.
r := w.ctl.walk(plan753)
r.Delivery.Go = nil
r.Delivery.Merges = []WalkMerge{{Repository: cat747, Commit: merge307}}
w.h.WalkMoved(w.ctl.put(r))
said, state := saidOf(w.h.WalksWaiting(), plan753)
if state != Unheld || !strings.Contains(said, id307+" (delivered)") {
t.Fatalf("`stalled walks` reads %s %q; want unheld, naming %s (delivered)", state, said, id307)
}
w.settleAll()
w.later(matchGrace + time.Minute)
w.settleAll()
said, _ = saidOf(w.h.Stalled(), plan753)
if !strings.Contains(said, "every delivery it carries ended: ") || !strings.Contains(said, id307+" (delivered)") {
t.Fatalf("`stalled` reads %q", said)
}
}
// m299 is a member of #301's group ordered before it, stopped for #336 as #301 was, its merge on another walk.
func m299() *Delivery {
at := at747
return &Delivery{ID: IDOf(cat747, "e299e299e299e299"), Repository: cat747, Commit: "e299e299e299e299", Number: 299,
Base: "main", Branch: g747, Group: g747, State: Stopped, StoppedFor: id336, Since: at.Add(11 * time.Minute),
Created: at.Add(-3 * time.Hour), MergedAs: "e299aaaaaaaaaaaa", MergedAt: at.Add(5 * time.Second),
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "gitea"),
Transitions: []Transition{{At: at.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner,
Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
}
// **A word the controller refuses is never silent** (review of issue 753): the controller keeps refusing it, and past
// the grace `stalled` lists the walk, with the refusal.
func TestARefusedWordIsListedByStalled(t *testing.T) {
w := night753Over(t, groupStop753(), nil, func(c *fakeController) { c.refuseDeliver = true })
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go: %v", w.ctl.delivers)
}
w.later(matchGrace + time.Minute)
w.settleAll()
said, state := saidOf(w.h.Stalled(), plan753)
if state != Published || !strings.Contains(said, "the controller refused its word: the controller refuses the word") {
t.Fatalf("`stalled` reads %s %q", state, said)
}
// Once the controller takes it, the walk is let go.
w.ctl.refuseDeliver = false
w.settleAll()
if !containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was not let go once the controller took the word: %v", w.ctl.delivers)
}
}
// **A walk held back for another stopped change is never silent** (review of issue 753): #304 was stopped by a person,
// and the controller refuses to leave its merge out or to stop the walk, so the word #301 owes is never sent; past the
// grace `stalled` lists the walk, saying why.
func TestAWalkHeldForAPersonsStopIsListedByStalled(t *testing.T) {
at := at747
m305 := &Delivery{ID: IDOf(cat747, "f305f305f305f305"), Repository: cat747, Commit: "f305f305f305f305", Number: 305,
Base: "main", Branch: "fix/other", State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour),
MergedAs: "f305aaaaaaaaaaaa", MergedAt: at.Add(35 * time.Minute), Check: &Verdict{Gate: "pass"},
Plan: planMoving(cat747, "gitea"), Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Ready,
To: Stopped, Event: EvStop, By: "jochen", Why: "stopped by hand"}}}
w := night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: m305.MergedAs, Carried: merge307}},
func(c *fakeController) { c.refuseWithout, c.refuseStop = true, true }, m305)
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk carrying a person's stop was let go: %v", w.ctl.delivers)
}
w.later(matchGrace + time.Minute)
w.settleAll()
said, _ := saidOf(w.h.Stalled(), plan753)
if !strings.Contains(said, "owe it (novox/hq issue 753)") || !strings.Contains(said, "f305aaaaaaaaaaaa") ||
!strings.Contains(said, "whose delivery was stopped") {
t.Fatalf("`stalled` reads %q", said)
}
}
@@ -8,12 +8,12 @@ import (
// novox/hq issue 783: mesh-catalog commit b5417c28 was pushed while #305 (from branch adr-359-mesh-mail-733) was
// open, so this owner recorded its delivery as #305's. #305 then closed unmerged (its base branch had been
// deleted), and #315, opened from the same branch and head, merged it as 0914a33f. The delivery stayed closed
// deleted), and #315, opened from the same branch and head, merged it as 0914a33f. The delivery stayed stopped
// ("the pull request closed unmerged"), though it recorded merged_as 0914a33f: a rule did not hold — a delivery
// keyed by commit follows the pull request that merges that commit, and a closed duplicate does not end it — and
// the walk that carried the merge waited 64 minutes for this owner's word until a person's `plans go` released it.
//
// A delivery closed only because its pull request closed unmerged, with no merge recorded, does not end its
// A delivery stopped only because its pull request closed unmerged, with no merge recorded, does not end its
// commit's delivery for another pull request with the same head: it is started over, under the same id, for the
// pull request that announces or merges it next.
@@ -26,22 +26,22 @@ const (
branch783 = "adr-359-mesh-mail-733"
)
// TestADeliveryClosedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit is the issue's own shape: #305
// announced, closed unmerged (closed); #315 announced from the same head, checked and readied; #315 merges it,
// TestADeliveryStoppedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit is the issue's own shape: #305
// announced, closed unmerged (stopped); #315 announced from the same head, checked and readied; #315 merges it,
// and the walk that answers that merge is let go — the delivery follows #315, never stuck on #305's stop.
func TestADeliveryClosedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit(t *testing.T) {
func TestADeliveryStoppedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783)
w.h.PullUpdated(pr(catalog783, 305, head783, branch783, "src/a.go"))
w.h.PullClosed(pr(catalog783, 305, head783, branch783))
if w.state(id) != Closed {
t.Fatalf("#305 closed unmerged left it %s; want closed", w.state(id))
if w.state(id) != Stopped {
t.Fatalf("#305 closed unmerged left it %s; want stopped", w.state(id))
}
w.h.PullUpdated(pr(catalog783, 315, head783, branch783, "src/a.go"))
if d := w.delivery(id); d.State == Closed || d.Number != 315 {
t.Fatalf("#315, the same head, left it %s, Number %d; want started over for #315, not closed", d.State, d.Number)
if d := w.delivery(id); d.State == Stopped || d.Number != 315 {
t.Fatalf("#315, the same head, left it %s, Number %d; want started over for #315, not stopped", d.State, d.Number)
}
if d := w.delivery(id); !strings.Contains(d.Transitions[len(d.Transitions)-1].Why, "#305 with the same head closed unmerged") {
t.Fatalf("the start-over's transition names what it replaces: %q", d.Transitions[len(d.Transitions)-1].Why)
@@ -52,79 +52,100 @@ func TestADeliveryClosedByAClosedDuplicateFollowsThePullRequestThatMergesItsComm
t.Fatalf("a passing verdict for #315 left it %s", d.State)
}
walk := w.ctl.put(aWalk("plan-783", catalog783, merge783, true))
w.h.WalkMoved(walk)
w.h.PullMerged(merged(catalog783, 315, head783, merge783))
if d := w.delivery(id); d.State != Merged || d.Number != 315 {
t.Fatalf("#315's merge left it %s, Number %d; want merged under #315", d.State, d.Number)
if d := w.delivery(id); d.State != Published || d.Number != 315 {
t.Fatalf("#315's merge left it %s, Number %d; want published under #315", d.State, d.Number)
}
w.settleAll()
if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-783" {
t.Fatalf("the walk that answers #315's merge was not let go: %v", w.ctl.delivers)
}
if d := w.delivery(id); d.State != Delivering {
t.Fatalf("let go, it is %s; want delivering", d.State)
}
}
// TestAMergeWithoutAnAnnouncementStartsOverAClosedDuplicateToo is decision 4: the merging pull request is never
// TestAMergeWithoutAnAnnouncementStartsOverAStoppedDuplicateToo is decision 4: the merging pull request is never
// announced on its own (heard only as `pull.merged`) — the start-over happens in PullMerged itself, not only in
// PullUpdated, so a merge heard without the open announcement is not lost either.
func TestAMergeWithoutAnAnnouncementStartsOverAClosedDuplicateToo(t *testing.T) {
func TestAMergeWithoutAnAnnouncementStartsOverAStoppedDuplicateToo(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783b)
w.h.PullUpdated(pr(catalog783, 306, head783b, branch783+"-b", "src/b.go"))
w.h.PullClosed(pr(catalog783, 306, head783b, branch783+"-b"))
if w.state(id) != Closed {
t.Fatalf("#306 closed unmerged left it %s; want closed", w.state(id))
if w.state(id) != Stopped {
t.Fatalf("#306 closed unmerged left it %s; want stopped", w.state(id))
}
walk := w.ctl.put(aWalk("plan-783b", catalog783, merge783b, true))
w.h.WalkMoved(walk)
m := merged(catalog783, 316, head783b, merge783b)
m.HeadChecks = map[string]string{"mesh/check": "success"}
w.h.PullMerged(m)
if d := w.delivery(id); d.State == Closed || d.Number != 316 {
if d := w.delivery(id); d.State == Stopped || d.Number != 316 {
t.Fatalf("#316's merge, heard with no announcement of its own, left it %s, Number %d; want started over for #316",
d.State, d.Number)
}
if d := w.delivery(id); d.State != Merged {
t.Fatalf("#316's merge left it %s; want merged", d.State)
w.settleAll()
if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-783b" {
t.Fatalf("the walk that answers #316's merge was not let go: %v", w.ctl.delivers)
}
}
// TestAClosedDeliveryAnnouncedAgainByItsOwnPullRequestStaysClosed guards decision 3's "any other closed
// TestAStoppedDeliveryAnnouncedAgainByItsOwnPullRequestStaysStopped guards decision 3's "any other stopped
// delivery ... is untouched": the same pull request announced again after its own close is the ordinary
// 'announced again' path, never a start-over, and stays closed.
func TestAClosedDeliveryAnnouncedAgainByItsOwnPullRequestStaysClosed(t *testing.T) {
// 'announced again' path, never a start-over, and stays stopped.
func TestAStoppedDeliveryAnnouncedAgainByItsOwnPullRequestStaysStopped(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783c)
w.h.PullUpdated(pr(catalog783, 307, head783c, branch783+"-c", "src/c.go"))
w.h.PullClosed(pr(catalog783, 307, head783c, branch783+"-c"))
if w.state(id) != Closed {
t.Fatalf("#307 closed unmerged left it %s; want closed", w.state(id))
if w.state(id) != Stopped {
t.Fatalf("#307 closed unmerged left it %s; want stopped", w.state(id))
}
w.h.PullUpdated(pr(catalog783, 307, head783c, branch783+"-c", "src/c.go"))
if d := w.delivery(id); d.State != Closed || d.Number != 307 {
t.Fatalf("#307 announced again by itself left it %s, Number %d; want it to stay closed", d.State, d.Number)
if d := w.delivery(id); d.State != Stopped || d.Number != 307 {
t.Fatalf("#307 announced again by itself left it %s, Number %d; want it to stay stopped", d.State, d.Number)
}
}
// TestAMergeHeardWithoutAnAnnouncementStartsOverAClosedDuplicateAsMadeFromTheForgesWord: a delivery started over from a
// merge heard without its open announcement takes FromForge exactly as a first-heard merge does. History is gone with the
// states after the merge (ADR 0539 §17): a merge heard long after it happened is merged, or held when no check passed.
func TestAMergeHeardWithoutAnAnnouncementStartsOverAClosedDuplicateAsMadeFromTheForgesWord(t *testing.T) {
// TestAMergeHeardLongAfterStartingOverAStoppedDuplicateIsHistoryLikeAnyOtherReplayedMerge is review round 1's
// decision: a delivery started over from a merge heard without its open announcement (as in decision 4, above)
// takes FromForge exactly as a first-heard merge does, so one replayed from the bus's history long after it
// happened still falls under the history rule (novox/hq issue 313) and is never held for a person.
func TestAMergeHeardLongAfterStartingOverAStoppedDuplicateIsHistoryLikeAnyOtherReplayedMerge(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783d)
w.h.PullUpdated(pr(catalog783, 308, head783d, branch783+"-d", "src/d.go"))
w.h.PullClosed(pr(catalog783, 308, head783d, branch783+"-d"))
if w.state(id) != Closed {
t.Fatalf("#308 closed unmerged left it %s; want closed", w.state(id))
if w.state(id) != Stopped {
t.Fatalf("#308 closed unmerged left it %s; want stopped", w.state(id))
}
m := merged(catalog783, 318, head783d, merge783d)
m.MergedAt = w.now.Add(-48 * time.Hour).UTC().Format(time.RFC3339)
m.HeadChecks = map[string]string{"mesh/check": "success"}
m.MergedAt = w.now.Add(-historyLate - time.Minute).UTC().Format(time.RFC3339)
w.h.PullMerged(m)
d := w.delivery(id)
if d.State != Merged || d.Number != 318 {
t.Fatalf("#318's merge, heard with no announcement of its own, left it %s, Number %d; want merged under #318", d.State, d.Number)
if d.State == Stopped || d.Number != 318 {
t.Fatalf("#318's merge, heard long after it happened with no announcement of its own, left it %s, Number %d; "+
"want started over for #318", d.State, d.Number)
}
if !d.FromForge {
t.Fatal("a merge heard without its own announcement is made from the forge's word: FromForge is false")
}
if d.History == "" {
t.Fatalf("a merge heard %s after it happened is history, never held for a person; History is empty", historyLate+time.Minute)
}
if d.State == Held {
t.Fatalf("history is never held for a person; state is %s, HeldWhy %q", d.State, d.HeldWhy)
}
}
@@ -9,8 +9,8 @@ import (
// mesh/delivery-group says success there; pending would keep its later members from ever merging.
func TestReplay902(t *testing.T) {
for state, want := range map[string]string{
"ready": "success", "merging": "success", "merged": "success",
"checking": "pending", "rejected": "failure", "held": "error", "closed": "error",
"ready": "success", "delivering": "success", "delivered": "success",
"checking": "pending", "rejected": "failure", "failed": "failure", "stopped": "error",
} {
if got := groupStatusState(state); got != want {
t.Errorf("a group %s says %s on mesh/delivery-group, want %s", state, got, want)
@@ -41,8 +41,8 @@ func TestReplay902TheNextMemberOfADeliveringGroupReadsSuccess(t *testing.T) {
st, _ := GroupState(g, w.h.membersOf(g))
s := w.forge.statuses["novox/mesh-controller@"+ctlHead+" mesh/delivery-group"]
t.Logf("state=%s status=%q", st, s)
if st != "merging" || !strings.HasPrefix(s, "success") {
t.Fatalf("want merging/success, got %s %q", st, s)
if st != "delivering" || !strings.HasPrefix(s, "success") {
t.Fatalf("want delivering/success, got %s %q", st, s)
}
}
@@ -53,6 +53,7 @@ func TestReplay902AClosedGroupComposedAgainAndFailingSaysNoSuccess(t *testing.T)
w.h.PullUpdated(pr("novox/mesh-controller", 1, newHead, "feat/x", "internal/a.go"))
w.h.Checked(verdict("novox/mesh-controller", 1, newHead, "pass"))
w.settleAll()
t.Logf("checks asked: %v", w.ctl.checks)
w.h.Checked(CheckedEvent{Group: "feat/x", ID: "check-2", Verdict: "fail", Summary: "no longer compose"})
w.settleAll()
g := w.h.groups["feat/x"]
@@ -2,8 +2,7 @@
// runtime launches. It owns the delivery — one commit in one repository, from its pull request's head to
// every machine — and the delivery group above it: their states in one compiled table, kept in its own state
// on the bus, every transition said as an event, noted on the commit and shown on the pull request through
// the forge's holder. It asks the Approve of a merge and merges once; it asks the controller nothing and derives nothing
// after the merge (ADR 0539 §17). stdout
// the forge's holder. It never sends to a machine: it asks the controller, through its seat's verbs. stdout
// is the MCP channel; what this module says, it says on stderr.
package main
@@ -31,15 +30,17 @@ type listening struct {
func (l *listening) set(s string) { l.mu.Lock(); l.now = s; l.mu.Unlock() }
func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.now }
// The events this owner follows: the forge's pull requests. The forge's status events say that a check answered (events.go).
// The events this owner follows: the forge's pull requests, and the controller's walks.
// The forge's status events say that a check answered (events.go).
var followed = []string{"gitea.pull.updated", "gitea.pull.merged", "gitea.pull.closed",
"gitea.forge.status", "gitea.forge.pull-request.edited", "gitea.forge.pull-request.closed",
"gitea.forge.status", "gitea.forge.pull-request.edited", "gitea.forge.pull-request.closed", ControllerSeat + ".plan-moved",
forgePattern, ConfigChangedEvent}
// forgePattern is the forge bridge's messages on FORGE (novox/hq ADR 0437 §8): the gate follows a merge and the refusal of one.
const forgePattern = "mesh-delivery-forge-bridge.forge.**"
func main() {
occupancy := &busOccupancy{}
settingsFile := os.Getenv("MESH_DELIVERY_GATE_SETTINGS")
var saidMu sync.Mutex
lastSaid := ""
@@ -57,20 +58,25 @@ func main() {
return s
}
h := &Holder{
Settings: settings,
Store: kvStore{},
Forge: toolForge{ask: stdio.Ask},
Emit: func(event string, body any) error { return stdio.Emit(event, body) },
Logf: logf,
Operator: kvOperator{},
Merges: kvMerges{},
Source: kvSource{},
Owed: kvOwed{},
Pulls: kvPulls{},
Settings: settings,
Store: kvStore{},
Controller: seatController{ask: stdio.Ask},
Forge: toolForge{ask: stdio.Ask},
Emit: func(event string, body any) error { return stdio.Emit(event, body) },
Logf: logf,
Operator: kvOperator{},
Merges: kvMerges{},
Source: kvSource{},
Owed: kvOwed{},
Pulls: kvPulls{},
// The gate acts only while it holds the seat, and carries today's state over when it first does (seat.go).
Seat: occupancy,
Carry: carryFromBus,
}
h.OnHold = func() { go readOnHolding(h) }
h.init()
l := &listening{now: "not yet: starting"}
go run(h, l)
go run(h, l, occupancy)
if err := announceKept(stdio.Keeps); err != nil {
logf("%v", err)
os.Exit(1)
@@ -81,14 +87,15 @@ func main() {
}
}
// run takes the events and the operator's answers. The state is read back at the start (start), and what the gate may have
// missed meanwhile is read once then (readOnStart): the forge's open pull requests, the operator's asks — and it acts on
// events from then on (novox/hq ADR 0406 §1, issue 975): no tick, no forge poll, no reconcile loop. Each step is retried,
// each failure said, never given up on quietly.
func run(h *Holder, l *listening) {
// run takes the events and the operator's answers, which a gate that does not hold the seat acknowledges and leaves
// alone, and watches who has the seat (watchSeat). The state is read back, after today's is carried, when the gate first
// holds the seat (SeatChanged), and what it may have missed meanwhile is read once then (readOnHolding): the controller's
// walks, the forge's open pull requests, the operator's asks — and it acts on events from then on (novox/hq ADR 0406 §1,
// issue 975): no tick, no forge poll, no reconcile loop. Each step is retried, each failure said, never given up on quietly.
func run(h *Holder, l *listening, occupancy *busOccupancy) {
time.Sleep(500 * time.Millisecond) // Serve first: the state is reached through it
loadKept(h, l)
go h.Work()
go watchSeat(h, l, occupancy)
for _, pattern := range followed {
pattern := pattern
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
@@ -131,35 +138,33 @@ func run(h *Holder, l *listening) {
time.Sleep(wait)
}
}()
l.set("listening")
logf("[mesh-delivery] listening: %s", strings.Join(followed, ", "))
readOnStart(h)
}
// loadKept reads the kept deliveries and groups back, retrying until it can: no event is taken before.
func loadKept(h *Holder, l *listening) {
// readOnHolding reads once, now that the gate holds the seat and events arrive, what happened while it was away: the
// forge's open pull requests (heads nobody checks) are read by the first look, the asks the router closed meanwhile are read
// here, and the controller's walks, until they have been read once; plan-moved events carry every move after.
func readOnHolding(h *Holder) {
h.wantForge()
h.hearAsks()
h.Poke()
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
err := h.Load()
err := h.Reconcile()
if err == nil {
break
}
l.set("not yet: " + err.Error())
logf("[mesh-delivery] the kept state cannot be read yet (%v); asking again in %s", err, wait)
logf("[mesh-delivery] the controller's walks cannot be read yet (%v); asking again in %s", err, wait)
time.Sleep(wait)
}
}
// readOnStart reads once what happened while the gate was away: the forge's open pull requests (heads nobody checks) are
// read by the first look, and the asks the router closed meanwhile are read here.
func readOnStart(h *Holder) {
h.wantForge()
h.hearAsks()
h.SourceCatchUp() // a merge missed or marked unwritten while the gate was away reaches source.main, and so the checker (issue 1726)
h.Poke()
}
// handle takes one event, then asks for a look at every step the event makes possible (Work, React). An event that cannot be read is
// said and acknowledged: read again it would fail again.
// said and acknowledged: read again it would fail again, and the controller's walks are read back by comparison anyway.
func handle(h *Holder, e stdio.Envelope) error {
if !h.Holding() {
return nil // not the holder of the seat: the event is acknowledged and taken by no one (seat.go)
}
taken := take(h, e)
h.Poke()
return taken
@@ -222,6 +227,13 @@ func take(h *Holder, e stdio.Envelope) error {
return nil
}
h.ConfigChanged(c)
case ControllerSeat + ".plan-moved":
var w Walk
if err := json.Unmarshal(e.Body, &w); err != nil {
h.Refuse(fmt.Errorf("a walk is not readable: %v", err))
return nil
}
h.WalkMoved(w)
}
return nil
}
@@ -288,23 +300,51 @@ func tools(h *Holder, l *listening) []stdio.Tool {
"repository": str("owner/repository"), "group": str("a group's id (its branch name)"),
"all": map[string]any{"type": "string", "enum": []string{"true", "false"}}}},
Run: func(a map[string]any) (any, error) {
return map[string]any{
// Headed by the week's delivery times (novox/hq ADR 0282 decision 6).
return map[string]any{"times": h.TimesLine(),
"deliveries": h.Deliveries(strArg(a, "state"), strArg(a, "repository"), strArg(a, "group"), boolArg(a, "all"))}, nil
}},
{Name: seat + "times",
Description: "Delivery time (novox/hq ADR 0282): from a merge to every machine of its walk running the build. " +
"Each class's budget (a leaf module three minutes, a core module ten), the last days' median and worst, " +
"how many within and over, the newest delivery of each class, every delivery over its budget with its " +
"longest phase, and the delivered ones whose time is not known and why. Given a delivery's id, its time " +
"phase by phase: window, queued, word, build, send-first, judgement, send-rest, between-tiers, apply — " +
"each measured, unknown (never a zero) or none.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"days": str("how many days back (default 7)"), "id": str("one delivery's id: its phases")}},
Run: func(a map[string]any) (any, error) {
days, err := intArg(a, "days")
if err != nil {
return nil, err
}
return h.Times(days, strArg(a, "id"))
}},
{Name: seat + "show",
Description: "One delivery or group whole: every transition with when and why, and its group. " +
"What the merge moved is source.main; whether it reached the machines is the view's.",
Description: "One delivery or group whole: its delivery plan, every transition with when and why, the " +
"machine steps of its walk and its group.",
Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("a delivery's id, or a group's")},
"required": []string{"id"}},
Run: func(a map[string]any) (any, error) { return h.Show(strArg(a, "id")) }},
{Name: seat + "groups",
Description: "Every delivery group: its members, its composed check and its state, derived from its members. " +
"A group has no order: it is merged in one pass, and moves source.main once (novox/hq ADR 0418, ADR 0539 clause 17); what " +
"reached the machines is the delivery view's. On mesh/check its check is the checker's one verdict at every member's head " +
"(ADR 0447); composed_gate says its members' gate is the controller's composed check's (ADR 0403), for a base still on it.",
"A group has no order: it is merged in one pass and delivered by one plan, whose tiers come from the module " +
"graph (novox/hq ADR 0418). On mesh/check its check is the checker's one verdict at every member's head (ADR 0447); " +
"composed_gate says its members' gate is the controller's composed check's (ADR 0403), for a base still on it.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"all": map[string]any{"type": "string", "enum": []string{"true", "false"}}}},
Run: func(a map[string]any) (any, error) { return h.Groups(boolArg(a, "all")), nil }},
{Name: seat + "what-if",
Description: "The delivery plan a change would have, asked of the controller's planner and kept nowhere.",
Input: map[string]any{"type": "object", "properties": map[string]any{"repository": str("owner/repository"),
"paths": str("the files it changes, comma-separated"), "base": str("the branch it merges into (default main)")},
"required": []string{"repository", "paths"}},
Run: func(a map[string]any) (any, error) {
if err := need(a, "repository", "paths"); err != nil {
return nil, err
}
return h.WhatIf(strArg(a, "repository"), strArg(a, "base"), strings.Split(strArg(a, "paths"), ","))
}},
{Name: seat + "checks",
Description: "What the mesh's checks said of a pull request's head or of one commit: each of the commit's " +
"statuses (mesh/check, mesh/delivery, …) with its state, description and when it " +
@@ -326,12 +366,20 @@ func tools(h *Holder, l *listening) []stdio.Tool {
}},
{Name: seat + "table",
Description: "The state table every delivery runs by: each transition with its guard, each state's bound " +
"and what healer H2 may do once it has passed; ",
"and what healer H2 may do once it has passed; and the machine steps' table.",
Run: func(map[string]any) (any, error) { return TableText(), nil }},
{Name: seat + "stalled",
Description: "Every delivery held past its state's bound, with the transition the table lets healer H2 take. " +
"Also a merge refused or not made past its bound, a group half merged, and a head with no merge check.",
Run: func(a map[string]any) (any, error) { return h.Stalled(), nil }},
"With walks: true, instead each walk waiting for this owner's word and what holds it — a held delivery, " +
"its reason and the verb that releases it, or the deliveries not at their turn (novox/hq issue 529).",
Input: map[string]any{"type": "object", "properties": map[string]any{
"walks": map[string]any{"type": "boolean", "description": "the waiting walks and what holds each, instead"}}},
Run: func(a map[string]any) (any, error) {
if walks, _ := a["walks"].(bool); walks {
return h.WalksWaiting(), nil
}
return h.Stalled(), nil
}},
{Name: seat + "recheck",
Description: "Refused for every delivery: the checker checks a head again on every move of the head, its trunk or its " +
"group, and nothing is asked of anyone here (novox/hq ADR 0447, issue 1318). Kept so a caller is told where to read.",
@@ -343,9 +391,34 @@ func tools(h *Holder, l *listening) []stdio.Tool {
}
return h.Recheck(strArg(a, "id"), strArg(a, "why"), aPerson)
}},
{Name: seat + "release",
Description: "A person's word that a held delivery goes on: it starts delivering. A delivery held for a failed backup asks the controller to take the backups again first. With why.",
Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("the delivery's id"),
"why": str("why, kept with the transition")}, "required": []string{"id", "why"}},
Run: func(a map[string]any) (any, error) {
if err := need(a, "id", "why"); err != nil {
return nil, err
}
return h.Release(strArg(a, "id"), strArg(a, "why"), aPerson)
}},
{Name: seat + "stop",
Description: "Stop a delivery that is not final, with why: its walk is ended through the controller, and " +
"in a delivering group every member after it is stopped too, naming it. When its walk carries other " +
"deliveries, it first answers what ending the walk would take down, and acts only on a way: `again` " +
"ends the walk and walks the others again without this merge, `walk` stops the whole walk (novox/hq ADR 0299).",
Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("the delivery's id"),
"why": str("why, kept with the transition"),
"way": str("on a walk shared with other deliveries: again (the others walked again without it) or walk " +
"(the whole walk stopped)")}, "required": []string{"id", "why"}},
Run: func(a map[string]any) (any, error) {
if err := need(a, "id", "why"); err != nil {
return nil, err
}
return h.StopWay(strArg(a, "id"), strArg(a, "why"), aPerson, strArg(a, "way"))
}},
{Name: seat + "close",
Description: "Healer H2's verb: for a proposed delivery past its bound, take the transition the table " +
"names for that state. Refused for any other.",
Description: "Healer H2's verb: for a delivery held past its bound, read its walk again and take the " +
"transition the table names for that state. Refused for any other.",
Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("the delivery's id"),
"why": str("the condition it answers")}, "required": []string{"id", "why"}},
Run: func(a map[string]any) (any, error) {
@@ -354,16 +427,40 @@ func tools(h *Holder, l *listening) []stdio.Tool {
}
return h.Close(strArg(a, "id"), strArg(a, "why"))
}},
{Name: seat + "retire-history",
Description: "Retire held deliveries that are history, on a person's word, with why: each made from the " +
"forge's word for a merge on the trunk heard 15 minutes or more after it happened — replayed when this " +
"owner first held the seat — with no walk of it open. Never released, delivered or put back: retired, " +
"final. One by id, refused when the rule does not take it; or every held one (of one repository when " +
"named), answering how many it retired and naming those it left held. dry \"true\" answers what it " +
"would do and changes nothing.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"why": str("why, kept with every transition"), "id": str("one delivery's id; every held one when empty"),
"repository": str("owner/repository: only its held deliveries"),
"dry": map[string]any{"type": "string", "enum": []string{"true", "false"}}},
"required": []string{"why"}},
Run: func(a map[string]any) (any, error) {
if err := need(a, "why"); err != nil {
return nil, err
}
return h.RetireHistory(strArg(a, "id"), strArg(a, "repository"), strArg(a, "why"), aPerson,
boolArg(a, "dry"))
}},
{Name: "delivery_status",
Description: "Whether this owner works: how many deliveries in each state, the groups, what is owed to " +
"the forge and the bus and not yet done, how many owed effects were tried since it started (done, failed " +
"and tried again, given up) and when the last was, whether its events arrive, and the last refusals.",
"and tried again, given up) and when the last was, walks with no delivery yet, whether its events arrive, when " +
"the controller's walks were last read or why they could not be, and the last refusals.",
Run: func(map[string]any) (any, error) { return h.Status(l.get()), nil }},
}
// A verb that changes a delivery asks for a look at what it makes possible (react.go).
for i := range list {
run := list[i].Run
ofSeat := strings.HasPrefix(list[i].Name, seat)
list[i].Run = func(a map[string]any) (any, error) {
if ofSeat && !h.Holding() {
return nil, h.notHolder() // a verb of the seat is answered by its holder only
}
out, err := run(a)
h.Poke()
return out, err
@@ -10,8 +10,6 @@ import (
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The manifest says what the code does: the seat it claims and the verbs it serves, the events it follows
@@ -70,19 +68,13 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) {
if len(m.Seats) != 0 {
t.Fatal("the manifest declares a seat; mesh-delivery is the mesh's own")
}
// The eight verbs the seat keeps (ADR 0539 §17): times, what-if, release, stop and retire-history went with the plans.
if want := []string{"deliveries", "show", "groups", "checks", "table", "stalled", "recheck", "close"}; !reflect.DeepEqual(m.Claims[0].Serves, want) {
t.Fatalf("serves %v, wanted %v", m.Claims[0].Serves, want)
// The thirteen verbs of the seat (ADR 0437 §3, §11).
if len(m.Claims[0].Serves) != 13 {
t.Fatalf("serves %d verbs, wanted 13: %v", len(m.Claims[0].Serves), m.Claims[0].Serves)
}
// R6-C1: no controller verb is invoked and no plan event is consumed.
for _, c := range m.Invokes {
if strings.HasPrefix(c, "seat:mesh-controller.") {
t.Errorf("the manifest still invokes %s (ADR 0539 §17)", c)
}
}
for _, c := range m.Consumes {
if strings.Contains(c, "plan-moved") || strings.HasPrefix(c, "mesh-controller.") {
t.Errorf("the manifest still consumes %s (ADR 0539 §17)", c)
for _, never := range []string{"mesh-controller.checked", "seat:mesh-controller.delivery-check", "seat:mesh-controller.queue"} {
if slices.Contains(m.Consumes, never) || slices.Contains(m.Invokes, never) {
t.Errorf("the manifest still names %s (issue 1318)", never)
}
}
if !reflect.DeepEqual(m.Consumes, followed) || !slices.Contains(m.Consumes, "mesh-delivery-forge-bridge.forge.**") {
@@ -96,14 +88,20 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) {
if !reflect.DeepEqual(buckets, []string{"deliveries", "groups", "pulls", "asks", "merges", "owed", "source"}) {
t.Fatalf("state %v", buckets)
}
// What it reads: the teller's answers and the checker's verdicts (issue 1302). The carry from today's module's buckets, the
// seat's holder and the machines' haves went with the seat's hand-over (ADR 0539 §17).
if !reflect.DeepEqual(m.Reads, []string{"mesh-delivery-teller.answers", "mesh-delivery-checker.verdicts"}) {
// What it reads: the carry from today's module's three buckets, the teller's answers, the checker's verdicts (issue
// 1302), the seat's holder and the machines' haves (ADR 0450 §6 and its insight of 2026-10-20).
if !reflect.DeepEqual(m.Reads, []string{"mesh-delivery.deliveries", "mesh-delivery.groups", "mesh-delivery.pulls",
"mesh-delivery-teller.answers", "mesh-delivery-checker.verdicts", "mesh-controller.assignments", "node-engine.have"}) {
t.Fatalf("reads %v", m.Reads)
}
// It calls no controller verb and reads the forge with three tools; the forge's writes are the bridge's (ADR 0437 §8). It
// uses no operator-channel: the teller asks (ADR 0437 §6).
want := []string{"gitea.gitea_commit_statuses", "gitea.gitea_contains", "gitea.gitea_open_pulls"}
// It calls exactly the controller's five verbs (ADR 0450 §7; no delivery-check, no queue and no forget-checks since issue 1318, record D5) and reads the forge with
// three tools; the forge's
// writes are the bridge's (ADR 0437 §8). It uses no operator-channel: the teller asks (ADR 0437 §6).
want := []string{}
for _, v := range []string{"deliver", "delivery-stop", "delivery-backups-again", "delivery-walks", "plans"} {
want = append(want, "seat:"+ControllerSeat+"."+v)
}
want = append(want, "gitea.gitea_commit_statuses", "gitea.gitea_contains", "gitea.gitea_open_pulls")
if len(m.Uses) != 0 {
t.Fatalf("uses %v: the teller asks, not the gate", m.Uses)
}
@@ -142,7 +140,7 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) {
}
}
// The forge's answers are read whatever wraps them.
// The forge's and the controller's answers are read whatever wraps them.
func TestAnAnswerIsReadWhateverWrapsIt(t *testing.T) {
for _, c := range []struct {
raw string
@@ -198,46 +196,12 @@ func TestARowWrittenWithTheReadyMarkStillReads(t *testing.T) {
"ready":{"head":"` + readyHead + `","by":"lead@shanks","marked_at":"2026-10-17T10:01:00Z","seen":"2026-10-17T10:02:00Z",
"last_try":"0001-01-01T00:00:00Z","asked_at":"2026-10-17T10:03:00Z","ask_id":"merge-7-111111111111-1","heads":{"novox/app@111111111111":"` + readyHead + `"}}}`
store.deliveries["novox/app@111111111111"] = []byte(row)
w := newWorldOver(t, store, newFakeForge(), time.Date(2026, 10, 17, 11, 0, 0, 0, time.UTC))
w := newWorldOver(t, store, newFakeController(), newFakeForge(), time.Date(2026, 10, 17, 11, 0, 0, 0, time.UTC))
d := w.delivery("novox/app@111111111111")
if d.Ready == nil || d.Ready.AskID != "merge-7-111111111111-1" || d.Ready.Head != readyHead {
t.Fatalf("the old row is not read: %+v", d.Ready)
}
}
// R6-C3 (ADR 0539 §17): the gate holds the seat as the SDK's table of release 6 has it. Every verb of the table that is not
// optional is served; the verbs release 6 removed are optional rows the SDK answers by name, and the gate serves none of them;
// and the SDK's own test of whether a module can hold a seat (CanHold) finds nothing missing or unpromised.
func TestTheGateHoldsTheSeatAsTheSDKsTableOfRelease6HasIt(t *testing.T) {
_, raw := readManifest(t)
manifest, err := catalogue.ParseManifest([]byte(raw))
if err != nil {
t.Fatalf("the SDK this module pins cannot read its manifest: %v", err)
}
seat, ok := catalogue.SeatNamed("mesh-delivery")
if !ok {
t.Fatal("the SDK this module pins names no seat mesh-delivery")
}
if err := catalogue.CanHold(manifest, seat); err != nil {
t.Fatalf("the gate cannot hold the seat of the SDK it pins: %v", err)
}
served := map[string]bool{}
for _, tool := range tools(&Holder{}, &listening{}) {
if _, verb, ok := strings.Cut(tool.Name, "."); ok {
served[verb] = true
}
}
for removed := range catalogue.RemovedDeliveryVerbs {
if served[removed] {
t.Errorf("the gate serves %s, which release 6 removed: the SDK answers it by name", removed)
}
if slices.Contains(manifest.Claims[0].Serves, removed) {
t.Errorf("the manifest serves %s, which release 6 removed", removed)
}
}
for _, verb := range []string{"deliveries", "show", "groups", "checks", "table", "stalled", "recheck", "close"} {
if !served[verb] {
t.Errorf("the gate does not serve %s, which the seat keeps", verb)
}
if !MemberOf(d).Ready {
t.Fatalf("an ask waiting is a ready member")
}
}
@@ -435,21 +435,16 @@ func gateMerges(d *Delivery) bool {
// happens outside it.
func (h *Holder) settleForgeMerges() []func() {
var out []func()
again := map[*Delivery]bool{}
if h.catchUp {
h.catchUp = false
again = h.newestInSource()
}
for _, d := range h.deliveries {
if d.MergedAs == "" || (d.InSource && !again[d]) || gateMerges(d) {
if d.MergedAs == "" || d.InSource || gateMerges(d) {
continue
}
if d.Base != MergeBranch || d.replayed() || h.newerMergeOf(d) {
if d.Base != MergeBranch || d.History != "" || h.newerMergeOf(d) {
d.InSource = true // not the trunk, or not its head: nothing to write
h.keep(d)
continue
}
id, repository, base, commit, at, group := d.ID, d.Repository, d.Base, d.MergedAs, mergedWhen(d), firstOf(d.Branch, d.Repository)
id, repository, base, commit, at, group := d.ID, d.Repository, d.Base, d.MergedAs, d.MergedAt, firstOf(d.Branch, d.Repository)
out = append(out, func() {
err := h.Source.Move(func(prev *sdk.Source) (sdk.Source, bool) {
next := sdk.Source{Group: group, Heads: map[string]string{repository: commit}}
@@ -486,21 +481,11 @@ func (h *Holder) settleForgeMerges() []func() {
return out
}
// replayLate is how long after its merge a merge heard only from the forge's word is a replay: the forge announces a merge
// within seconds, so one heard this late was replayed from the bus's history, and the source already holds what came after.
const replayLate = 15 * time.Minute
// replayed is whether the merge was heard from the forge's word replayLate or more after it happened (novox/hq issue 313,
// kept for source.main's sake after the history state went, ADR 0539 §17): never written to source.main.
func (d *Delivery) replayed() bool {
return d.FromForgesWord() && !d.MergedAt.IsZero() && d.Created.Sub(d.MergedAt) >= replayLate
}
// mergeOf is a delivery's merge into its trunk as the gate knows it: the forge's word (MergedAs), or else the bridge's
// (Ready.MergeCommit, Ready.Merged), which comes first for a merge the gate handed. Empty when it is not merged.
func mergeOf(d *Delivery) (string, time.Time) {
if d.MergedAs != "" {
return d.MergedAs, mergedWhen(d)
return d.MergedAs, d.MergedAt
}
if d.Ready != nil && d.Ready.Merged != nil && d.Ready.MergeCommit != "" {
return d.Ready.MergeCommit, *d.Ready.Merged
@@ -512,7 +497,7 @@ func mergeOf(d *Delivery) (string, time.Time) {
func (h *Holder) newerMergeOf(d *Delivery) bool {
for _, o := range h.deliveries {
if commit, at := mergeOf(o); o != d && o.Repository == d.Repository && o.Base == d.Base && commit != "" &&
!sameCommit(commit, d.MergedAs) && at.After(mergedWhen(d)) {
!sameCommit(commit, d.MergedAs) && at.After(d.MergedAt) {
return true
}
}
@@ -533,43 +518,3 @@ func (h *Holder) olderHead(repository, base, head string, at time.Time) bool {
}
return at.After(h.Now().Add(-keptFinal))
}
// mergedWhen is when a merged delivery's merge landed: its merge time (the forge's), else when the gate made the delivery. A
// zero time made every other merge of the repository look newer, so such a merge never moved source.main (issue 1726).
func mergedWhen(d *Delivery) time.Time {
if !d.MergedAt.IsZero() {
return d.MergedAt
}
return d.Created
}
// SourceCatchUp asks the next look to weigh again, once, the newest merge into the trunk of each repository though it is
// marked in the source (issue 1726, mesh-catalog #474, kept here because no seat holder arrives in release 6 to run it): what
// the gate missed while it was not running, or marked without writing, reaches source.main. The write never moves a repository
// back (olderHead), and one already there is not written again.
func (h *Holder) SourceCatchUp() {
h.mu.Lock()
h.catchUp = true
h.mu.Unlock()
h.Poke()
}
// newestInSource is, for each repository, its newest merge into the trunk heard from the forge, marked in the source. Under the lock.
func (h *Holder) newestInSource() map[*Delivery]bool {
newest := map[string]*Delivery{}
for _, d := range h.deliveries {
if d.MergedAs == "" || !d.InSource || gateMerges(d) || d.Base != MergeBranch || d.replayed() {
continue
}
if o := newest[d.Repository]; o == nil || mergedWhen(d).After(mergedWhen(o)) {
newest[d.Repository] = d
}
}
out := map[*Delivery]bool{}
for _, d := range newest {
if !h.newerMergeOf(d) {
out[d] = true
}
}
return out
}
@@ -3,6 +3,7 @@ package main
import (
"encoding/json"
"fmt"
"maps"
"regexp"
"sort"
"strings"
@@ -48,6 +49,9 @@ type Delivery struct {
Since time.Time `json:"since"`
Created time.Time `json:"created"`
// Plan is its delivery plan, as the controller's planner computed it (the build plan, the deploy plan,
// the steps that are not an ordinary send).
Plan *DeliveryPlan `json:"plan,omitempty"`
// Check is its own check's verdict.
Check *Verdict `json:"check,omitempty"`
// Required are the statuses its base requires, as the forge last said them (gitea_open_pulls): mesh/check makes it
@@ -84,8 +88,53 @@ type Delivery struct {
// A merge the gate handed to the bridge on an Approve is moved there by finish instead (merges.go).
InSource bool `json:"in_source,omitempty"`
HeldWhy string `json:"held_why,omitempty"`
// BackupsAgainAfter is the revision of its walk when a person released it for a failed backup: a record at that
// revision or before still holds the failure the controller was asked to clear, and does not hold it again
// (novox/hq ADR 0404, issue 1068).
BackupsAgainAfter int64 `json:"backups_again_after,omitempty"`
// FromForge is a delivery made from the forge's word at its merge: its head was never heard by this owner.
FromForge bool `json:"from_forge,omitempty"`
// History is why it is history, once it is (novox/hq issue 313).
FromForge bool `json:"from_forge,omitempty"`
History string `json:"history,omitempty"`
// CoveredBy says which later merges into the same trunk delivered every module it moves, once they did
// (novox/hq issue 319); Lifted why what held it for a person no longer holds (issue 316).
CoveredBy string `json:"covered_by,omitempty"`
Lifted string `json:"lifted,omitempty"`
// Stranded says why a merge whose walk ended done before its word, without moving every module it moves, is
// stopped: nothing later delivered them either (novox/hq issue 474). Set once the cover test says no.
Stranded string `json:"stranded,omitempty"`
// CoverTold is whether the forge was told that such a stop was covered since: its status, its view and its note
// (novox/hq issue 480). Kept, so a restart tells it once, and a stop covered before this was built is told too.
CoverTold bool `json:"cover_told,omitempty"`
// DeliveredBy says, for a delivery that failed or was stopped, the later finished walk of its repository whose
// commit contains its merge and which built every module it moves — each it did not build, its own walk said
// unchanged: its change reached the machines after all (novox/hq issue 747). Its state stays final, as a covered
// stop's does (issue 480), and it reads covered: its group counts it as delivered.
DeliveredBy string `json:"delivered_by,omitempty"`
// StoppedFor is, for a member its group stopped, the member whose end stopped it (novox/hq issue 747): once that
// member's change reached the machines, the stop has lapsed, and holds no walk of its repository back.
StoppedFor string `json:"stopped_for,omitempty"`
// HeldBack is why the walk it waits on is not let go: it carries a merge whose delivery was stopped, and is
// asked to leave it out (novox/hq ADR 0299). Said by `waits`, and by `stalled` past its bound.
HeldBack string `json:"held_back,omitempty"`
// ContainedIn is the forge's answer, by a later merge commit, to whether that commit contains this one's
// merge (gitea_contains): asked once per pair, and kept (ADR 0270).
ContainedIn map[string]bool `json:"contained_in,omitempty"`
// AnsweredBy is the commit of its repository the walk it follows carries for it: its own merge, or a later merge
// of its branch that contains it (novox/hq ADR 0276). Unwalked says it is held because no walk answered it.
AnsweredBy string `json:"answered_by,omitempty"`
// Carrier says, for a merge a later one carries, which: its delivery and walk, in words.
Carrier string `json:"carrier,omitempty"`
Unwalked bool `json:"unwalked,omitempty"`
// Walk is the walk that delivers it, as last read: its own trunk commit's, or one that carries a later merge
// of its branch; Steps the machines it reached.
Walk *WalkSeen `json:"walk,omitempty"`
Steps []Step `json:"steps,omitempty"`
// Time is its delivery time, class and phases (novox/hq ADR 0282), once its walk's end is known.
Time *DeliveryTime `json:"delivery_time,omitempty"`
// LetGo is when this owner asked the controller to start its walk, and why.
LetGo time.Time `json:"let_go"`
LetGoWhy string `json:"let_go_why,omitempty"`
// Ready is what this owner did with the operator's Approve of its merge into main, and with the merge (novox/hq ADR
// 0421, ready.go). The json key is the one the ready mark's record had, so rows written before still read.
@@ -198,6 +247,75 @@ func (v *Verdict) Passes() bool {
return gate && repo
}
// DeliveryPlan is a delivery plan as the controller's planner says it (mesh-controller link.ChangePlan).
type DeliveryPlan struct {
Repository string `json:"repository"`
Base string `json:"base"`
Head string `json:"head"`
Moved []string `json:"moved,omitempty"`
Dependents []string `json:"dependents,omitempty"`
New []string `json:"new,omitempty"`
Unread []string `json:"unread,omitempty"`
Tiers [][]string `json:"tiers,omitempty"`
Machines []MachinePlan `json:"machines,omitempty"`
Steps []string `json:"steps,omitempty"`
Summary string `json:"summary"`
// Class is the class of the slowest-budgeted module it moves, as the planner says it: core, leaf, or none when
// it moves no module (novox/hq ADR 0313 §2). Empty in a plan from a controller that did not say it.
Class string `json:"class,omitempty"`
// Trunk is the trunk commit the head was merged into when its verdict judged it (novox/hq ADR 0323 §3).
Trunk string `json:"trunk,omitempty"`
}
// receiving are the machines the plan sends a module to, in the plan's order; none when the plan names no
// machines, as a plan kept before they were said may not.
func (p *DeliveryPlan) receiving(module string) []string {
if p == nil {
return nil
}
var out []string
for _, m := range p.Machines {
if containsString(m.Receives, module) {
out = append(out, m.Machine)
}
}
return out
}
// MachinePlan is one machine's part of a plan.
type MachinePlan struct {
Machine string `json:"machine"`
Receives []string `json:"receives,omitempty"`
Waits []string `json:"waits,omitempty"`
}
// waitsForItsOwnWalk is whether it is published again after the walk that carried a later commit for it failed.
func (d *Delivery) waitsForItsOwnWalk() bool {
return d.State == Published && d.Walk != nil && d.Walk.State == walkFailed && d.carriedLater()
}
// carriedLater is whether the walk it follows carries a later merge of its branch for it, not its own.
func (d *Delivery) carriedLater() bool {
return d.AnsweredBy != "" && d.MergedAs != "" && !sameCommit(d.AnsweredBy, d.MergedAs)
}
// MovesNothing is whether the delivery's plan moves no module the mesh holds: nothing for a walk to do. A
// module it adds is not moved: the controller builds and registers it and sends it nowhere, and opens no walk
// for it (novox/hq issues 300 and 319). A delivery with no plan known is not said to move nothing.
func (d *Delivery) MovesNothing() bool {
return d.Plan != nil && len(d.Plan.Moved) == 0
}
// Step is one machine's part of a delivering walk.
type Step struct {
Machine string `json:"machine"`
Module string `json:"module"`
Build string `json:"build,omitempty"`
State StepState `json:"state"`
Since time.Time `json:"since"`
Why string `json:"why,omitempty"`
}
// Effect is one thing owed outside the state: an event said, a line appended to a commit's note, the view
// on the pull request refreshed, a status set.
type Effect struct {
@@ -235,8 +353,124 @@ func (d *Delivery) FromForgesWord() bool {
strings.Contains(d.Transitions[0].Why, "so it is made from the forge's word")
}
// shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose
// pull request the forge may no longer hold — or holds under its number another one.
// coveredStop is whether a delivery that ended stopped or failed was covered since, and reads covered. A stop
// because its walk ended done before its word without building its modules (novox/hq issue 474) is covered when a
// later merge into its trunk that contains it was delivered, carrying every module it moves to every machine running
// it (ADR 0270's cover test, kept as CoveredBy; issue 480). Any stop or failure is covered when a later finished walk
// of its repository whose commit contains its merge built its modules (DeliveredBy, novox/hq issue 747). Its state
// stays final, as a covered superseded member stays superseded (issue 442), and it reads covered, by that merge or
// walk. A person's stop with a cover found only while it was held keeps its error.
func (d *Delivery) coveredStop() bool {
switch d.State {
case Stopped:
return d.Stranded != "" && d.CoveredBy != "" || d.DeliveredBy != ""
case Failed, Superseded:
return d.DeliveredBy != ""
}
return false
}
// strandedEnded is whether a stop for a walk that built none of its modules (novox/hq issue 474), found covered by no
// merge, waits for nothing that can come: every module its walk left unbuilt is one no later merge can carry, because
// the pull request removed it from the catalogue (its module.json is among the removed paths) or its plan sends it to
// no machine ("sent nowhere"). It is a reading of the stop, as covered is (issue 480): the state stays stopped, and
// `stalled` no longer lists it, so the condition that listed it ends by itself (issue 1581).
func (d *Delivery) strandedEnded() bool {
if d.State != Stopped || d.Stranded == "" || d.CoveredBy != "" {
return false
}
missing := d.notMovedBy()
for _, m := range missing {
if !d.cannotBeCarried(m) {
return false
}
}
return len(missing) > 0
}
// cannotBeCarried is whether no later merge can deliver a module this delivery's plan moved: it was removed, or the
// plan sends it nowhere.
func (d *Delivery) cannotBeCarried(module string) bool {
if d.Plan == nil {
return false
}
for _, p := range d.Removed {
if strings.HasSuffix(p, "/"+module+"/module.json") {
return true
}
}
return strings.Contains(d.Plan.Summary, ", sent nowhere") && len(d.Plan.receiving(module)) == 0
}
// strandedEndedSaid is how an ended stop reads, in `show`.
func (d *Delivery) strandedEndedSaid() string {
return "ended: it waited for a later merge to carry " + strings.Join(d.notMovedBy(), ", ") +
", and none can: the module was removed from the catalogue or is built for no machine (novox/hq issue 1581)"
}
// coveredStopSaid is how a covered stop or failure reads: on the forge's status, in its view and note, and in `show`.
func (d *Delivery) coveredStopSaid() string {
if d.State == Superseded {
return "delivered after its walk was superseded: " + d.DeliveredBy
}
ended := "its stop"
if d.State == Failed {
ended = "its failure"
}
if d.Stranded != "" && d.CoveredBy != "" {
return "covered after " + ended + ": a later merge that contains it was delivered: " + d.CoveredBy
}
return "covered after " + ended + ": a later walk that contains it was delivered: " + d.DeliveredBy
}
// memberReached is whether a group counts a member as delivered: its change reached the machines, or had nothing to
// reach them with — delivered, history or covered, a superseded member a later delivered walk carried (issue 442),
// or a stop or failure covered since (issues 480 and 747).
func memberReached(d *Delivery) bool {
return d.State.Reached() || d.State == Superseded && d.CoveredBy != "" || d.coveredStop()
}
// stoppedFor is the member whose end stopped this one in its group: kept since novox/hq issue 747, and read from the
// stop's reason for a stop made before ("its group <id> stopped: <member> …").
func (d *Delivery) stoppedFor() string {
if d.StoppedFor != "" || d.State != Stopped || d.Group == "" {
return d.StoppedFor
}
prefix := "its group " + d.Group + " stopped: "
for i := len(d.Transitions) - 1; i >= 0; i-- {
t := d.Transitions[i]
if t.Event != EvStop {
continue
}
if t.By != byOwner || !strings.HasPrefix(t.Why, prefix) {
return ""
}
member, _, _ := strings.Cut(strings.TrimPrefix(t.Why, prefix), " ")
return strings.TrimRight(member, ",;:")
}
return ""
}
// stopLapsed is whether a member its group stopped for another member is stopped for nothing any more: that member's
// change reached the machines since — delivered, or its failure covered by a later walk (novox/hq issue 747). Its own
// change is still to deliver, by a later walk of its repository that contains it; its stop holds none back. find
// gives a delivery by its id.
func stopLapsed(d *Delivery, find func(string) *Delivery) bool {
if d.State != Stopped || d.coveredStop() {
return false
}
by := d.stoppedFor()
if by == "" || by == d.ID {
return false
}
o := find(by)
return o != nil && memberReached(o)
}
func (d *Delivery) shownOnItsPullRequest() bool {
return d.Number > 0
return d.Number > 0 && d.History == "" && d.State != Retired
}
// Owner and Repo split the repository.
@@ -331,6 +565,479 @@ func AfterIn(body string) []string {
return out
}
// The walk: the controller's plan of one trunk commit, as its `plan-moved` and `delivery-walks` say it
// (mesh-controller inventory.Plan). Read, never written here.
// The states of a walk as the controller keeps them.
const (
walkBuilding = "building"
walkRolling = "rolling"
walkDone = "done"
walkFailed = "failed"
walkSuperseded = "superseded"
)
// Walk is a walk whole, as the controller says it.
type Walk struct {
ID string `json:"id"`
Repository string `json:"repository"`
Branch string `json:"branch,omitempty"`
Commit string `json:"commit"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
State string `json:"state"`
Tier int `json:"tier"`
Tiers [][]string `json:"tiers"`
Modules map[string]*WalkModule `json:"modules"`
Note string `json:"note,omitempty"`
Revision int64 `json:"revision"`
Release *WalkRelease `json:"release,omitempty"`
Delivery *WalkDelivery `json:"delivery,omitempty"`
// Backups is what the backups the controller asks before it sends anything came to (novox/hq ADR 0404, issue
// 1068); absent when the plan moves no backed-up item.
Backups *WalkBackups `json:"backups,omitempty"`
// Phases is the walk's time from its merge, phase by phase, as the controller works it out (novox/hq ADR
// 0282): what a delivery's time is kept from. Absent from a controller that does not say it.
Phases *WalkPhases `json:"phases,omitempty"`
}
// WalkBackups is the backups of a walk before its send, as the controller's plan record says them: state pending,
// done or failed (failed as soon as any entry failed), each backup asked, and the machines with no backup holder.
type WalkBackups struct {
State string `json:"state"`
Asked []WalkBackupAsked `json:"asked,omitempty"`
NoHolder []string `json:"no_holder,omitempty"`
}
// WalkBackupAsked is one backup asked of a machine for a module: its state, and the backup's own words.
type WalkBackupAsked struct {
Machine string `json:"machine"`
Module string `json:"module"`
State string `json:"state"`
AskedAt *time.Time `json:"asked_at,omitempty"`
Words string `json:"words,omitempty"`
}
// WalkBackupFailure is one backup that failed or stayed silent past its bound, as a delivery keeps it.
type WalkBackupFailure struct {
Machine string `json:"machine"`
Module string `json:"module"`
Words string `json:"words"`
}
// WalkModule is one module of a walk.
type WalkModule struct {
State string `json:"state,omitempty"`
SentAt *time.Time `json:"sent_at,omitempty"`
First []string `json:"first,omitempty"`
FirstAt *time.Time `json:"first_at,omitempty"`
Commit string `json:"commit,omitempty"`
Why string `json:"why,omitempty"`
Build string `json:"build,omitempty"`
Gate *WalkGate `json:"gate,omitempty"`
GatedBy string `json:"gated_by,omitempty"`
// Made is what the walk's build of it did (novox/hq ADR 0350 §4): "reused from <check build>", naming the pull
// request's head, the trunk commit and the digests, or "built: <why>". Empty from a controller before it.
Made string `json:"made,omitempty"`
}
// WalkGate is a module's judging on its first machine.
type WalkGate struct {
Machines []string `json:"machines"`
From string `json:"from,omitempty"`
To string `json:"to,omitempty"`
Since *time.Time `json:"since,omitempty"`
Passes int `json:"passes,omitempty"`
Verdict string `json:"verdict,omitempty"`
Why string `json:"why,omitempty"`
JudgedAt *time.Time `json:"judged_at,omitempty"`
Rollback string `json:"rollback,omitempty"`
Carried []WalkCarry `json:"carried,omitempty"`
}
// WalkCarry is one build a gated send carried.
type WalkCarry struct {
Module string `json:"module"`
Node string `json:"node"`
From string `json:"from,omitempty"`
To string `json:"to"`
Build string `json:"build,omitempty"`
}
// WalkRelease is a backlog walk's way through the machines.
type WalkRelease struct {
Order []string `json:"order"`
Next int `json:"next"`
Gate *WalkGate `json:"gate,omitempty"`
Done []string `json:"done,omitempty"`
By string `json:"by,omitempty"`
}
// WalkDelivery is what a walk waits for, and which merges it answers (novox/hq ADR 0276).
type WalkDelivery struct {
Awaits string `json:"awaits"`
Go *time.Time `json:"go,omitempty"`
By string `json:"by,omitempty"`
Why string `json:"why,omitempty"`
Stopped string `json:"stopped,omitempty"`
StoppedWhy string `json:"stopped_why,omitempty"`
// Merges are every merge the walk answers: each merge commit, its repository, and the commit of that
// repository the walk carries — the merge itself, or a later merge of its branch that contains it. Absent
// from a controller that does not say it: the walk then answers the merge of its own commit, and a merge
// the forge says its commit contains.
Merges []WalkMerge `json:"merges,omitempty"`
// TakenOverBy is, on a walk folded before it started, the walk that took it over and answers every merge
// it answered.
TakenOverBy string `json:"taken_over_by,omitempty"`
// WalkedAgainBy is, on a walk stopped without some of its merges, the batch that walks its other merges again,
// and LeftOut the merges it left out (novox/hq ADR 0299): a delivery of a merge left out ends stopped with it,
// one of a merge walked again follows that batch's walk.
WalkedAgainBy string `json:"walked_again_by,omitempty"`
LeftOut []WalkLeftOut `json:"left_out,omitempty"`
// Batch is set while the record is a batch, not yet a walk: its state `assembling` or `queued`. A batch, once
// cut, is the walk under the same id: its next record is `building`. A merge this owner reads in a batch is
// not held as unwalked while the batch is heard of.
Batch *WalkBatch `json:"batch,omitempty"`
// Unchanged are the modules the walk did not build because their build source holds none of the files its
// merges changed, each with why (novox/hq issue 611, ADR 0267 §5): the build the mesh holds is theirs. A delivery
// plan that read them whole names them; this says they matched what runs, not that the walk left them unbuilt.
// Absent from a controller that does not say it.
Unchanged map[string]string `json:"unchanged,omitempty"`
}
// WalkMerge is one merge a walk answers.
type WalkMerge struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
// Carried is the commit of the repository the walk carries for it: Commit itself, or a later merge of its
// branch that contains it. Empty means Commit.
Carried string `json:"carried,omitempty"`
}
// WalkLeftOut is a merge a stop left out (novox/hq ADR 0299): a merge stopped, or a later merge of its
// repository's branch that contains it, Contains naming the stopped merge.
type WalkLeftOut struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
Contains string `json:"contains,omitempty"`
// Shares are the modules it moves that a stopped merge moves too, when that is why it was left out: Contains
// then names that stopped merge, which it does not contain.
Shares []string `json:"shares,omitempty"`
}
// leftOut is the merge of a repository a stop left out, when it did.
func (w Walk) leftOut(repository, merge string) (WalkLeftOut, bool) {
if w.Delivery == nil || merge == "" {
return WalkLeftOut{}, false
}
for _, o := range w.Delivery.LeftOut {
if strings.EqualFold(o.Repository, repository) && sameCommit(o.Commit, merge) {
return o, true
}
}
return WalkLeftOut{}, false
}
// WalkBatch is a batch's window: when it closes unless another merge arrives, when it closes at the latest,
// and, once closed, the walk it waits behind.
type WalkBatch struct {
ClosesAt *time.Time `json:"closes_at,omitempty"`
AtMost *time.Time `json:"at_most,omitempty"`
Behind string `json:"behind,omitempty"`
}
// The states a batch's record is in before it is a walk (novox/hq ADR 0276).
const (
walkAssembling = "assembling"
walkQueued = "queued"
)
// IsBatch is whether the record is a batch still to be cut into a walk.
func (w Walk) IsBatch() bool { return w.State == walkAssembling || w.State == walkQueued }
// supersededNote reads the walk that took a superseded walk over from its note, for a controller that does not
// say TakenOverBy: "superseded at tier N by plan-… (…)".
var supersededNote = regexp.MustCompile(`^superseded at tier \d+ by (plan-[0-9A-Za-z_-]+)`)
// takenOverBy is the walk that took this one over, said or read from its note; empty for none.
func (w Walk) takenOverBy() string {
if w.State != walkSuperseded {
return ""
}
if w.Delivery != nil && w.Delivery.TakenOverBy != "" {
return w.Delivery.TakenOverBy
}
if m := supersededNote.FindStringSubmatch(w.Note); m != nil {
return m[1]
}
return ""
}
// carriedFor is the commit the walk carries for a merge of a repository, and whether it answers that merge by
// what it says: the merge named among its merges, or its own commit.
func (w Walk) carriedFor(repository, merge string) (string, bool) {
if w.Delivery != nil {
for _, m := range w.Delivery.Merges {
if strings.EqualFold(m.Repository, repository) && sameCommit(m.Commit, merge) {
if m.Carried != "" {
return m.Carried, true
}
return m.Commit, true
}
}
}
if strings.EqualFold(w.Repository, repository) && sameCommit(w.Commit, merge) {
return w.Commit, true
}
return "", false
}
// WalkSeen is what a delivery keeps of its walk.
type WalkSeen struct {
ID string `json:"id"`
Commit string `json:"commit"`
State string `json:"state"`
Note string `json:"note,omitempty"`
Revision int64 `json:"revision"`
Waits bool `json:"waits,omitempty"`
WaitedFor string `json:"waited_for,omitempty"`
GoBy string `json:"go_by,omitempty"`
StoppedBy string `json:"stopped_by,omitempty"`
// StoppedWhy is why it was stopped; WalkedAgainBy, on a walk stopped without some merges, the batch that walks
// this delivery's merge again; Contains, when this delivery's merge was left out for containing a stopped
// merge, that merge (novox/hq ADR 0299).
StoppedWhy string `json:"stopped_why,omitempty"`
WalkedAgainBy string `json:"walked_again_by,omitempty"`
Contains string `json:"contains,omitempty"`
// Shares are, when this delivery's merge was left out for moving modules a stopped merge moves, those modules:
// built at the branch, which holds the stopped merge, they would deliver it (ADR 0299).
Shares []string `json:"shares,omitempty"`
Tier int `json:"tier"`
Tiers int `json:"tiers"`
Updated time.Time `json:"updated"`
// Branch is the branch the walk's commit was merged into; empty in a record kept before it was read.
Branch string `json:"branch,omitempty"`
// Created is when the controller made the walk: a delivery follows the newest walk that answers it.
Created time.Time `json:"created"`
// TakenOverBy is the walk that took a superseded one over (novox/hq ADR 0276): its deliveries follow it.
TakenOverBy string `json:"taken_over_by,omitempty"`
// Moved are the modules the walk sent, as its record names them; MovedSaid says they were read, which a
// record kept before novox/hq issue 474 did not. What a walk that ended before its word delivered is read
// from them.
Moved []string `json:"moved,omitempty"`
MovedSaid bool `json:"moved_said,omitempty"`
// Unchanged are the modules the walk said matched what runs, with why (novox/hq issue 611): not built, and
// nothing for it to deliver.
Unchanged map[string]string `json:"unchanged,omitempty"`
// Built is, per module, what the walk's build of it did (novox/hq ADR 0350 §4): "reused from <check build>" or
// "built: <why>" — what `show` says beside the build phase.
Built map[string]string `json:"built,omitempty"`
// BackupsFailed are the backups before the send that failed, each with the backup's words (ADR 0404).
BackupsFailed []WalkBackupFailure `json:"backups_failed,omitempty"`
}
// backupWords says the failed backups: the machine, the module and the backup's own words.
func (w *WalkSeen) backupWords() string {
var parts []string
for _, f := range w.BackupsFailed {
parts = append(parts, fmt.Sprintf("%s on %s: %s", f.Module, f.Machine, f.Words))
}
return "a backup before the send failed, and nothing was sent: " + strings.Join(parts, "; ")
}
// Seen is a walk as a delivery keeps it.
func (w Walk) Seen() *WalkSeen {
s := &WalkSeen{ID: w.ID, Commit: w.Commit, State: w.State, Note: w.Note, Revision: w.Revision, Tier: w.Tier,
Tiers: len(w.Tiers), Updated: w.Updated}
if d := w.Delivery; d != nil {
s.WaitedFor = d.Awaits
s.Waits = d.Awaits != "" && d.Go == nil
s.GoBy = d.By
s.StoppedBy, s.StoppedWhy = d.Stopped, d.StoppedWhy
if len(d.Unchanged) > 0 {
s.Unchanged = maps.Clone(d.Unchanged)
}
}
s.Branch = w.Branch
s.Created = w.Created
s.TakenOverBy = w.takenOverBy()
// Only a module sent, or passed on its first machine, was built for the machines: one the record names with
// why it was not sent, or a walk that never started, built nothing (review of novox/hq issue 474).
for name, m := range w.Modules {
if m != nil && (m.SentAt != nil || m.Gate != nil && m.Gate.Verdict == "passed") {
s.Moved = append(s.Moved, name)
}
if m != nil && m.Made != "" {
if s.Built == nil {
s.Built = map[string]string{}
}
s.Built[name] = m.Made
}
}
sort.Strings(s.Moved)
s.MovedSaid = true
if w.Backups != nil {
for _, a := range w.Backups.Asked {
if a.State == "failed" {
s.BackupsFailed = append(s.BackupsFailed, WalkBackupFailure{Machine: a.Machine, Module: a.Module, Words: a.Words})
}
}
}
return s
}
// endedUnstarted is whether a published delivery's walk ended done without ever being let go: a batch's walk the
// controller judged to move nothing it holds is done when it is cut (novox/hq issue 474). No start can come for it,
// so its record settles the delivery.
func (d *Delivery) endedUnstarted() bool {
return d.State == Published && d.Walk != nil && d.Walk.State == walkDone && !d.Walk.Started()
}
// walkStopped is whether the walk it follows was stopped through this owner, and walkedAgain whether its merge is
// walked again by another walk since (novox/hq ADR 0299).
func (d *Delivery) walkStopped() bool {
return d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy != ""
}
func (d *Delivery) walkedAgain() bool { return d.walkStopped() && d.Walk.WalkedAgainBy != "" }
// stoppedWords is why a delivery ends with the walk it follows stopped: the stop, and the stopped merge its own
// contains when it was left out for that.
func (d *Delivery) stoppedWords() string {
s := fmt.Sprintf("its walk %s was stopped by %s", d.Walk.ID, d.Walk.StoppedBy)
if d.Walk.StoppedWhy != "" {
s += ": " + d.Walk.StoppedWhy
}
switch {
case len(d.Walk.Shares) > 0:
s += fmt.Sprintf("; its merge moves %s, as the stopped merge %s does: built at the branch, which holds that "+
"merge, it would deliver it, so it was left out with it (novox/hq ADR 0299)", strings.Join(d.Walk.Shares, ", "),
shortOf(d.Walk.Contains, 12))
case d.Walk.Contains != "":
s += fmt.Sprintf("; its merge contains the stopped merge %s, so it cannot be walked without delivering it, "+
"and was left out with it (novox/hq ADR 0299)", shortOf(d.Walk.Contains, 12))
}
return s
}
// walkMovedItsModules is whether the walk it follows answered every module its plan moves, and whether that is
// known: each built, or said unchanged — its build source holds none of the files the walk's merges changed, so the
// build the mesh holds is its build (novox/hq issue 611) — from the modules the walk's record names, or, for a
// record kept before they were, from a walk of no tier, which moved nothing. Unknown when its plan is not known.
func (d *Delivery) walkMovedItsModules() (moved, known bool) {
if d.Walk == nil || d.Plan == nil {
return false, false
}
if !d.Walk.MovedSaid && d.Walk.Tiers > 0 {
return false, false
}
return len(d.notMovedBy()) == 0, true
}
// notMovedBy is the modules its plan moves that the walk it follows neither built nor said unchanged: what a walk
// that ended without building them left undelivered.
func (d *Delivery) notMovedBy() []string {
var out []string
if d.Walk == nil || d.Plan == nil {
return nil
}
for _, m := range d.Plan.Moved {
if _, unchanged := d.Walk.Unchanged[m]; !containsString(d.Walk.Moved, m) && !unchanged {
out = append(out, m)
}
}
return out
}
// unchangedOfIts is the modules its plan moves that the walk it follows said matched what runs, sorted (novox/hq
// issue 611).
func (d *Delivery) unchangedOfIts() []string {
var out []string
if d.Walk == nil || d.Plan == nil {
return nil
}
for _, m := range d.Plan.Moved {
if _, unchanged := d.Walk.Unchanged[m]; unchanged && !containsString(d.Walk.Moved, m) {
out = append(out, m)
}
}
sort.Strings(out)
return out
}
// builtNoneOfIts is whether the walk it follows built none of the modules its plan moves and said each unchanged:
// its merge moves nothing, whichever commit the walk carried (ADR 0270 §1).
func (d *Delivery) builtNoneOfIts() bool {
return d.Plan != nil && len(d.Plan.Moved) > 0 && len(d.unchangedOfIts()) == len(d.Plan.Moved)
}
// Started is whether the walk is past its wait: it waited for nobody, or was let go — whatever came of it
// since. A walk superseded or stopped while it waited never started.
func (s *WalkSeen) Started() bool { return s != nil && !s.Waits && (s.WaitedFor == "" || s.GoBy != "") }
// Waited is whether it waited for a word at all.
func (s *WalkSeen) Waited() bool { return s != nil && s.WaitedFor != "" }
// LetGoByAPerson is whether a person's `plans go` started it.
func (s *WalkSeen) LetGoByAPerson() bool { return s != nil && strings.HasPrefix(s.GoBy, "a person") }
// Open is whether the controller still works it.
func (w Walk) Open() bool { return w.State == walkBuilding || w.State == walkRolling }
// The steps that say a module reached every machine running it: sent to all at once, or to the rest after its
// first machines passed.
const (
stepEveryMachine = "every machine running it"
stepTheRest = "the rest running it"
)
// StepsOf are the machine steps a walk's record says, for one delivery: every module it moved, on its first
// machines from its gate, and the rest once sent.
func StepsOf(w Walk) []Step {
var out []Step
names := make([]string, 0, len(w.Modules))
for n := range w.Modules {
names = append(names, n)
}
sort.Strings(names)
for _, name := range names {
m := w.Modules[name]
if m == nil || m.FirstAt == nil {
if m != nil && m.SentAt != nil && m.FirstAt == nil && m.Why == "" {
out = append(out, Step{Machine: stepEveryMachine, Module: name, Build: m.Build,
State: StepSent, Since: *m.SentAt})
}
continue
}
gate := m.Gate
state, why, since := StepSent, "", *m.FirstAt
if gate != nil {
switch {
case gate.Rollback == "rolled-back":
state, why = StepRolledBack, gate.Why
case gate.Verdict == "failed":
state, why = StepFailed, gate.Why
case gate.Verdict == "passed":
state, why = StepPassed, gate.Why
case gate.Since != nil:
state = StepJudging
why = fmt.Sprintf("%d healthy judging(s) so far", gate.Passes)
}
if gate.JudgedAt != nil {
since = *gate.JudgedAt
}
}
for _, machine := range m.First {
out = append(out, Step{Machine: machine, Module: name, Build: m.Build, State: state, Since: since, Why: why})
}
if m.SentAt != nil && state == StepPassed {
out = append(out, Step{Machine: stepTheRest, Module: name, Build: m.Build, State: StepSent,
Since: *m.SentAt})
}
}
return out
}
// Group is a delivery group: two or more deliveries sharing a head branch name, one level only.
type Group struct {
ID string `json:"id"`
@@ -411,3 +1118,13 @@ func mustJSON(v any) json.RawMessage {
raw, _ := json.Marshal(v)
return raw
}
// supersededByItsWalk is whether a delivery ended superseded because its walk was: its last move was a supersession
// from published or delivering, not a newer head of its pull request's (issue 1786).
func (d *Delivery) supersededByItsWalk() bool {
if d.State != Superseded || len(d.Transitions) == 0 {
return false
}
t := d.Transitions[len(d.Transitions)-1]
return t.Event == EvSuperseded && (t.From == Published || t.From == Delivering)
}
@@ -0,0 +1,80 @@
package main
import (
"strings"
"testing"
)
// **Where today's module stands** (novox/hq ADR 0450 §6 as corrected, issues 1347 and 1369; review round 1, G1). Fixture
// sources: an assignment is mesh-controller internal/chain/records.go Assignment (a key is never removed: an unassignment writes
// `ended`, a pending one has a `state`); a have is mesh-sdk go/delivery Have, whose phase `removed` the node-engine writes for a
// module its declaration no longer holds.
const (
standing = `{"node":"novox","module":"mesh-delivery","assigned":"2026-10-01T10:00:00Z","build":"b1","since":"2026-10-01T10:00:00Z"}`
ended = `{"node":"novox","module":"mesh-delivery","assigned":"2026-10-01T10:00:00Z","build":"b1","since":"2026-10-01T10:00:00Z","ended":"2026-10-20T03:00:00Z"}`
pending = `{"node":"novox","module":"mesh-delivery","state":"waiting"}`
haveRun = `{"v":1,"build":"b1","phase":"running","health":"ok"}`
haveGone = `{"v":1,"build":"b1","phase":"removed","health":"unknown"}`
haveFailed = `{"v":1,"build":"b1","phase":"failed","health":"failed"}`
)
func bucket(current bool, kv ...string) seatBucket {
b := seatBucket{values: map[string][]byte{}, current: current}
for i := 0; i+1 < len(kv); i += 2 {
b.values[kv[i]] = []byte(kv[i+1])
}
return b
}
func TestAnEndedAssignmentKeepsItsKeyAndIsNoOccupant(t *testing.T) {
who, err := occupantsOf(bucket(true, "novox.mesh-delivery", ended), bucket(true, "novox.mesh-delivery", haveGone))
if err != nil || len(who) != 0 {
t.Fatalf("an ended assignment and a removed have gave %v, %v; wanted a free seat", who, err)
}
if who, _ := occupantsOf(bucket(true, "novox.mesh-delivery", standing), bucket(true, "novox.mesh-delivery", haveGone)); len(who) != 1 || !strings.Contains(who[0], "assigned on novox") {
t.Fatalf("a standing assignment gave %v", who)
}
// A pending assignment, waiting for its build, is no assignment (every reader of the chain says so).
if who, _ := occupantsOf(bucket(true, "novox.mesh-delivery", pending), bucket(true, "novox.mesh-delivery", haveGone)); len(who) != 0 {
t.Fatalf("a pending assignment gave %v", who)
}
// Other modules' keys are no business of the gate's.
if who, _ := occupantsOf(bucket(true, "novox.mesh-delivery-gate", standing), bucket(true, "novox.mesh-delivery", haveGone)); len(who) != 0 {
t.Fatalf("another module's assignment gave %v", who)
}
}
func TestAHaveKeyCountsUnlessItsPhaseIsRemoved(t *testing.T) {
for _, have := range []string{haveRun, haveFailed, `{"v":1,"phase":"applying","health":"unknown"}`, `not json`} {
who, err := occupantsOf(bucket(true), bucket(true, "novox.mesh-delivery", have))
if err != nil || len(who) != 1 || !strings.Contains(who[0], "node-engine of novox") {
t.Fatalf("the have %s gave %v, %v; wanted an occupant", have, who, err)
}
}
// One machine removed it, another runs it still.
who, _ := occupantsOf(bucket(true), bucket(true, "novox.mesh-delivery", haveGone, "ace.mesh-delivery", haveRun))
if len(who) != 1 || !strings.Contains(who[0], "ace") {
t.Fatalf("a module removed on one machine and running on another gave %v", who)
}
}
// An empty read says nothing of the module: the seat is not free until a node-engine said the module removed.
func TestAnEmptyReadOfTheHaveKeysDoesNotFreeTheSeat(t *testing.T) {
who, err := occupantsOf(bucket(true), bucket(true))
if err != nil || len(who) != 1 || !strings.Contains(who[0], "no have key") {
t.Fatalf("an empty read gave %v, %v; wanted the seat held back", who, err)
}
if who, _ := occupantsOf(bucket(true), bucket(true, "novox.node", `{"v":1}`, "novox.mesh-delivery-gate", haveRun)); len(who) != 1 {
t.Fatalf("have keys of other modules gave %v", who)
}
}
// A watch that has not delivered its current values yet says nothing: the seat is not judged.
func TestTheSeatIsNotJudgedBeforeBothWatchesAreCurrent(t *testing.T) {
for _, c := range [][2]bool{{false, true}, {true, false}, {false, false}} {
who, err := occupantsOf(bucket(c[0]), bucket(c[1], "novox.mesh-delivery", haveGone))
if err == nil || len(who) != 0 {
t.Fatalf("watches current %v gave %v, %v; wanted an error", c, who, err)
}
}
}
@@ -28,7 +28,7 @@ func TestADeliveryOnMeshCheckGoesReadyAskedAndMergedOverARealBus(t *testing.T) {
defer rt.mu.Unlock()
return len(rt.subs) >= len(followed)
})
rt.untilListening()
rt.untilHolding()
const head = "2222222222222222222222222222222222222222"
const trunk = "3333333333333333333333333333333333333333"
@@ -67,7 +67,11 @@ func TestADeliveryOnMeshCheckGoesReadyAskedAndMergedOverARealBus(t *testing.T) {
if len(handed.Pulls) != 1 || handed.Pulls[0].Head.Head != head || handed.Pulls[0].State != sdk.MergeMerging {
t.Fatalf("the merge handed is %+v", handed)
}
rt.noControllerAsk()
for _, verb := range []string{"delivery-check", "forget-checks"} {
if n := len(rt.asksOf("seat:" + ControllerSeat + "." + verb)); n != 0 {
t.Fatalf("the controller was asked %s %d times on a mesh/check base", verb, n)
}
}
// No mesh/merge-gate is owed on the head.
keys, _ := b.kvs["owed"].Keys(ctx)
for _, k := range keys {

Some files were not shown because too many files have changed in this diff Show More