buildstep: keep a check ask's cache layer out of every step's reach, move through an os.Root, and layer the build cache alone

Review of 62a50eb: the layer lay in the workspace, which every step binds read-write,
so a change's command could swap a kept directory for a symlink and the move, run
outside the sandbox, would write into the shared cache (ADR 0541 §2). The layer now
lies beside the workspace (Workspaces.CacheLayer, removed by Release and Sweep),
Command refuses a layer within reach of the workspace, a writable directory or a
cache, and every move goes through an os.Root on the layer and keeps no symlink. Only
LayerCaches (the build cache) are layered; a module cache keeps its throwaway layer
(issue 1921).

go/buildstep/askrun.go

@@ -170,8 +170,8 @@ func (s AskSteps) check(ctx context.Context, job delivery.CheckJob) (any, error)
 	}
 	defer s.release(f.Workspace)
 	answer := delivery.CheckAnswer{Fetch: r}
-	if len(f.Conflicts) > 0 {
-		return answer, nil // the asker reads the conflict from the fetch's answer: a verdict, not an error
+	if len(f.Conflicts) > 0 || job.MergeOnly {
+		return answer, nil // the asker reads the merge or the conflict from the fetch's answer: a verdict, not an error
 	}
 	if s.Test == nil {
 		return nil, Refuse("this build module tests nothing")

go/buildstep/askrun_test.go

@@ -235,7 +235,7 @@ test {"level":"unit","runner":"workspace","workspace":"/w"}`},
 }
 
 // A Node check tests each module directory, the one whose tests changed first; a conflict ends the ask with the fetch
-// alone, a verdict for the asker.
+// alone, a verdict for the asker; so does a merge-only ask of a head that merges.
 func TestRunAskChecksNodeModulesAndConflicts(t *testing.T) {
 	r := &recorder{fetch: map[string]any{"workspace": "/w", "modules": []string{"a", "b"}, "changed": []string{"b/test/x.test.ts"}},
 		tests: map[string]bool{"a": true, "b": true}}
@@ -252,6 +252,12 @@ func TestRunAskChecksNodeModulesAndConflicts(t *testing.T) {
 	if err != nil || len(r.calls) != 1 || len(out.(delivery.CheckAnswer).Levels) != 0 {
 		t.Fatalf("a conflict: %v %v", err, r.calls)
 	}
+	// A merge-only ask answers the fetch of a head that merges and runs no level.
+	r = &recorder{fetch: map[string]any{"workspace": "/w", "merge": "m1"}, tests: map[string]bool{"a": true}}
+	out, err = RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit}, MergeOnly: true, Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
+	if err != nil || len(r.calls) != 1 || len(out.(delivery.CheckAnswer).Levels) != 0 || !strings.Contains(string(out.(delivery.CheckAnswer).Fetch), `"m1"`) {
+		t.Fatalf("merge only: %v %v %v", err, r.calls, out)
+	}
 }
 
 // Issue 1921: a failure in the packages whose tests changed does not stop the level: the other packages still run, so two

go/buildstep/capacity.go

@@ -22,20 +22,7 @@ func NodeCapacity(path, node string, logf func(string, ...any)) func() int {
 		}
 	}
 	return func() int {
-		if path == "" {
-			return 0
-		}
-		raw, err := os.ReadFile(path)
-		if err != nil {
-			say("the build capacity cannot be read, so %s pulls nothing: %v", node, err)
-			return 0
-		}
-		var s map[string]json.RawMessage
-		if err := json.Unmarshal(raw, &s); err != nil {
-			say("the settings file %s is not JSON, so %s pulls nothing: %v", path, node, err)
-			return 0
-		}
-		n, err := capacityOf(s[CapacitySetting])
+		n, err := ReadCapacity(path)
 		if err != nil {
 			say("%v, so %s pulls nothing", err, node)
 			return 0
@@ -44,6 +31,22 @@ func NodeCapacity(path, node string, logf func(string, ...any)) func() int {
 	}
 }
 
+// ReadCapacity is the capacity the settings file at path holds, and why it cannot be read when it cannot (then 0).
+func ReadCapacity(path string) (int, error) {
+	if path == "" {
+		return 0, nil
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return 0, fmt.Errorf("the build capacity cannot be read: %w", err)
+	}
+	var s map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &s); err != nil {
+		return 0, fmt.Errorf("the settings file %s is not JSON: %w", path, err)
+	}
+	return capacityOf(s[CapacitySetting])
+}
+
 // capacityOf is the capacity a config value holds: absent or null is 0; anything but a whole number of at least 0 is
 // refused, naming it.
 func capacityOf(value json.RawMessage) (int, error) {

go/buildstep/isolate.go

@@ -3,10 +3,14 @@ package buildstep
 import (
 	"context"
 	"fmt"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
+	"strconv"
 	"strings"
+	"sync"
 
 	"git.novox.be/novox/mesh-sdk/go/forgekey"
 )
@@ -29,16 +33,31 @@ const Bubblewrap = "bwrap"
 //   - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step
 //     downloaded) read-only;
 //   - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone
-//     (WriteCaches), so a candidate's code never changes what a later build reads.
+//     (WriteCaches), so a candidate's code never changes what a later build reads;
+//   - with a CacheLayer, what one command writes into the LayerCaches kept for the later commands of the same ask (issue
+//     1921, the try speed-up): the vet, the unit tests and the integration compile of one ask compile each package once.
 type Isolation struct {
 	Workspace   string
 	Writable    []string
 	ReadOnly    []string
 	Caches      []string
 	WriteCaches bool
+	// CacheLayer is a directory of the ask's own, gone with its workspace (Workspaces.CacheLayer): each command writes
+	// the LayerCaches into a layer of its own beneath it (overlayfs refuses an upper layer two mounts share, and an
+	// ask's commands run side by side), which Isolated moves into the ask's kept layer when the command ends; every
+	// later command reads the kept layer above the shared cache. The shared cache stays unchanged. It must lie outside
+	// the Workspace and every Writable directory, so no command reaches it (a command that could would make the move,
+	// which runs outside the sandbox, write where it points). Empty: what a command writes is thrown away.
+	CacheLayer string
+	// LayerCaches are the Caches kept in the CacheLayer: a build cache, whose entries are whole files written by
+	// rename. Every other cache (a module cache, with read-only directories and half-written modules) keeps a
+	// throwaway layer.
+	LayerCaches []string
 	Env         []string
 	// Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback.
 	Hosts string
+	// run is the directory of this command's own writable layers under CacheLayer; Isolated makes one per command.
+	run string
 }
 
 // etcNeeded is what of /etc a program in the step may read: who the users are, where the libraries are, the
@@ -91,13 +110,16 @@ func (iso Isolation) command(env []string, name string, args ...string) ([]strin
 	for _, group := range []struct {
 		what string
 		dirs []string
-	}{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}} {
+	}{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}, {"cache layer", layerDirs(iso.CacheLayer)}} {
 		for _, d := range group.dirs {
 			if err := checkDir(group.what, d); err != nil {
 				return nil, err
 			}
 		}
 	}
+	if err := iso.checkLayer(); err != nil {
+		return nil, err
+	}
 	words := append(append([]string{}, iso.Env...), env...)
 	home := false
 	for _, w := range words {
@@ -113,7 +135,7 @@ func (iso Isolation) command(env []string, name string, args ...string) ([]strin
 	for _, d := range iso.ReadOnly {
 		argv = append(argv, "--ro-bind", d, d)
 	}
-	argv = append(argv, cacheArgs(iso.Caches, iso.WriteCaches)...)
+	argv = append(argv, iso.cacheArgs()...)
 	argv = append(argv, "--bind", iso.Workspace, iso.Workspace)
 	for _, d := range iso.Writable {
 		argv = append(argv, "--bind", d, d)
@@ -134,19 +156,156 @@ func (iso Isolation) command(env []string, name string, args ...string) ([]strin
 
 // cacheArgs binds the shared warm caches (S18): for a trunk build read-write, so the next build is warm; for any other
 // read-only beneath an overlay whose writes go to a tmpfs of the step's own, so the step writes as a build does and
-// nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace).
-func cacheArgs(caches []string, write bool) []string {
+// nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace). A cache of
+// LayerCaches, with a CacheLayer, has the ask's kept layer above the shared cache and the command's writes go to its own
+// layer on disk (--overlay), which Isolated keeps for the ask when the command ends.
+func (iso Isolation) cacheArgs() []string {
 	var argv []string
-	for _, c := range caches {
-		if write {
+	for i, c := range iso.Caches {
+		switch {
+		case iso.WriteCaches:
 			argv = append(argv, "--bind", c, c)
-			continue
+		case iso.layered(c):
+			upper, work := iso.runLayer(i)
+			argv = append(argv, "--overlay-src", c, "--overlay-src", keptLayer(iso.CacheLayer, i), "--overlay", upper, work, c)
+		default:
+			argv = append(argv, "--overlay-src", c, "--tmp-overlay", c)
 		}
-		argv = append(argv, "--overlay-src", c, "--tmp-overlay", c)
 	}
 	return argv
 }
 
+// layered says whether the cache c is kept in the ask's cache layer.
+func (iso Isolation) layered(c string) bool {
+	return iso.CacheLayer != "" && !iso.WriteCaches && slices.Contains(iso.LayerCaches, c)
+}
+
+// checkLayer refuses a cache layer a command could reach (in or above the workspace or a writable directory, or a
+// cache), and a layered cache that is not one of the caches.
+func (iso Isolation) checkLayer() error {
+	if iso.CacheLayer == "" {
+		return nil
+	}
+	for _, d := range append(append([]string{iso.Workspace}, iso.Writable...), iso.Caches...) {
+		if within(iso.CacheLayer, d) || within(d, iso.CacheLayer) {
+			return fmt.Errorf("buildstep: the cache layer %s is within reach of the step's %s", iso.CacheLayer, d)
+		}
+	}
+	for _, c := range iso.LayerCaches {
+		if !slices.Contains(iso.Caches, c) {
+			return fmt.Errorf("buildstep: the layered cache %s is not one of the caches", c)
+		}
+	}
+	return nil
+}
+
+// within says whether p is dir or lies beneath it.
+func within(p, dir string) bool { return p == dir || strings.HasPrefix(p, dir+"/") }
+
+func layerDirs(layer string) []string {
+	if layer == "" {
+		return nil
+	}
+	return []string{layer}
+}
+
+// keptRel is where, in the cache layer, the ask keeps what its commands wrote into the cache i; keptLayer is its path.
+func keptRel(i int) string                 { return filepath.Join("kept", strconv.Itoa(i)) }
+func keptLayer(layer string, i int) string { return filepath.Join(layer, keptRel(i)) }
+
+// runLayer is the upper and work directory of this command's writes into the cache i. Command, which makes no
+// directory, names the layer "run"; Isolated gives each command one of its own.
+func (iso Isolation) runLayer(i int) (upper, work string) {
+	run := iso.run
+	if run == "" {
+		run = filepath.Join(iso.CacheLayer, "run")
+	}
+	return filepath.Join(run, strconv.Itoa(i), "upper"), filepath.Join(run, strconv.Itoa(i), "work")
+}
+
+// layerLocks hold one mutex per cache layer, so two commands ending at once keep their writes one after the other.
+var layerLocks sync.Map
+
+// prepareLayer makes the kept layers and a run directory of the command's own under the cache layer.
+func (iso *Isolation) prepareLayer() error {
+	if iso.CacheLayer == "" || iso.WriteCaches || len(iso.LayerCaches) == 0 {
+		return nil
+	}
+	if err := iso.checkLayer(); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Join(iso.CacheLayer, "runs"), 0o700); err != nil {
+		return err
+	}
+	run, err := os.MkdirTemp(filepath.Join(iso.CacheLayer, "runs"), "run-")
+	if err != nil {
+		return err
+	}
+	iso.run = run
+	for i, c := range iso.Caches {
+		if !iso.layered(c) {
+			continue
+		}
+		upper, work := iso.runLayer(i)
+		for _, d := range []string{keptLayer(iso.CacheLayer, i), upper, work} {
+			if err := os.MkdirAll(d, 0o700); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// keepLayer moves what the command wrote into the layered caches into the ask's kept layers, and removes its run
+// directory. A file the command wrote replaces the kept one (a cache's newest write wins); a whiteout (a file the
+// command deleted), a symlink or any other entry that is not a directory or a regular file is not kept. Every move goes
+// through an os.Root on the cache layer, so no symlink leads it outside the layer. A write that cannot be kept only
+// means a later command builds it again.
+func (iso Isolation) keepLayer() {
+	if iso.run == "" {
+		return
+	}
+	defer os.RemoveAll(iso.run)
+	mu, _ := layerLocks.LoadOrStore(iso.CacheLayer, &sync.Mutex{})
+	mu.(*sync.Mutex).Lock()
+	defer mu.(*sync.Mutex).Unlock()
+	root, err := os.OpenRoot(iso.CacheLayer)
+	if err != nil {
+		return
+	}
+	defer root.Close()
+	runRel, err := filepath.Rel(iso.CacheLayer, iso.run)
+	if err != nil {
+		return
+	}
+	for i, c := range iso.Caches {
+		if !iso.layered(c) {
+			continue
+		}
+		upper := filepath.Join(runRel, strconv.Itoa(i), "upper")
+		fs.WalkDir(root.FS(), upper, func(p string, d fs.DirEntry, err error) error {
+			if err != nil || p == upper {
+				return nil
+			}
+			rel, _ := filepath.Rel(upper, p)
+			to := filepath.Join(keptRel(i), rel)
+			switch {
+			case d.IsDir():
+				if fi, err := root.Lstat(to); err == nil && !fi.IsDir() {
+					root.Remove(to)
+				}
+				root.MkdirAll(to, 0o700)
+			case d.Type().IsRegular():
+				if fi, err := root.Lstat(to); err == nil && !fi.Mode().IsRegular() {
+					return nil // never replace what is not a file the layer kept
+				}
+				root.Rename(p, to)
+			}
+			return nil
+		})
+	}
+}
+
 // loopbackHosts is the hosts file of a step: the loopback and nothing else.
 const loopbackHosts = "127.0.0.1 localhost\n::1 localhost\n"
 
@@ -166,6 +325,10 @@ func Isolated(run forgekey.Runner, iso Isolation) forgekey.Runner {
 		}
 		iso := iso
 		iso.Hosts = hosts
+		if err := iso.prepareLayer(); err != nil {
+			return forgekey.Ran{Status: 1, Err: "the step's cache layer: " + err.Error()}
+		}
+		defer iso.keepLayer()
 		argv, err := iso.command(env, name, args...)
 		if err != nil {
 			return forgekey.Ran{Status: 1, Err: err.Error()}

go/buildstep/isolate_caches_test.go

@@ -1,6 +1,9 @@
 package buildstep
 
-import "testing"
+import (
+	"strings"
+	"testing"
+)
 
 // The shared warm caches are read under a throwaway writable layer (novox/hq ADR 0448; issue 1309): a candidate's step
 // writes into them as a build does, and nothing it writes outlives it or reaches a later build; only a trunk build
@@ -27,4 +30,46 @@ func TestACandidatesStepWritesTheCachesOnlyIntoAThrowawayLayer(t *testing.T) {
 			t.Errorf("a trunk build does not write the cache %s: %v", c, argv)
 		}
 	}
+	// 1921/S2: with a cache layer, a command reads the ask's kept layer above the shared cache and writes into a layer
+	// on disk of the ask's own, never into the shared cache and never into a throwaway tmpfs.
+	// Only the layered caches (the build cache) are; the module cache keeps its throwaway layer.
+	iso = Isolation{Workspace: "/w/ws1", Caches: []string{"/cache/gomod", "/cache/gobuild"}, CacheLayer: "/w/ws1.cachelayer",
+		LayerCaches: []string{"/cache/gobuild"}}
+	argv, err = iso.Command("go", "vet")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !holds(argv, "--overlay-src", "/cache/gobuild", "--overlay-src", keptLayer(iso.CacheLayer, 1), "--overlay") {
+		t.Errorf("the build cache is not read beneath the ask's kept layer: %v", argv)
+	}
+	if up := overlayOf(argv, "/cache/gobuild"); up == "" || !strings.HasPrefix(up, "/w/ws1.cachelayer/") {
+		t.Errorf("the build cache is written into %q, not a layer of the ask's own: %v", up, argv)
+	}
+	if !holds(argv, "--overlay-src", "/cache/gomod", "--tmp-overlay", "/cache/gomod") || overlayOf(argv, "/cache/gomod") != "" {
+		t.Errorf("the module cache is not under a throwaway layer: %v", argv)
+	}
+	// A layer a command could reach is refused: in the workspace (bound read-write), in a writable directory, in a
+	// cache, or above one of them; so is a relative one, and a layered cache that is no cache.
+	for _, bad := range []Isolation{
+		{Workspace: "/w/ws1", CacheLayer: "/w/ws1/cachelayer"},
+		{Workspace: "/w/ws1", CacheLayer: "/w"},
+		{Workspace: "/w/ws1", Writable: []string{"/out"}, CacheLayer: "/out/layer"},
+		{Workspace: "/w/ws1", Caches: []string{"/cache/gobuild"}, CacheLayer: "/cache/gobuild/layer"},
+		{Workspace: "/w/ws1", CacheLayer: "cachelayer"},
+		{Workspace: "/w/ws1", Caches: []string{"/cache/gobuild"}, CacheLayer: "/w/ws1.cachelayer", LayerCaches: []string{"/cache/other"}},
+	} {
+		if _, err := bad.Command("go"); err == nil {
+			t.Errorf("the cache layer %s of %+v was taken", bad.CacheLayer, bad)
+		}
+	}
+}
+
+// overlayOf answers the upper directory a command line gives the cache c, "" when it gives none.
+func overlayOf(argv []string, c string) string {
+	for i := 0; i+3 < len(argv); i++ {
+		if argv[i] == "--overlay" && argv[i+3] == c {
+			return argv[i+1]
+		}
+	}
+	return ""
 }

go/buildstep/isolate_integration_test.go

@@ -63,4 +63,37 @@ func TestARunStepReachesNothingAndLeavesTheCachesUnchanged(t *testing.T) {
 	if b, err := os.ReadFile(filepath.Join(workspace, "out")); r.Status != 0 || err != nil || string(b) != "built\n" {
 		t.Fatalf("the step's write to its workspace did not stay (status %d, %s%s): %q %v", r.Status, r.Stderr, r.Err, b, err)
 	}
+
+	// 1921/S2: with the ask's cache layer, what one command writes into the cache the next command of the ask reads,
+	// above what the shared cache holds, and the shared cache stays unchanged.
+	layer := filepath.Join(filepath.Dir(workspace), "ws1.cachelayer")
+	asked := Isolated(forgekey.ExecRunnerFor(0), Isolation{Workspace: workspace, Caches: []string{cacheDir},
+		CacheLayer: layer, LayerCaches: []string{cacheDir}})
+	r = asked(context.Background(), []string{"PATH=/usr/bin:/bin"}, "/bin/sh", "-c", "cat "+filepath.Join(cacheDir, "warm")+" && mkdir -p "+filepath.Join(cacheDir, "ab")+" && echo compiled > "+filepath.Join(cacheDir, "ab", "one-d"))
+	if r.Status != 0 || r.Err != "" || strings.TrimSpace(r.Stdout) != "warm" {
+		t.Fatalf("the first command of the ask did not read the shared cache and write its own (status %d, %s): %s%s", r.Status, r.Err, r.Stdout, r.Stderr)
+	}
+	r = asked(context.Background(), []string{"PATH=/usr/bin:/bin"}, "/bin/cat", filepath.Join(cacheDir, "ab", "one-d"), filepath.Join(cacheDir, "warm"))
+	if r.Status != 0 || r.Err != "" || r.Stdout != "compiled\nwarm\n" {
+		t.Fatalf("the next command of the ask did not find what the first wrote (status %d, %s): %q %s", r.Status, r.Err, r.Stdout, r.Stderr)
+	}
+	// A command cannot reach the layer (it lies outside every directory the step binds), and a symlink it writes into
+	// the cache is not kept, so nothing it does makes the keeping write outside the layer.
+	r = asked(context.Background(), []string{"PATH=/usr/bin:/bin"}, "/bin/sh", "-c",
+		"ls "+layer+" >/dev/null 2>&1 && echo reached; ln -s "+cacheDir+" "+filepath.Join(cacheDir, "esc")+" && mkdir "+filepath.Join(cacheDir, "cd")+" && echo x > "+filepath.Join(cacheDir, "cd", "f"))
+	if r.Status != 0 || r.Err != "" || strings.Contains(r.Stdout, "reached") {
+		t.Fatalf("the command reached the cache layer or could not write the cache (status %d, %s): %q %s", r.Status, r.Err, r.Stdout, r.Stderr)
+	}
+	if fi, err := os.Lstat(filepath.Join(keptLayer(layer, 0), "esc")); err == nil {
+		t.Fatalf("a symlink the command wrote was kept: %v", fi.Mode())
+	}
+	if b, err := os.ReadFile(filepath.Join(keptLayer(layer, 0), "cd", "f")); err != nil || string(b) != "x\n" {
+		t.Fatalf("the command's file was not kept: %q %v", b, err)
+	}
+	if entries, _ := os.ReadDir(cacheDir); len(entries) != 1 || entries[0].Name() != "warm" {
+		t.Fatalf("the ask's commands wrote the shared cache: %v", entries)
+	}
+	if err := os.RemoveAll(layer); err != nil {
+		t.Fatalf("the ask's cache layer cannot be removed: %v", err)
+	}
 }

go/buildstep/isolate_test.go

@@ -2,7 +2,10 @@ package buildstep
 
 import (
 	"context"
+	"os"
+	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
 
 	"git.novox.be/novox/mesh-sdk/go/forgekey"
@@ -112,4 +115,63 @@ func TestIsolatedRunsEachCommandThroughBubblewrap(t *testing.T) {
 	if hosts == "" {
 		t.Fatalf("no hosts file was bound: %v", ran)
 	}
+
+	// 1921/S2: with a cache layer, what one command wrote into the caches is kept for the ask's later commands, the
+	// shared cache stays unchanged, two commands at once write into layers of their own (overlayfs refuses an upper
+	// layer two mounts share), and a command's own layer is gone when it ends.
+	root := t.TempDir()
+	shared := filepath.Join(root, "gobuild")
+	if err := os.MkdirAll(shared, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	layer := filepath.Join(root, "ws1.cachelayer")
+	var mu sync.Mutex
+	uppers := map[string]bool{}
+	gate := make(chan struct{})
+	var started sync.WaitGroup
+	started.Add(2)
+	writer := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
+		up := overlayOf(args, shared)
+		mu.Lock()
+		uppers[up] = true
+		mu.Unlock()
+		started.Done()
+		<-gate // both commands are running before either ends
+		os.MkdirAll(filepath.Join(up, "ab"), 0o700)
+		os.WriteFile(filepath.Join(up, "ab", args[len(args)-1]+"-d"), []byte("compiled"), 0o600)
+		return forgekey.Ran{}
+	}
+	// A symlink in the kept layer (none is ever kept; planted here) leads no move outside the layer: the kept directory
+	// replaces it, and the shared cache it points to stays unchanged.
+	if err := os.MkdirAll(keptLayer(layer, 0), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink(shared, filepath.Join(keptLayer(layer, 0), "ab")); err != nil {
+		t.Fatal(err)
+	}
+	run := Isolated(writer, Isolation{Workspace: filepath.Join(root, "ws1"), Caches: []string{shared}, CacheLayer: layer, LayerCaches: []string{shared}})
+	var wg sync.WaitGroup
+	for _, pkg := range []string{"one", "two"} {
+		wg.Add(1)
+		go func() { defer wg.Done(); run(context.Background(), nil, "go", "build", pkg) }()
+	}
+	started.Wait()
+	close(gate)
+	wg.Wait()
+	if len(uppers) != 2 {
+		t.Fatalf("two commands at once wrote into %d layers, want 2 of their own: %v", len(uppers), uppers)
+	}
+	for _, pkg := range []string{"one", "two"} {
+		if b, err := os.ReadFile(filepath.Join(keptLayer(layer, 0), "ab", pkg+"-d")); err != nil || string(b) != "compiled" {
+			t.Errorf("what the build of %s compiled is not kept for the ask: %q %v", pkg, b, err)
+		}
+	}
+	for up := range uppers {
+		if _, err := os.Stat(up); !os.IsNotExist(err) {
+			t.Errorf("the command's own layer %s outlived it: %v", up, err)
+		}
+	}
+	if entries, _ := os.ReadDir(shared); len(entries) != 0 {
+		t.Errorf("the shared cache was written: %v", entries)
+	}
 }

go/buildstep/pull.go

@@ -98,6 +98,12 @@ type Puller struct {
 	// again, and the worker it came from is not looked at again until the next pull's wait, so a node lacking the need
 	// does not take the asker's next copy at once. Nil has nothing.
 	Has func(need string) bool
+	// SettingsFile is the module's settings file. With it set the puller wakes whenever the file is written (a file event,
+	// WatchSettings), and reads Capacity from it itself, keeping why it cannot be read (CapacityNote); a parked puller at
+	// capacity 0 therefore wakes when the node-engine refills the file, and a failed config follow is tried again then.
+	SettingsFile string
+	// watching, when set, is called each time the settings file's watch is (re)armed: a test waits on it, not on a sleep.
+	watching func()
 
 	mu      sync.Mutex
 	inHand  int
@@ -108,6 +114,14 @@ type Puller struct {
 	// watched is the capacity the config watch handed last, used in place of Capacity once hasWatched.
 	watched    int
 	hasWatched bool
+	// capFollow and hoursFollow are the state of the two config follows: followed once WatchConfig took them, trying
+	// while a start is in flight; followFault is the last failure said, so each failure is said once.
+	capFollowed, capTrying, hoursFollowed, hoursTrying bool
+	followFault                                        string
+	// fileFault is why the settings file's capacity cannot be read, "" when it can.
+	fileFault string
+	// capWhy is why the config watch's last value could not be read, "" when it could.
+	capWhy string
 	// hours is the build window the config watch handed last, and hoursFault why its value cannot be read; used in place
 	// of BuildHours once hasHours.
 	hours      *BuildWindow
@@ -248,6 +262,21 @@ func (p *Puller) roomNow() (ok bool, capacity, inHand int) {
 	return !p.paused && p.inHand < capacity, capacity, p.inHand
 }
 
+// CapacityNote says when the puller takes nothing for want of a capacity: why, and whether it is a fault (a setting that
+// cannot be read or is refused, so a person must look) and not a choice (a capacity set to 0). "" when it pulls.
+func (p *Puller) CapacityNote() (why string, fault bool) {
+	c := p.CapacityDetail()
+	if c.Effective > 0 {
+		return "", false
+	}
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.fileFault != "" && !p.hasWatched {
+		return "the " + p.Language + " build capacity of " + p.Node + " is 0 because it cannot be read: " + p.fileFault, true
+	}
+	return "the " + p.Language + " build capacity of " + p.Node + " is " + c.Why, p.hasWatched && p.capWhy != ""
+}
+
 // CapacityNow is the capacity the puller judges by (CapacityDetail's Effective).
 func (p *Puller) CapacityNow() int {
 	return p.CapacityDetail().Effective
@@ -265,6 +294,15 @@ func (p *Puller) CapacityDetail() delivery.Capacity {
 	switch {
 	case has:
 		configured, from = watched, "mesh-config, as the config watch handed it"
+	case p.SettingsFile != "":
+		n, err := ReadCapacity(p.SettingsFile)
+		configured = n
+		p.mu.Lock()
+		p.fileFault = ""
+		if err != nil {
+			p.fileFault = err.Error()
+		}
+		p.mu.Unlock()
 	case p.Capacity != nil:
 		configured = p.Capacity()
 	}
@@ -386,8 +424,8 @@ func (p *Puller) Pull(ctx context.Context) error {
 		// node kept from building is held by its capacity (0), not by a pause.
 		p.logf("the pause of %s cannot be read: %v; pulling as not paused", p.Node, err)
 	}
-	go p.watchCapacity() // WatchConfigServed waits for Serve, which a module starts after its pull
-	go p.watchHours()
+	p.follow() // WatchConfigServed waits for Serve, which a module starts after its pull
+	go p.watchSettings(ctx)
 	wait := p.PullWait
 	if wait <= 0 {
 		wait = DefaultPullWait
@@ -414,6 +452,7 @@ func (p *Puller) Pull(ctx context.Context) error {
 			case <-edge:
 			}
 			stop()
+			p.follow() // a follow that failed is tried again at the next event
 			_, capacity, inHand = p.roomNow()
 			p.logf("the %s puller on %s wakes: %d of capacity %d in hand, paused %v", p.Language, p.Node, inHand, capacity, p.Paused())
 			continue
@@ -432,6 +471,7 @@ func (p *Puller) Pull(ctx context.Context) error {
 			case <-ctx.Done():
 				return ctx.Err()
 			case <-p.wake:
+				p.follow()
 			case <-time.After(wait):
 			}
 			continue
@@ -620,11 +660,18 @@ func (p *Puller) watchCapacity() {
 	if p.WatchConfig == nil {
 		return
 	}
+	p.mu.Lock()
+	if p.capFollowed || p.capTrying {
+		p.mu.Unlock()
+		return
+	}
+	p.capTrying = true
+	p.mu.Unlock()
 	err := p.WatchConfig(CapacitySetting, func(c stdio.ProvisionChange) error {
 		n, why := p.capacityFrom(c)
 		p.mu.Lock()
 		before, had := p.watched, p.hasWatched
-		p.watched, p.hasWatched = n, true
+		p.watched, p.hasWatched, p.capWhy = n, true, why
 		p.mu.Unlock()
 		switch {
 		case why != "":
@@ -635,9 +682,31 @@ func (p *Puller) watchCapacity() {
 		p.Wake()
 		return nil
 	})
+	p.followed(&p.capFollowed, &p.capTrying, err, "build capacity")
+}
+
+// follow starts each config follow that is not running: at the start, and at every wake after (a file event, a resume, a
+// step's end), so a follow that failed is tried again by the next event and not given up for good.
+func (p *Puller) follow() {
+	go p.watchCapacity()
+	go p.watchHours()
+}
+
+// followed records the end of one follow's start. A failure is said once per distinct error, with that it is tried again.
+func (p *Puller) followed(done, trying *bool, err error, what string) {
+	p.mu.Lock()
+	*trying = false
+	*done = err == nil
+	said := err != nil && err.Error() == p.followFault
 	if err != nil {
-		p.logf("the %s build capacity of %s cannot be followed on the bus, so it is read from the settings file when the "+
-			"puller wakes: %v", p.Language, p.Node, err)
+		p.followFault = err.Error()
+	} else {
+		p.followFault = ""
+	}
+	p.mu.Unlock()
+	if err != nil && !said {
+		p.logf("the %s %s of %s cannot be followed on the bus, so it is read from the settings file; the follow is tried "+
+			"again at the next event: %v", p.Language, what, p.Node, err)
 	}
 }
 
@@ -648,6 +717,13 @@ func (p *Puller) watchHours() {
 	if p.WatchConfig == nil || p.BuildHours == nil {
 		return
 	}
+	p.mu.Lock()
+	if p.hoursFollowed || p.hoursTrying {
+		p.mu.Unlock()
+		return
+	}
+	p.hoursTrying = true
+	p.mu.Unlock()
 	err := p.WatchConfig(BuildHoursSetting, func(c stdio.ProvisionChange) error {
 		setting := ""
 		var w *BuildWindow
@@ -672,9 +748,7 @@ func (p *Puller) watchHours() {
 		p.Wake()
 		return nil
 	})
-	if err != nil {
-		p.logf("the %s build hours of %s cannot be followed on the bus, so they are read from the settings file: %v", p.Language, p.Node, err)
-	}
+	p.followed(&p.hoursFollowed, &p.hoursTrying, err, "build hours")
 }
 
 // WatchConfigServed is stdio.WatchConfig once Serve has the runtime's channel (stdio.Served): a build module starts its

go/buildstep/pull_test.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -301,6 +303,71 @@ func TestPullAtCapacityZeroTakesNothing(t *testing.T) {
 		w.set(t, "2")
 		waitFor(t, "the ask taken once the capacity moved to 2", func() bool { a, _, _ := src.Fates(); return len(a) == 1 })
 	})
+	t.Run("settings refill wakes a parked puller and retries a failed follow", func(t *testing.T) {
+		// 2026-10-28: a replaced builder held the old "shanks=2,..." in its settings file, its config follow timed out, and
+		// the puller parked at 0 with nothing to wake it when the node-engine refilled the file.
+		src, st := NewFakeAsks(), newSteps()
+		_ = src.PublishAsk(testAsk(delivery.AskDelivery))
+		file := filepath.Join(t.TempDir(), "settings.json")
+		if err := os.WriteFile(file, []byte(`{"capacity":"shanks=2,ace=2"}`), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		var mu sync.Mutex
+		attempts := 0
+		watch := func(key string, handle func(stdio.ProvisionChange) error) error {
+			mu.Lock()
+			defer mu.Unlock()
+			attempts++
+			if attempts == 1 {
+				return errors.New("nats: timeout")
+			}
+			return nil // followed; no value handed yet
+		}
+		armed := make(chan struct{}, 8)
+		var parked atomic.Int32
+		parks := func() int32 { return parked.Load() }
+		logf := func(format string, args ...any) {
+			if strings.Contains(format, "parks") {
+				parked.Add(1)
+			}
+		}
+		p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, PullWait: 10 * time.Millisecond,
+			SettingsFile: file, WatchConfig: watch, Logf: logf, watching: func() { armed <- struct{}{} },
+			Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return nil, nil }}
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		go p.Pull(ctx)
+		waitFor(t, "the parked puller to say why", func() bool { why, fault := p.CapacityNote(); return fault && why != "" })
+		if src.Waiting(delivery.AskDelivery) != 1 {
+			t.Fatal("a node whose capacity cannot be read took an ask")
+		}
+		<-armed // the file watch is in place
+		tmp := file + ".new"
+		if err := os.WriteFile(tmp, []byte(`{"capacity":2}`), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.Rename(tmp, file); err != nil {
+			t.Fatal(err)
+		}
+		waitFor(t, "the ask taken once the file was refilled", func() bool { a, _, _ := src.Fates(); return len(a) == 1 })
+		// and the directory itself replaced (IN_IGNORED): the watch is armed again and a later refill still wakes the puller
+		dir := filepath.Dir(file)
+		before := parks()
+		if err := os.RemoveAll(dir); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.Mkdir(dir, 0o700); err != nil {
+			t.Fatal(err)
+		}
+		<-armed // armed again
+		waitFor(t, "the puller parked with no settings file", func() bool { return parks() > before })
+		_ = src.PublishAsk(testAsk(delivery.AskDelivery))
+		if err := os.WriteFile(file, []byte(`{"capacity":2}`), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		waitFor(t, "the second ask taken after the directory was replaced", func() bool { a, _, _ := src.Fates(); return len(a) == 2 })
+		waitFor(t, "the failed follow tried again", func() bool { mu.Lock(); defer mu.Unlock(); return attempts >= 2 })
+	})
 	for _, c := range []struct {
 		name  string
 		watch func(string, func(stdio.ProvisionChange) error) error

go/buildstep/watch_settings_linux.go

@@ -0,0 +1,112 @@
+//go:build linux
+
+package buildstep
+
+import (
+	"context"
+	"encoding/binary"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+const settingsMask = syscall.IN_CLOSE_WRITE | syscall.IN_MOVED_TO | syscall.IN_CREATE
+
+// watchSettings wakes the puller each time its settings file is written or replaced (inotify on the file's directory, so
+// a rename over it counts): the event the node-engine's refill of the file is, which a parked puller at capacity 0
+// otherwise never sees. No timer: it blocks on the kernel's event. When the kernel drops the watch (IN_IGNORED: the
+// directory was replaced or removed) it is added again, said once; while the directory is missing the nearest existing
+// ancestor is watched for the directory's return. A watch that cannot start is said once.
+func (p *Puller) watchSettings(ctx context.Context) {
+	if p.SettingsFile == "" {
+		return
+	}
+	fd, err := syscall.InotifyInit1(syscall.IN_CLOEXEC | syscall.IN_NONBLOCK)
+	if err != nil {
+		p.logf("the settings file %s cannot be watched, so a refill of it wakes the puller only at another event: %v", p.SettingsFile, err)
+		return
+	}
+	f := os.NewFile(uintptr(fd), "inotify")
+	defer f.Close()
+	dir, base := filepath.Split(p.SettingsFile)
+	dir = filepath.Clean(dir)
+	// arm watches dir, or when it is missing its nearest existing ancestor; it returns the watch descriptor and whether it
+	// is the directory's own.
+	arm := func() (int, bool, error) {
+		wd, err := syscall.InotifyAddWatch(fd, dir, settingsMask)
+		if err == nil {
+			return wd, true, nil
+		}
+		for up := filepath.Dir(dir); ; up = filepath.Dir(up) {
+			wd, aerr := syscall.InotifyAddWatch(fd, up, settingsMask)
+			if aerr == nil {
+				// the directory may have come back between the two calls: look once more, now that the ancestor is watched
+				if own, oerr := syscall.InotifyAddWatch(fd, dir, settingsMask); oerr == nil {
+					return own, true, nil
+				}
+				return wd, false, nil
+			}
+			if up == filepath.Dir(up) {
+				return 0, false, err
+			}
+		}
+	}
+	wd, own, err := arm()
+	if err != nil {
+		p.logf("the settings file %s cannot be watched, so a refill of it wakes the puller only at another event: %v", p.SettingsFile, err)
+		return
+	}
+	if p.watching != nil {
+		p.watching()
+	}
+	go func() { <-ctx.Done(); f.Close() }()
+	buf := make([]byte, 4096)
+	for {
+		n, err := f.Read(buf)
+		if err != nil {
+			return
+		}
+		hit, rearm := false, false
+		for off := 0; off+16 <= n; {
+			ewd := int(int32(binary.LittleEndian.Uint32(buf[off:])))
+			mask := binary.LittleEndian.Uint32(buf[off+4:])
+			l := int(binary.LittleEndian.Uint32(buf[off+12:]))
+			name := buf[off+16 : off+16+l]
+			for i, c := range name {
+				if c == 0 {
+					name = name[:i]
+					break
+				}
+			}
+			switch {
+			case mask&syscall.IN_IGNORED != 0 && ewd == wd:
+				rearm = true
+			case ewd == wd && own && string(name) == base:
+				hit = true
+			case ewd == wd && !own:
+				rearm = true // something appeared where the directory was: look again
+			}
+			off += 16 + l
+		}
+		if rearm {
+			nwd, nown, err := arm()
+			if err != nil {
+				p.logf("the settings file %s is no longer watched, so a refill of it wakes the puller only at another event: %v", p.SettingsFile, err)
+				return
+			}
+			if own || nown { // said once per loss of the watch on the directory itself
+				if own {
+					p.logf("the watch of the settings file %s was dropped (its directory was replaced); watching again", p.SettingsFile)
+				}
+			}
+			wd, own = nwd, nown
+			if p.watching != nil {
+				p.watching()
+			}
+			hit = true // the file may have been refilled meanwhile
+		}
+		if hit {
+			p.Wake()
+		}
+	}
+}

go/buildstep/watch_settings_other.go

@@ -0,0 +1,8 @@
+//go:build !linux
+
+package buildstep
+
+import "context"
+
+// watchSettings watches nothing off Linux: the puller wakes at its other events.
+func (p *Puller) watchSettings(ctx context.Context) {}

go/buildstep/workspaces.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
@@ -116,6 +117,14 @@ func (w *Workspaces) touch(id string) {
 	_ = os.Chtimes(filepath.Join(w.Dir, id), now, now) // a workspace not made yet has nothing to mark
 }
 
+// cacheLayerSuffix names a workspace's cache layer beside it (Isolation.CacheLayer): never inside it, which every step
+// of the workspace binds read-write.
+const cacheLayerSuffix = ".cachelayer"
+
+// CacheLayer is the directory of the workspace id's cache layer: beside the workspace, out of every step's reach, and
+// removed with it by Release and Sweep.
+func (w *Workspaces) CacheLayer(id string) string { return filepath.Join(w.Dir, id+cacheLayerSuffix) }
+
 // Release removes the workspace at once: its caller's chain of steps has ended. One already gone is no error; one a
 // running step holds is refused and kept; an id that is not a workspace's is refused.
 func (w *Workspaces) Release(id string) error {
@@ -130,6 +139,9 @@ func (w *Workspaces) Release(id string) error {
 	if err := os.RemoveAll(filepath.Join(w.Dir, id)); err != nil {
 		return fmt.Errorf("the workspace %s could not be removed: %w", id, err)
 	}
+	if err := os.RemoveAll(w.CacheLayer(id)); err != nil {
+		return fmt.Errorf("the cache layer of the workspace %s could not be removed: %w", id, err)
+	}
 	return nil
 }
 
@@ -144,6 +156,13 @@ func (w *Workspaces) Sweep(now time.Time, bound time.Duration) []string {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	for _, e := range entries {
+		if id, ok := strings.CutSuffix(e.Name(), cacheLayerSuffix); ok && workspaceID.MatchString(id) && w.held[id] == 0 {
+			// a cache layer whose workspace is gone (its removal failed or was cut short) goes too
+			if _, err := os.Stat(filepath.Join(w.Dir, id)); os.IsNotExist(err) {
+				os.RemoveAll(filepath.Join(w.Dir, e.Name()))
+			}
+			continue
+		}
 		if !e.IsDir() || !workspaceID.MatchString(e.Name()) || w.held[e.Name()] > 0 {
 			continue
 		}
@@ -152,6 +171,7 @@ func (w *Workspaces) Sweep(now time.Time, bound time.Duration) []string {
 			continue
 		}
 		if os.RemoveAll(filepath.Join(w.Dir, e.Name())) == nil {
+			os.RemoveAll(w.CacheLayer(e.Name()))
 			removed = append(removed, e.Name())
 		}
 	}

go/buildstep/workspaces_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 )
@@ -68,8 +69,15 @@ func gone(p string) bool { _, err := os.Stat(p); return os.IsNotExist(err) }
 func TestReleaseRemovesAWorkspaceAtOnce(t *testing.T) {
 	w := &Workspaces{Dir: t.TempDir()}
 	p := makeWorkspace(t, w, "abc1", 0)
-	if err := w.Release("abc1"); err != nil || !gone(p) {
-		t.Fatalf("Release = %v, gone %v", err, gone(p))
+	layer := w.CacheLayer("abc1") // issue 1921/S2: the cache layer lies beside the workspace and goes with it
+	if filepath.Dir(layer) != w.Dir || strings.HasPrefix(layer, p+"/") {
+		t.Fatalf("the cache layer %s does not lie beside the workspace %s", layer, p)
+	}
+	if err := os.MkdirAll(filepath.Join(layer, "kept", "0"), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Release("abc1"); err != nil || !gone(p) || !gone(layer) {
+		t.Fatalf("Release = %v, gone %v, its cache layer gone %v", err, gone(p), gone(layer))
 	}
 	if err := w.Release("abc1"); err != nil {
 		t.Errorf("a second release = %v, want nil", err)
@@ -98,11 +106,21 @@ func TestSweepRemovesUnheldWorkspacesPastTheBound(t *testing.T) {
 	if err := os.WriteFile(filepath.Join(w.Dir, "not-a-workspace.txt"), nil, 0o600); err != nil {
 		t.Fatal(err)
 	}
+	// issue 1921/S2: a swept workspace's cache layer goes with it, and so does a layer whose workspace is gone; a held
+	// workspace's stays
+	for _, id := range []string{"old1", "held1", "orphan1"} {
+		if err := os.MkdirAll(w.CacheLayer(id), 0o700); err != nil {
+			t.Fatal(err)
+		}
+	}
 	done := w.Hold("held1")
 	removed := w.Sweep(time.Now(), 2*time.Hour)
 	if !gone(old) || gone(heldOld) || gone(recent) {
 		t.Errorf("after the sweep: old gone %v, held kept %v, recent kept %v", gone(old), !gone(heldOld), !gone(recent))
 	}
+	if !gone(w.CacheLayer("old1")) || !gone(w.CacheLayer("orphan1")) || gone(w.CacheLayer("held1")) {
+		t.Errorf("after the sweep: old1's layer gone %v, orphan1's gone %v, held1's kept %v", gone(w.CacheLayer("old1")), gone(w.CacheLayer("orphan1")), !gone(w.CacheLayer("held1")))
+	}
 	if len(removed) != 1 || removed[0] != "old1" {
 		t.Errorf("Sweep answered %v, want [old1]", removed)
 	}

go/compose/frombus/answer.go

@@ -99,6 +99,7 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st
 		for _, m := range sortedKeys(failed) {
 			a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: failed[m]})
 		}
+		heldAsToday(&a, s.HeldToday)
 		a.Verdict = delivery.VerdictOf(a)
 		return a
 	}
@@ -203,10 +204,31 @@ func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]st
 		a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: failed[m]})
 	}
 	sort.Slice(a.Refusals, func(i, j int) bool { return a.Refusals[i].Module < a.Refusals[j].Module })
+	heldAsToday(&a, s.HeldToday)
 	a.Verdict = delivery.VerdictOf(a)
 	return a
 }
 
+// heldAsToday moves each refusal of a module the node holds today (Scope.HeldToday) into Held: the candidate holds it as
+// the node does, which is no change (issue 1126: a seed reproduces today, and today holds it too).
+func heldAsToday(a *delivery.CandidateAnswer, today []string) {
+	if len(today) == 0 {
+		return
+	}
+	kept := a.Refusals[:0]
+	for _, r := range a.Refusals {
+		if slices.Contains(today, r.Module) {
+			a.Held = append(a.Held, r)
+		} else {
+			kept = append(kept, r)
+		}
+	}
+	a.Refusals = kept
+	if len(a.Refusals) == 0 {
+		a.Refusals = nil
+	}
+}
+
 // maxCulprits bounds the modules one refusal is tried on, each a whole composition.
 const maxCulprits = 8
 

go/compose/frombus/answer_rows_test.go

@@ -91,6 +91,19 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) {
 	if lemursOK := slices.ContainsFunc(a.Added, func(m delivery.ModuleAdded) bool { return m.Module == "lemurs" }); !lemursOK {
 		t.Errorf("lemurs, which composes, is not answered beside the refusals: %+v", a)
 	}
+	// Issue 1126: a module the node holds today that the candidate holds too is held as today, no refusal: with every
+	// refused module held today the verdict is no refusal, and each is named under Held.
+	today := Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b), HeldToday: []string{"api", "spiegel", "web"}},
+		map[string]string{"lemurs": fp("e"), "web": fp("1"), "api": fp("2"), "spiegel": fp("3")}, nil, "p-h1", 3)
+	if today.Verdict == delivery.CandidateRefused || len(today.Refusals) != 0 || len(today.Held) != 3 || today.Held[0].Why == "" {
+		t.Errorf("held as today: %+v", today)
+	}
+	// A refused module the node does not hold today stays a refusal: holding only api today leaves spiegel and web refused.
+	part := Answer(ctx, b, Mesh{}, Scope{Node: "n1", Placements: given(b), HeldToday: []string{"api"}},
+		map[string]string{"lemurs": fp("e"), "web": fp("1"), "api": fp("2"), "spiegel": fp("3")}, nil, "p-h1", 3)
+	if part.Verdict != delivery.CandidateRefused || len(part.Refusals) != 2 || len(part.Held) != 1 || part.Held[0].Module != "api" {
+		t.Errorf("held as today in part: %+v", part)
+	}
 
 	// The fold: spiegel runs inside the node's own composition at a build of the old form. Its resources are carried as
 	// they run, each after the one it runs after: the node's own is the one it runs, and the node composes itself.

go/compose/frombus/frombus.go

@@ -134,6 +134,10 @@ type Scope struct {
 	Placements map[string]*delivery.Placement
 	InOwn      []string
 	Running    []delivery.Composed
+	// HeldToday is, in an answer, the modules the node holds today ("node" for its own composition): their have's phase
+	// is held, or they are held inside its own composition. A candidate that holds one too holds it as today, which is
+	// said under CandidateAnswer.Held and is no refusal (issue 1126).
+	HeldToday []string
 	// SameValue, in an answer, says whether two sealed values open to the same value (the node opens them with its own
 	// key), so a differing resource shows a value sealed again as such (delivery.CompareComposedWith). Nil shows them
 	// as differing.

go/compose/judge/judge.go

@@ -5,9 +5,11 @@
 package judge
 
 import (
+	"encoding/json"
 	"fmt"
 	"reflect"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -121,7 +123,7 @@ func Intended(current, laidInput *compose.Input, currentPlaced map[string]delive
 // shown in the ask's brief. The seed alone fails on any difference.
 func JudgeAnswers(p delivery.PlacementProposal, intended map[string]map[string]bool, answers map[string]*delivery.CandidateAnswer,
 	revisions map[string]uint64, online map[string]bool, pulsing map[string]bool, now time.Time) delivery.PlacementVerdict {
-	v := delivery.PlacementVerdict{At: now.UTC(), Intended: map[string][]string{}, Unintended: map[string][]string{},
+	v := delivery.PlacementVerdict{At: now.UTC(), Intended: map[string][]string{}, Unintended: map[string][]string{}, Carried: map[string][]string{},
 		Answers: map[string]delivery.CandidateAnswer{}}
 	for node, set := range intended {
 		for m := range set {
@@ -150,7 +152,8 @@ func JudgeAnswers(p delivery.PlacementProposal, intended map[string]map[string]b
 		want := intended[node]
 		differs := func(module, words string) {
 			if p.Seed {
-				v.Fails = append(v.Fails, fmt.Sprintf("%s %s with the seed, which must be the same as what runs", node, words))
+				// The seed carries what its nodes found, each named in the brief for the operator who approves it.
+				v.Carried[node] = append(v.Carried[node], words)
 				return
 			}
 			if !want[module] {
@@ -158,7 +161,7 @@ func JudgeAnswers(p delivery.PlacementProposal, intended map[string]map[string]b
 			}
 		}
 		for _, d := range a.Differs {
-			differs(d.Module, fmt.Sprintf("composes %s differently (%s)", d.Module, d.Why))
+			differs(d.Module, differenceWords(d))
 		}
 		for _, d := range a.Added {
 			differs(d.Module, "would run "+d.Module)
@@ -174,6 +177,9 @@ func JudgeAnswers(p delivery.PlacementProposal, intended map[string]map[string]b
 	if len(v.Unintended) == 0 {
 		v.Unintended = nil
 	}
+	if len(v.Carried) == 0 {
+		v.Carried = nil
+	}
 	if len(v.Answers) == 0 {
 		v.Answers = nil
 	}
@@ -209,7 +215,7 @@ func JudgeAnswers(p delivery.PlacementProposal, intended map[string]map[string]b
 }
 
 // Brief is the verdict as the ask's brief shows it, by node (§5): what the proposal changes and what each node answered
-// beside it, marked, with the proposal's why.
+// beside it, marked, and what stays held as the node holds it today, with the proposal's why.
 func Brief(p delivery.PlacementProposal, v delivery.PlacementVerdict) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "%s (proposed by %s)", p.Why, p.By)
@@ -223,6 +229,14 @@ func Brief(p delivery.PlacementProposal, v delivery.PlacementVerdict) string {
 	for _, n := range p.Nodes() {
 		nodes[n] = true
 	}
+	for n, a := range v.Answers {
+		if len(a.Held) > 0 {
+			nodes[n] = true
+		}
+	}
+	for n := range v.Carried {
+		nodes[n] = true
+	}
 	names := make([]string, 0, len(nodes))
 	for n := range nodes {
 		names = append(names, n)
@@ -238,6 +252,117 @@ func Brief(p delivery.PlacementProposal, v delivery.PlacementVerdict) string {
 		if m := v.Unintended[n]; len(m) > 0 {
 			fmt.Fprintf(&b, "; also differs, not computed: %s", strings.Join(m, ", "))
 		}
+		if m := v.Carried[n]; len(m) > 0 {
+			fmt.Fprintf(&b, "; the seed carries: %s", strings.Join(m, "; "))
+		}
+		if held := v.Answers[n].Held; len(held) > 0 {
+			var names []string
+			for _, h := range held {
+				names = append(names, h.Module)
+			}
+			fmt.Fprintf(&b, "; stays held as today: %s", strings.Join(names, ", "))
+		}
 	}
 	return b.String()
 }
+
+// differenceWords names one module's difference: the module, the first field that differs in it with its value as it
+// runs and as the seed composes it, each cut to about 60 characters ("(value cut)" where a body shown is cut before the
+// field), and how many fields more; or the resource and why when no body is shown.
+func differenceWords(d delivery.ModuleDiffers) string {
+	for _, b := range d.Bodies {
+		if len(b.Paths) == 0 {
+			continue
+		}
+		path := b.Paths[0]
+		words := fmt.Sprintf("%s: %s %s → %s", d.Module, fieldName(b.Running, b.Composed, path),
+			valueAt(b.Running, path), valueAt(b.Composed, path))
+		if n := len(b.Paths) - 1; n > 0 {
+			words += fmt.Sprintf(" (and %d more fields)", n)
+		}
+		return words
+	}
+	if d.Resource != "" {
+		return fmt.Sprintf("%s: %s (%s)", d.Module, d.Resource, d.Why)
+	}
+	return fmt.Sprintf("%s: %s", d.Module, d.Why)
+}
+
+// shownValue bounds a value named in a brief.
+const shownValue = 60
+
+// walk follows a differing path (delivery.DifferingPaths: keys joined by dots, a key may hold dots, and a JSON value
+// written in a string is read as JSON) through a body, answering the keys it passed and the value at its end.
+func walk(body, path string) (keys []string, value any, ok bool) {
+	var v any
+	if json.Unmarshal([]byte(body), &v) != nil {
+		return nil, nil, false
+	}
+	rest := path
+	for rest != "" {
+		if s, isString := v.(string); isString {
+			var inner any
+			if json.Unmarshal([]byte(s), &inner) != nil {
+				return nil, nil, false
+			}
+			v = inner
+		}
+		switch t := v.(type) {
+		case map[string]any:
+			best := ""
+			for k := range t {
+				if (rest == k || strings.HasPrefix(rest, k+".")) && len(k) > len(best) {
+					best = k
+				}
+			}
+			if best == "" {
+				return nil, nil, false
+			}
+			keys, v, rest = append(keys, best), t[best], strings.TrimPrefix(strings.TrimPrefix(rest, best), ".")
+		case []any:
+			seg, after, _ := strings.Cut(rest, ".")
+			n, err := strconv.Atoi(seg)
+			if err != nil || n < 0 || n >= len(t) {
+				return nil, nil, false
+			}
+			keys, v, rest = append(keys, seg), t[n], after
+		default:
+			return nil, nil, false
+		}
+	}
+	return keys, v, true
+}
+
+// valueAt is the value at path in a shown body, cut to shownValue, or why it is not shown.
+func valueAt(body, path string) string {
+	if body == "" {
+		return "(none)"
+	}
+	_, v, ok := walk(body, path)
+	if !ok {
+		return "(value cut)"
+	}
+	s, isString := v.(string)
+	if !isString {
+		raw, _ := json.Marshal(v)
+		s = string(raw)
+	}
+	if r := []rune(s); len(r) > shownValue {
+		s = string(r[:shownValue-1]) + "…"
+	}
+	return s
+}
+
+// fieldName is the differing field as a person reads it: its parent's name and its own, `PARENT[key]`, from whichever
+// body reaches it; the path's end when neither does.
+func fieldName(running, composed, path string) string {
+	for _, body := range []string{running, composed} {
+		if keys, _, ok := walk(body, path); ok && len(keys) > 0 {
+			if len(keys) == 1 {
+				return keys[0]
+			}
+			return keys[len(keys)-2] + "[" + keys[len(keys)-1] + "]"
+		}
+	}
+	return path
+}

go/compose/judge/judge_test.go

@@ -47,11 +47,17 @@ func TestJudge(t *testing.T) {
 		{"a module added where intended", false, map[string]*delivery.CandidateAnswer{"anchor": {Proposal: "p-one", Revision: 7, Verdict: delivery.CandidateDiffers,
 			Added: []delivery.ModuleAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
 		{"a difference not computed passes, marked", false, map[string]*delivery.CandidateAnswer{"anchor": differs(7, "mail"), "laptop": same(8)}, nil, delivery.VerdictPass, "", "mail"},
-		{"the seed with a difference", true, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": differs(8, "web")}, nil, delivery.VerdictFail, "laptop composes web differently", ""},
+		{"the seed carries a difference, named", true, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
+			Verdict: delivery.CandidateDiffers, Differs: []delivery.ModuleDiffers{{Module: "node-tools", Resource: "node-tools.runtime", Why: "the resource differs",
+				Bodies: []delivery.ResourceBodies{{ID: "node-tools.runtime", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf", "env.MESH_TOOL_MODULES"},
+					Running:  `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
+					Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}","MESH_TOOL_MODULES":"… (cut at 8192 bytes)`}}}}}}, nil, delivery.VerdictPass, "", ""},
 		{"a refusal", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8, Verdict: delivery.CandidateRefused,
 			Refusals: []delivery.ModuleRefused{{Module: "web", Why: "no own secret"}}}}, nil, delivery.VerdictFail, "laptop refuses", ""},
 		{"a node not answering, within its bound", false, map[string]*delivery.CandidateAnswer{"anchor": same(7)}, map[string]bool{"laptop": true}, delivery.VerdictWaits, "laptop (offline)", ""},
 		{"an answer of an earlier revision, past its bound", false, map[string]*delivery.CandidateAnswer{"anchor": same(6), "laptop": same(8)}, nil, delivery.VerdictFail, "anchor did not answer", ""},
+		{"a module held as today passes, shown in the brief", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
+			Verdict: delivery.CandidateSame, Held: []delivery.ModuleRefused{{Module: "spiegel", Why: "a required value is unset"}}}}, nil, delivery.VerdictPass, "", ""},
 		{"its later answer", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
 	} {
 		q := p
@@ -61,6 +67,21 @@ func TestJudge(t *testing.T) {
 		if v.State != row.state || !strings.Contains(all, row.words) {
 			t.Errorf("%s: %s, %q", row.name, v.State, all)
 		}
+		if row.seed && row.state == delivery.VerdictPass && !strings.Contains(Brief(q, v), "the seed carries: node-tools: MESH_OWNED_PATHS[/etc/resolv.conf] networkmanager (node-uplink) → (value cut) (and 1 more fields)") {
+			t.Errorf("%s: the brief does not name it: %s", row.name, Brief(q, v))
+		}
+		if row.seed && row.state == delivery.VerdictPass {
+			whole := row.answers["laptop"].Differs[0]
+			whole.Bodies = []delivery.ResourceBodies{{ID: "r", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf"},
+				Running:  `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
+				Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}"}}`}}
+			if got := differenceWords(whole); got != "node-tools: MESH_OWNED_PATHS[/etc/resolv.conf] networkmanager (node-uplink) → systemd-resolved (node-resolver)" {
+				t.Errorf("the difference is named %q", got)
+			}
+		}
+		if strings.HasPrefix(row.name, "a module held as today") && !strings.Contains(Brief(q, v), "stays held as today: spiegel") {
+			t.Errorf("%s: the brief does not show it: %s", row.name, Brief(q, v))
+		}
 		if row.marked != "" && (!slices.Contains(v.Unintended["anchor"], row.marked) || !strings.Contains(Brief(q, v), "not computed: "+row.marked)) {
 			t.Errorf("%s: the difference is not marked: %+v, %s", row.name, v.Unintended, Brief(q, v))
 		}

go/delivery/placement_candidate.go

@@ -3,6 +3,7 @@ package delivery
 import (
 	"encoding/json"
 	"fmt"
+	"slices"
 	"strings"
 )
 
@@ -121,6 +122,9 @@ type CandidateAnswer struct {
 	Added    []ModuleAdded   `json:"added,omitempty"`
 	Removed  []string        `json:"removed,omitempty"`
 	Refusals []ModuleRefused `json:"refusals,omitempty"`
+	// Held is each module the candidate cannot compose that the node holds today too, with why: held as today, which
+	// changes nothing and counts for no verdict (issue 1126: a seed reproduces today).
+	Held []ModuleRefused `json:"held,omitempty"`
 	// Refused is why the node could not compose with the candidate at all; Refusals is then empty.
 	Refused string `json:"refused,omitempty"`
 	// Returns is, for a proposal that takes the node's placement key away, what the node does then: it returns to the
@@ -233,7 +237,7 @@ func (a CandidateAnswer) check() error {
 			return fmt.Errorf("delivery: a module the candidate adds is not named")
 		}
 	}
-	for _, d := range a.Refusals {
+	for _, d := range append(slices.Clone(a.Refusals), a.Held...) {
 		if d.Module == "" || d.Why == "" {
 			return fmt.Errorf("delivery: a candidate refusal names no module or no why")
 		}

go/delivery/placement_proposal.go

@@ -356,15 +356,19 @@ func NewProposalID(at time.Time) string {
 // is shown by node: Intended is what the holder computed the proposal changes, Unintended what a node answered beside
 // it, marked as such in the brief.
 type PlacementVerdict struct {
-	State      string                     `json:"state"`
-	At         time.Time                  `json:"at"`
-	Fails      []string                   `json:"fails,omitempty"`
-	Silent     []string                   `json:"silent,omitempty"`
-	Until      time.Time                  `json:"until,omitempty"`
-	Intended   map[string][]string        `json:"intended,omitempty"`
-	Unintended map[string][]string        `json:"unintended,omitempty"`
-	Answers    map[string]CandidateAnswer `json:"answers,omitempty"`
-	Notes      []string                   `json:"notes,omitempty"`
+	State      string              `json:"state"`
+	At         time.Time           `json:"at"`
+	Fails      []string            `json:"fails,omitempty"`
+	Silent     []string            `json:"silent,omitempty"`
+	Until      time.Time           `json:"until,omitempty"`
+	Intended   map[string][]string `json:"intended,omitempty"`
+	Unintended map[string][]string `json:"unintended,omitempty"`
+	// Carried is, for the seed, by node each difference a node found that the seed carries, named for the operator who
+	// approves it knowingly ("<module>: <resource> at <path>"): the seed fails on no difference, only on a refusal or
+	// silence (issue 1126).
+	Carried map[string][]string        `json:"carried,omitempty"`
+	Answers map[string]CandidateAnswer `json:"answers,omitempty"`
+	Notes   []string                   `json:"notes,omitempty"`
 }
 
 // PlacementProposal is an open proposal, the value of its key in PlacementProposalsBucket.

go/delivery/stepasks.go

@@ -345,6 +345,11 @@ type CheckJob struct {
 	// has to list them. The build step hands them to its test tool as the argument NodeNamesArg, and the build module
 	// gives them to the tests in NodeNamesEnv.
 	NodeNames []string `json:"node-names,omitempty"`
+	// MergeOnly says the build module answers the fetch alone, its merge or the files in conflict, and runs no level: the
+	// checker asks it of a delivery group's members after one member's check failed, so one verdict names every member
+	// that does not merge into its trunk instead of the first. Levels stay named, so a build module from before it (which
+	// ignores it) still takes the ask; the checker reads only the fetch of such an answer.
+	MergeOnly bool `json:"merge-only,omitempty"`
 }
 
 // NodeNamesArg is the test tool's argument that carries a check's node names, and NodeNamesEnv the environment