Keep the content of every file given by digest on the node, so an object the controller's sweep removed never refuses a composition the node already had (issue 924)

On novox at 01:16 on 2026-10-29, after the node began composing its own modules, mesh-vault's apply failed while the
vault ran: its own composition carries the export as it runs (mesh-sdk frombus KeptRuns) by a digest whose object the
controller's sweep (mesh-controller cmd/mesh-controller/declaration_sweep.go, sweepNode and sweepDeclarationFiles)
had removed, since the sweep keeps only what the controller's compositions in composed-<node> and the wants name.
The node may only read its bucket (ADR 0486 §2), so the node keeps what it had: a file here that verifies, or an
object fetched, goes to <state dir>/objects under the object's name, and a kept one is used before the bus is asked.
An object unused for 14 days is let go. A fetch that still fails says the keep never held it, before its cause.

Quick fix weighed: re-put the object from the controller's store. Thorough (chosen): the node never depends on the
bucket for content it once had, for every module, since the sweep and a node composing its own can always disagree.
The whole-apply test gains three rows that fail on the old code; the test count stays.

cmd/mesh-host/file_references_test.go

@@ -82,6 +82,37 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
 	if err != nil || string(got) != string(body) {
 		t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body))
 	}
+
+	// Issue 924 (novox, 2026-10-29 01:16): the controller's sweep removed an object a composition the node ran still named,
+	// and the module's apply failed while it ran. Content the node once had is its own keep's, and the bus is not asked.
+	gone := aFilesLink{err: fmt.Errorf("the object %s was deleted from the bucket declaration-files-novox", name)}
+	never := []byte(strings.Repeat("never had here\n", 6000))
+	for _, row := range []struct {
+		name    string
+		raw     []byte
+		onDisk  []byte // nil removes the file
+		refused string // "" applies
+	}{
+		{"the file here drifted, its object deleted", raw, []byte("the module wrote it since"), ""},
+		{"the file here is gone, its object deleted", raw, nil, ""},
+		{"never had here, its object deleted", referenceDeclaration(target, never), nil, "never held it"},
+	} {
+		if row.onDisk == nil {
+			_ = os.Remove(target)
+		} else if err := os.WriteFile(target, row.onDisk, 0o600); err != nil {
+			t.Fatal(err)
+		}
+		report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, row.raw, nil, nil, nil)
+		if row.refused == "" {
+			got, _ := os.ReadFile(target)
+			if report.Refused != "" || string(got) != string(body) {
+				t.Errorf("924: %s: refused %q, the file holds %d bytes; want it applied from the node's own keep",
+					row.name, report.Refused, len(got))
+			}
+		} else if !strings.Contains(report.Refused, row.refused) || !strings.Contains(report.Refused, "was deleted") {
+			t.Errorf("924: %s: refused %q; want a refusal saying the bucket and the keep both lack it", row.name, report.Refused)
+		}
+	}
 }
 
 // 1695/AC8: the hand-run `apply <file>` and `reconcile` have no bus: a reference whose file here verifies is applied

cmd/mesh-host/main.go

@@ -516,7 +516,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 
 	// A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has
 	// no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty.
-	if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil {
+	if err := apply.ResolveFileReferencesKept(ctx, d, link.FilesFrom(ctx), objectKeep(opts)); err != nil {
 		return fmt.Errorf("a file of this declaration could not be had by its digest: %w", err)
 	}
 
@@ -1664,6 +1664,15 @@ func announceOr(say link.Announce) func(string) {
 // run by hand beside this service — the lock beside the state does the same (store.Lock).
 var applying sync.Mutex
 
+// objectKeep is the node's own keep of the objects its compositions name by digest, beside its state (apply.ObjectKeep,
+// issue 924): a composition the node once applied never again needs the controller's bucket to hold its files.
+func objectKeep(opts options) *apply.ObjectKeep {
+	if opts.state == "" {
+		return nil
+	}
+	return apply.KeepObjectsIn(filepath.Join(filepath.Dir(opts.state), "objects"))
+}
+
 // appliedModule is how the apply of one module's declaration came out.
 type appliedModule struct {
 	// Refused is why nothing was applied, when nothing was.
@@ -1703,7 +1712,7 @@ func applyModuleHeld(ctx context.Context, opts options, declared *declaration.De
 
 	// **A file given by digest is fetched and verified before anything of the composition is applied** (novox/hq ADR
 	// 0486 §1, issue 1695): one object missing or differing refuses the module whole.
-	if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil {
+	if err := apply.ResolveFileReferencesKept(ctx, declared, link.FilesFrom(ctx), objectKeep(opts)); err != nil {
 		return appliedModule{Refused: "a file of this composition could not be had by its digest: " + err.Error()}
 	}
 

internal/apply/objectkeep.go

@@ -0,0 +1,111 @@
+package apply
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+)
+
+// ObjectKeep is the node-engine's own keep of the content of files given by digest (novox/hq ADR 0486 §1, issue 924): one
+// file per object, named as the object is, beside the node's state. **What a node once had it never asks the bus for
+// again**, so neither the controller's sweep of the bucket (an object is removed a day after no composition of the
+// controller names it) nor a bus that does not answer can refuse a composition the node already ran. The node's account
+// may only read its bucket (ADR 0486 §2), so the keep is the node's, not the bucket's: nothing is written back.
+//
+// An object unused for KeptObjectsFor is let go when another is kept, so the keep holds what the node applied lately and
+// not every export it ever ran. Every failure to keep is silent and costs only a fetch: the keep is a copy.
+type ObjectKeep struct {
+	Dir string
+	// Now is the clock; nil is time.Now.
+	Now func() time.Time
+}
+
+// KeptObjectsFor is how long an object of the keep is held after it was last used.
+const KeptObjectsFor = 14 * 24 * time.Hour
+
+// KeepObjectsIn is the keep at dir; an empty dir keeps nothing.
+func KeepObjectsIn(dir string) *ObjectKeep {
+	if dir == "" {
+		return nil
+	}
+	return &ObjectKeep{Dir: dir}
+}
+
+func (k *ObjectKeep) now() time.Time {
+	if k.Now != nil {
+		return k.Now()
+	}
+	return time.Now()
+}
+
+// get is the kept content of the object name when it is what f names, and marks it used.
+func (k *ObjectKeep) get(name string, f *declaration.File) ([]byte, bool) {
+	if k == nil {
+		return nil, false
+	}
+	path := filepath.Join(k.Dir, name)
+	body, err := os.ReadFile(path)
+	if err != nil || int64(len(body)) != f.Size || f.Verify(body) != nil {
+		return nil, false
+	}
+	now := k.now()
+	_ = os.Chtimes(path, now, now)
+	return body, true
+}
+
+// put keeps body under name, unless it is kept already (then it is marked used), and lets go of what was unused too long.
+func (k *ObjectKeep) put(name string, body []byte) {
+	if k == nil {
+		return
+	}
+	path := filepath.Join(k.Dir, name)
+	now := k.now()
+	if info, err := os.Stat(path); err == nil && info.Size() == int64(len(body)) {
+		_ = os.Chtimes(path, now, now)
+		return
+	}
+	if err := os.MkdirAll(k.Dir, 0o700); err != nil {
+		return
+	}
+	tmp, err := os.CreateTemp(k.Dir, ".put-*")
+	if err != nil {
+		return
+	}
+	_, werr := tmp.Write(body)
+	cerr := tmp.Close()
+	if werr != nil || cerr != nil || os.Chmod(tmp.Name(), 0o600) != nil || os.Rename(tmp.Name(), path) != nil {
+		_ = os.Remove(tmp.Name())
+		return
+	}
+	_ = os.Chtimes(path, now, now)
+	k.prune(now)
+}
+
+// prune lets go of every object unused for KeptObjectsFor.
+func (k *ObjectKeep) prune(now time.Time) {
+	entries, err := os.ReadDir(k.Dir)
+	if err != nil {
+		return
+	}
+	for _, e := range entries {
+		info, err := e.Info()
+		if err != nil || !info.Mode().IsRegular() {
+			continue
+		}
+		if now.Sub(info.ModTime()) >= KeptObjectsFor {
+			_ = os.Remove(filepath.Join(k.Dir, e.Name()))
+		}
+	}
+}
+
+// never says, in a failed fetch's error before its cause (a transient one's detail keeps the tail), that the node's own
+// keep did not hold the object either.
+func (k *ObjectKeep) never() string {
+	if k == nil {
+		return ""
+	}
+	return fmt.Sprintf(" (this node's own keep of objects, %s, never held it)", k.Dir)
+}

internal/apply/references.go

@@ -27,13 +27,23 @@ type FetchObject func(ctx context.Context, name string, limit int64) ([]byte, er
 // unchanged declaration, and a node that is asleep and has no bus, apply as they always did. One object is fetched
 // once however many files name its digest.
 func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetch FetchObject) error {
+	return ResolveFileReferencesKept(ctx, d, fetch, nil)
+}
+
+// ResolveFileReferencesKept is ResolveFileReferences with the node's own keep of objects (ObjectKeep): every content had
+// (a file here that verifies, or an object of the bus) is kept, and a kept one is used before the bus is asked. **A node
+// depends on its bucket only for content it never had** (issue 924, 2026-10-29: novox's own composition named the
+// export mesh-vault runs by a digest whose object the controller's sweep had removed, so the vault's apply failed while
+// it ran; the bucket's objects are kept only while a composition of the controller names them, and a node composing its
+// own names objects the controller no longer knows of). A nil keep keeps nothing.
+func ResolveFileReferencesKept(ctx context.Context, d *declaration.Declaration, fetch FetchObject, keep *ObjectKeep) error {
 	fetched := map[string][]byte{}
 	for i, r := range d.Resources {
 		f, ok := r.(*declaration.File)
 		if !ok || !f.IsReference() {
 			continue
 		}
-		body, err := referencedContent(ctx, f, fetch, fetched)
+		body, err := referencedContent(ctx, f, fetch, fetched, keep)
 		if err != nil {
 			return err
 		}
@@ -50,31 +60,37 @@ func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetc
 }
 
 func referencedContent(ctx context.Context, f *declaration.File, fetch FetchObject,
-	fetched map[string][]byte) ([]byte, error) {
+	fetched map[string][]byte, keep *ObjectKeep) ([]byte, error) {
 	if body, ok := fetched[f.Digest]; ok {
 		return body, nil
 	}
-	if body, ok := alreadyThere(f); ok {
-		fetched[f.Digest] = body
-		return body, nil
-	}
 	name, err := declaration.ObjectName(f.Digest)
 	if err != nil {
 		return nil, fmt.Errorf("file %s (%s): %w", f.ID, f.Path, err)
 	}
+	if body, ok := alreadyThere(f); ok {
+		fetched[f.Digest] = body
+		keep.put(name, body)
+		return body, nil
+	}
+	if body, ok := keep.get(name, f); ok {
+		fetched[f.Digest] = body
+		return body, nil
+	}
 	if fetch == nil {
 		return nil, fmt.Errorf("file %s (%s) is given by digest %s and is not on this machine, and this node-engine "+
 			"has no link to the bus to fetch it from", f.ID, f.Path, f.Digest)
 	}
 	body, err := fetch(ctx, name, f.Size)
 	if err != nil {
-		return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket: %w",
-			f.ID, f.Path, name, err)
+		return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket%s: %w",
+			f.ID, f.Path, name, keep.never(), err)
 	}
 	if err := f.Verify(body); err != nil {
 		return nil, fmt.Errorf("%s (%s): %w; nothing was applied", f.ID, f.Path, err)
 	}
 	fetched[f.Digest] = body
+	keep.put(name, body)
 	return body, nil
 }