From 2e681825ab9ae062364daeb0ed87cedc4b6b0a88 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 02:01:59 +0100 Subject: [PATCH] Keep the content of every file given by digest on the node, so an object the controller's sweep removed never refuses a composition the node already had (issue 924) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On novox at 01:16 on 2026-10-29, after the node began composing its own modules, mesh-vault's apply failed while the vault ran: its own composition carries the export as it runs (mesh-sdk frombus KeptRuns) by a digest whose object the controller's sweep (mesh-controller cmd/mesh-controller/declaration_sweep.go, sweepNode and sweepDeclarationFiles) had removed, since the sweep keeps only what the controller's compositions in composed- and the wants name. The node may only read its bucket (ADR 0486 §2), so the node keeps what it had: a file here that verifies, or an object fetched, goes to /objects under the object's name, and a kept one is used before the bus is asked. An object unused for 14 days is let go. A fetch that still fails says the keep never held it, before its cause. Quick fix weighed: re-put the object from the controller's store. Thorough (chosen): the node never depends on the bucket for content it once had, for every module, since the sweep and a node composing its own can always disagree. The whole-apply test gains three rows that fail on the old code; the test count stays. --- cmd/mesh-host/file_references_test.go | 31 +++++++ cmd/mesh-host/main.go | 13 ++- internal/apply/objectkeep.go | 111 ++++++++++++++++++++++++++ internal/apply/references.go | 32 ++++++-- 4 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 internal/apply/objectkeep.go diff --git a/cmd/mesh-host/file_references_test.go b/cmd/mesh-host/file_references_test.go index a8303fb..21444ea 100644 --- a/cmd/mesh-host/file_references_test.go +++ b/cmd/mesh-host/file_references_test.go @@ -82,6 +82,37 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) { if err != nil || string(got) != string(body) { t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body)) } + + // Issue 924 (novox, 2026-10-29 01:16): the controller's sweep removed an object a composition the node ran still named, + // and the module's apply failed while it ran. Content the node once had is its own keep's, and the bus is not asked. + gone := aFilesLink{err: fmt.Errorf("the object %s was deleted from the bucket declaration-files-novox", name)} + never := []byte(strings.Repeat("never had here\n", 6000)) + for _, row := range []struct { + name string + raw []byte + onDisk []byte // nil removes the file + refused string // "" applies + }{ + {"the file here drifted, its object deleted", raw, []byte("the module wrote it since"), ""}, + {"the file here is gone, its object deleted", raw, nil, ""}, + {"never had here, its object deleted", referenceDeclaration(target, never), nil, "never held it"}, + } { + if row.onDisk == nil { + _ = os.Remove(target) + } else if err := os.WriteFile(target, row.onDisk, 0o600); err != nil { + t.Fatal(err) + } + report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, row.raw, nil, nil, nil) + if row.refused == "" { + got, _ := os.ReadFile(target) + if report.Refused != "" || string(got) != string(body) { + t.Errorf("924: %s: refused %q, the file holds %d bytes; want it applied from the node's own keep", + row.name, report.Refused, len(got)) + } + } else if !strings.Contains(report.Refused, row.refused) || !strings.Contains(report.Refused, "was deleted") { + t.Errorf("924: %s: refused %q; want a refusal saying the bucket and the keep both lack it", row.name, report.Refused) + } + } } // 1695/AC8: the hand-run `apply ` and `reconcile` have no bus: a reference whose file here verifies is applied diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index d8b3610..7d1e63a 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -516,7 +516,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw // A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has // no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty. - if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil { + if err := apply.ResolveFileReferencesKept(ctx, d, link.FilesFrom(ctx), objectKeep(opts)); err != nil { return fmt.Errorf("a file of this declaration could not be had by its digest: %w", err) } @@ -1646,6 +1646,15 @@ func announceOr(say link.Announce) func(string) { // run by hand beside this service — the lock beside the state does the same (store.Lock). var applying sync.Mutex +// objectKeep is the node's own keep of the objects its compositions name by digest, beside its state (apply.ObjectKeep, +// issue 924): a composition the node once applied never again needs the controller's bucket to hold its files. +func objectKeep(opts options) *apply.ObjectKeep { + if opts.state == "" { + return nil + } + return apply.KeepObjectsIn(filepath.Join(filepath.Dir(opts.state), "objects")) +} + // appliedModule is how the apply of one module's declaration came out. type appliedModule struct { // Refused is why nothing was applied, when nothing was. @@ -1685,7 +1694,7 @@ func applyModuleHeld(ctx context.Context, opts options, declared *declaration.De // **A file given by digest is fetched and verified before anything of the composition is applied** (novox/hq ADR // 0486 §1, issue 1695): one object missing or differing refuses the module whole. - if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil { + if err := apply.ResolveFileReferencesKept(ctx, declared, link.FilesFrom(ctx), objectKeep(opts)); err != nil { return appliedModule{Refused: "a file of this composition could not be had by its digest: " + err.Error()} } diff --git a/internal/apply/objectkeep.go b/internal/apply/objectkeep.go new file mode 100644 index 0000000..d5be5df --- /dev/null +++ b/internal/apply/objectkeep.go @@ -0,0 +1,111 @@ +package apply + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "git.novox.be/novox/mesh-sdk/go/declaration" +) + +// ObjectKeep is the node-engine's own keep of the content of files given by digest (novox/hq ADR 0486 §1, issue 924): one +// file per object, named as the object is, beside the node's state. **What a node once had it never asks the bus for +// again**, so neither the controller's sweep of the bucket (an object is removed a day after no composition of the +// controller names it) nor a bus that does not answer can refuse a composition the node already ran. The node's account +// may only read its bucket (ADR 0486 §2), so the keep is the node's, not the bucket's: nothing is written back. +// +// An object unused for KeptObjectsFor is let go when another is kept, so the keep holds what the node applied lately and +// not every export it ever ran. Every failure to keep is silent and costs only a fetch: the keep is a copy. +type ObjectKeep struct { + Dir string + // Now is the clock; nil is time.Now. + Now func() time.Time +} + +// KeptObjectsFor is how long an object of the keep is held after it was last used. +const KeptObjectsFor = 14 * 24 * time.Hour + +// KeepObjectsIn is the keep at dir; an empty dir keeps nothing. +func KeepObjectsIn(dir string) *ObjectKeep { + if dir == "" { + return nil + } + return &ObjectKeep{Dir: dir} +} + +func (k *ObjectKeep) now() time.Time { + if k.Now != nil { + return k.Now() + } + return time.Now() +} + +// get is the kept content of the object name when it is what f names, and marks it used. +func (k *ObjectKeep) get(name string, f *declaration.File) ([]byte, bool) { + if k == nil { + return nil, false + } + path := filepath.Join(k.Dir, name) + body, err := os.ReadFile(path) + if err != nil || int64(len(body)) != f.Size || f.Verify(body) != nil { + return nil, false + } + now := k.now() + _ = os.Chtimes(path, now, now) + return body, true +} + +// put keeps body under name, unless it is kept already (then it is marked used), and lets go of what was unused too long. +func (k *ObjectKeep) put(name string, body []byte) { + if k == nil { + return + } + path := filepath.Join(k.Dir, name) + now := k.now() + if info, err := os.Stat(path); err == nil && info.Size() == int64(len(body)) { + _ = os.Chtimes(path, now, now) + return + } + if err := os.MkdirAll(k.Dir, 0o700); err != nil { + return + } + tmp, err := os.CreateTemp(k.Dir, ".put-*") + if err != nil { + return + } + _, werr := tmp.Write(body) + cerr := tmp.Close() + if werr != nil || cerr != nil || os.Chmod(tmp.Name(), 0o600) != nil || os.Rename(tmp.Name(), path) != nil { + _ = os.Remove(tmp.Name()) + return + } + _ = os.Chtimes(path, now, now) + k.prune(now) +} + +// prune lets go of every object unused for KeptObjectsFor. +func (k *ObjectKeep) prune(now time.Time) { + entries, err := os.ReadDir(k.Dir) + if err != nil { + return + } + for _, e := range entries { + info, err := e.Info() + if err != nil || !info.Mode().IsRegular() { + continue + } + if now.Sub(info.ModTime()) >= KeptObjectsFor { + _ = os.Remove(filepath.Join(k.Dir, e.Name())) + } + } +} + +// never says, in a failed fetch's error before its cause (a transient one's detail keeps the tail), that the node's own +// keep did not hold the object either. +func (k *ObjectKeep) never() string { + if k == nil { + return "" + } + return fmt.Sprintf(" (this node's own keep of objects, %s, never held it)", k.Dir) +} diff --git a/internal/apply/references.go b/internal/apply/references.go index ad6bed5..64ce30c 100644 --- a/internal/apply/references.go +++ b/internal/apply/references.go @@ -27,13 +27,23 @@ type FetchObject func(ctx context.Context, name string, limit int64) ([]byte, er // unchanged declaration, and a node that is asleep and has no bus, apply as they always did. One object is fetched // once however many files name its digest. func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetch FetchObject) error { + return ResolveFileReferencesKept(ctx, d, fetch, nil) +} + +// ResolveFileReferencesKept is ResolveFileReferences with the node's own keep of objects (ObjectKeep): every content had +// (a file here that verifies, or an object of the bus) is kept, and a kept one is used before the bus is asked. **A node +// depends on its bucket only for content it never had** (issue 924, 2026-10-29: novox's own composition named the +// export mesh-vault runs by a digest whose object the controller's sweep had removed, so the vault's apply failed while +// it ran; the bucket's objects are kept only while a composition of the controller names them, and a node composing its +// own names objects the controller no longer knows of). A nil keep keeps nothing. +func ResolveFileReferencesKept(ctx context.Context, d *declaration.Declaration, fetch FetchObject, keep *ObjectKeep) error { fetched := map[string][]byte{} for i, r := range d.Resources { f, ok := r.(*declaration.File) if !ok || !f.IsReference() { continue } - body, err := referencedContent(ctx, f, fetch, fetched) + body, err := referencedContent(ctx, f, fetch, fetched, keep) if err != nil { return err } @@ -50,31 +60,37 @@ func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetc } func referencedContent(ctx context.Context, f *declaration.File, fetch FetchObject, - fetched map[string][]byte) ([]byte, error) { + fetched map[string][]byte, keep *ObjectKeep) ([]byte, error) { if body, ok := fetched[f.Digest]; ok { return body, nil } - if body, ok := alreadyThere(f); ok { - fetched[f.Digest] = body - return body, nil - } name, err := declaration.ObjectName(f.Digest) if err != nil { return nil, fmt.Errorf("file %s (%s): %w", f.ID, f.Path, err) } + if body, ok := alreadyThere(f); ok { + fetched[f.Digest] = body + keep.put(name, body) + return body, nil + } + if body, ok := keep.get(name, f); ok { + fetched[f.Digest] = body + return body, nil + } if fetch == nil { return nil, fmt.Errorf("file %s (%s) is given by digest %s and is not on this machine, and this node-engine "+ "has no link to the bus to fetch it from", f.ID, f.Path, f.Digest) } body, err := fetch(ctx, name, f.Size) if err != nil { - return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket: %w", - f.ID, f.Path, name, err) + return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket%s: %w", + f.ID, f.Path, name, keep.never(), err) } if err := f.Verify(body); err != nil { return nil, fmt.Errorf("%s (%s): %w; nothing was applied", f.ID, f.Path, err) } fetched[f.Digest] = body + keep.put(name, body) return body, nil }