novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Keep the content of every file given by digest on the node, so an object the controller's sweep removed never refuses a composition the node already had (issue 924)

Browse Source 
On novox at 01:16 on 2026-10-29, after the node began composing its own modules, mesh-vault's apply failed while the
vault ran: its own composition carries the export as it runs (mesh-sdk frombus KeptRuns) by a digest whose object the
controller's sweep (mesh-controller cmd/mesh-controller/declaration_sweep.go, sweepNode and sweepDeclarationFiles)
had removed, since the sweep keeps only what the controller's compositions in composed-<node> and the wants name.
The node may only read its bucket (ADR 0486 §2), so the node keeps what it had: a file here that verifies, or an
object fetched, goes to <state dir>/objects under the object's name, and a kept one is used before the bus is asked.
An object unused for 14 days is let go. A fetch that still fails says the keep never held it, before its cause.

Quick fix weighed: re-put the object from the controller's store. Thorough (chosen): the node never depends on the
bucket for content it once had, for every module, since the sweep and a node composing its own can always disagree.
The whole-apply test gains three rows that fail on the old code; the test count stays.
This commit is contained in:
jschoubben
2026-10-29 02:01:59 +01:00
parent fbea777263
commit 2e681825ab
4 changed files with 177 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/apply/objectkeep.go
+111
View File
@@ -0,0 +1,111 @@
package apply
import (
"fmt"
"os"
"path/filepath"
"time"
"git.novox.be/novox/mesh-sdk/go/declaration"
)
// ObjectKeep is the node-engine's own keep of the content of files given by digest (novox/hq ADR 0486 §1, issue 924): one
// file per object, named as the object is, beside the node's state. **What a node once had it never asks the bus for
// again**, so neither the controller's sweep of the bucket (an object is removed a day after no composition of the
// controller names it) nor a bus that does not answer can refuse a composition the node already ran. The node's account
// may only read its bucket (ADR 0486 §2), so the keep is the node's, not the bucket's: nothing is written back.
//
// An object unused for KeptObjectsFor is let go when another is kept, so the keep holds what the node applied lately and
// not every export it ever ran. Every failure to keep is silent and costs only a fetch: the keep is a copy.
type ObjectKeep struct {
Dir string
// Now is the clock; nil is time.Now.
Now func() time.Time
}
// KeptObjectsFor is how long an object of the keep is held after it was last used.
const KeptObjectsFor = 14 * 24 * time.Hour
// KeepObjectsIn is the keep at dir; an empty dir keeps nothing.
func KeepObjectsIn(dir string) *ObjectKeep {
if dir == "" {
return nil
}
return &ObjectKeep{Dir: dir}
}
func (k *ObjectKeep) now() time.Time {
if k.Now != nil {
return k.Now()
}
return time.Now()
}
// get is the kept content of the object name when it is what f names, and marks it used.
func (k *ObjectKeep) get(name string, f *declaration.File) ([]byte, bool) {
if k == nil {
return nil, false
}
path := filepath.Join(k.Dir, name)
body, err := os.ReadFile(path)
if err != nil || int64(len(body)) != f.Size || f.Verify(body) != nil {
return nil, false
}
now := k.now()
_ = os.Chtimes(path, now, now)
return body, true
}
// put keeps body under name, unless it is kept already (then it is marked used), and lets go of what was unused too long.
func (k *ObjectKeep) put(name string, body []byte) {
if k == nil {
return
}
path := filepath.Join(k.Dir, name)
now := k.now()
if info, err := os.Stat(path); err == nil && info.Size() == int64(len(body)) {
_ = os.Chtimes(path, now, now)
return
}
if err := os.MkdirAll(k.Dir, 0o700); err != nil {
return
}
tmp, err := os.CreateTemp(k.Dir, ".put-*")
if err != nil {
return
}
_, werr := tmp.Write(body)
cerr := tmp.Close()
if werr != nil || cerr != nil || os.Chmod(tmp.Name(), 0o600) != nil || os.Rename(tmp.Name(), path) != nil {
_ = os.Remove(tmp.Name())
return
}
_ = os.Chtimes(path, now, now)
k.prune(now)
}
// prune lets go of every object unused for KeptObjectsFor.
func (k *ObjectKeep) prune(now time.Time) {
entries, err := os.ReadDir(k.Dir)
if err != nil {
return
}
for _, e := range entries {
info, err := e.Info()
if err != nil || !info.Mode().IsRegular() {
continue
}
if now.Sub(info.ModTime()) >= KeptObjectsFor {
_ = os.Remove(filepath.Join(k.Dir, e.Name()))
}
}
}
// never says, in a failed fetch's error before its cause (a transient one's detail keeps the tail), that the node's own
// keep did not hold the object either.
func (k *ObjectKeep) never() string {
if k == nil {
return ""
}
return fmt.Sprintf(" (this node's own keep of objects, %s, never held it)", k.Dir)
}
internal/apply/references.go
+24 -8
View File
@@ -27,13 +27,23 @@ type FetchObject func(ctx context.Context, name string, limit int64) ([]byte, er
// unchanged declaration, and a node that is asleep and has no bus, apply as they always did. One object is fetched
// once however many files name its digest.
func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetch FetchObject) error {
return ResolveFileReferencesKept(ctx, d, fetch, nil)
}
// ResolveFileReferencesKept is ResolveFileReferences with the node's own keep of objects (ObjectKeep): every content had
// (a file here that verifies, or an object of the bus) is kept, and a kept one is used before the bus is asked. **A node
// depends on its bucket only for content it never had** (issue 924, 2026-10-29: novox's own composition named the
// export mesh-vault runs by a digest whose object the controller's sweep had removed, so the vault's apply failed while
// it ran; the bucket's objects are kept only while a composition of the controller names them, and a node composing its
// own names objects the controller no longer knows of). A nil keep keeps nothing.
func ResolveFileReferencesKept(ctx context.Context, d *declaration.Declaration, fetch FetchObject, keep *ObjectKeep) error {
fetched := map[string][]byte{}
for i, r := range d.Resources {
f, ok := r.(*declaration.File)
if !ok || !f.IsReference() {
continue
}
body, err := referencedContent(ctx, f, fetch, fetched)
body, err := referencedContent(ctx, f, fetch, fetched, keep)
if err != nil {
return err
}
@@ -50,31 +60,37 @@ func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetc
}
func referencedContent(ctx context.Context, f *declaration.File, fetch FetchObject,
fetched map[string][]byte) ([]byte, error) {
fetched map[string][]byte, keep *ObjectKeep) ([]byte, error) {
if body, ok := fetched[f.Digest]; ok {
return body, nil
}
if body, ok := alreadyThere(f); ok {
fetched[f.Digest] = body
return body, nil
}
name, err := declaration.ObjectName(f.Digest)
if err != nil {
return nil, fmt.Errorf("file %s (%s): %w", f.ID, f.Path, err)
}
if body, ok := alreadyThere(f); ok {
fetched[f.Digest] = body
keep.put(name, body)
return body, nil
}
if body, ok := keep.get(name, f); ok {
fetched[f.Digest] = body
return body, nil
}
if fetch == nil {
return nil, fmt.Errorf("file %s (%s) is given by digest %s and is not on this machine, and this node-engine "+
"has no link to the bus to fetch it from", f.ID, f.Path, f.Digest)
}
body, err := fetch(ctx, name, f.Size)
if err != nil {
return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket: %w",
f.ID, f.Path, name, err)
return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket%s: %w",
f.ID, f.Path, name, keep.never(), err)
}
if err := f.Verify(body); err != nil {
return nil, fmt.Errorf("%s (%s): %w; nothing was applied", f.ID, f.Path, err)
}
fetched[f.Digest] = body
keep.put(name, body)
return body, nil
}