mesh-host/internal/store/save.go

package store

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"
)

// What a node knows is written by one writer at a time, completely or not at all, and never loses a record without
// saying so (novox/hq issue 1585). On novox, 2026-11-04, the node-engine's state had lost the records of two archives
// (nats's program snapshot and its tools bundle) that a save of 2026-10-24 had written into its temporary file and
// never renamed into place, and about 260 such temporary files (~80 MB) lay beside it: the process ended between
// writing a save and renaming it, again and again, because the engine stopped without waiting for the apply in flight
// (cmd/mesh-host, stopBetweenApplies). The temporary files and the lost records are the same fault.

// tempPattern is the name a save writes before it renames the file into place.
const tempPattern = ".state-*.json"

// saving is the one writer of this process: a save holds it from its read of what was saved before to its rename and
// the directory's flush, and Quiesce holds it for good.
var saving sync.Mutex

// Said is where a save says loudly what must not pass in silence: a record the state held before and this save would
// drop although nothing forgot it. The journal when the engine runs as a service.
var Said = func(line string) { fmt.Fprintln(os.Stderr, line) }

// Save writes the state, atomically and durably: the whole file to a temporary one beside it, flushed, renamed over
// the old one, and the directory flushed so the rename itself survives a power cut.
//
// Atomic because the alternative has a failure mode with no floor: a host interrupted while
// writing loses the record of everything it owns, and then owns nothing it can clean up.
//
// Before it writes, it reads what was saved before and says, through Said, each resource or hold that state recorded
// and this one would drop although this copy never forgot it (Forget, Release): a save from a copy read before
// another writer saved, or a copy that lost a record on the way, is said at once rather than found weeks later when a
// file the host put there reads as foreign. The save still goes ahead: it is what the apply found on the machine, and
// refusing it would lose that too.
func Save(path string, s State) error {
	saving.Lock()
	defer saving.Unlock()
	return save(path, s)
}

func save(path string, s State) error {
	s.UpdatedAt = time.Now().UTC()

	dir := filepath.Dir(path)
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return fmt.Errorf("making room for the node's state: %w", err)
	}

	if before, err := Load(path); err == nil {
		for _, lost := range Dropped(before, s) {
			Said(fmt.Sprintf("the node's state is saved without the record of %s, which the state saved before held "+
				"and nothing forgot: what it put on this machine is no longer known as the mesh's (issue 1585)", lost))
		}
	}

	raw, err := json.MarshalIndent(s, "", "  ")
	if err != nil {
		return fmt.Errorf("encoding the node's state: %w", err)
	}
	raw = append(raw, '\n')

	tmp, err := os.CreateTemp(dir, tempPattern)
	if err != nil {
		return fmt.Errorf("writing the node's state: %w", err)
	}
	defer os.Remove(tmp.Name())

	if _, err := tmp.Write(raw); err != nil {
		tmp.Close()
		return fmt.Errorf("writing the node's state: %w", err)
	}
	// Flushed before the rename: a rename is atomic, and a rename of a file whose contents are
	// still in the page cache is atomically the wrong thing.
	if err := tmp.Sync(); err != nil {
		tmp.Close()
		return fmt.Errorf("flushing the node's state: %w", err)
	}
	if err := tmp.Close(); err != nil {
		return fmt.Errorf("closing the node's state: %w", err)
	}
	if err := os.Chmod(tmp.Name(), 0o600); err != nil {
		return fmt.Errorf("securing the node's state: %w", err)
	}
	if err := os.Rename(tmp.Name(), path); err != nil {
		return fmt.Errorf("replacing the node's state: %w", err)
	}
	// And the directory, so the new name is on disk and not only the new contents.
	if d, err := os.Open(dir); err == nil {
		syncErr := d.Sync()
		d.Close()
		if syncErr != nil {
			return fmt.Errorf("flushing the directory of the node's state: %w", syncErr)
		}
	}
	return nil
}

// Dropped is each resource and hold before recorded that after does not, and that after never forgot: "resource <id>"
// or "hold <id>", sorted.
func Dropped(before, after State) []string {
	kept := map[string]bool{}
	for _, r := range after.Resources {
		kept["resource "+r.ID] = true
	}
	for _, h := range after.Held {
		kept["hold "+h.ID] = true
	}
	var out []string
	for _, r := range before.Resources {
		if k := "resource " + r.ID; !kept[k] && !after.forgotten[k] {
			out = append(out, k)
		}
	}
	for _, h := range before.Held {
		if k := "hold " + h.ID; !kept[k] && !after.forgotten[k] {
			out = append(out, k)
		}
	}
	sort.Strings(out)
	return out
}

// forget notes that this copy dropped a record on purpose.
func (s *State) forget(what string) {
	if s.forgotten == nil {
		s.forgotten = map[string]bool{}
	}
	s.forgotten[what] = true
}

// Quiesce waits for a save in flight to end and lets no other start in this process, for the engine's stop: a process
// that ends while a save writes leaves a temporary file and the state before it (issue 1585). It returns what lets saves
// start again, for a test; a stopping engine never calls it.
func Quiesce() (release func()) {
	saving.Lock()
	return saving.Unlock
}

// RemoveLeftovers removes the temporary files of saves that never reached their rename, beside the state at path, and
// names each it removed. Under the machine's apply lock, so no save of another process is in flight: every writer of
// the state takes it (Lock).
func RemoveLeftovers(path string) ([]string, error) {
	unlock, err := Lock(path, nil)
	if err != nil {
		return nil, err
	}
	defer unlock()
	saving.Lock()
	defer saving.Unlock()
	found, err := filepath.Glob(filepath.Join(filepath.Dir(path), tempPattern))
	if err != nil {
		return nil, err
	}
	var removed []string
	var failed []string
	for _, f := range found {
		if err := os.Remove(f); err != nil {
			failed = append(failed, fmt.Sprintf("%s: %v", filepath.Base(f), err))
			continue
		}
		removed = append(removed, filepath.Base(f))
	}
	sort.Strings(removed)
	if len(failed) > 0 {
		return removed, fmt.Errorf("leftover saves of the node's state could not be removed: %s", strings.Join(failed, "; "))
	}
	return removed, nil
}