package store import ( "encoding/json" "fmt" "os" "path/filepath" "sort" "strings" "sync" "time" ) // What a node knows is written by one writer at a time, completely or not at all, and never loses a record without // saying so (novox/hq issue 1585). On novox, 2026-11-04, the node-engine's state had lost the records of two archives // (nats's program snapshot and its tools bundle) that a save of 2026-10-24 had written into its temporary file and // never renamed into place, and about 260 such temporary files (~80 MB) lay beside it: the process ended between // writing a save and renaming it, again and again, because the engine stopped without waiting for the apply in flight // (cmd/mesh-host, stopBetweenApplies). The temporary files and the lost records are the same fault. // tempPattern is the name a save writes before it renames the file into place. const tempPattern = ".state-*.json" // saving is the one writer of this process: a save holds it from its read of what was saved before to its rename and // the directory's flush, and Quiesce holds it for good. var saving sync.Mutex // Said is where a save says loudly what must not pass in silence: a record the state held before and this save would // drop although nothing forgot it. The journal when the engine runs as a service. var Said = func(line string) { fmt.Fprintln(os.Stderr, line) } // Save writes the state, atomically and durably: the whole file to a temporary one beside it, flushed, renamed over // the old one, and the directory flushed so the rename itself survives a power cut. // // Atomic because the alternative has a failure mode with no floor: a host interrupted while // writing loses the record of everything it owns, and then owns nothing it can clean up. // // Before it writes, it reads what was saved before and says, through Said, each resource or hold that state recorded // and this one would drop although this copy never forgot it (Forget, Release): a save from a copy read before // another writer saved, or a copy that lost a record on the way, is said at once rather than found weeks later when a // file the host put there reads as foreign. The save still goes ahead: it is what the apply found on the machine, and // refusing it would lose that too. func Save(path string, s State) error { saving.Lock() defer saving.Unlock() return save(path, s) } func save(path string, s State) error { s.UpdatedAt = time.Now().UTC() dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("making room for the node's state: %w", err) } if before, err := Load(path); err == nil { for _, lost := range Dropped(before, s) { Said(fmt.Sprintf("the node's state is saved without the record of %s, which the state saved before held "+ "and nothing forgot: what it put on this machine is no longer known as the mesh's (issue 1585)", lost)) } } raw, err := json.MarshalIndent(s, "", " ") if err != nil { return fmt.Errorf("encoding the node's state: %w", err) } raw = append(raw, '\n') tmp, err := os.CreateTemp(dir, tempPattern) if err != nil { return fmt.Errorf("writing the node's state: %w", err) } defer os.Remove(tmp.Name()) if _, err := tmp.Write(raw); err != nil { tmp.Close() return fmt.Errorf("writing the node's state: %w", err) } // Flushed before the rename: a rename is atomic, and a rename of a file whose contents are // still in the page cache is atomically the wrong thing. if err := tmp.Sync(); err != nil { tmp.Close() return fmt.Errorf("flushing the node's state: %w", err) } if err := tmp.Close(); err != nil { return fmt.Errorf("closing the node's state: %w", err) } if err := os.Chmod(tmp.Name(), 0o600); err != nil { return fmt.Errorf("securing the node's state: %w", err) } if err := os.Rename(tmp.Name(), path); err != nil { return fmt.Errorf("replacing the node's state: %w", err) } // And the directory, so the new name is on disk and not only the new contents. if d, err := os.Open(dir); err == nil { syncErr := d.Sync() d.Close() if syncErr != nil { return fmt.Errorf("flushing the directory of the node's state: %w", syncErr) } } return nil } // Dropped is each resource and hold before recorded that after does not, and that after never forgot: "resource " // or "hold ", sorted. func Dropped(before, after State) []string { kept := map[string]bool{} for _, r := range after.Resources { kept["resource "+r.ID] = true } for _, h := range after.Held { kept["hold "+h.ID] = true } var out []string for _, r := range before.Resources { if k := "resource " + r.ID; !kept[k] && !after.forgotten[k] { out = append(out, k) } } for _, h := range before.Held { if k := "hold " + h.ID; !kept[k] && !after.forgotten[k] { out = append(out, k) } } sort.Strings(out) return out } // forget notes that this copy dropped a record on purpose. func (s *State) forget(what string) { if s.forgotten == nil { s.forgotten = map[string]bool{} } s.forgotten[what] = true } // Quiesce waits for a save in flight to end and lets no other start in this process, for the engine's stop: a process // that ends while a save writes leaves a temporary file and the state before it (issue 1585). It returns what lets saves // start again, for a test; a stopping engine never calls it. func Quiesce() (release func()) { saving.Lock() return saving.Unlock } // RemoveLeftovers removes the temporary files of saves that never reached their rename, beside the state at path, and // names each it removed. Under the machine's apply lock, so no save of another process is in flight: every writer of // the state takes it (Lock). func RemoveLeftovers(path string) ([]string, error) { unlock, err := Lock(path, nil) if err != nil { return nil, err } defer unlock() saving.Lock() defer saving.Unlock() found, err := filepath.Glob(filepath.Join(filepath.Dir(path), tempPattern)) if err != nil { return nil, err } var removed []string var failed []string for _, f := range found { if err := os.Remove(f); err != nil { failed = append(failed, fmt.Sprintf("%s: %v", filepath.Base(f), err)) continue } removed = append(removed, filepath.Base(f)) } sort.Strings(removed) if len(failed) > 0 { return removed, fmt.Errorf("leftover saves of the node's state could not be removed: %s", strings.Join(failed, "; ")) } return removed, nil }