mesh-host/internal/store/store.go

// Package store is what this node knows about itself, and it is authoritative while
// disconnected.
//
// Not a cache of the control plane. novox/hq ADR 0004 makes disconnection an ordinary
// situation rather than an exception, and this is what makes it ordinary: a machine shut for a
// week comes back and reconciles, it does not come back and ask what it is.
//
// Its first job arrives with the first apply rather than with the link (ADR 0005): the host
// removes what it previously applied and is no longer declared, and it can only know that
// because it wrote it down.
package store

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"sort"
	"time"
)

// DefaultPath is where a node keeps what it knows. Under /var/lib because it survives a
// reboot and is not configuration — nothing generates this, the host writes it.
const DefaultPath = "/var/lib/mesh-host/state.json"

// Applied is one resource the host put on this machine, and what it did.
//
// Recorded AFTER the resource was applied and read back, never before (novox/hq ADR 0018).
// A record written up front restates the request in a new place and inherits none of the
// authority of having happened.
type Applied struct {
	ID   string `json:"id"`
	Type string `json:"type"`
	// Origin is who asked for this: the bundle this host carries, or the mesh.
	//
	// Recorded because the two must not remove each other. A node raises its own substrate from
	// the bundle before any mesh exists, then enrols and is sent declarations — and a
	// declaration naming two resources would otherwise remove the store, the broker and the
	// control plane, which is 04-ISSUES/010 and happened on the first end-to-end run.
	//
	// Empty means carried, for state written before this field existed: everything a host had
	// applied at that point came from its bundle.
	Origin string `json:"origin,omitempty"`

	// Holds are the machine's own ports this resource occupies.
	//
	// **So the mesh can assign around what it did not put here** (novox/hq ADR 0038). A node
	// raises its substrate from the bundle before any mesh exists, so the control plane has never
	// heard of the store, the broker or the control plane's own container — and a module assigned
	// afterwards would be given a port one of them already holds, and would be told so by a
	// container runtime rather than by anything that could have prevented it.
	//
	// Recorded per resource rather than counted per machine, because what a machine happens to
	// have open right now is a moving target, and what its declaration binds is not.
	Holds []int `json:"holds,omitempty"`
	// Target is what was changed — a path, a unit — so removal knows what to undo without
	// re-reading a declaration that may no longer exist.
	Target    string    `json:"target"`
	AppliedAt time.Time `json:"applied_at"`

	// Wrote is a digest of what this host last put there, for resources where that is a
	// meaningful question.
	//
	// Without it, a file that does not match the declaration has two possible explanations and
	// the host cannot tell them apart: the mesh changed what it wants, or somebody edited the
	// machine. Both end with the file being rewritten, so the outcome is identical — and a
	// person who edits a managed file watches their change vanish every few minutes with nothing
	// anywhere saying why.
	Wrote string `json:"wrote,omitempty"`
}

// State is the whole of what a node knows about what it has done.
type State struct {
	// Resources, keyed by identity, in the order they were applied. Order matters for removal:
	// undoing in reverse is the only ordering the host can derive without deciding anything.
	Resources []Applied `json:"resources"`
	UpdatedAt time.Time `json:"updated_at"`
}

// Find returns what was applied under an identity.
func (s State) Find(id string) (Applied, bool) {
	for _, r := range s.Resources {
		if r.ID == id {
			return r, true
		}
	}
	return Applied{}, false
}

// IDs returns every identity the host has applied, sorted.
func (s State) IDs() []string {
	out := make([]string, 0, len(s.Resources))
	for _, r := range s.Resources {
		out = append(out, r.ID)
	}
	sort.Strings(out)
	return out
}

// Load reads the state. A node that has never applied anything has an empty state, which is a
// fact rather than an error — the first apply on a fresh machine is the ordinary case.
//
// A state file that exists and cannot be read IS an error, and a loud one: continuing with an
// empty state would make the host believe it owns nothing, and it would then remove nothing it
// should and re-apply everything it need not.
func Load(path string) (State, error) {
	raw, err := os.ReadFile(path)
	if errors.Is(err, fs.ErrNotExist) {
		return State{}, nil
	}
	if err != nil {
		return State{}, fmt.Errorf("reading what this node knows about itself (%s): %w", path, err)
	}

	var s State
	if err := json.Unmarshal(raw, &s); err != nil {
		return State{}, fmt.Errorf(
			"what this node knows about itself is unreadable (%s): %w\n"+
				"Refusing rather than starting empty: an empty state would mean the host "+
				"believes it owns nothing, so it would remove nothing it should and re-apply "+
				"everything it need not", path, err)
	}
	return s, nil
}

// Save writes the state, atomically.
//
// Atomic because the alternative has a failure mode with no floor: a host interrupted while
// writing loses the record of everything it owns, and then owns nothing it can clean up.
func Save(path string, s State) error {
	s.UpdatedAt = time.Now().UTC()

	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return fmt.Errorf("making room for the node's state: %w", err)
	}

	raw, err := json.MarshalIndent(s, "", "  ")
	if err != nil {
		return fmt.Errorf("encoding the node's state: %w", err)
	}
	raw = append(raw, '\n')

	tmp, err := os.CreateTemp(filepath.Dir(path), ".state-*.json")
	if err != nil {
		return fmt.Errorf("writing the node's state: %w", err)
	}
	defer os.Remove(tmp.Name())

	if _, err := tmp.Write(raw); err != nil {
		tmp.Close()
		return fmt.Errorf("writing the node's state: %w", err)
	}
	// Flushed before the rename: a rename is atomic, and a rename of a file whose contents are
	// still in the page cache is atomically the wrong thing.
	if err := tmp.Sync(); err != nil {
		tmp.Close()
		return fmt.Errorf("flushing the node's state: %w", err)
	}
	if err := tmp.Close(); err != nil {
		return fmt.Errorf("closing the node's state: %w", err)
	}
	if err := os.Chmod(tmp.Name(), 0o600); err != nil {
		return fmt.Errorf("securing the node's state: %w", err)
	}
	if err := os.Rename(tmp.Name(), path); err != nil {
		return fmt.Errorf("replacing the node's state: %w", err)
	}
	return nil
}

// Record adds or replaces what is known about one resource, preserving order.
func (s *State) Record(a Applied) {
	for i, existing := range s.Resources {
		if existing.ID == a.ID {
			s.Resources[i] = a
			return
		}
	}
	s.Resources = append(s.Resources, a)
}

// Forget drops a resource from what the node owns.
func (s *State) Forget(id string) {
	kept := s.Resources[:0]
	for _, r := range s.Resources {
		if r.ID != id {
			kept = append(kept, r)
		}
	}
	s.Resources = kept
}

// Orphans returns what the host applied and the declaration no longer names, newest first.
//
// Reverse order because undoing in the order things were made undoes a directory before the
// file inside it. Reversing is the only ordering the host can derive without deciding
// anything, which is the line novox/hq ADR 0005 draws.
func (s State) Orphans(declared map[string]bool, origin string) []Applied {
	var out []Applied
	for i := len(s.Resources) - 1; i >= 0; i-- {
		r := s.Resources[i]
		// Only this origin's own. A mesh declaration says nothing about what the bundle raised,
		// and a bundle says nothing about what the mesh assigned — so neither may remove the
		// other's by omission, which is the only way either could express removal.
		if originOf(r) != origin {
			continue
		}
		if !declared[r.ID] {
			out = append(out, r)
		}
	}
	return out
}

// Origins a resource can have.
const (
	// Carried is the bundle this host was built with.
	OriginCarried = "carried"
	// Declared is the mesh, over the link.
	OriginDeclared = "declared"
)

// originOf reads a record's origin, treating absence as carried.
//
// State written before origins existed was all bundle-applied: a host had no other way to be
// told anything. Guessing wrong in the other direction would have a first upgrade remove the
// substrate, which is the fault this field exists to prevent.
func originOf(r Applied) string {
	if r.Origin == "" {
		return OriginCarried
	}
	return r.Origin
}