mesh-host/internal/store/store.go

// Package store is what this node knows about itself, and it is authoritative while
// disconnected.
//
// Not a cache of the control plane. novox/hq ADR 0004 makes disconnection an ordinary
// situation rather than an exception, and this is what makes it ordinary: a machine shut for a
// week comes back and reconciles, it does not come back and ask what it is.
//
// Its first job arrives with the first apply rather than with the link (ADR 0005): the host
// removes what it previously applied and is no longer declared, and it can only know that
// because it wrote it down.
package store

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"sort"
	"time"
)

// DefaultPath is where a node keeps what it knows. Under /var/lib because it survives a
// reboot and is not configuration — nothing generates this, the host writes it.
const DefaultPath = "/var/lib/mesh-host/state.json"

// Applied is one resource the host put on this machine, and what it did.
//
// Recorded AFTER the resource was applied and read back, never before (novox/hq ADR 0018).
// A record written up front restates the request in a new place and inherits none of the
// authority of having happened.
type Applied struct {
	ID   string `json:"id"`
	Type string `json:"type"`
	// Origin is who asked for this: the bundle this host carries, or the mesh.
	//
	// Recorded because the two must not remove each other. A node raises its own foundation from
	// the bundle before any mesh exists, then enrols and is sent declarations — and a
	// declaration naming two resources would otherwise remove the store, the broker and the
	// control plane, which is 04-ISSUES/010 and happened on the first end-to-end run.
	//
	// Empty means carried, for state written before this field existed: everything a host had
	// applied at that point came from its bundle.
	Origin string `json:"origin,omitempty"`

	// Holds are the machine's own ports this resource occupies.
	//
	// **So the mesh can assign around what it did not put here** (novox/hq ADR 0038). A node
	// raises its foundation from the bundle before any mesh exists, so the control plane has never
	// heard of the store, the broker or the control plane's own container — and a module assigned
	// afterwards would be given a port one of them already holds, and would be told so by a
	// container runtime rather than by anything that could have prevented it.
	//
	// Recorded per resource rather than counted per machine, because what a machine happens to
	// have open right now is a moving target, and what its declaration binds is not.
	Holds []int `json:"holds,omitempty"`
	// Target is what was changed — a path, a unit — so removal knows what to undo without
	// re-reading a declaration that may no longer exist.
	Target    string    `json:"target"`
	AppliedAt time.Time `json:"applied_at"`

	// Wrote is a digest of what this host last put there, for resources where that is a
	// meaningful question.
	//
	// Without it, a file that does not match the declaration has two possible explanations and
	// the host cannot tell them apart: the mesh changed what it wants, or somebody edited the
	// machine. Both end with the file being rewritten, so the outcome is identical — and a
	// person who edits a managed file watches their change vanish every few minutes with nothing
	// anywhere saying why.
	Wrote string `json:"wrote,omitempty"`

	// Kept is where the original of a file this host wrote over was kept (novox/hq ADR 0100):
	// by the keep on its first write, or by the hold that was released when its module was taken.
	// Recorded rather than only reported, because a file once written whole and now written into
	// (novox/hq issue 128) is given back its original with the mesh's region in it — and a path
	// said once in a log line is not a path the host can find again.
	Kept string `json:"kept,omitempty"`

	// Stateless is, for a service, that its unit's lifecycle was never the mesh's (novox/hq ADR
	// 0117) — kept here because removal happens once the declaration that said so is gone, and a
	// service removed as if it had a state is stopped: the machine's network manager, for one.
	Stateless bool `json:"stateless,omitempty"`

	// Found is, for a service, the state its unit was in when this host first applied it — before
	// the mesh started, stopped, enabled or disabled anything. Removal gives that back and nothing
	// more (novox/hq ADR 0118): a unit that was running before the mesh arrived keeps running
	// when its declaration goes; one the mesh started is stopped again. Absent on a record written
	// before the host kept it, and then the unit is left exactly as it is.
	Found *FoundUnit `json:"found,omitempty"`

	// Into is set for a file written into rather than over (novox/hq ADR 0102): the format, what
	// each of the mesh's keys held before it set them, which of them were absent, and whether the
	// file itself was — so undeclaring it gives the machine back exactly what it had.
	Into *Into `json:"into,omitempty"`

	// Reads is, for a container, the digest of each file it was created reading — its env-files
	// and the files mounted into it — by path (novox/hq 04-ISSUES/103).
	//
	// A container takes those in once, when it is created, and the digest of the whole is in the
	// container's spec label; this is the same information kept per file, so that when the spec
	// no longer matches the host can say WHICH file changed rather than only that something did.
	Reads map[string]string `json:"reads,omitempty"`
}

// Into is what a file written into held before the mesh's keys, or before the mesh's block.
type Into struct {
	Format  string                     `json:"format"`
	Before  map[string]json.RawMessage `json:"before,omitempty"`
	Absent  []string                   `json:"absent,omitempty"`
	Created bool                       `json:"created,omitempty"`
	// Added is, for each key whose declared value is a list, exactly the members the mesh added
	// to the machine's list — never a member that was already there. Undeclared, only these go,
	// and drift is judged on these alone (novox/hq ADR 0102).
	Added map[string][]json.RawMessage `json:"added,omitempty"`

	// Region is, for a file written into a block (novox/hq issue 128), what the lines between the
	// mesh's markers held before the mesh wrote them — nil when there was no region, which is
	// different from a region that was there and empty. Undeclared, a recorded region is put back
	// and an unrecorded one is taken out, markers and all. It is the block's "what each key held
	// before": the one thing the host needs to give the file back.
	Region *string `json:"region,omitempty"`
	// Separated says the host put a blank line between the region and the machine's lines when it
	// added the region — before it at the end, after it at the start, as At says — so taking the
	// region out takes that line with it and nothing of the operator's.
	Separated bool `json:"separated,omitempty"`
	// At is where the host added the region: "start", or empty for the end.
	At string `json:"at,omitempty"`
	// Ended says the machine's last line had no line end and the host gave it one to add the
	// region after it, so taking the region out takes that line end too.
	Ended bool `json:"ended,omitempty"`
}

// State is the whole of what a node knows about what it has done.
type State struct {
	// Resources, keyed by identity, in the order they were applied. Order matters for removal:
	// undoing in reverse is the only ordering the host can derive without deciding anything.
	Resources []Applied `json:"resources"`
	UpdatedAt time.Time `json:"updated_at"`

	// Held is what this host found on the machine and is keeping as it is, until the module
	// declaring it is taken (novox/hq ADR 0100). Never a Resource: nothing here was applied, so
	// nothing here is ever removed as an orphan — what is held is not the host's to remove, even
	// when its module is unassigned.
	Held []Held `json:"held,omitempty"`

	// Firewall is the firewall found on this machine when it was first adopted, and whether the
	// mesh has since retired it (novox/hq ADR 0100). Nil on a node that was never adopted.
	Firewall *FoundFirewall `json:"firewall,omitempty"`

	// Mode is the node's mode as this host last recorded it: adopted or converged (novox/hq ADR
	// 0100). Empty on a machine nothing has said a mode to yet. Recorded from every declaration
	// the mesh sends and at genesis from what the operator said, so a declaration that says the
	// other mode can be refused before it is applied (novox/hq issue 104).
	Mode string `json:"mode,omitempty"`

	// FoundFirst is, by resource id, what a service's unit was found as by an apply of it that did
	// not finish — kept apart from Resources, because a record follows the fact and this apply's
	// fact never came (novox/hq ADR 0118).
	//
	// **The capture is the one reading that cannot be taken again.** A first apply that enabled a
	// unit and then failed to start it leaves no record; without this, the next apply would find
	// the unit enabled, take that for what the machine had, and undeclaring would leave enabled a
	// unit the mesh enabled. So what is found is written the moment it is read, whatever the apply
	// of the resource then does, and a later apply reads it here before it reads the machine.
	// Dropped once a record carrying it is written, and when its resource is no longer declared.
	FoundFirst map[string]PendingFound `json:"found_first,omitempty"`

	// Genesis is the bundle this host consumed raising the foundation, if it has. Once recorded,
	// the bundle carried in the binary is not applied again: what genesis applied was rewritten
	// for this machine, and the mesh has said more since (novox/hq issue 104).
	Genesis *Genesis `json:"genesis,omitempty"`

	// Retired is each found tunnel configuration the mesh removed from where its unit reads it,
	// once the private network's take of that tunnel was proven (novox/hq ADR 0119).
	//
	// **Not a hold, and never released with one.** The hold on the found configuration ends at the
	// retirement — what it held for has been replaced, and the node stops reporting it — so without
	// this the next apply would find no hold and no file and read the take as one whose
	// configuration vanished before it could be kept. It is kept whether or not the private
	// network stays declared: undeclaring brings nothing back (ADR 0118), and a private network
	// assigned again finds the tunnel's configuration retired rather than missing.
	Retired []Retired `json:"retired,omitempty"`
}

// Retired is a found configuration the mesh removed once what replaced it was proven (novox/hq
// ADR 0119): where it was, under which hold it had been kept, and where its original still is.
type Retired struct {
	ID   string `json:"id"`
	Path string `json:"path"`
	// Kept is the original as found (novox/hq ADR 0100) — the record of what the predecessor was,
	// and a person's way back if one is ever wanted. The mesh never copies it back. It is the FIRST
	// original, and stays so however often the file comes back: a retirement repeated never moves
	// it. Digest is what it holds.
	Kept   string `json:"kept"`
	Digest string `json:"digest,omitempty"`
	// Extra is where what was at the path when it was last retired is kept, when that differed from
	// the first original — rewritten since it was found, or put back with other content — and
	// ExtraDigest what it holds. One copy per distinct content: a file that comes back as it was
	// last retired keeps nothing more.
	Extra       string    `json:"extra,omitempty"`
	ExtraDigest string    `json:"extra_digest,omitempty"`
	At          time.Time `json:"at"`
	// Again is how many times the configuration came back after it was retired, and was retired
	// again (novox/hq ADR 0119).
	Again int `json:"again,omitempty"`
}

// Modes a node can be in (novox/hq ADR 0100).
const (
	ModeAdopted   = "adopted"
	ModeConverged = "converged"
)

// Genesis is the bundle a host consumed raising this machine's foundation.
type Genesis struct {
	// Digest is sha256 of the exact bytes applied.
	Digest string    `json:"digest"`
	At     time.Time `json:"at"`
	// Rewritten is true when the bytes applied were the carried bundle rewritten for this
	// machine — its foundation ports, root credentials, and adoption — so the bundle the binary
	// carries is not what was applied.
	Rewritten bool `json:"rewritten,omitempty"`
}

// FoundFirewall is what the host found filtering this machine, and what it did about it.
type FoundFirewall struct {
	// Kind is ufw or none: an unsupported kind is refused adoption, never recorded.
	Kind string `json:"kind"`
	// WasActive is whether it was in force when found — which is what converging the node
	// retires, and returning it to adopted restores.
	WasActive bool `json:"was_active,omitempty"`
	// DisabledByMesh is set when converging retired it, so returning to adopted enables it again
	// and nothing else ever does.
	DisabledByMesh bool `json:"disabled_by_mesh,omitempty"`
	// Forward is each family's forward policy as it was before the mesh disabled the firewall,
	// by the tool that sets it — recorded before, so a retirement retried puts back what the
	// machine had.
	Forward map[string]string `json:"forward,omitempty"`
	FoundAt time.Time         `json:"found_at"`
}

// Held is one thing found on an adopted node — a file, directory or container present at a declared
// path or name, or a service's unit, with no record of this host having made it — kept as it was
// found; or what would reach one: a container mounting found data, an action run in a held
// container (novox/hq ADR 0100, ADR 0103).
type Held struct {
	ID     string `json:"id"`
	Module string `json:"module"`
	Kind   string `json:"kind"`
	Target string `json:"target"`
	// Since is when it was first found. It stays held from then until its module is taken, even
	// if it disappears: a vanished file is reported, not recreated.
	Since time.Time `json:"since"`

	// A file's content as found, by digest; its mode and owner; and where the original was kept
	// before anything else could happen to it.
	Digest string `json:"digest,omitempty"`
	Mode   string `json:"mode,omitempty"`
	Owner  string `json:"owner,omitempty"`
	Kept   string `json:"kept,omitempty"`

	// A container's id as found, and whether it was running — or a service's unit, whether it
	// was running.
	Container string `json:"container,omitempty"`
	Running   bool   `json:"running,omitempty"`

	// Why says what was found when it is not the resource's own target: the path or volume a
	// container would mount, or the held container an action would run in (novox/hq ADR 0103).
	Why string `json:"why,omitempty"`

	// Changed is what something other than the mesh has done to it since it was found —
	// rewritten, stopped, replaced or gone — and empty while it is as found. Reported, never
	// reverted: that is how a predecessor still writing is caught.
	Changed   string    `json:"changed,omitempty"`
	ChangedAt time.Time `json:"changed_at,omitempty"`
}

// Recorded reports whether this host has a record, of any origin, of putting something of this
// kind at this target. What it has a record of is not found: it wrote it, in this life of the node
// or an earlier one — including a foundation raised from the bundle and adopted as modules later
// (novox/hq ADR 0078).
func (s State) Recorded(kind, target string) bool {
	for _, r := range s.Resources {
		if r.Type == kind && r.Target == target {
			return true
		}
	}
	return false
}

// At returns what this host has a record of putting at a target of this kind, under any id and of
// any origin — Recorded, with the record.
//
// By target rather than by id because the id a thing was declared under may change while the thing
// does not: the bundle's `store` becomes a module's `postgres.server` for the same container, and
// the file that container reads is the same file under either id. Both ids may then hold a record
// for the one target — the bundle's is never removed by the mesh's declaration — and the most
// recently applied is the one that says what is there now.
func (s State) At(kind, target string) (Applied, bool) {
	var latest Applied
	found := false
	for _, r := range s.Resources {
		if r.Type != kind || r.Target != target {
			continue
		}
		if !found || r.AppliedAt.After(latest.AppliedAt) {
			latest, found = r, true
		}
	}
	return latest, found
}

// HeldAt returns what is held under a resource id.
func (s State) HeldAt(id string) (Held, bool) {
	for _, h := range s.Held {
		if h.ID == id {
			return h, true
		}
	}
	return Held{}, false
}

// RecordHeld adds or replaces what is held under one id, preserving order.
func (s *State) RecordHeld(h Held) {
	for i, existing := range s.Held {
		if existing.ID == h.ID {
			s.Held[i] = h
			return
		}
	}
	s.Held = append(s.Held, h)
}

// Release drops a hold, once its module is taken and the host has converged what was held.
func (s *State) Release(id string) {
	kept := s.Held[:0]
	for _, h := range s.Held {
		if h.ID != id {
			kept = append(kept, h)
		}
	}
	s.Held = kept
	if len(s.Held) == 0 {
		s.Held = nil
	}
}

// RetiredAt returns the retirement of the found configuration at a path, if the mesh retired one.
func (s State) RetiredAt(path string) (Retired, bool) {
	for _, r := range s.Retired {
		if r.Path == path {
			return r, true
		}
	}
	return Retired{}, false
}

// RecordRetired adds or replaces the retirement of the configuration at one path.
func (s *State) RecordRetired(r Retired) {
	for i, existing := range s.Retired {
		if existing.Path == r.Path {
			s.Retired[i] = r
			return
		}
	}
	s.Retired = append(s.Retired, r)
}

// Find returns what was applied under an identity.
func (s State) Find(id string) (Applied, bool) {
	for _, r := range s.Resources {
		if r.ID == id {
			return r, true
		}
	}
	return Applied{}, false
}

// IDs returns every identity the host has applied, sorted.
func (s State) IDs() []string {
	out := make([]string, 0, len(s.Resources))
	for _, r := range s.Resources {
		out = append(out, r.ID)
	}
	sort.Strings(out)
	return out
}

// Load reads the state. A node that has never applied anything has an empty state, which is a
// fact rather than an error — the first apply on a fresh machine is the ordinary case.
//
// A state file that exists and cannot be read IS an error, and a loud one: continuing with an
// empty state would make the host believe it owns nothing, and it would then remove nothing it
// should and re-apply everything it need not.
func Load(path string) (State, error) {
	raw, err := os.ReadFile(path)
	if errors.Is(err, fs.ErrNotExist) {
		return State{}, nil
	}
	if err != nil {
		return State{}, fmt.Errorf("reading what this node knows about itself (%s): %w", path, err)
	}

	var s State
	if err := json.Unmarshal(raw, &s); err != nil {
		return State{}, fmt.Errorf(
			"what this node knows about itself is unreadable (%s): %w\n"+
				"Refusing rather than starting empty: an empty state would mean the host "+
				"believes it owns nothing, so it would remove nothing it should and re-apply "+
				"everything it need not", path, err)
	}
	return s, nil
}

// Save writes the state, atomically.
//
// Atomic because the alternative has a failure mode with no floor: a host interrupted while
// writing loses the record of everything it owns, and then owns nothing it can clean up.
func Save(path string, s State) error {
	s.UpdatedAt = time.Now().UTC()

	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return fmt.Errorf("making room for the node's state: %w", err)
	}

	raw, err := json.MarshalIndent(s, "", "  ")
	if err != nil {
		return fmt.Errorf("encoding the node's state: %w", err)
	}
	raw = append(raw, '\n')

	tmp, err := os.CreateTemp(filepath.Dir(path), ".state-*.json")
	if err != nil {
		return fmt.Errorf("writing the node's state: %w", err)
	}
	defer os.Remove(tmp.Name())

	if _, err := tmp.Write(raw); err != nil {
		tmp.Close()
		return fmt.Errorf("writing the node's state: %w", err)
	}
	// Flushed before the rename: a rename is atomic, and a rename of a file whose contents are
	// still in the page cache is atomically the wrong thing.
	if err := tmp.Sync(); err != nil {
		tmp.Close()
		return fmt.Errorf("flushing the node's state: %w", err)
	}
	if err := tmp.Close(); err != nil {
		return fmt.Errorf("closing the node's state: %w", err)
	}
	if err := os.Chmod(tmp.Name(), 0o600); err != nil {
		return fmt.Errorf("securing the node's state: %w", err)
	}
	if err := os.Rename(tmp.Name(), path); err != nil {
		return fmt.Errorf("replacing the node's state: %w", err)
	}
	return nil
}

// Record adds or replaces what is known about one resource, preserving order.
func (s *State) Record(a Applied) {
	for i, existing := range s.Resources {
		if existing.ID == a.ID {
			s.Resources[i] = a
			return
		}
	}
	s.Resources = append(s.Resources, a)
}

// Forget drops a resource from what the node owns.
func (s *State) Forget(id string) {
	kept := s.Resources[:0]
	for _, r := range s.Resources {
		if r.ID != id {
			kept = append(kept, r)
		}
	}
	s.Resources = kept
}

// Orphans returns what the host applied and the declaration no longer names, newest first.
//
// Reverse order because undoing in the order things were made undoes a directory before the
// file inside it. Reversing is the only ordering the host can derive without deciding
// anything, which is the line novox/hq ADR 0005 draws.
func (s State) Orphans(declared map[string]bool, origin string) []Applied {
	var out []Applied
	for i := len(s.Resources) - 1; i >= 0; i-- {
		r := s.Resources[i]
		// Only this origin's own. A mesh declaration says nothing about what the bundle raised,
		// and a bundle says nothing about what the mesh assigned — so neither may remove the
		// other's by omission, which is the only way either could express removal.
		if originOf(r) != origin {
			continue
		}
		if !declared[r.ID] {
			out = append(out, r)
		}
	}
	return out
}

// Origins a resource can have.
const (
	// Carried is the bundle this host was built with.
	OriginCarried = "carried"
	// Declared is the mesh, over the link.
	OriginDeclared = "declared"
)

// originOf reads a record's origin, treating absence as carried.
//
// State written before origins existed was all bundle-applied: a host had no other way to be
// told anything. Guessing wrong in the other direction would have a first upgrade remove the
// foundation, which is the fault this field exists to prevent.
func originOf(r Applied) string {
	if r.Origin == "" {
		return OriginCarried
	}
	return r.Origin
}

// FoundUnit is a service's unit as the host first found it.
type FoundUnit struct {
	// Unit is which unit this was read from. What was found about one unit says nothing about
	// another, so a service whose declaration moves to a different unit is read again for that one
	// (novox/hq ADR 0118). Empty on what was kept before this was: the record's Target then says.
	Unit string `json:"unit,omitempty"`
	// State is "running" or "stopped".
	State string `json:"state"`
	// Boot is "enabled" or "disabled" — or empty when the declaration never set it, and the host
	// never touched it.
	Boot string `json:"boot,omitempty"`
}

// PendingFound is a unit as found by an apply of its service that has not yet been recorded, and
// who asked for that apply — so only a declaration from the same origin can say it is gone.
type PendingFound struct {
	FoundUnit
	Origin string `json:"origin,omitempty"`
}

// KeepFound writes down what an unfinished apply found a service's unit as.
func (s *State) KeepFound(id, origin string, f FoundUnit) {
	if s.FoundFirst == nil {
		s.FoundFirst = map[string]PendingFound{}
	}
	s.FoundFirst[id] = PendingFound{FoundUnit: f, Origin: origin}
}

// DropFound forgets what was found for one resource: its record now carries it, or it is gone.
func (s *State) DropFound(id string) {
	delete(s.FoundFirst, id)
	if len(s.FoundFirst) == 0 {
		s.FoundFirst = nil
	}
}

// DropFoundUndeclared forgets what was found for every resource of this origin the declaration no
// longer names. Only this origin's, for the reason Orphans gives: a mesh declaration's silence says
// nothing about what the bundle applies, nor the other way round.
func (s *State) DropFoundUndeclared(declared map[string]bool, origin string) {
	for id, p := range s.FoundFirst {
		if !declared[id] && originOf(Applied{Origin: p.Origin}) == origin {
			s.DropFound(id)
		}
	}
}