// Package store is what this node knows about itself, and it is authoritative while // disconnected. // // Not a cache of the control plane. novox/hq ADR 0004 makes disconnection an ordinary // situation rather than an exception, and this is what makes it ordinary: a machine shut for a // week comes back and reconciles, it does not come back and ask what it is. // // Its first job arrives with the first apply rather than with the link (ADR 0005): the host // removes what it previously applied and is no longer declared, and it can only know that // because it wrote it down. package store import ( "encoding/json" "errors" "fmt" "io/fs" "os" "path/filepath" "sort" "time" ) // DefaultPath is where a node keeps what it knows. Under /var/lib because it survives a // reboot and is not configuration — nothing generates this, the host writes it. const DefaultPath = "/var/lib/mesh-host/state.json" // Applied is one resource the host put on this machine, and what it did. // // Recorded AFTER the resource was applied and read back, never before (novox/hq ADR 0018). // A record written up front restates the request in a new place and inherits none of the // authority of having happened. type Applied struct { ID string `json:"id"` Type string `json:"type"` // Origin is who asked for this: the bundle this host carries, or the mesh. // // Recorded because the two must not remove each other. A node raises its own foundation from // the bundle before any mesh exists, then enrols and is sent declarations — and a // declaration naming two resources would otherwise remove the store, the broker and the // control plane, which is 04-ISSUES/010 and happened on the first end-to-end run. // // Empty means carried, for state written before this field existed: everything a host had // applied at that point came from its bundle. Origin string `json:"origin,omitempty"` // Holds are the machine's own ports this resource occupies. // // **So the mesh can assign around what it did not put here** (novox/hq ADR 0038). A node // raises its foundation from the bundle before any mesh exists, so the control plane has never // heard of the store, the broker or the control plane's own container — and a module assigned // afterwards would be given a port one of them already holds, and would be told so by a // container runtime rather than by anything that could have prevented it. // // Recorded per resource rather than counted per machine, because what a machine happens to // have open right now is a moving target, and what its declaration binds is not. Holds []int `json:"holds,omitempty"` // Target is what was changed — a path, a unit — so removal knows what to undo without // re-reading a declaration that may no longer exist. Target string `json:"target"` AppliedAt time.Time `json:"applied_at"` // Wrote is a digest of what this host last put there, for resources where that is a // meaningful question. // // Without it, a file that does not match the declaration has two possible explanations and // the host cannot tell them apart: the mesh changed what it wants, or somebody edited the // machine. Both end with the file being rewritten, so the outcome is identical — and a // person who edits a managed file watches their change vanish every few minutes with nothing // anywhere saying why. Wrote string `json:"wrote,omitempty"` } // State is the whole of what a node knows about what it has done. type State struct { // Resources, keyed by identity, in the order they were applied. Order matters for removal: // undoing in reverse is the only ordering the host can derive without deciding anything. Resources []Applied `json:"resources"` UpdatedAt time.Time `json:"updated_at"` } // Find returns what was applied under an identity. func (s State) Find(id string) (Applied, bool) { for _, r := range s.Resources { if r.ID == id { return r, true } } return Applied{}, false } // IDs returns every identity the host has applied, sorted. func (s State) IDs() []string { out := make([]string, 0, len(s.Resources)) for _, r := range s.Resources { out = append(out, r.ID) } sort.Strings(out) return out } // Load reads the state. A node that has never applied anything has an empty state, which is a // fact rather than an error — the first apply on a fresh machine is the ordinary case. // // A state file that exists and cannot be read IS an error, and a loud one: continuing with an // empty state would make the host believe it owns nothing, and it would then remove nothing it // should and re-apply everything it need not. func Load(path string) (State, error) { raw, err := os.ReadFile(path) if errors.Is(err, fs.ErrNotExist) { return State{}, nil } if err != nil { return State{}, fmt.Errorf("reading what this node knows about itself (%s): %w", path, err) } var s State if err := json.Unmarshal(raw, &s); err != nil { return State{}, fmt.Errorf( "what this node knows about itself is unreadable (%s): %w\n"+ "Refusing rather than starting empty: an empty state would mean the host "+ "believes it owns nothing, so it would remove nothing it should and re-apply "+ "everything it need not", path, err) } return s, nil } // Save writes the state, atomically. // // Atomic because the alternative has a failure mode with no floor: a host interrupted while // writing loses the record of everything it owns, and then owns nothing it can clean up. func Save(path string, s State) error { s.UpdatedAt = time.Now().UTC() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return fmt.Errorf("making room for the node's state: %w", err) } raw, err := json.MarshalIndent(s, "", " ") if err != nil { return fmt.Errorf("encoding the node's state: %w", err) } raw = append(raw, '\n') tmp, err := os.CreateTemp(filepath.Dir(path), ".state-*.json") if err != nil { return fmt.Errorf("writing the node's state: %w", err) } defer os.Remove(tmp.Name()) if _, err := tmp.Write(raw); err != nil { tmp.Close() return fmt.Errorf("writing the node's state: %w", err) } // Flushed before the rename: a rename is atomic, and a rename of a file whose contents are // still in the page cache is atomically the wrong thing. if err := tmp.Sync(); err != nil { tmp.Close() return fmt.Errorf("flushing the node's state: %w", err) } if err := tmp.Close(); err != nil { return fmt.Errorf("closing the node's state: %w", err) } if err := os.Chmod(tmp.Name(), 0o600); err != nil { return fmt.Errorf("securing the node's state: %w", err) } if err := os.Rename(tmp.Name(), path); err != nil { return fmt.Errorf("replacing the node's state: %w", err) } return nil } // Record adds or replaces what is known about one resource, preserving order. func (s *State) Record(a Applied) { for i, existing := range s.Resources { if existing.ID == a.ID { s.Resources[i] = a return } } s.Resources = append(s.Resources, a) } // Forget drops a resource from what the node owns. func (s *State) Forget(id string) { kept := s.Resources[:0] for _, r := range s.Resources { if r.ID != id { kept = append(kept, r) } } s.Resources = kept } // Orphans returns what the host applied and the declaration no longer names, newest first. // // Reverse order because undoing in the order things were made undoes a directory before the // file inside it. Reversing is the only ordering the host can derive without deciding // anything, which is the line novox/hq ADR 0005 draws. func (s State) Orphans(declared map[string]bool, origin string) []Applied { var out []Applied for i := len(s.Resources) - 1; i >= 0; i-- { r := s.Resources[i] // Only this origin's own. A mesh declaration says nothing about what the bundle raised, // and a bundle says nothing about what the mesh assigned — so neither may remove the // other's by omission, which is the only way either could express removal. if originOf(r) != origin { continue } if !declared[r.ID] { out = append(out, r) } } return out } // Origins a resource can have. const ( // Carried is the bundle this host was built with. OriginCarried = "carried" // Declared is the mesh, over the link. OriginDeclared = "declared" ) // originOf reads a record's origin, treating absence as carried. // // State written before origins existed was all bundle-applied: a host had no other way to be // told anything. Guessing wrong in the other direction would have a first upgrade remove the // foundation, which is the fault this field exists to prevent. func originOf(r Applied) string { if r.Origin == "" { return OriginCarried } return r.Origin }