// Package store is what this node knows about itself, and it is authoritative while // disconnected. // // Not a cache of the control plane. novox/hq ADR 0004 makes disconnection an ordinary // situation rather than an exception, and this is what makes it ordinary: a machine shut for a // week comes back and reconciles, it does not come back and ask what it is. // // Its first job arrives with the first apply rather than with the link (ADR 0005): the host // removes what it previously applied and is no longer declared, and it can only know that // because it wrote it down. package store import ( "encoding/json" "errors" "fmt" "io/fs" "os" "path/filepath" "sort" "time" ) // DefaultPath is where a node keeps what it knows. Under /var/lib because it survives a // reboot and is not configuration — nothing generates this, the host writes it. const DefaultPath = "/var/lib/mesh-host/state.json" // Applied is one resource the host put on this machine, and what it did. // // Recorded AFTER the resource was applied and read back, never before (novox/hq ADR 0018). // A record written up front restates the request in a new place and inherits none of the // authority of having happened. type Applied struct { ID string `json:"id"` Type string `json:"type"` // Origin is who asked for this: the bundle this host carries, or the mesh. // // Recorded because the two must not remove each other. A node raises its own foundation from // the bundle before any mesh exists, then enrols and is sent declarations — and a // declaration naming two resources would otherwise remove the store, the broker and the // control plane, which is 04-ISSUES/010 and happened on the first end-to-end run. // // Empty means carried, for state written before this field existed: everything a host had // applied at that point came from its bundle. Origin string `json:"origin,omitempty"` // Holds are the machine's own ports this resource occupies. // // **So the mesh can assign around what it did not put here** (novox/hq ADR 0038). A node // raises its foundation from the bundle before any mesh exists, so the control plane has never // heard of the store, the broker or the control plane's own container — and a module assigned // afterwards would be given a port one of them already holds, and would be told so by a // container runtime rather than by anything that could have prevented it. // // Recorded per resource rather than counted per machine, because what a machine happens to // have open right now is a moving target, and what its declaration binds is not. Holds []int `json:"holds,omitempty"` // Target is what was changed — a path, a unit — so removal knows what to undo without // re-reading a declaration that may no longer exist. Target string `json:"target"` AppliedAt time.Time `json:"applied_at"` // Wrote is a digest of what this host last put there, for resources where that is a // meaningful question. // // Without it, a file that does not match the declaration has two possible explanations and // the host cannot tell them apart: the mesh changed what it wants, or somebody edited the // machine. Both end with the file being rewritten, so the outcome is identical — and a // person who edits a managed file watches their change vanish every few minutes with nothing // anywhere saying why. Wrote string `json:"wrote,omitempty"` // Into is set for a file written into rather than over (novox/hq ADR 0102): the format, what // each of the mesh's keys held before it set them, which of them were absent, and whether the // file itself was — so undeclaring it gives the machine back exactly what it had. Into *Into `json:"into,omitempty"` } // Into is what a file written into held before the mesh's keys. type Into struct { Format string `json:"format"` Before map[string]json.RawMessage `json:"before,omitempty"` Absent []string `json:"absent,omitempty"` Created bool `json:"created,omitempty"` } // State is the whole of what a node knows about what it has done. type State struct { // Resources, keyed by identity, in the order they were applied. Order matters for removal: // undoing in reverse is the only ordering the host can derive without deciding anything. Resources []Applied `json:"resources"` UpdatedAt time.Time `json:"updated_at"` // Held is what this host found on the machine and is keeping as it is, until the module // declaring it is taken (novox/hq ADR 0100). Never a Resource: nothing here was applied, so // nothing here is ever removed as an orphan — what is held is not the host's to remove, even // when its module is unassigned. Held []Held `json:"held,omitempty"` // Firewall is the firewall found on this machine when it was first adopted, and whether the // mesh has since retired it (novox/hq ADR 0100). Nil on a node that was never adopted. Firewall *FoundFirewall `json:"firewall,omitempty"` } // FoundFirewall is what the host found filtering this machine, and what it did about it. type FoundFirewall struct { // Kind is ufw or none: an unsupported kind is refused adoption, never recorded. Kind string `json:"kind"` // WasActive is whether it was in force when found — which is what converging the node // retires, and returning it to adopted restores. WasActive bool `json:"was_active,omitempty"` // DisabledByMesh is set when converging retired it, so returning to adopted enables it again // and nothing else ever does. DisabledByMesh bool `json:"disabled_by_mesh,omitempty"` FoundAt time.Time `json:"found_at"` } // Held is one thing found on an adopted node — a file, directory or container present at a declared // path or name, or a service's unit, with no record of this host having made it — kept as it was // found; or what would reach one: a container mounting found data, an action run in a held // container (novox/hq ADR 0100, ADR 0103). type Held struct { ID string `json:"id"` Module string `json:"module"` Kind string `json:"kind"` Target string `json:"target"` // Since is when it was first found. It stays held from then until its module is taken, even // if it disappears: a vanished file is reported, not recreated. Since time.Time `json:"since"` // A file's content as found, by digest; its mode and owner; and where the original was kept // before anything else could happen to it. Digest string `json:"digest,omitempty"` Mode string `json:"mode,omitempty"` Owner string `json:"owner,omitempty"` Kept string `json:"kept,omitempty"` // A container's id as found, and whether it was running — or a service's unit, whether it // was running. Container string `json:"container,omitempty"` Running bool `json:"running,omitempty"` // Why says what was found when it is not the resource's own target: the path or volume a // container would mount, or the held container an action would run in (novox/hq ADR 0103). Why string `json:"why,omitempty"` // Changed is what something other than the mesh has done to it since it was found — // rewritten, stopped, replaced or gone — and empty while it is as found. Reported, never // reverted: that is how a predecessor still writing is caught. Changed string `json:"changed,omitempty"` ChangedAt time.Time `json:"changed_at,omitempty"` } // Recorded reports whether this host has a record, of any origin, of putting something of this // kind at this target. What it has a record of is not found: it wrote it, in this life of the node // or an earlier one — including a foundation raised from the bundle and adopted as modules later // (novox/hq ADR 0078). func (s State) Recorded(kind, target string) bool { for _, r := range s.Resources { if r.Type == kind && r.Target == target { return true } } return false } // HeldAt returns what is held under a resource id. func (s State) HeldAt(id string) (Held, bool) { for _, h := range s.Held { if h.ID == id { return h, true } } return Held{}, false } // RecordHeld adds or replaces what is held under one id, preserving order. func (s *State) RecordHeld(h Held) { for i, existing := range s.Held { if existing.ID == h.ID { s.Held[i] = h return } } s.Held = append(s.Held, h) } // Release drops a hold, once its module is taken and the host has converged what was held. func (s *State) Release(id string) { kept := s.Held[:0] for _, h := range s.Held { if h.ID != id { kept = append(kept, h) } } s.Held = kept if len(s.Held) == 0 { s.Held = nil } } // Find returns what was applied under an identity. func (s State) Find(id string) (Applied, bool) { for _, r := range s.Resources { if r.ID == id { return r, true } } return Applied{}, false } // IDs returns every identity the host has applied, sorted. func (s State) IDs() []string { out := make([]string, 0, len(s.Resources)) for _, r := range s.Resources { out = append(out, r.ID) } sort.Strings(out) return out } // Load reads the state. A node that has never applied anything has an empty state, which is a // fact rather than an error — the first apply on a fresh machine is the ordinary case. // // A state file that exists and cannot be read IS an error, and a loud one: continuing with an // empty state would make the host believe it owns nothing, and it would then remove nothing it // should and re-apply everything it need not. func Load(path string) (State, error) { raw, err := os.ReadFile(path) if errors.Is(err, fs.ErrNotExist) { return State{}, nil } if err != nil { return State{}, fmt.Errorf("reading what this node knows about itself (%s): %w", path, err) } var s State if err := json.Unmarshal(raw, &s); err != nil { return State{}, fmt.Errorf( "what this node knows about itself is unreadable (%s): %w\n"+ "Refusing rather than starting empty: an empty state would mean the host "+ "believes it owns nothing, so it would remove nothing it should and re-apply "+ "everything it need not", path, err) } return s, nil } // Save writes the state, atomically. // // Atomic because the alternative has a failure mode with no floor: a host interrupted while // writing loses the record of everything it owns, and then owns nothing it can clean up. func Save(path string, s State) error { s.UpdatedAt = time.Now().UTC() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return fmt.Errorf("making room for the node's state: %w", err) } raw, err := json.MarshalIndent(s, "", " ") if err != nil { return fmt.Errorf("encoding the node's state: %w", err) } raw = append(raw, '\n') tmp, err := os.CreateTemp(filepath.Dir(path), ".state-*.json") if err != nil { return fmt.Errorf("writing the node's state: %w", err) } defer os.Remove(tmp.Name()) if _, err := tmp.Write(raw); err != nil { tmp.Close() return fmt.Errorf("writing the node's state: %w", err) } // Flushed before the rename: a rename is atomic, and a rename of a file whose contents are // still in the page cache is atomically the wrong thing. if err := tmp.Sync(); err != nil { tmp.Close() return fmt.Errorf("flushing the node's state: %w", err) } if err := tmp.Close(); err != nil { return fmt.Errorf("closing the node's state: %w", err) } if err := os.Chmod(tmp.Name(), 0o600); err != nil { return fmt.Errorf("securing the node's state: %w", err) } if err := os.Rename(tmp.Name(), path); err != nil { return fmt.Errorf("replacing the node's state: %w", err) } return nil } // Record adds or replaces what is known about one resource, preserving order. func (s *State) Record(a Applied) { for i, existing := range s.Resources { if existing.ID == a.ID { s.Resources[i] = a return } } s.Resources = append(s.Resources, a) } // Forget drops a resource from what the node owns. func (s *State) Forget(id string) { kept := s.Resources[:0] for _, r := range s.Resources { if r.ID != id { kept = append(kept, r) } } s.Resources = kept } // Orphans returns what the host applied and the declaration no longer names, newest first. // // Reverse order because undoing in the order things were made undoes a directory before the // file inside it. Reversing is the only ordering the host can derive without deciding // anything, which is the line novox/hq ADR 0005 draws. func (s State) Orphans(declared map[string]bool, origin string) []Applied { var out []Applied for i := len(s.Resources) - 1; i >= 0; i-- { r := s.Resources[i] // Only this origin's own. A mesh declaration says nothing about what the bundle raised, // and a bundle says nothing about what the mesh assigned — so neither may remove the // other's by omission, which is the only way either could express removal. if originOf(r) != origin { continue } if !declared[r.ID] { out = append(out, r) } } return out } // Origins a resource can have. const ( // Carried is the bundle this host was built with. OriginCarried = "carried" // Declared is the mesh, over the link. OriginDeclared = "declared" ) // originOf reads a record's origin, treating absence as carried. // // State written before origins existed was all bundle-applied: a host had no other way to be // told anything. Guessing wrong in the other direction would have a first upgrade remove the // foundation, which is the fault this field exists to prevent. func originOf(r Applied) string { if r.Origin == "" { return OriginCarried } return r.Origin }