package link import ( "strconv" "strings" "time" ) // The wire formats shared with the control plane, which defines them separately because this // binary requires nothing present and does not import it. A test on each side asserts the field // names, so a rename breaks both at once rather than on a real machine months later. // Routing keys a node may publish. Its broker account is scoped to this exchange and its own // queue, so it can say these things and nothing else. const ( KeyReport = "report" KeyAlive = "alive" ) // Alive is a node saying nothing except that it is there. // // novox/hq 09-the-node-lifecycle: *how long it has been disconnected is a fact the mesh must // hold, and nothing holds it today. Without it, a node running last month's assignments looks // exactly like one that is current.* // // Separate from a report because the two happen at completely different rates: a node is alive // constantly and applies something rarely, and reading one as the other would make a quiet node // look like a stale one. type Alive struct { Node string `json:"node"` // IntervalSeconds is how often this node says it is there (novox/hq to-be 45 §3, S1): the // controller's watchdog is bound to three of them, so the bound moves with the interval rather // than with a number the controller guessed. A controller older than that ignores it. IntervalSeconds int `json:"interval_seconds"` } // Signed is a declaration and the signature over it. // // novox/hq ADR 0004: the transport is verified once at connect, and **each declaration is // verified by its signature, every time**. The two are different questions — a node connects to // the broker and takes instruction from the control plane behind it, and pinning only the first // would make the second transitive. // // The signature is over Declaration exactly as it arrived, bytes unchanged. Re-encoding before // verifying would mean checking a signature over something other than what was sent, and any // difference in key order or spacing would break it — so the raw message is what is signed and // what is checked. type Signed struct { Declaration []byte `json:"declaration"` Signature []byte `json:"signature"` } // Report is what a node says after applying, and it is a statement rather than a write. // // A node states; the context that owns the data writes (novox/hq ADR 0006). The difference is the // security boundary: something that can write cannot be prevented from writing anything, and // something that can only state has its blast radius bounded by what this struct can say. type Report struct { Node string `json:"node"` // Applied is what this machine now owns, by resource id. Applied []string `json:"applied,omitempty"` // Failed says what could not be applied, and why, in words for a person. Failed map[string]string `json:"failed,omitempty"` // Refused is set when the declaration was rejected whole rather than applied in part. Refused string `json:"refused,omitempty"` // Superseded names the newer declaration this one was set aside for, unapplied. // // A machine asked to be five successive things becomes the last one (novox/hq issue 031): // when several declarations are waiting, the host applies the newest and acknowledges the // rest without applying them. Each of those is still reported, because silence reads as a // machine that ignored an instruction and "applied" would be a lie — this is the third word. Superseded string `json:"superseded,omitempty"` // Carried are the machine's ports held by what this host raised from its own bundle. // // **So the mesh can assign around what it did not put here** (novox/hq ADR 0038). A node // raises its foundation before any mesh exists, so the control plane has never heard of the // store or the broker — and would hand a module a port one of them holds, discovering it only // when a container runtime refused to start. // // A node *states* and the mesh writes, which is the whole shape of this message: this is the // machine saying what is true of it, not asking for anything. Carried []int `json:"carried,omitempty"` // Declared is the digest of the declaration this report is about — sha256 of the exact bytes // the mesh sent, which the mesh recorded when it sent them. // // **Which declaration, not when.** The mesh compared its send time to this report's arrival // to decide whether a machine had caught up, and lost the race it invited: an apply started // under the previous declaration finishes after the next one is sent, its report lands newer // than the send, and the machine reads as caught up with words it has not read yet. Clocks // cannot answer "which"; the digest is the answer itself. Declared string `json:"declared,omitempty"` // Order is where the declaration this report is about stands — its `epoch` and `sequence`, as // the declaration carried them — so the mesh orders reports by what they are about rather than // by when they arrived (novox/hq to-be 45 §6). On the wire as two top-level keys beside // `declared`; absent for a declaration that claimed no order, and for a report about none. Order // ReportSequence is this node-engine's own order among everything it reports: one higher for // every report it makes, kept on disk so it goes on increasing across restarts and self-updates // (to-be 45 §6). With the declaration's order it lets the mesh refuse an older report about the // same declaration — a reconcile's account overtaking the apply that followed it (novox/hq issue // 267) is then refused by number rather than set aside by digest. A report said again because // it never reached the mesh (issue 264) carries the number it was made with. // // Zero claims no order: every report an older host makes, and a one-shot report a command makes // (a rekey). **Its presence also says this host reads a declaration's `epoch`**, which is how the // mesh knows it may send one: an older host decodes a declaration strictly and refuses a key it // does not know. ReportSequence int64 `json:"report_sequence,omitempty"` // OlderThan is set on a report refusing a declaration older than one this node has applied: the // order of the newer declaration it holds (to-be 45 §6, rule 2). The refused declaration is the // report's own Declared and Order, and Refused says it in words. Applying it would make the // machine into something a controller that lost its lease said, after one that holds it. OlderThan *Order `json:"older_than,omitempty"` // RefusedOlder is how many declarations this node-engine has refused as older, ever — kept on // disk with the report sequence. On every report, so a count missed with a lost refusal is still // read from the next one: what the mesh's stale-writer watchdog (to-be 45 §3, S13) counts. RefusedOlder int64 `json:"refused_older,omitempty"` // Held is what this adopted node found and is keeping as it was until its module is taken // (novox/hq ADR 0100). Without it an adopted node reads as converged. Held []Held `json:"held,omitempty"` // Firewall is the firewall found on this machine — "ufw" or "none" — and empty on a node that // was never asked, which is every converged one. Firewall string `json:"firewall,omitempty"` // Reachable is what can be reached on this machine now: every listening socket and every // published container port. Only an adopted node reports it; it is what converging the node // previews, so nothing closes without being named first. Reachable []Reach `json:"reachable,omitempty"` // Tunnel is what this adopted node says about the tunnel it found and carried (novox/hq ADR // 0105): which interface, its port, range and peer count, whether the found interface is down // and the mesh's up in its place, and where the found configuration's original was kept. Tunnel *CarriedTunnel `json:"tunnel,omitempty"` // Filters is what filters this machine now, every table and chain that refuses traffic with its // owner — the mesh's, the found firewall's, the container runtime's own, a ban list, or other // (novox/hq ADR 0168). Every node reports it, adopted or converged, so the mesh can say // truthfully what filters a converged machine and name what it did not write. Filters []Filter `json:"filters,omitempty"` // FoundFirewall is the state of the firewall a converged machine was found with: whether it is // in force now, and how it came to be inactive — the mesh disabled it, or a reconcile found it so // (ADR 0168). Nil on a machine found with none, and on an adopted one, where Firewall says it. FoundFirewall *FoundFirewall `json:"found_firewall,omitempty"` // Windows is the maintenance windows open on this machine when it reported (novox/hq issue 224, // ADR 0189): a scheduled step holding its module's containers still. A machine whose store is // stopped at 03:31 because its collector is running is working, and without this it reads as // broken. Windows []Window `json:"windows,omitempty"` // Strays is what runs on the machine that the mesh neither wrote nor holds (novox/hq ADR // 0163): containers nobody declared and nobody holds, the ones a cutover leaves behind. Strays []Stray `json:"strays,omitempty"` // Profile is what this machine can do, detected again by the apply that reports (novox/hq // ADR 0161) — the same shape enrolment sends — so a capability gained or lost since enrolment, // a network manager switched, reaches the mesh at the next push rather than never. Profile map[string]any `json:"profile,omitempty"` // Host is the version of the host that produced this report (novox/hq ADR 0141). // // Without it nothing can say a machine is behind, so "every machine current with its source" // could not include the host — the one component the mesh did not deliver. It is a fact the // machine states about itself, like the firewall it found and the links that face outside. Host string `json:"host,omitempty"` // Outward is the links on this machine that face outside it — the ones carrying a default // route (novox/hq ADR 0140). Every node reports it, adopted or converged, because a converged // node's filter is written around it. // // **It replaces a list of addresses.** The filter used to block everything passing through the // machine and then allow the machine's own containers back by naming the ranges they sit on. // A range describes one machine and goes stale in silence; the link carrying the default route // is read afresh on every report and does not change when a module is added or removed. // // Empty means this machine has no route off itself. The mesh then composes no filter for it and // leaves the one it has, rather than writing a rule around a link with no name — a rule set // that does not load is a machine filtering nothing while its unit reports success. Outward []string `json:"outward,omitempty"` // Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq // ADR 0105). Not an account of the machine: a report carrying one says nothing else. Rekey *Rekey `json:"rekey,omitempty"` // Rollbacks is what this machine's witnesses decided about a core build that was not healthy // in bound (novox/hq to-be 45 §8, ADR 0227 rule 8): the node-engine's launcher about the // engine, the engine about the controller and the node tools. **Said on every report while it // stands** — while the build it judged is still the one the mesh asks this machine to run — so // a report lost, or a controller restarted, never makes a rollback silent. Empty once a newer // build has proved itself. Rollbacks []Rollback `json:"rollbacks,omitempty"` // Witness is the version of the witness contract this node-engine keeps (to-be 45 §8): its // presence says it reads a process's `witness` and `not-reversible`, which a strict decoder // older than it refuses — so the controller sends either only to a machine whose report carries // it, as it sends `epoch` only where `report_sequence` was seen (ADR 0229). Witness int `json:"witness,omitempty"` // Health is this node-engine's word on every long-running resource it runs for a module (novox/hq // ADR 0240, to-be 48 §4): its state, since when, its failing streak and the restarts it counted. // **Its presence says this engine judges**: a health with no resources is a machine that runs nothing // long-lived, and no health at all is an engine older than the judging — which the controller reads // as "not known", never as healthy and never as a reason to raise anything. Fresh on every report // this engine makes, and said between reports on HealthSubject when it changes. Health *Health `json:"health,omitempty"` } // LivenessContract is the version of the health statement this host keeps (ADR 0240 Phase A: liveness, // judged with no declaration). const LivenessContract = 1 // Health is one statement of every long-running resource's state on this machine (to-be 48 §4). Said in // every report, as an event on each change, and again every minute while anything is not healthy — so a // lost statement is not a lost fault. type Health struct { // Contract is LivenessContract. Contract int `json:"contract"` // At is when the engine looked, on this machine's clock: the order of its statements. A controller // keeps the newest it heard and refuses an older one arriving late. At time.Time `json:"at"` // Resources are every long-running resource of a module this machine runs, by module and id. Resources []ResourceHealth `json:"resources"` } // The states a long-running resource is said in (ADR 0240 §4). const ( StateHealthy = "healthy" StateUnhealthy = "unhealthy" // StateStarting is inside its grace after a start: not yet a verdict either way. StateStarting = "starting" // StateHeld is held still by an open maintenance window (ADR 0189): neither alive nor dead. StateHeld = "held" // StateUnknown is a resource whose state could not be read. StateUnknown = "unknown" ) // The reasons an unhealthy resource is said with, when it is liveness that judged it. const ( ReasonDown = "down" ReasonRestarting = "restarting" ) // ResourceHealth is one long-running resource's state (to-be 48 §4). type ResourceHealth struct { // Module owns the resource; Resource is its id in the declaration. Module string `json:"module"` Resource string `json:"resource"` // Kind is container, service or process; Target the container's name or the unit. Kind string `json:"kind"` Target string `json:"target"` State string `json:"state"` // Reason is why it is not healthy: down, restarting, or what could not be read. Reason string `json:"reason,omitempty"` Since time.Time `json:"since"` // Streak is how many looks in a row have found it unhealthy. Streak int `json:"streak,omitempty"` // Restarts is how many restarts the engine counted after a grace, kept across recreates and across // its own restarts: the runtime's own count is lost on every recreate. Restarts int `json:"restarts,omitempty"` } // HealthSaid is the health event: a machine's statement between its reports, on HealthSubject. type HealthSaid struct { Node string `json:"node"` Health Health `json:"health"` } // WitnessContract is the version of the witness contract this host keeps: the fields above, the // process fields `witness` and `not-reversible`, and what each witness reads (internal/witness). const WitnessContract = 1 // The core components a witness judges (to-be 45 §8), as a rollback names them. const ( ComponentEngine = "node-engine" ComponentController = "controller" ComponentNodeTools = "node-tools" ) // What a witness concluded, as a rollback says it. const ( // RolledBack is the previous build restored and running. RolledBack = "rolled-back" // NotReversible is a build that failed its health and was declared not reversible: the // previous build would run against what the new one already changed (a migration that ran), so // nothing was restored. The failing build is left as it is, and this is urgent. NotReversible = "not-reversible" // NothingToRestore is a build that failed its health with no previous build kept to go back to. NothingToRestore = "nothing-to-restore" // RestoreFailed is a restoration that was attempted and did not complete. RestoreFailed = "restore-failed" // Unwitnessed is a build whose health could not be judged at all within the bound — the // witness could not ask (no grant, no lease bucket) — so it is neither proved nor rolled back. Unwitnessed = "unwitnessed" // Halted is the node-engine's launcher giving up: the build it would go back to fails too, so // the fault is the machine's and not a build's. Halted = "halted" ) // Rollback is one witness's verdict on one core build (to-be 45 §8). The controller raises // `core...` from it; RolledBack, NotReversible, RestoreFailed and Halted // are urgent. type Rollback struct { // Component is which core component: node-engine, controller or node-tools. Component string `json:"component"` // From is the build that was judged: a host version for the node-engine, the bundle's digest for // a process. From string `json:"from"` // To is the build restored, and empty when none was. To string `json:"to,omitempty"` // Outcome is one of the words above. Outcome string `json:"outcome"` // Why is what the witness saw, in words for a person. Why string `json:"why"` At time.Time `json:"at"` } // Order is where a declaration stands among everything the mesh has sent this node (novox/hq to-be // 45 §6): the controller's lease **epoch** it was sent under, and its **sequence** — one higher for // every send to this node (04-ISSUES/107). The declaration carries both inside what is signed, as // top-level `epoch` and `sequence` beside `declaration`; a report carries back the pair of the // declaration it is about. // // Zero in either is "no order claimed", never "first": every declaration an older controller sent, // and the bundle genesis applies. type Order struct { Epoch int64 `json:"epoch,omitempty"` Sequence int64 `json:"sequence,omitempty"` } // Older says whether a declaration of this order is older than one of order `than`, which this node // has applied — the one rule by which a node-engine refuses a declaration (to-be 45 §6: "older epoch; // same epoch, lower sequence"). // // **Only when both claim an epoch.** A declaration with none is an older controller's — or a // controller rolled back to a build from before the lease — and one kept with none is what this host // held before any controller had a lease. Neither has an order to compare, and refusing on a guess // would strand the machine the moment the controller is older than the host: without an epoch, today's // behaviour stands. A higher epoch is a new lease holder and is never older, whatever its sequence. func (o Order) Older(than Order) bool { if o.Epoch <= 0 || than.Epoch <= 0 { return false } if o.Epoch != than.Epoch { return o.Epoch < than.Epoch } return o.Sequence > 0 && than.Sequence > 0 && o.Sequence < than.Sequence } // Supersedes says whether a declaration of order `o`, arriving after one of order `before`, takes its // place among what is waiting to be applied. By epoch when both claim one and they differ, then by // sequence when both claim one, and by arrival when they do not — what the drain had to go on before // declarations said where they stand (04-ISSUES/107). func (o Order) Supersedes(before Order) bool { if o.Epoch > 0 && before.Epoch > 0 && o.Epoch != before.Epoch { return o.Epoch > before.Epoch } if o.Sequence > 0 && before.Sequence > 0 { return o.Sequence >= before.Sequence } return true } // Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a // Stringer promoted onto every report would print each one as its order alone. func (o Order) Words() string { switch { case o.Epoch > 0: return "epoch " + strconv.FormatInt(o.Epoch, 10) + ", sequence " + strconv.FormatInt(o.Sequence, 10) case o.Sequence > 0: return "sequence " + strconv.FormatInt(o.Sequence, 10) + ", no epoch" } return "no order" } // CarriedTunnel is this node's account of the tunnel it took over. State is one of the Carried // states below; Note is what the host did about it, when it did something. type CarriedTunnel struct { Interface string `json:"interface"` Port int `json:"port"` Range string `json:"range"` Peers int `json:"peers"` State string `json:"state"` Note string `json:"note,omitempty"` Kept string `json:"kept,omitempty"` } // The states a carried tunnel can be in: the found interface still up and the mesh's not; the // found one down and the mesh's up with its key; or the found one down and the mesh's not up — the // one state where the peers reach nothing, said as its own word so nothing reads it as either of // the others. const ( CarriedNotTaken = "not-taken" CarriedTaken = "taken" CarriedDown = "down" ) // Rekey is this node saying it took a found tunnel's key as its overlay key after enrolling // (novox/hq ADR 0105): the path for a node that enrolled before the mesh knew to take a tunnel // over, since re-enrolling would rotate every key it holds. Signed with the identity key over // RekeyProof, so a report forged on a stolen broker account cannot move this node's overlay key. type Rekey struct { // Previous is the overlay key this node held until now, as the mesh records it; the mesh // refuses a rekey naming another, which is how a replayed one is refused. Previous string `json:"previous"` OverlayKey string `json:"overlay_key"` Tunnel *Tunnel `json:"tunnel"` Proof []byte `json:"proof"` } // RekeyProof is what a node signs when it rekeys — the node, the key it leaves, the key it takes // and the tunnel it took it from — so a proof cannot be moved to another node or another tunnel. // Byte for byte the mesh's own (mesh-controller internal/link RekeyProof). func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte { var t Tunnel if tunnel != nil { t = *tunnel } peers := make([]string, 0, len(t.Peers)) for _, p := range t.Peers { peers = append(peers, p.PublicKey+"@"+p.Address) } return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" + t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" + t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ",")) } // Held is one file or container found on an adopted node and kept as it was. type Held struct { ID string `json:"id"` Module string `json:"module"` Kind string `json:"kind"` Target string `json:"target"` Since time.Time `json:"since"` // Changed is what something other than the mesh did to it since — rewritten, stopped, // replaced or gone — and empty while it is as found. Changed string `json:"changed,omitempty"` // Kept is where a file's original was kept. Kept string `json:"kept,omitempty"` // Facts is the found thing beside what the module declares — what a take compares (novox/hq // ADR 0163). The same shape the host keeps; the controller reads it as data. Facts map[string]any `json:"facts,omitempty"` } // A Window is one scheduled step holding containers still (novox/hq issue 224): the step's id, the // containers by runtime name, since when, and the latest moment the host believes it. type Window struct { Step string `json:"step"` Holds []string `json:"holds"` Since time.Time `json:"since"` Until time.Time `json:"until"` } // A Stray is a container the mesh neither wrote nor holds (ADR 0163). type Stray struct { Kind string `json:"kind"` Name string `json:"name"` Detail string `json:"detail,omitempty"` } // Reach is one thing reachable on the machine: a listening socket, or a published container port. type Reach struct { Protocol string `json:"protocol"` Address string `json:"address"` Port int `json:"port"` // By is what holds it — a process, or a container's name. By string `json:"by,omitempty"` // Published is a container port the runtime publishes, reached on the forwarded path; its // container's own port is ContainerPort. Published bool `json:"published,omitempty"` ContainerPort int `json:"container-port,omitempty"` } // A Filter is one place on the machine that refuses traffic, with its owner (novox/hq ADR 0168): // the same shape the host's firewall package reads, carried as data. type Filter struct { Where string `json:"where"` Owner string `json:"owner"` Refuses string `json:"refuses"` } // FoundFirewall is the state of a converged machine's found firewall (ADR 0168). type FoundFirewall struct { Kind string `json:"kind"` Active bool `json:"active"` RetiredBy string `json:"retired_by,omitempty"` }