package lease import ( "fmt" "strings" "time" ) // The controller's rollback witness: the contract between the controller and the node-engine that // placed it (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0236). Its other half is mesh-host's // internal/witness/contract.go and the Report's `rollbacks` and `witness` (mesh-host internal/link); // the two are held field for field, and a change on either side is a change to both. // // **The component being replaced is never the only witness of its successor.** A new controller that // starts and does nothing, crashes, or cannot reach the bus cannot say so, and cannot put back the build // before it. The node-engine on the machine running the controller can: it placed the new bundle, it // keeps the previous one beside it, and it reads one key on the bus to judge the new one by. // // **What the host reads.** A direct get of the lease bucket's one key — LeaseReadSubject — answered with // the Holder below as JSON (unknown fields ignored, `build` not read). The node principal of every // machine assigned the controller is granted that one subject, and every machine's node principal the // runtime's PING on its own machine (broker.WitnessSubjects); replies come to its own inbox. // // **When the host calls a new controller healthy** (HeldBySince, as the host has it): the holder's host // is this machine, it took the key at or after the moment the host started the new build (less Skew), // and it renewed it within the key's age. Asked every WitnessEvery, within ControllerWithin of the // start; not met by then, the host stops the new build, starts the one it kept, and says so in its // reports (Rollback). A key absent, deleted or purged is held by nobody. // // **What the controller adds, and the host does not read.** Health: whether the holder says it is // ready — its self-check ran and `status` answered in bound. The host's bound is the lease alone, sixty // seconds; the controller's own gate (gate.go) also asks for ready within ten minutes and, failing that, // sends the previous controller build itself, which the host applies as any declaration. // // **What the controller does with what the host says.** Each Rollback a report carries is a condition // `core...` — urgent for rolled-back, not-reversible, restore-failed and halted; // a warning for nothing-to-restore and unwitnessed — kept while reports carry it and cleared by the first // report from that machine without it. A rolled-back controller or node tools build is marked failed at // its gate and the module's registered build is put back to the one running, so nothing sends it again. // Witness bounds, as the host has them (mesh-host internal/witness). const ( // ControllerWithin: the new controller holds the lease within this of starting. ControllerWithin = 60 * time.Second // NodeToolsWithin: the node tools answer PING within this of starting, each PING within PingWithin. NodeToolsWithin = 60 * time.Second PingWithin = 5 * time.Second // WitnessEvery is how often the host asks. WitnessEvery = 5 * time.Second // Skew is how far the controller's clock and the host's may disagree about when it took the lease: // one machine, one clock — a margin, not a tolerance. Skew = 2 * time.Second // FreshWithin is the key's age: a holder not renewed within it is not holding it. FreshWithin = 15 * time.Second // ReadyWithin is the controller's own bound for saying it is ready (Health), which its gate judges. ReadyWithin = 10 * time.Minute ) // LeaseReadSubject is the one subject the witness of the controller publishes to read the lease: a // direct get of the key `holder`. func LeaseReadSubject(bucket string) string { return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + Key } // PingSubject is the subject a machine's witness asks its own node tools on: the services protocol's // PING to the runtime, whose instance is the machine's name. func PingSubject(node string) string { return "$SRV.PING." + NodeToolsService + "." + node } // NodeToolsService is the runtime's name on the services protocol. const NodeToolsService = "node-tools" // Health is what the holding controller says of itself in every write of the lease's key. The host // does not read it; the controller's gate does. A controller that says nothing is not ready. type Health struct { // Ready is the controller's health definition met (to-be 45 §8): it holds the lease, its self-check // has run once, and in that run `status` answered in full within ten seconds (D9). Ready bool `json:"ready"` // ReadyAt is when it first became ready; zero while it is not. ReadyAt time.Time `json:"ready_at,omitempty"` // Started is when this process started. Started time.Time `json:"started"` // DoctorRan is when its self-check last finished a run; zero before the first. DoctorRan time.Time `json:"doctor_ran,omitempty"` // Why says what is missing while it is not ready, in the mesh's words. Why string `json:"why,omitempty"` } // The witness contract version a host keeps (Report.Witness): its presence says the host reads a // process's `witness` and `not-reversible`, which an older, strict host refuses — so the controller // sends them only to a machine whose report carries it. const WitnessContract = 1 // The core components a witness judges, as a Rollback names them. const ( ComponentEngine = "node-engine" ComponentController = "controller" ComponentNodeTools = "node-tools" ) // What a witness concluded, as a Rollback says it. const ( OutcomeRolledBack = "rolled-back" OutcomeNotReversible = "not-reversible" OutcomeNothingToRestore = "nothing-to-restore" OutcomeRestoreFailed = "restore-failed" OutcomeUnwitnessed = "unwitnessed" OutcomeHalted = "halted" ) // Urgent says whether a witness's outcome needs the operator now. func Urgent(outcome string) bool { switch outcome { case OutcomeRolledBack, OutcomeNotReversible, OutcomeRestoreFailed, OutcomeHalted: return true } return false } // Rollback is one witness's verdict on one core build, as the node-engine says it in its report // (`rollbacks`) — on every report while it stands, until a newer build of that component is declared to // it and proves itself. type Rollback struct { // Component is node-engine, controller or node-tools. Component string `json:"component"` // From is the build judged: a host version for the node-engine, a bundle's digest for a process. From string `json:"from"` // To is the build restored; empty when none was. To string `json:"to,omitempty"` // Outcome is one of the Outcome words. Outcome string `json:"outcome"` Why string `json:"why"` At time.Time `json:"at"` } // ModuleOf is the module a core component is delivered as. func ModuleOf(component string) string { switch component { case ComponentController: return "mesh-controller" case ComponentEngine: return "mesh-host" case ComponentNodeTools: return NodeToolsService } return "" } // HeldBySince is the host's judgement of a new controller (mesh-host witness.ControllerLease.HeldBySince, // kept here so both sides test one rule): the lease is held by a controller on machine `host` that took // it at or after `since`, less Skew, and renewed it within the key's age. func (h Holder) HeldBySince(host string, since, now time.Time) (bool, string) { last := h.Taken if h.Renewed.After(last) { last = h.Renewed } switch { case h.Instance == "": return false, "the lease names no holder" case host != "" && !sameMachine(h.Host, host): return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", h.Instance, h.Host) case h.Taken.Before(since.Add(-Skew)): return false, fmt.Sprintf("the lease is held by %s, taken before the new build started", h.Instance) case now.Sub(last) > FreshWithin: return false, fmt.Sprintf("the lease names %s and was last renewed %s ago", h.Instance, now.Sub(last).Round(time.Second)) } return true, fmt.Sprintf("%s holds the lease, epoch %d", h.Instance, h.Epoch) } // sameMachine compares two hostnames as names, so a short name and its fully qualified form agree. func sameMachine(a, b string) bool { short := func(s string) string { s = strings.ToLower(strings.TrimSpace(s)) if i := strings.IndexByte(s, '.'); i > 0 { s = s[:i] } return s } return short(a) == short(b) }