// Package witness is the node-engine watching a core build it placed — the controller on the control // node, the node tools on every machine — and restoring the build before it when the new one is not // healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the component being replaced is never the // only witness of its successor). // // **The contract is this file.** What the engine reads to call a build healthy, from where, in what // shape, within which bound — said once, here, and held by contract_test.go. The controller's side // writes what is read here (mesh-controller internal/lease Holder); a change on either side is a // change to this file and its test. package witness import ( "encoding/json" "fmt" "strings" "time" "github.com/novox/mesh-host/internal/link" ) // The two witnesses, as a process declares which watches it (`witness`). A process that says none is // placed as ever; one that says nothing is judged by its name's default (Default). const ( // ByLease: healthy when the controller this machine started holds the controller's lease. ByLease = "lease" // ByPing: healthy when this machine's runtime answers the services protocol's PING. ByPing = "ping" // ByNone: not judged. ByNone = "none" ) // The controller's lease, as the host reads it (novox/hq ADR 0229): the key `holder` in the bucket // `mesh-controller_lease`, whose keys live fifteen seconds unless renewed, renewed every five. const ( LeaseBucket = "mesh-controller_lease" LeaseKey = "holder" // LeaseAge is the bucket's age for its key. A holder whose last renewal is older than this is // not holding it, whatever the key still says. LeaseAge = 15 * time.Second // Skew is how far the controller's clock and this host's may disagree about when it took the // lease. One machine, one clock, in practice: a margin, not a tolerance. Skew = 2 * time.Second ) // Bounds (to-be 45 §8). A build is given Within of time the witness could ask in; asked every Every. const ( // ControllerWithin: the controller holds the lease within sixty seconds of starting. ControllerWithin = 60 * time.Second // NodeToolsWithin: the runtime is announced and answering within sixty seconds of starting, // each PING answered within PingWithin. NodeToolsWithin = 60 * time.Second PingWithin = 5 * time.Second Every = 5 * time.Second // GiveUp is how long a witness goes on when it cannot ask at all before it says the build is // unwitnessed: the grant missing, the bucket missing, the bus away the whole time. GiveUp = 30 * time.Minute ) // ControllerLease is the lease's value as the controller writes it — mesh-controller internal/lease // Holder, field for field by its JSON names. Only what the witness reads is required; a field the // controller adds later is ignored here. type ControllerLease struct { // Instance names one controller process: "controller@ pid since ". Instance string `json:"instance"` // Host is the machine it runs on, as its own hostname says. Host string `json:"host,omitempty"` // Build is the controller's build as it knows it; not read here — the toolchain stamps no // version, so it says "development build" — and a bundle's identity is its digest, which the // host already knows. Build string `json:"build,omitempty"` Epoch uint64 `json:"epoch,omitempty"` // Taken is when this instance took the key, Renewed when it last renewed it, by its own clock. Taken time.Time `json:"taken"` Renewed time.Time `json:"renewed"` } // ParseLease reads the key's value. func ParseLease(value []byte) (ControllerLease, error) { var l ControllerLease if err := json.Unmarshal(value, &l); err != nil { return ControllerLease{}, fmt.Errorf("the lease's holder is not the controller's lease value: %w", err) } return l, nil } // HeldBySince says whether the lease is held by a controller on machine `host` that took it at or // after `since` — the controller this machine started then, and not the one before it — and is // held now. Why says what it saw when it is not. func (l ControllerLease) HeldBySince(host string, since, now time.Time) (bool, string) { switch { case l.Instance == "": return false, "the lease names no holder" case host != "" && !sameMachine(l.Host, host): return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", l.Instance, l.Host) case l.Taken.Before(since.Add(-Skew)): return false, fmt.Sprintf("the lease is held by %s, taken %s — before the new build started at %s", l.Instance, l.Taken.UTC().Format(time.RFC3339), since.UTC().Format(time.RFC3339)) case now.Sub(latest(l.Taken, l.Renewed)) > LeaseAge: return false, fmt.Sprintf("the lease names %s and was last renewed %s ago, past its %s", l.Instance, now.Sub(latest(l.Taken, l.Renewed)).Round(time.Second), LeaseAge) } return true, fmt.Sprintf("%s holds the lease, epoch %d", l.Instance, l.Epoch) } // sameMachine compares two hostnames as names, so a short name and its fully qualified form agree. func sameMachine(a, b string) bool { short := func(s string) string { s = strings.ToLower(strings.TrimSpace(s)) if i := strings.IndexByte(s, '.'); i > 0 { s = s[:i] } return s } return short(a) == short(b) } func latest(a, b time.Time) time.Time { if b.After(a) { return b } return a } // NodeToolsService is the runtime's name on the services protocol, and its instance is this // machine's node name: what `$SRV.PING..` asks, so only this machine's runtime can // answer (mesh-tools node-tools internal/runtime, announce.Service{Name: module, ID: node}). const NodeToolsService = "node-tools" // Default is the witness a process is judged by when it names none: the two core processes the mesh // composes, by the names it composes them under, and nothing else. func Default(process string) string { switch process { case "mesh-controller": return ByLease case NodeToolsService: return ByPing } return ByNone } // Within is the bound for a witness. func Within(by string) time.Duration { if by == ByLease { return ControllerWithin } return NodeToolsWithin } // Component names what a witness judges, as a rollback says it. func Component(by string) string { if by == ByLease { return link.ComponentController } return link.ComponentNodeTools }