mesh-controller/internal/lease/witness.go

package lease

import (
	"fmt"
	"strings"
	"time"
)

// The controller's rollback witness: the contract between the controller and the node-engine that
// placed it (novox/hq to-be 45 §8, ADR 0227 rule 8, ADR 0236). Its other half is mesh-host's
// internal/witness/contract.go and the Report's `rollbacks` and `witness` (mesh-host internal/link);
// the two are held field for field, and a change on either side is a change to both.
//
// **The component being replaced is never the only witness of its successor.** A new controller that
// starts and does nothing, crashes, or cannot reach the bus cannot say so, and cannot put back the build
// before it. The node-engine on the machine running the controller can: it placed the new bundle, it
// keeps the previous one beside it, and it reads one key on the bus to judge the new one by.
//
// **What the host reads.** A direct get of the lease bucket's one key — LeaseReadSubject — answered with
// the Holder below as JSON (unknown fields ignored, `build` not read). The node principal of every
// machine assigned the controller is granted that one subject, and every machine's node principal the
// runtime's PING on its own machine (broker.WitnessSubjects); replies come to its own inbox.
//
// **When the host calls a new controller healthy** (HeldBySince, as the host has it): the holder's host
// is this machine, it took the key at or after the moment the host started the new build (less Skew),
// and it renewed it within the key's age. Asked every WitnessEvery, within ControllerWithin of the
// start; not met by then, the host stops the new build, starts the one it kept, and says so in its
// reports (Rollback). A key absent, deleted or purged is held by nobody.
//
// **What the controller adds, and the host does not read.** Health: whether the holder says it is
// ready — its self-check ran and `status` answered in bound. The host's bound is the lease alone, sixty
// seconds; the controller's own gate (gate.go) also asks for ready within ten minutes and, failing that,
// sends the previous controller build itself, which the host applies as any declaration.
//
// **What the controller does with what the host says.** Each Rollback a report carries is a condition
// `core.<component>.<node>.<outcome>` — urgent for rolled-back, not-reversible, restore-failed and halted;
// a warning for nothing-to-restore and unwitnessed — kept while reports carry it and cleared by the first
// report from that machine without it. A rolled-back controller or node tools build is marked failed at
// its gate and the module's registered build is put back to the one running, so nothing sends it again.

// Witness bounds, as the host has them (mesh-host internal/witness).
const (
	// ControllerWithin: the new controller holds the lease within this of starting.
	ControllerWithin = 60 * time.Second
	// NodeToolsWithin: the node tools answer PING within this of starting, each PING within PingWithin.
	NodeToolsWithin = 60 * time.Second
	PingWithin      = 5 * time.Second
	// WitnessEvery is how often the host asks.
	WitnessEvery = 5 * time.Second
	// Skew is how far the controller's clock and the host's may disagree about when it took the lease:
	// one machine, one clock — a margin, not a tolerance.
	Skew = 2 * time.Second
	// FreshWithin is the key's age: a holder not renewed within it is not holding it.
	FreshWithin = 15 * time.Second
	// ReadyWithin is the controller's own bound for saying it is ready (Health), which its gate judges.
	ReadyWithin = 10 * time.Minute
)

// LeaseReadSubject is the one subject the witness of the controller publishes to read the lease: a
// direct get of the key `holder`.
func LeaseReadSubject(bucket string) string {
	return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + Key
}

// PingSubject is the subject a machine's witness asks its own node tools on: the services protocol's
// PING to the runtime, whose instance is the machine's name.
func PingSubject(node string) string { return "$SRV.PING." + NodeToolsService + "." + node }

// NodeToolsService is the runtime's name on the services protocol.
const NodeToolsService = "node-tools"

// Health is what the holding controller says of itself in every write of the lease's key. The host
// does not read it; the controller's gate does. A controller that says nothing is not ready.
type Health struct {
	// Ready is the controller's health definition met (to-be 45 §8): it holds the lease, its self-check
	// has run once, and in that run `status` answered in full within ten seconds (D9).
	Ready bool `json:"ready"`
	// ReadyAt is when it first became ready; zero while it is not.
	ReadyAt time.Time `json:"ready_at,omitempty"`
	// Started is when this process started.
	Started time.Time `json:"started"`
	// DoctorRan is when its self-check last finished a run; zero before the first.
	DoctorRan time.Time `json:"doctor_ran,omitempty"`
	// Why says what is missing while it is not ready, in the mesh's words.
	Why string `json:"why,omitempty"`
}

// The witness contract version a host keeps (Report.Witness): its presence says the host reads a
// process's `witness` and `not-reversible`, which an older, strict host refuses — so the controller
// sends them only to a machine whose report carries it.
const WitnessContract = 1

// The core components a witness judges, as a Rollback names them.
const (
	ComponentEngine     = "node-engine"
	ComponentController = "controller"
	ComponentNodeTools  = "node-tools"
)

// What a witness concluded, as a Rollback says it.
const (
	OutcomeRolledBack       = "rolled-back"
	OutcomeNotReversible    = "not-reversible"
	OutcomeNothingToRestore = "nothing-to-restore"
	OutcomeRestoreFailed    = "restore-failed"
	OutcomeUnwitnessed      = "unwitnessed"
	OutcomeHalted           = "halted"
)

// Urgent says whether a witness's outcome needs the operator now.
func Urgent(outcome string) bool {
	switch outcome {
	case OutcomeRolledBack, OutcomeNotReversible, OutcomeRestoreFailed, OutcomeHalted:
		return true
	}
	return false
}

// Rollback is one witness's verdict on one core build, as the node-engine says it in its report
// (`rollbacks`) — on every report while it stands, until a newer build of that component is declared to
// it and proves itself.
type Rollback struct {
	// Component is node-engine, controller or node-tools.
	Component string `json:"component"`
	// From is the build judged: a host version for the node-engine, a bundle's digest for a process.
	From string `json:"from"`
	// To is the build restored; empty when none was.
	To string `json:"to,omitempty"`
	// Outcome is one of the Outcome words.
	Outcome string    `json:"outcome"`
	Why     string    `json:"why"`
	At      time.Time `json:"at"`
}

// ModuleOf is the module a core component is delivered as.
func ModuleOf(component string) string {
	switch component {
	case ComponentController:
		return "mesh-controller"
	case ComponentEngine:
		return "mesh-host"
	case ComponentNodeTools:
		return NodeToolsService
	}
	return ""
}

// HeldBySince is the host's judgement of a new controller (mesh-host witness.ControllerLease.HeldBySince,
// kept here so both sides test one rule): the lease is held by a controller on machine `host` that took
// it at or after `since`, less Skew, and renewed it within the key's age.
func (h Holder) HeldBySince(host string, since, now time.Time) (bool, string) {
	last := h.Taken
	if h.Renewed.After(last) {
		last = h.Renewed
	}
	switch {
	case h.Instance == "":
		return false, "the lease names no holder"
	case host != "" && !sameMachine(h.Host, host):
		return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", h.Instance, h.Host)
	case h.Taken.Before(since.Add(-Skew)):
		return false, fmt.Sprintf("the lease is held by %s, taken before the new build started", h.Instance)
	case now.Sub(last) > FreshWithin:
		return false, fmt.Sprintf("the lease names %s and was last renewed %s ago", h.Instance,
			now.Sub(last).Round(time.Second))
	}
	return true, fmt.Sprintf("%s holds the lease, epoch %d", h.Instance, h.Epoch)
}

// sameMachine compares two hostnames as names, so a short name and its fully qualified form agree.
func sameMachine(a, b string) bool {
	short := func(s string) string {
		s = strings.ToLower(strings.TrimSpace(s))
		if i := strings.IndexByte(s, '.'); i > 0 {
			s = s[:i]
		}
		return s
	}
	return short(a) == short(b)
}