mesh-host/internal/witness/contract.go

// Package witness is the node-engine watching a core build it placed — the controller on the control
// node, the node tools on every machine — and restoring the build before it when the new one is not
// healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the component being replaced is never the
// only witness of its successor).
//
// **The contract is this file.** What the engine reads to call a build healthy, from where, in what
// shape, within which bound — said once, here, and held by contract_test.go. The controller's side
// writes what is read here (mesh-controller internal/lease Holder); a change on either side is a
// change to this file and its test.
package witness

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/novox/mesh-host/internal/link"
)

// The two witnesses, as a process declares which watches it (`witness`). A process that says none is
// placed as ever; one that says nothing is judged by its name's default (Default).
const (
	// ByLease: healthy when the controller this machine started holds the controller's lease.
	ByLease = "lease"
	// ByPing: healthy when this machine's runtime answers the services protocol's PING.
	ByPing = "ping"
	// ByNone: not judged.
	ByNone = "none"
)

// The controller's lease, as the host reads it (novox/hq ADR 0229): the key `holder` in the bucket
// `mesh-controller_lease`, whose keys live fifteen seconds unless renewed, renewed every five.
const (
	LeaseBucket = "mesh-controller_lease"
	LeaseKey    = "holder"
	// LeaseAge is the bucket's age for its key. A holder whose last renewal is older than this is
	// not holding it, whatever the key still says.
	LeaseAge = 15 * time.Second
	// Skew is how far the controller's clock and this host's may disagree about when it took the
	// lease. One machine, one clock, in practice: a margin, not a tolerance.
	Skew = 2 * time.Second
)

// Bounds (to-be 45 §8). A build is given Within of time the witness could ask in; asked every Every.
const (
	// ControllerWithin: the controller holds the lease within sixty seconds of starting.
	ControllerWithin = 60 * time.Second
	// NodeToolsWithin: the runtime is announced and answering within sixty seconds of starting,
	// each PING answered within PingWithin.
	NodeToolsWithin = 60 * time.Second
	PingWithin      = 5 * time.Second
	Every           = 5 * time.Second
	// GiveUp is how long a witness goes on when it cannot ask at all before it says the build is
	// unwitnessed: the grant missing, the bucket missing, the bus away the whole time.
	GiveUp = 30 * time.Minute
)

// ControllerLease is the lease's value as the controller writes it — mesh-controller internal/lease
// Holder, field for field by its JSON names. Only what the witness reads is required; a field the
// controller adds later is ignored here.
type ControllerLease struct {
	// Instance names one controller process: "controller@<machine> pid <pid> since <RFC 3339>".
	Instance string `json:"instance"`
	// Host is the machine it runs on, as its own hostname says.
	Host string `json:"host,omitempty"`
	// Build is the controller's build as it knows it; not read here — the toolchain stamps no
	// version, so it says "development build" — and a bundle's identity is its digest, which the
	// host already knows.
	Build string `json:"build,omitempty"`
	Epoch uint64 `json:"epoch,omitempty"`
	// Taken is when this instance took the key, Renewed when it last renewed it, by its own clock.
	Taken   time.Time `json:"taken"`
	Renewed time.Time `json:"renewed"`
}

// ParseLease reads the key's value.
func ParseLease(value []byte) (ControllerLease, error) {
	var l ControllerLease
	if err := json.Unmarshal(value, &l); err != nil {
		return ControllerLease{}, fmt.Errorf("the lease's holder is not the controller's lease value: %w", err)
	}
	return l, nil
}

// HeldBySince says whether the lease is held by a controller on machine `host` that took it at or
// after `since` — the controller this machine started then, and not the one before it — and is
// held now. Why says what it saw when it is not.
func (l ControllerLease) HeldBySince(host string, since, now time.Time) (bool, string) {
	switch {
	case l.Instance == "":
		return false, "the lease names no holder"
	case host != "" && !sameMachine(l.Host, host):
		return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", l.Instance, l.Host)
	case l.Taken.Before(since.Add(-Skew)):
		return false, fmt.Sprintf("the lease is held by %s, taken %s — before the new build started at %s",
			l.Instance, l.Taken.UTC().Format(time.RFC3339), since.UTC().Format(time.RFC3339))
	case now.Sub(latest(l.Taken, l.Renewed)) > LeaseAge:
		return false, fmt.Sprintf("the lease names %s and was last renewed %s ago, past its %s",
			l.Instance, now.Sub(latest(l.Taken, l.Renewed)).Round(time.Second), LeaseAge)
	}
	return true, fmt.Sprintf("%s holds the lease, epoch %d", l.Instance, l.Epoch)
}

// sameMachine compares two hostnames as names, so a short name and its fully qualified form agree.
func sameMachine(a, b string) bool {
	short := func(s string) string {
		s = strings.ToLower(strings.TrimSpace(s))
		if i := strings.IndexByte(s, '.'); i > 0 {
			s = s[:i]
		}
		return s
	}
	return short(a) == short(b)
}

func latest(a, b time.Time) time.Time {
	if b.After(a) {
		return b
	}
	return a
}

// NodeToolsService is the runtime's name on the services protocol, and its instance is this
// machine's node name: what `$SRV.PING.<service>.<node>` asks, so only this machine's runtime can
// answer (mesh-tools node-tools internal/runtime, announce.Service{Name: module, ID: node}).
const NodeToolsService = "node-tools"

// Default is the witness a process is judged by when it names none: the two core processes the mesh
// composes, by the names it composes them under, and nothing else.
func Default(process string) string {
	switch process {
	case "mesh-controller":
		return ByLease
	case NodeToolsService:
		return ByPing
	}
	return ByNone
}

// Within is the bound for a witness.
func Within(by string) time.Duration {
	if by == ByLease {
		return ControllerWithin
	}
	return NodeToolsWithin
}

// Component names what a witness judges, as a rollback says it.
func Component(by string) string {
	if by == ByLease {
		return link.ComponentController
	}
	return link.ComponentNodeTools
}