Witness the node tools alone: the controller's lease names no build any more

The controller ended by its unassignment (issue 1126, ADR 0590), so no build is judged by the lease: the lease
witness, its contract and the direct get the witness asked it with are removed, and its tests now prove each rollback
path on the node tools' PING.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
2026-11-02 11:22:21 +01:00
parent 84737f4cb2
commit ff4dc71ce4
11 changed files with 89 additions and 425 deletions
+1 -2
View File
@@ -146,9 +146,8 @@ func launcherReplaced(pid int, proc, path string) bool {
// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) {
host, _ := os.Hostname()
w := &witness.Watcher{
Root: apply.DaemonRoot(), Node: node, Host: host,
Root: apply.DaemonRoot(), Node: node,
Asker: func() link.Asker { return queue.Asker() },
Run: witness.Runner(apply.ExecRunner),
Hold: func(f func()) {
+2 -3
View File
@@ -488,9 +488,8 @@ const WitnessContract = 1
// The core components a witness judges (to-be 45 §8), as a rollback names them.
const (
ComponentEngine = "node-engine"
ComponentController = "controller"
ComponentNodeTools = "node-tools"
ComponentEngine = "node-engine"
ComponentNodeTools = "node-tools"
)
// What a witness concluded, as a rollback says it.
+1 -1
View File
@@ -27,7 +27,7 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
[]string{"protocol", "address", "port", "by", "published", "container-port"}},
// novox/hq to-be 45 §8: a witness's verdicts, said on every report while they stand, and the
// witness contract this host keeps.
{Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentController}}, Witness: WitnessContract},
{Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentNodeTools}}, Witness: WitnessContract},
[]string{"node", "rollbacks", "witness"}},
{Rollback{Component: ComponentNodeTools, From: "sha256:b", To: "sha256:a", Outcome: RolledBack, Why: "w"},
[]string{"component", "from", "to", "outcome", "why", "at"}},
+3 -5
View File
@@ -3,16 +3,15 @@
# docker run -d --rm --name w -p 14224:4222 -v $PWD/internal/link/testdata:/c:ro nats:2.11-alpine -js -c /c/witness-grants.conf
# MESH_TEST_NATS_GRANTS=nats://127.0.0.1:14224 go test ./internal/link/ -run TestNatsWitness
#
# `node.anchor` is a machine's host with exactly the two subjects the witness asks added to what a
# host already has (its inbox); `node.bare` is a host without them. `runtime` stands in for the tool
# runtime, `admin` for the controller writing the lease.
# `node.anchor` is a machine's host with exactly the subject the witness asks added to what a host
# already has (its inbox); `node.bare` is a host without it. `runtime` stands in for the tool runtime.
jetstream: enabled
accounts {
MESH {
jetstream: enabled
users = [
{ user: "node.anchor", password: "a", permissions: {
publish: { allow: ["$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder", "$SRV.PING.node-tools.anchor"] }
publish: { allow: ["$SRV.PING.node-tools.anchor"] }
subscribe: { allow: ["_INBOX.node.anchor.>"] }
} }
{ user: "node.bare", password: "b", permissions: {
@@ -23,7 +22,6 @@ accounts {
subscribe: { allow: ["$SRV.PING.node-tools.>"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "admin", password: "x" }
]
}
}
+4 -46
View File
@@ -10,14 +10,11 @@ import (
"github.com/nats-io/nats.go"
)
// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the controller and the
// node tools it placed on this machine, by what the bus says about them rather than by what they
// say about themselves.
// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the node tools it placed on this
// machine, by what the bus says about them rather than by what they say about themselves.
//
// **Two questions, both read-only, both on the host's own link.** The controller's lease key, read
// by JetStream's direct get — one subject, no stream information, nothing written; and this
// machine's tool runtime, asked the NATS services protocol's PING by its name and this machine's —
// so only this machine's runtime can answer it.
// **One question, read-only, on the host's own link**: this machine's tool runtime, asked the NATS services protocol's
// PING by its name and this machine's, so only this machine's runtime can answer it.
// ErrCannotAsk is a question the bus did not let this host put, or did not answer: no grant for it,
// no such bucket, the bus slow. **It says nothing about the build being judged**, so a witness
@@ -30,20 +27,11 @@ var ErrNoAnswer = errors.New("nothing answered")
// Asker is what a witness asks through. The link open now implements it; nil while there is none.
type Asker interface {
// ReadKey is a key-value bucket's current value for a key. Found false is nobody holding it —
// absent, deleted or expired; an error wrapping ErrCannotAsk is no reading at all.
ReadKey(ctx context.Context, bucket, key string) (value []byte, found bool, err error)
// Ping asks the service `service`'s instance `id` whether it is there. Nil is an answer;
// ErrNoAnswer is none in time; ErrCannotAsk is no asking.
Ping(ctx context.Context, service, id string) error
}
// DirectGetSubject is the one subject a host asks a bucket's key on: JetStream's direct get of the
// bucket's stream, for the key's own subject. What a host's grant names exactly — no wildcard.
func DirectGetSubject(bucket, key string) string {
return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + key
}
// PingSubject is the services protocol's PING of one instance of one service.
func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id }
@@ -58,36 +46,6 @@ func (q *Session) Asker() Asker {
return nil
}
func (l *natsLink) ReadKey(ctx context.Context, bucket, key string) ([]byte, bool, error) {
subject := DirectGetSubject(bucket, key)
before := l.conn.LastError()
msg, err := l.conn.RequestWithContext(ctx, subject, nil)
switch {
case errors.Is(err, nats.ErrNoResponders):
return nil, false, fmt.Errorf("%w: the bus has no bucket %s that answers a direct get", ErrCannotAsk, bucket)
case err != nil:
return nil, false, fmt.Errorf("%w: reading %s from %s: %v%s", ErrCannotAsk, key, bucket, err, l.refusal(subject, before))
}
if msg.Header != nil {
switch status := msg.Header.Get("Status"); status {
case "":
case "404":
return nil, false, nil
default:
return nil, false, fmt.Errorf("%w: reading %s from %s: the bus answered %s %s", ErrCannotAsk, key,
bucket, status, msg.Header.Get("Description"))
}
switch msg.Header.Get("KV-Operation") {
case "DEL", "PURGE":
return nil, false, nil
}
}
if len(msg.Data) == 0 {
return nil, false, nil
}
return msg.Data, true, nil
}
func (l *natsLink) Ping(ctx context.Context, service, id string) error {
subject := PingSubject(service, id)
before := l.conn.LastError()
+2 -62
View File
@@ -2,7 +2,6 @@ package link
import (
"context"
"encoding/json"
"errors"
"os"
"testing"
@@ -12,7 +11,7 @@ import (
)
// What a witness asks, asked of a real server with the grants the mesh composes (novox/hq to-be 45 §8):
// whether the direct get and the PING work as the two subjects named, and whether a host without them
// whether the PING works as the subject named, and whether a host without them
// is told apart — "cannot ask", never "the build did not answer". See testdata/witness-grants.conf.
func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS_GRANTS")
@@ -28,63 +27,14 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
t.Cleanup(conn.Close)
return conn
}
admin := dial("admin", "x", "_INBOX.admin")
js, err := admin.JetStream()
if err != nil {
t.Fatal(err)
}
_ = js.DeleteKeyValue("mesh-controller_lease")
kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_lease", TTL: 15 * time.Second, History: 1})
if err != nil {
t.Fatal(err)
}
anchor := &natsLink{conn: dial("node.anchor", "a", "_INBOX.node.anchor")}
bare := &natsLink{conn: dial("node.bare", "b", "_INBOX.node.bare")}
ask := func() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), 2*time.Second)
}
// Nobody holds it.
ctx, cancel := ask()
if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found {
t.Fatalf("an empty lease read as %v, %v", found, err)
}
cancel()
// Held.
if _, err := kv.Put("holder", []byte(`{"instance":"controller@anchor pid 1 since now","epoch":3}`)); err != nil {
t.Fatal(err)
}
ctx, cancel = ask()
value, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder")
cancel()
if err != nil || !found {
t.Fatalf("a held lease read as %v, %v", found, err)
}
var holder struct {
Instance string `json:"instance"`
}
if json.Unmarshal(value, &holder); holder.Instance == "" {
t.Fatalf("the lease's value is %s", value)
}
// Given back.
if err := kv.Delete("holder"); err != nil {
t.Fatal(err)
}
ctx, cancel = ask()
if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found {
t.Fatalf("a lease given back read as %v, %v", found, err)
}
cancel()
// A host without the grant cannot ask — which is not "nobody holds it".
ctx, cancel = ask()
if _, _, err := bare.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) {
t.Fatalf("a host without the grant read the lease as %v, want it unable to ask", err)
}
cancel()
// No runtime on the bus: not there to answer.
ctx, cancel = ask()
ctx, cancel := ask()
if err := anchor.Ping(ctx, "node-tools", "anchor"); !errors.Is(err, ErrNoAnswer) {
t.Fatalf("a PING nobody serves is %v, want no answer", err)
}
@@ -112,14 +62,4 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
t.Fatalf("a host without the grant heard %v, want it unable to ask", err)
}
cancel()
// No lease bucket at all — a controller from before the lease: cannot ask, not "nobody holds it".
if err := js.DeleteKeyValue("mesh-controller_lease"); err != nil {
t.Fatal(err)
}
ctx, cancel = ask()
if _, _, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) {
t.Fatalf("a bus with no lease bucket read as %v, want unable to ask", err)
}
cancel()
}
+12 -113
View File
@@ -1,155 +1,54 @@
// Package witness is the node-engine watching a core build it placed — the controller on the control
// node, the node tools on every machine — and restoring the build before it when the new one is not
// healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the component being replaced is never the
// only witness of its successor).
// Package witness is the node-engine watching a core build it placed — the node tools on every machine — and
// restoring the build before it when the new one is not healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the
// component being replaced is never the only witness of its successor).
//
// **The contract is this file.** What the engine reads to call a build healthy, from where, in what
// shape, within which bound — said once, here, and held by contract_test.go. The controller's side
// writes what is read here (mesh-controller internal/lease Holder); a change on either side is a
// change to this file and its test.
// shape, within which bound — said once, here, and held by contract_test.go.
package witness
import (
"encoding/json"
"fmt"
"strings"
"time"
"github.com/novox/mesh-host/internal/link"
)
// The two witnesses, as a process declares which watches it (`witness`). A process that says none is
// placed as ever; one that says nothing is judged by its name's default (Default).
// The witnesses, as a process declares which watches it (`witness`). A process that says none is placed as ever; one
// that says nothing is judged by its name's default (Default).
const (
// ByLease: healthy when the controller this machine started holds the controller's lease.
ByLease = "lease"
// ByPing: healthy when this machine's runtime answers the services protocol's PING.
ByPing = "ping"
// ByNone: not judged.
ByNone = "none"
)
// The controller's lease, as the host reads it (novox/hq ADR 0229): the key `holder` in the bucket
// `mesh-controller_lease`, whose keys live fifteen seconds unless renewed, renewed every five.
const (
LeaseBucket = "mesh-controller_lease"
LeaseKey = "holder"
// LeaseAge is the bucket's age for its key. A holder whose last renewal is older than this is
// not holding it, whatever the key still says.
LeaseAge = 15 * time.Second
// Skew is how far the controller's clock and this host's may disagree about when it took the
// lease. One machine, one clock, in practice: a margin, not a tolerance.
Skew = 2 * time.Second
)
// Bounds (to-be 45 §8). A build is given Within of time the witness could ask in; asked every Every.
const (
// ControllerWithin: the controller holds the lease within sixty seconds of starting.
ControllerWithin = 60 * time.Second
// NodeToolsWithin: the runtime is announced and answering within sixty seconds of starting,
// each PING answered within PingWithin.
NodeToolsWithin = 60 * time.Second
PingWithin = 5 * time.Second
Every = 5 * time.Second
// GiveUp is how long a witness goes on when it cannot ask at all before it says the build is
// unwitnessed: the grant missing, the bucket missing, the bus away the whole time.
// unwitnessed: the grant missing, the bus away the whole time.
GiveUp = 30 * time.Minute
)
// ControllerLease is the lease's value as the controller writes it — mesh-controller internal/lease
// Holder, field for field by its JSON names. Only what the witness reads is required; a field the
// controller adds later is ignored here.
type ControllerLease struct {
// Instance names one controller process: "controller@<machine> pid <pid> since <RFC 3339>".
Instance string `json:"instance"`
// Host is the machine it runs on, as its own hostname says.
Host string `json:"host,omitempty"`
// Build is the controller's build as it knows it; not read here — the toolchain stamps no
// version, so it says "development build" — and a bundle's identity is its digest, which the
// host already knows.
Build string `json:"build,omitempty"`
Epoch uint64 `json:"epoch,omitempty"`
// Taken is when this instance took the key, Renewed when it last renewed it, by its own clock.
Taken time.Time `json:"taken"`
Renewed time.Time `json:"renewed"`
}
// ParseLease reads the key's value.
func ParseLease(value []byte) (ControllerLease, error) {
var l ControllerLease
if err := json.Unmarshal(value, &l); err != nil {
return ControllerLease{}, fmt.Errorf("the lease's holder is not the controller's lease value: %w", err)
}
return l, nil
}
// HeldBySince says whether the lease is held by a controller on machine `host` that took it at or
// after `since` — the controller this machine started then, and not the one before it — and is
// held now. Why says what it saw when it is not.
func (l ControllerLease) HeldBySince(host string, since, now time.Time) (bool, string) {
switch {
case l.Instance == "":
return false, "the lease names no holder"
case host != "" && !sameMachine(l.Host, host):
return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", l.Instance, l.Host)
case l.Taken.Before(since.Add(-Skew)):
return false, fmt.Sprintf("the lease is held by %s, taken %s — before the new build started at %s",
l.Instance, l.Taken.UTC().Format(time.RFC3339), since.UTC().Format(time.RFC3339))
case now.Sub(latest(l.Taken, l.Renewed)) > LeaseAge:
return false, fmt.Sprintf("the lease names %s and was last renewed %s ago, past its %s",
l.Instance, now.Sub(latest(l.Taken, l.Renewed)).Round(time.Second), LeaseAge)
}
return true, fmt.Sprintf("%s holds the lease, epoch %d", l.Instance, l.Epoch)
}
// sameMachine compares two hostnames as names, so a short name and its fully qualified form agree.
func sameMachine(a, b string) bool {
short := func(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
if i := strings.IndexByte(s, '.'); i > 0 {
s = s[:i]
}
return s
}
return short(a) == short(b)
}
func latest(a, b time.Time) time.Time {
if b.After(a) {
return b
}
return a
}
// NodeToolsService is the runtime's name on the services protocol, and its instance is this
// machine's node name: what `$SRV.PING.<service>.<node>` asks, so only this machine's runtime can
// answer (mesh-tools node-tools internal/runtime, announce.Service{Name: module, ID: node}).
const NodeToolsService = "node-tools"
// Default is the witness a process is judged by when it names none: the two core processes the mesh
// composes, by the names it composes them under, and nothing else.
// Default is the witness a process is judged by when it names none: the core process the mesh composes on every
// machine, by the name it composes it under, and nothing else.
func Default(process string) string {
switch process {
case "mesh-controller":
return ByLease
case NodeToolsService:
if process == NodeToolsService {
return ByPing
}
return ByNone
}
// Within is the bound for a witness.
func Within(by string) time.Duration {
if by == ByLease {
return ControllerWithin
}
return NodeToolsWithin
}
func Within(string) time.Duration { return NodeToolsWithin }
// Component names what a witness judges, as a rollback says it.
func Component(by string) string {
if by == ByLease {
return link.ComponentController
}
return link.ComponentNodeTools
}
func Component(string) string { return link.ComponentNodeTools }
+9 -84
View File
@@ -9,82 +9,7 @@ import (
"github.com/novox/mesh-host/internal/link"
)
// The contract, held. Each of these is a line the controller's side relies on or writes; a change on
// either side changes this test (novox/hq to-be 45 §8).
// The lease is read where the controller keeps it (ADR 0229), on the one subject a host's grant names.
func TestTheLeaseIsReadWhereTheControllerKeepsIt(t *testing.T) {
if LeaseBucket != "mesh-controller_lease" || LeaseKey != "holder" {
t.Fatalf("the lease is read from %s/%s; the controller keeps it in mesh-controller_lease/holder",
LeaseBucket, LeaseKey)
}
if got := link.DirectGetSubject(LeaseBucket, LeaseKey); got != "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder" {
t.Fatalf("the host asks %s for the lease; its grant names exactly the direct get of the key", got)
}
if LeaseAge != 15*time.Second {
t.Fatalf("the lease's age is %s; the controller's bucket keeps a key fifteen seconds", LeaseAge)
}
}
// What the controller writes — mesh-controller internal/lease Holder, by its JSON names — is what is
// read. The value below is the shape that Holder marshals to.
func TestTheLeaseValueIsTheControllersHolder(t *testing.T) {
written := `{"instance":"controller@anchor pid 4242 since 2026-10-06T10:00:00Z","host":"anchor",` +
`"build":"development build","epoch":57,"taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:31Z"}`
l, err := ParseLease([]byte(written))
if err != nil {
t.Fatal(err)
}
if l.Instance == "" || l.Host != "anchor" || l.Epoch != 57 || l.Taken.IsZero() || l.Renewed.IsZero() {
t.Fatalf("the lease was not read whole: %+v", l)
}
// And a field the controller adds later does not stop it being read.
if _, err := ParseLease([]byte(`{"instance":"i","taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:01Z","bundle":"sha256:x"}`)); err != nil {
t.Fatalf("a lease value with a field this host does not know was refused: %v", err)
}
}
// Healthy is: held now, on this machine, by an instance that took it after the new build started.
func TestTheNewControllerHoldsTheLeaseOnlyWhenItTookItAfterItStarted(t *testing.T) {
started := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
now := started.Add(30 * time.Second)
held := ControllerLease{Instance: "controller@anchor pid 2 since …", Host: "anchor.example",
Taken: started.Add(3 * time.Second), Renewed: now.Add(-2 * time.Second), Epoch: 58}
for _, c := range []struct {
name string
l ControllerLease
want bool
says string
}{
{"held by the new instance", held, true, "holds the lease"},
{"nobody", ControllerLease{}, false, "no holder"},
{"the old instance, taken before the restart", func() ControllerLease {
l := held
l.Taken = started.Add(-time.Hour)
return l
}(), false, "before the new build started"},
{"another machine's controller", func() ControllerLease {
l := held
l.Host = "home-server"
return l
}(), false, "not this machine"},
{"taken and no longer renewed", func() ControllerLease {
l := held
l.Renewed = now.Add(-20 * time.Second)
return l
}(), false, "past its"},
{"taken within the clocks' skew of the start", func() ControllerLease {
l := held
l.Taken = started.Add(-time.Second)
return l
}(), true, "holds the lease"},
} {
got, why := c.l.HeldBySince("anchor", started, now)
if got != c.want || !strings.Contains(why, c.says) {
t.Errorf("%s: healthy %v (%s), want %v saying %q", c.name, got, why, c.want, c.says)
}
}
}
// The contract, held (novox/hq to-be 45 §8).
// The runtime is asked by its own name and this machine's: only this machine's runtime can answer.
func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) {
@@ -93,23 +18,23 @@ func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) {
}
}
// What is judged by default is the mesh's two core processes, by the names the controller composes
// them under, and nothing else.
func TestOnlyTheCoreProcessesAreJudgedByDefault(t *testing.T) {
for name, want := range map[string]string{"mesh-controller": ByLease, "node-tools": ByPing, "greeter": ByNone} {
// What is judged by default is the mesh's core process on every machine, by the name it is composed under, and
// nothing else: the controller ended (issue 1126), so its name is judged as any other.
func TestOnlyTheCoreProcessIsJudgedByDefault(t *testing.T) {
for name, want := range map[string]string{"mesh-controller": ByNone, "node-tools": ByPing, "greeter": ByNone} {
if got := Default(name); got != want {
t.Errorf("%s is judged by %q, want %q", name, got, want)
}
}
if ControllerWithin != 60*time.Second || NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second {
t.Fatal("the bounds are to-be 45 §8's: the lease within sixty seconds, a PING answered within five")
if NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second {
t.Fatal("the bounds are to-be 45 §8's: the runtime within sixty seconds, a PING answered within five")
}
}
// A verdict on the wire names its component as the controller's condition does.
func TestAVerdictNamesItsComponent(t *testing.T) {
raw, _ := json.Marshal(link.Rollback{Component: Component(ByLease), Outcome: link.RolledBack})
if !strings.Contains(string(raw), `"component":"controller"`) || Component(ByPing) != "node-tools" {
raw, _ := json.Marshal(link.Rollback{Component: Component(ByPing), Outcome: link.RolledBack})
if !strings.Contains(string(raw), `"component":"node-tools"`) {
t.Fatalf("a verdict names its component as %s", raw)
}
}
+3 -3
View File
@@ -85,11 +85,11 @@ func retiredBuilds(t *testing.T, root, name string) []string {
// after its build was moved into a 0700 directory, and then deleted, could not run its own verbs.
func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T) {
root := t.TempDir()
name := "mesh-controller"
name := "node-tools"
start := time.Date(2026, 10, 7, 2, 0, 0, 0, time.UTC)
// Build A placed and running.
p, err := Place(root, name, ByLease, buildA, "", "", start)
p, err := Place(root, name, ByPing, buildA, "", "", start)
if err != nil {
t.Fatal(err)
}
@@ -99,7 +99,7 @@ func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T)
}
// Build B placed while A still runs: A is moved aside, and reachable at its new path.
place(t, root, name, ByLease, buildB, "", "", start.Add(time.Minute))
place(t, root, name, ByPing, buildB, "", "", start.Add(time.Minute))
for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} {
info, err := os.Stat(dir)
if err != nil {
+5 -20
View File
@@ -14,15 +14,14 @@ import (
// concluded (to-be 45 §8).
//
// **Time counts only while it could ask.** A build's bound is spent only on looks that got an answer
// from the bus — the lease read, or the PING delivered — so a machine whose own link is down, or a
// from the bus — the PING delivered — so a machine whose own link is down, or a
// bus that refuses the question, never rolls a build back for the host's own trouble. A look that
// could not ask counts towards GiveUp instead, and at GiveUp the build is said to be unwitnessed:
// neither proved nor rolled back, and said.
type Watcher struct {
Root string
// Node is this machine's node name, the instance its runtime answers PING as; Host its hostname,
// as the controller's lease names its machine.
Node, Host string
// Node is this machine's node name, the instance its runtime answers PING as.
Node string
// Asker is the link open now, or nil.
Asker func() link.Asker
Run Runner
@@ -69,7 +68,7 @@ func (w *Watcher) look(ctx context.Context, s State) {
every = Every
}
now := w.now()
healthy, why, err := w.judge(ctx, s, now)
healthy, why, err := w.judge(ctx, s)
w.hold(func() {
// Read again in turn: an apply may have placed another build meanwhile, which starts its own
@@ -123,7 +122,7 @@ func (w *Watcher) look(ctx context.Context, s State) {
}
// judge asks once. An error is a look that could not ask; otherwise healthy, and why not when not.
func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, string, error) {
func (w *Watcher) judge(ctx context.Context, s State) (bool, string, error) {
var asker link.Asker
if w.Asker != nil {
asker = w.Asker()
@@ -134,20 +133,6 @@ func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, stri
asking, cancel := context.WithTimeout(ctx, PingWithin)
defer cancel()
switch s.Witness {
case ByLease:
value, found, err := asker.ReadKey(asking, LeaseBucket, LeaseKey)
if err != nil {
return false, "", err
}
if !found {
return false, "nobody holds the controller's lease", nil
}
lease, err := ParseLease(value)
if err != nil {
return false, err.Error(), nil
}
held, why := lease.HeldBySince(w.Host, s.Started, now)
return held, why, nil
case ByPing:
err := asker.Ping(asking, s.Process, w.Node)
switch {
+47 -86
View File
@@ -13,8 +13,8 @@ import (
"github.com/novox/mesh-host/internal/link"
)
// Each rollback path, induced against a real directory: a controller that never takes the lease, a
// runtime that never answers PING, a build declared not reversible; and a healthy update, which
// Each rollback path, induced against a real directory: a runtime that never answers PING, a build
// declared not reversible; and a healthy update, which
// deletes nothing before it is proved and the build before it once it is (novox/hq to-be 45 §8).
const (
@@ -62,18 +62,10 @@ func kept(root, name string) bool {
// asker answers as the build it is told is running would.
type asker struct {
mu sync.Mutex
lease func() ([]byte, bool, error)
ping func() error
asked int
}
func (a *asker) ReadKey(context.Context, string, string) ([]byte, bool, error) {
a.mu.Lock()
defer a.mu.Unlock()
a.asked++
return a.lease()
}
func (a *asker) Ping(context.Context, string, string) error {
a.mu.Lock()
defer a.mu.Unlock()
@@ -95,7 +87,7 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
}
func watcher(root string, a link.Asker, m *machine, clock *time.Time, verdicts *[]link.Rollback) *Watcher {
return &Watcher{Root: root, Node: "anchor", Host: "anchor",
return &Watcher{Root: root, Node: "anchor",
Asker: func() link.Asker {
if a == nil {
return nil
@@ -116,45 +108,37 @@ func look(w *Watcher, clock *time.Time, n int) {
}
}
func leaseHeldBy(instance string, taken time.Time, clock *time.Time) func() ([]byte, bool, error) {
return func() ([]byte, bool, error) {
return []byte(fmt.Sprintf(`{"instance":%q,"host":"anchor","epoch":9,"taken":%q,"renewed":%q}`,
instance, taken.Format(time.RFC3339Nano), clock.Add(-time.Second).Format(time.RFC3339Nano))), true, nil
}
}
// A controller that starts and never takes the lease: restored to the build before, once, said once.
func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) {
// A runtime that starts and never answers PING: restored to the build before, once, said once.
func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) {
root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
if running(t, root, "mesh-controller") != buildB || !kept(root, "mesh-controller") {
t.Fatal("the new controller is not running with the build before it kept beside it")
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
if running(t, root, "node-tools") != buildB || !kept(root, "node-tools") {
t.Fatal("the new runtime is not running with the build before it kept beside it")
}
// The old instance's lease, taken an hour ago, is all the bus ever shows.
a := &asker{lease: leaseHeldBy("controller@anchor pid 1 since earlier", clock.Add(-time.Hour), &clock)}
a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }}
m := &machine{}
var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, int(ControllerWithin/Every)-1)
if len(verdicts) != 0 || running(t, root, "mesh-controller") != buildB {
look(w, &clock, int(NodeToolsWithin/Every)-1)
if len(verdicts) != 0 || running(t, root, "node-tools") != buildB {
t.Fatalf("rolled back before its bound: %+v", verdicts)
}
look(w, &clock, 1)
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].From != buildB || verdicts[0].To != buildA ||
verdicts[0].Component != link.ComponentController {
t.Fatalf("the verdict is %+v, want controller rolled back from B to A", verdicts)
verdicts[0].Component != link.ComponentNodeTools {
t.Fatalf("the verdict is %+v, want node-tools rolled back from B to A", verdicts)
}
if running(t, root, "mesh-controller") != buildA {
t.Fatalf("the controller running is %s, want the build before", running(t, root, "mesh-controller"))
if running(t, root, "node-tools") != buildA {
t.Fatalf("the runtime running is %s, want the build before", running(t, root, "node-tools"))
}
if strings.Join(m.commands, "; ") != "systemctl stop mesh-controller.service; systemctl restart mesh-controller.service" {
if strings.Join(m.commands, "; ") != "systemctl stop node-tools.service; systemctl restart node-tools.service" {
t.Fatalf("the machine was asked %v", m.commands)
}
if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "before the new build started") {
if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "is on the bus") {
t.Fatalf("the verdict does not say why: %s", verdicts[0].Why)
}
@@ -168,81 +152,58 @@ func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) {
t.Fatalf("what every report says is %+v", s)
}
// The mesh still declares B: refused, A kept running, the verdict still standing.
p := place(t, root, "mesh-controller", ByLease, buildB, buildB, "", clock)
if p.Unpack || running(t, root, "mesh-controller") != buildA || !strings.Contains(p.Detail, "rolled back") {
t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "mesh-controller"))
p := place(t, root, "node-tools", ByPing, buildB, buildB, "", clock)
if p.Unpack || running(t, root, "node-tools") != buildA || !strings.Contains(p.Detail, "rolled back") {
t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "node-tools"))
}
if len(Standing(root)) != 1 || len(Trials(root)) != 0 {
t.Fatal("refusing the rolled-back build changed what stands or started a trial")
}
// A newer build is a new trial, and ends what was concluded once it is proved.
place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock)
if running(t, root, "mesh-controller") != buildC || len(Trials(root)) != 1 {
place(t, root, "node-tools", ByPing, buildC, buildB, "", clock)
if running(t, root, "node-tools") != buildC || len(Trials(root)) != 1 {
t.Fatal("a newer build after a rollback was not placed on trial")
}
if err := Proved(root, "mesh-controller"); err != nil {
if err := Proved(root, "node-tools"); err != nil {
t.Fatal(err)
}
if len(Standing(root)) != 0 || kept(root, "mesh-controller") {
if len(Standing(root)) != 0 || kept(root, "node-tools") {
t.Fatal("a newer build proved and the rollback still stands, or the build before it is still kept")
}
}
// A runtime that never answers PING: restored, once.
func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) {
root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }}
m := &machine{}
var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, int(NodeToolsWithin/Every)+20)
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].Component != link.ComponentNodeTools {
t.Fatalf("the verdict is %+v, want node-tools rolled back once", verdicts)
}
if running(t, root, "node-tools") != buildA || kept(root, "node-tools") {
t.Fatal("the runtime running is not the build before, or a copy of it is still kept")
}
}
// A healthy update: nothing deleted before it is proved; the build before it deleted once it is.
func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
// The controller placed by a host from before any witness: a record, and no state.
if err := os.MkdirAll(filepath.Join(root, "mesh-controller"), 0o755); err != nil {
// The runtime placed by a host from before any witness: a record, and no state.
if err := os.MkdirAll(filepath.Join(root, "node-tools"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "mesh-controller", "build"), []byte(buildA), 0o644); err != nil {
if err := os.WriteFile(filepath.Join(root, "node-tools", "build"), []byte(buildA), 0o644); err != nil {
t.Fatal(err)
}
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
started := clock
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }}
a := &asker{ping: func() error { return link.ErrNoAnswer }}
m := &machine{}
var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, 5)
if !kept(root, "mesh-controller") {
if !kept(root, "node-tools") {
t.Fatal("the build before was deleted while the new one was still on trial")
}
// A third build while the second is on trial: the one kept is still the one seen healthy.
place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock)
if s, _ := Load(root, "mesh-controller"); s.Previous != buildA {
place(t, root, "node-tools", ByPing, buildC, buildB, "", clock)
if s, _ := Load(root, "node-tools"); s.Previous != buildA {
t.Fatalf("the build kept to go back to is %s, want A — the last one seen healthy", s.Previous)
}
started = clock
a.lease = leaseHeldBy("controller@anchor pid 3 since now", started.Add(4*time.Second), &clock)
a.ping = func() error { return nil }
look(w, &clock, 2)
if len(verdicts) != 0 || len(m.commands) != 0 {
t.Fatalf("a healthy update was judged or acted on: %+v %v", verdicts, m.commands)
}
if kept(root, "mesh-controller") || len(Trials(root)) != 0 || running(t, root, "mesh-controller") != buildC {
if kept(root, "node-tools") || len(Trials(root)) != 0 || running(t, root, "node-tools") != buildC {
t.Fatal("the build before was not deleted once the new one was proved")
}
}
@@ -252,14 +213,14 @@ func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
place(t, root, "mesh-controller", ByLease, buildB, buildA, "migration 0073 cannot be undone", clock)
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
place(t, root, "node-tools", ByPing, buildB, buildA, "migration 0073 cannot be undone", clock)
a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }}
a := &asker{ping: func() error { return link.ErrNoAnswer }}
m := &machine{}
var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, int(ControllerWithin/Every)+20)
look(w, &clock, int(NodeToolsWithin/Every)+20)
if len(verdicts) != 1 || verdicts[0].Outcome != link.NotReversible || verdicts[0].To != "" {
t.Fatalf("the verdict is %+v, want not-reversible with nothing restored", verdicts)
@@ -267,9 +228,9 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
if !strings.Contains(verdicts[0].Why, "migration 0073 cannot be undone") {
t.Fatalf("the verdict does not say why it may not be rolled back: %s", verdicts[0].Why)
}
if len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB {
if len(m.commands) != 0 || running(t, root, "node-tools") != buildB {
t.Fatalf("the build before was started against the newer data: %v, running %s", m.commands,
running(t, root, "mesh-controller"))
running(t, root, "node-tools"))
}
if len(Standing(root)) != 1 {
t.Fatal("the not-reversible verdict does not stand")
@@ -280,24 +241,24 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
func TestAWitnessThatCannotAskNeverRollsBack(t *testing.T) {
root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
a := &asker{lease: func() ([]byte, bool, error) {
return nil, false, fmt.Errorf("%w: this host's grant does not name the lease", link.ErrCannotAsk)
a := &asker{ping: func() error {
return fmt.Errorf("%w: this host's grant does not name the PING", link.ErrCannotAsk)
}}
m := &machine{}
var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, int(GiveUp/Every)-1)
if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB {
if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "node-tools") != buildB {
t.Fatalf("a build was judged on questions that were never asked: %+v %v", verdicts, m.commands)
}
look(w, &clock, 1)
if len(verdicts) != 1 || verdicts[0].Outcome != link.Unwitnessed || len(m.commands) != 0 {
t.Fatalf("the verdict at GiveUp is %+v, want unwitnessed with nothing done", verdicts)
}
if !kept(root, "mesh-controller") {
if !kept(root, "node-tools") {
t.Fatal("the build before was deleted though the new one was never seen healthy")
}