Witness the node tools alone: the controller's lease names no build any more

The controller ended by its unassignment (issue 1126, ADR 0590), so no build is judged by the lease: the lease
witness, its contract and the direct get the witness asked it with are removed, and its tests now prove each rollback
path on the node tools' PING.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
2026-11-02 11:22:21 +01:00
parent 84737f4cb2
commit ff4dc71ce4
11 changed files with 89 additions and 425 deletions
+1 -2
View File
@@ -146,9 +146,8 @@ func launcherReplaced(pid int, proc, path string) bool {
// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8). // watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) { func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) {
host, _ := os.Hostname()
w := &witness.Watcher{ w := &witness.Watcher{
Root: apply.DaemonRoot(), Node: node, Host: host, Root: apply.DaemonRoot(), Node: node,
Asker: func() link.Asker { return queue.Asker() }, Asker: func() link.Asker { return queue.Asker() },
Run: witness.Runner(apply.ExecRunner), Run: witness.Runner(apply.ExecRunner),
Hold: func(f func()) { Hold: func(f func()) {
+2 -3
View File
@@ -488,9 +488,8 @@ const WitnessContract = 1
// The core components a witness judges (to-be 45 §8), as a rollback names them. // The core components a witness judges (to-be 45 §8), as a rollback names them.
const ( const (
ComponentEngine = "node-engine" ComponentEngine = "node-engine"
ComponentController = "controller" ComponentNodeTools = "node-tools"
ComponentNodeTools = "node-tools"
) )
// What a witness concluded, as a rollback says it. // What a witness concluded, as a rollback says it.
+1 -1
View File
@@ -27,7 +27,7 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
[]string{"protocol", "address", "port", "by", "published", "container-port"}}, []string{"protocol", "address", "port", "by", "published", "container-port"}},
// novox/hq to-be 45 §8: a witness's verdicts, said on every report while they stand, and the // novox/hq to-be 45 §8: a witness's verdicts, said on every report while they stand, and the
// witness contract this host keeps. // witness contract this host keeps.
{Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentController}}, Witness: WitnessContract}, {Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentNodeTools}}, Witness: WitnessContract},
[]string{"node", "rollbacks", "witness"}}, []string{"node", "rollbacks", "witness"}},
{Rollback{Component: ComponentNodeTools, From: "sha256:b", To: "sha256:a", Outcome: RolledBack, Why: "w"}, {Rollback{Component: ComponentNodeTools, From: "sha256:b", To: "sha256:a", Outcome: RolledBack, Why: "w"},
[]string{"component", "from", "to", "outcome", "why", "at"}}, []string{"component", "from", "to", "outcome", "why", "at"}},
+3 -5
View File
@@ -3,16 +3,15 @@
# docker run -d --rm --name w -p 14224:4222 -v $PWD/internal/link/testdata:/c:ro nats:2.11-alpine -js -c /c/witness-grants.conf # docker run -d --rm --name w -p 14224:4222 -v $PWD/internal/link/testdata:/c:ro nats:2.11-alpine -js -c /c/witness-grants.conf
# MESH_TEST_NATS_GRANTS=nats://127.0.0.1:14224 go test ./internal/link/ -run TestNatsWitness # MESH_TEST_NATS_GRANTS=nats://127.0.0.1:14224 go test ./internal/link/ -run TestNatsWitness
# #
# `node.anchor` is a machine's host with exactly the two subjects the witness asks added to what a # `node.anchor` is a machine's host with exactly the subject the witness asks added to what a host
# host already has (its inbox); `node.bare` is a host without them. `runtime` stands in for the tool # already has (its inbox); `node.bare` is a host without it. `runtime` stands in for the tool runtime.
# runtime, `admin` for the controller writing the lease.
jetstream: enabled jetstream: enabled
accounts { accounts {
MESH { MESH {
jetstream: enabled jetstream: enabled
users = [ users = [
{ user: "node.anchor", password: "a", permissions: { { user: "node.anchor", password: "a", permissions: {
publish: { allow: ["$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder", "$SRV.PING.node-tools.anchor"] } publish: { allow: ["$SRV.PING.node-tools.anchor"] }
subscribe: { allow: ["_INBOX.node.anchor.>"] } subscribe: { allow: ["_INBOX.node.anchor.>"] }
} } } }
{ user: "node.bare", password: "b", permissions: { { user: "node.bare", password: "b", permissions: {
@@ -23,7 +22,6 @@ accounts {
subscribe: { allow: ["$SRV.PING.node-tools.>"] } subscribe: { allow: ["$SRV.PING.node-tools.>"] }
allow_responses: { max: 1, ttl: "1m" } allow_responses: { max: 1, ttl: "1m" }
} } } }
{ user: "admin", password: "x" }
] ]
} }
} }
+4 -46
View File
@@ -10,14 +10,11 @@ import (
"github.com/nats-io/nats.go" "github.com/nats-io/nats.go"
) )
// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the controller and the // What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the node tools it placed on this
// node tools it placed on this machine, by what the bus says about them rather than by what they // machine, by what the bus says about them rather than by what they say about themselves.
// say about themselves.
// //
// **Two questions, both read-only, both on the host's own link.** The controller's lease key, read // **One question, read-only, on the host's own link**: this machine's tool runtime, asked the NATS services protocol's
// by JetStream's direct get — one subject, no stream information, nothing written; and this // PING by its name and this machine's, so only this machine's runtime can answer it.
// machine's tool runtime, asked the NATS services protocol's PING by its name and this machine's —
// so only this machine's runtime can answer it.
// ErrCannotAsk is a question the bus did not let this host put, or did not answer: no grant for it, // ErrCannotAsk is a question the bus did not let this host put, or did not answer: no grant for it,
// no such bucket, the bus slow. **It says nothing about the build being judged**, so a witness // no such bucket, the bus slow. **It says nothing about the build being judged**, so a witness
@@ -30,20 +27,11 @@ var ErrNoAnswer = errors.New("nothing answered")
// Asker is what a witness asks through. The link open now implements it; nil while there is none. // Asker is what a witness asks through. The link open now implements it; nil while there is none.
type Asker interface { type Asker interface {
// ReadKey is a key-value bucket's current value for a key. Found false is nobody holding it —
// absent, deleted or expired; an error wrapping ErrCannotAsk is no reading at all.
ReadKey(ctx context.Context, bucket, key string) (value []byte, found bool, err error)
// Ping asks the service `service`'s instance `id` whether it is there. Nil is an answer; // Ping asks the service `service`'s instance `id` whether it is there. Nil is an answer;
// ErrNoAnswer is none in time; ErrCannotAsk is no asking. // ErrNoAnswer is none in time; ErrCannotAsk is no asking.
Ping(ctx context.Context, service, id string) error Ping(ctx context.Context, service, id string) error
} }
// DirectGetSubject is the one subject a host asks a bucket's key on: JetStream's direct get of the
// bucket's stream, for the key's own subject. What a host's grant names exactly — no wildcard.
func DirectGetSubject(bucket, key string) string {
return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + key
}
// PingSubject is the services protocol's PING of one instance of one service. // PingSubject is the services protocol's PING of one instance of one service.
func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id } func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id }
@@ -58,36 +46,6 @@ func (q *Session) Asker() Asker {
return nil return nil
} }
func (l *natsLink) ReadKey(ctx context.Context, bucket, key string) ([]byte, bool, error) {
subject := DirectGetSubject(bucket, key)
before := l.conn.LastError()
msg, err := l.conn.RequestWithContext(ctx, subject, nil)
switch {
case errors.Is(err, nats.ErrNoResponders):
return nil, false, fmt.Errorf("%w: the bus has no bucket %s that answers a direct get", ErrCannotAsk, bucket)
case err != nil:
return nil, false, fmt.Errorf("%w: reading %s from %s: %v%s", ErrCannotAsk, key, bucket, err, l.refusal(subject, before))
}
if msg.Header != nil {
switch status := msg.Header.Get("Status"); status {
case "":
case "404":
return nil, false, nil
default:
return nil, false, fmt.Errorf("%w: reading %s from %s: the bus answered %s %s", ErrCannotAsk, key,
bucket, status, msg.Header.Get("Description"))
}
switch msg.Header.Get("KV-Operation") {
case "DEL", "PURGE":
return nil, false, nil
}
}
if len(msg.Data) == 0 {
return nil, false, nil
}
return msg.Data, true, nil
}
func (l *natsLink) Ping(ctx context.Context, service, id string) error { func (l *natsLink) Ping(ctx context.Context, service, id string) error {
subject := PingSubject(service, id) subject := PingSubject(service, id)
before := l.conn.LastError() before := l.conn.LastError()
+2 -62
View File
@@ -2,7 +2,6 @@ package link
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"os" "os"
"testing" "testing"
@@ -12,7 +11,7 @@ import (
) )
// What a witness asks, asked of a real server with the grants the mesh composes (novox/hq to-be 45 §8): // What a witness asks, asked of a real server with the grants the mesh composes (novox/hq to-be 45 §8):
// whether the direct get and the PING work as the two subjects named, and whether a host without them // whether the PING works as the subject named, and whether a host without them
// is told apart — "cannot ask", never "the build did not answer". See testdata/witness-grants.conf. // is told apart — "cannot ask", never "the build did not answer". See testdata/witness-grants.conf.
func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) { func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS_GRANTS") url := os.Getenv("MESH_TEST_NATS_GRANTS")
@@ -28,63 +27,14 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
t.Cleanup(conn.Close) t.Cleanup(conn.Close)
return conn return conn
} }
admin := dial("admin", "x", "_INBOX.admin")
js, err := admin.JetStream()
if err != nil {
t.Fatal(err)
}
_ = js.DeleteKeyValue("mesh-controller_lease")
kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_lease", TTL: 15 * time.Second, History: 1})
if err != nil {
t.Fatal(err)
}
anchor := &natsLink{conn: dial("node.anchor", "a", "_INBOX.node.anchor")} anchor := &natsLink{conn: dial("node.anchor", "a", "_INBOX.node.anchor")}
bare := &natsLink{conn: dial("node.bare", "b", "_INBOX.node.bare")} bare := &natsLink{conn: dial("node.bare", "b", "_INBOX.node.bare")}
ask := func() (context.Context, context.CancelFunc) { ask := func() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), 2*time.Second) return context.WithTimeout(context.Background(), 2*time.Second)
} }
// Nobody holds it.
ctx, cancel := ask()
if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found {
t.Fatalf("an empty lease read as %v, %v", found, err)
}
cancel()
// Held.
if _, err := kv.Put("holder", []byte(`{"instance":"controller@anchor pid 1 since now","epoch":3}`)); err != nil {
t.Fatal(err)
}
ctx, cancel = ask()
value, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder")
cancel()
if err != nil || !found {
t.Fatalf("a held lease read as %v, %v", found, err)
}
var holder struct {
Instance string `json:"instance"`
}
if json.Unmarshal(value, &holder); holder.Instance == "" {
t.Fatalf("the lease's value is %s", value)
}
// Given back.
if err := kv.Delete("holder"); err != nil {
t.Fatal(err)
}
ctx, cancel = ask()
if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found {
t.Fatalf("a lease given back read as %v, %v", found, err)
}
cancel()
// A host without the grant cannot ask — which is not "nobody holds it".
ctx, cancel = ask()
if _, _, err := bare.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) {
t.Fatalf("a host without the grant read the lease as %v, want it unable to ask", err)
}
cancel()
// No runtime on the bus: not there to answer. // No runtime on the bus: not there to answer.
ctx, cancel = ask() ctx, cancel := ask()
if err := anchor.Ping(ctx, "node-tools", "anchor"); !errors.Is(err, ErrNoAnswer) { if err := anchor.Ping(ctx, "node-tools", "anchor"); !errors.Is(err, ErrNoAnswer) {
t.Fatalf("a PING nobody serves is %v, want no answer", err) t.Fatalf("a PING nobody serves is %v, want no answer", err)
} }
@@ -112,14 +62,4 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
t.Fatalf("a host without the grant heard %v, want it unable to ask", err) t.Fatalf("a host without the grant heard %v, want it unable to ask", err)
} }
cancel() cancel()
// No lease bucket at all — a controller from before the lease: cannot ask, not "nobody holds it".
if err := js.DeleteKeyValue("mesh-controller_lease"); err != nil {
t.Fatal(err)
}
ctx, cancel = ask()
if _, _, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) {
t.Fatalf("a bus with no lease bucket read as %v, want unable to ask", err)
}
cancel()
} }
+12 -113
View File
@@ -1,155 +1,54 @@
// Package witness is the node-engine watching a core build it placed — the controller on the control // Package witness is the node-engine watching a core build it placed — the node tools on every machine — and
// node, the node tools on every machine — and restoring the build before it when the new one is not // restoring the build before it when the new one is not healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the
// healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the component being replaced is never the // component being replaced is never the only witness of its successor).
// only witness of its successor).
// //
// **The contract is this file.** What the engine reads to call a build healthy, from where, in what // **The contract is this file.** What the engine reads to call a build healthy, from where, in what
// shape, within which bound — said once, here, and held by contract_test.go. The controller's side // shape, within which bound — said once, here, and held by contract_test.go.
// writes what is read here (mesh-controller internal/lease Holder); a change on either side is a
// change to this file and its test.
package witness package witness
import ( import (
"encoding/json"
"fmt"
"strings"
"time" "time"
"github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/link"
) )
// The two witnesses, as a process declares which watches it (`witness`). A process that says none is // The witnesses, as a process declares which watches it (`witness`). A process that says none is placed as ever; one
// placed as ever; one that says nothing is judged by its name's default (Default). // that says nothing is judged by its name's default (Default).
const ( const (
// ByLease: healthy when the controller this machine started holds the controller's lease.
ByLease = "lease"
// ByPing: healthy when this machine's runtime answers the services protocol's PING. // ByPing: healthy when this machine's runtime answers the services protocol's PING.
ByPing = "ping" ByPing = "ping"
// ByNone: not judged. // ByNone: not judged.
ByNone = "none" ByNone = "none"
) )
// The controller's lease, as the host reads it (novox/hq ADR 0229): the key `holder` in the bucket
// `mesh-controller_lease`, whose keys live fifteen seconds unless renewed, renewed every five.
const (
LeaseBucket = "mesh-controller_lease"
LeaseKey = "holder"
// LeaseAge is the bucket's age for its key. A holder whose last renewal is older than this is
// not holding it, whatever the key still says.
LeaseAge = 15 * time.Second
// Skew is how far the controller's clock and this host's may disagree about when it took the
// lease. One machine, one clock, in practice: a margin, not a tolerance.
Skew = 2 * time.Second
)
// Bounds (to-be 45 §8). A build is given Within of time the witness could ask in; asked every Every. // Bounds (to-be 45 §8). A build is given Within of time the witness could ask in; asked every Every.
const ( const (
// ControllerWithin: the controller holds the lease within sixty seconds of starting.
ControllerWithin = 60 * time.Second
// NodeToolsWithin: the runtime is announced and answering within sixty seconds of starting, // NodeToolsWithin: the runtime is announced and answering within sixty seconds of starting,
// each PING answered within PingWithin. // each PING answered within PingWithin.
NodeToolsWithin = 60 * time.Second NodeToolsWithin = 60 * time.Second
PingWithin = 5 * time.Second PingWithin = 5 * time.Second
Every = 5 * time.Second Every = 5 * time.Second
// GiveUp is how long a witness goes on when it cannot ask at all before it says the build is // GiveUp is how long a witness goes on when it cannot ask at all before it says the build is
// unwitnessed: the grant missing, the bucket missing, the bus away the whole time. // unwitnessed: the grant missing, the bus away the whole time.
GiveUp = 30 * time.Minute GiveUp = 30 * time.Minute
) )
// ControllerLease is the lease's value as the controller writes it — mesh-controller internal/lease
// Holder, field for field by its JSON names. Only what the witness reads is required; a field the
// controller adds later is ignored here.
type ControllerLease struct {
// Instance names one controller process: "controller@<machine> pid <pid> since <RFC 3339>".
Instance string `json:"instance"`
// Host is the machine it runs on, as its own hostname says.
Host string `json:"host,omitempty"`
// Build is the controller's build as it knows it; not read here — the toolchain stamps no
// version, so it says "development build" — and a bundle's identity is its digest, which the
// host already knows.
Build string `json:"build,omitempty"`
Epoch uint64 `json:"epoch,omitempty"`
// Taken is when this instance took the key, Renewed when it last renewed it, by its own clock.
Taken time.Time `json:"taken"`
Renewed time.Time `json:"renewed"`
}
// ParseLease reads the key's value.
func ParseLease(value []byte) (ControllerLease, error) {
var l ControllerLease
if err := json.Unmarshal(value, &l); err != nil {
return ControllerLease{}, fmt.Errorf("the lease's holder is not the controller's lease value: %w", err)
}
return l, nil
}
// HeldBySince says whether the lease is held by a controller on machine `host` that took it at or
// after `since` — the controller this machine started then, and not the one before it — and is
// held now. Why says what it saw when it is not.
func (l ControllerLease) HeldBySince(host string, since, now time.Time) (bool, string) {
switch {
case l.Instance == "":
return false, "the lease names no holder"
case host != "" && !sameMachine(l.Host, host):
return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", l.Instance, l.Host)
case l.Taken.Before(since.Add(-Skew)):
return false, fmt.Sprintf("the lease is held by %s, taken %s — before the new build started at %s",
l.Instance, l.Taken.UTC().Format(time.RFC3339), since.UTC().Format(time.RFC3339))
case now.Sub(latest(l.Taken, l.Renewed)) > LeaseAge:
return false, fmt.Sprintf("the lease names %s and was last renewed %s ago, past its %s",
l.Instance, now.Sub(latest(l.Taken, l.Renewed)).Round(time.Second), LeaseAge)
}
return true, fmt.Sprintf("%s holds the lease, epoch %d", l.Instance, l.Epoch)
}
// sameMachine compares two hostnames as names, so a short name and its fully qualified form agree.
func sameMachine(a, b string) bool {
short := func(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
if i := strings.IndexByte(s, '.'); i > 0 {
s = s[:i]
}
return s
}
return short(a) == short(b)
}
func latest(a, b time.Time) time.Time {
if b.After(a) {
return b
}
return a
}
// NodeToolsService is the runtime's name on the services protocol, and its instance is this // NodeToolsService is the runtime's name on the services protocol, and its instance is this
// machine's node name: what `$SRV.PING.<service>.<node>` asks, so only this machine's runtime can // machine's node name: what `$SRV.PING.<service>.<node>` asks, so only this machine's runtime can
// answer (mesh-tools node-tools internal/runtime, announce.Service{Name: module, ID: node}). // answer (mesh-tools node-tools internal/runtime, announce.Service{Name: module, ID: node}).
const NodeToolsService = "node-tools" const NodeToolsService = "node-tools"
// Default is the witness a process is judged by when it names none: the two core processes the mesh // Default is the witness a process is judged by when it names none: the core process the mesh composes on every
// composes, by the names it composes them under, and nothing else. // machine, by the name it composes it under, and nothing else.
func Default(process string) string { func Default(process string) string {
switch process { if process == NodeToolsService {
case "mesh-controller":
return ByLease
case NodeToolsService:
return ByPing return ByPing
} }
return ByNone return ByNone
} }
// Within is the bound for a witness. // Within is the bound for a witness.
func Within(by string) time.Duration { func Within(string) time.Duration { return NodeToolsWithin }
if by == ByLease {
return ControllerWithin
}
return NodeToolsWithin
}
// Component names what a witness judges, as a rollback says it. // Component names what a witness judges, as a rollback says it.
func Component(by string) string { func Component(string) string { return link.ComponentNodeTools }
if by == ByLease {
return link.ComponentController
}
return link.ComponentNodeTools
}
+9 -84
View File
@@ -9,82 +9,7 @@ import (
"github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/link"
) )
// The contract, held. Each of these is a line the controller's side relies on or writes; a change on // The contract, held (novox/hq to-be 45 §8).
// either side changes this test (novox/hq to-be 45 §8).
// The lease is read where the controller keeps it (ADR 0229), on the one subject a host's grant names.
func TestTheLeaseIsReadWhereTheControllerKeepsIt(t *testing.T) {
if LeaseBucket != "mesh-controller_lease" || LeaseKey != "holder" {
t.Fatalf("the lease is read from %s/%s; the controller keeps it in mesh-controller_lease/holder",
LeaseBucket, LeaseKey)
}
if got := link.DirectGetSubject(LeaseBucket, LeaseKey); got != "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder" {
t.Fatalf("the host asks %s for the lease; its grant names exactly the direct get of the key", got)
}
if LeaseAge != 15*time.Second {
t.Fatalf("the lease's age is %s; the controller's bucket keeps a key fifteen seconds", LeaseAge)
}
}
// What the controller writes — mesh-controller internal/lease Holder, by its JSON names — is what is
// read. The value below is the shape that Holder marshals to.
func TestTheLeaseValueIsTheControllersHolder(t *testing.T) {
written := `{"instance":"controller@anchor pid 4242 since 2026-10-06T10:00:00Z","host":"anchor",` +
`"build":"development build","epoch":57,"taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:31Z"}`
l, err := ParseLease([]byte(written))
if err != nil {
t.Fatal(err)
}
if l.Instance == "" || l.Host != "anchor" || l.Epoch != 57 || l.Taken.IsZero() || l.Renewed.IsZero() {
t.Fatalf("the lease was not read whole: %+v", l)
}
// And a field the controller adds later does not stop it being read.
if _, err := ParseLease([]byte(`{"instance":"i","taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:01Z","bundle":"sha256:x"}`)); err != nil {
t.Fatalf("a lease value with a field this host does not know was refused: %v", err)
}
}
// Healthy is: held now, on this machine, by an instance that took it after the new build started.
func TestTheNewControllerHoldsTheLeaseOnlyWhenItTookItAfterItStarted(t *testing.T) {
started := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
now := started.Add(30 * time.Second)
held := ControllerLease{Instance: "controller@anchor pid 2 since …", Host: "anchor.example",
Taken: started.Add(3 * time.Second), Renewed: now.Add(-2 * time.Second), Epoch: 58}
for _, c := range []struct {
name string
l ControllerLease
want bool
says string
}{
{"held by the new instance", held, true, "holds the lease"},
{"nobody", ControllerLease{}, false, "no holder"},
{"the old instance, taken before the restart", func() ControllerLease {
l := held
l.Taken = started.Add(-time.Hour)
return l
}(), false, "before the new build started"},
{"another machine's controller", func() ControllerLease {
l := held
l.Host = "home-server"
return l
}(), false, "not this machine"},
{"taken and no longer renewed", func() ControllerLease {
l := held
l.Renewed = now.Add(-20 * time.Second)
return l
}(), false, "past its"},
{"taken within the clocks' skew of the start", func() ControllerLease {
l := held
l.Taken = started.Add(-time.Second)
return l
}(), true, "holds the lease"},
} {
got, why := c.l.HeldBySince("anchor", started, now)
if got != c.want || !strings.Contains(why, c.says) {
t.Errorf("%s: healthy %v (%s), want %v saying %q", c.name, got, why, c.want, c.says)
}
}
}
// The runtime is asked by its own name and this machine's: only this machine's runtime can answer. // The runtime is asked by its own name and this machine's: only this machine's runtime can answer.
func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) { func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) {
@@ -93,23 +18,23 @@ func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) {
} }
} }
// What is judged by default is the mesh's two core processes, by the names the controller composes // What is judged by default is the mesh's core process on every machine, by the name it is composed under, and
// them under, and nothing else. // nothing else: the controller ended (issue 1126), so its name is judged as any other.
func TestOnlyTheCoreProcessesAreJudgedByDefault(t *testing.T) { func TestOnlyTheCoreProcessIsJudgedByDefault(t *testing.T) {
for name, want := range map[string]string{"mesh-controller": ByLease, "node-tools": ByPing, "greeter": ByNone} { for name, want := range map[string]string{"mesh-controller": ByNone, "node-tools": ByPing, "greeter": ByNone} {
if got := Default(name); got != want { if got := Default(name); got != want {
t.Errorf("%s is judged by %q, want %q", name, got, want) t.Errorf("%s is judged by %q, want %q", name, got, want)
} }
} }
if ControllerWithin != 60*time.Second || NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second { if NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second {
t.Fatal("the bounds are to-be 45 §8's: the lease within sixty seconds, a PING answered within five") t.Fatal("the bounds are to-be 45 §8's: the runtime within sixty seconds, a PING answered within five")
} }
} }
// A verdict on the wire names its component as the controller's condition does. // A verdict on the wire names its component as the controller's condition does.
func TestAVerdictNamesItsComponent(t *testing.T) { func TestAVerdictNamesItsComponent(t *testing.T) {
raw, _ := json.Marshal(link.Rollback{Component: Component(ByLease), Outcome: link.RolledBack}) raw, _ := json.Marshal(link.Rollback{Component: Component(ByPing), Outcome: link.RolledBack})
if !strings.Contains(string(raw), `"component":"controller"`) || Component(ByPing) != "node-tools" { if !strings.Contains(string(raw), `"component":"node-tools"`) {
t.Fatalf("a verdict names its component as %s", raw) t.Fatalf("a verdict names its component as %s", raw)
} }
} }
+3 -3
View File
@@ -85,11 +85,11 @@ func retiredBuilds(t *testing.T, root, name string) []string {
// after its build was moved into a 0700 directory, and then deleted, could not run its own verbs. // after its build was moved into a 0700 directory, and then deleted, could not run its own verbs.
func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T) { func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T) {
root := t.TempDir() root := t.TempDir()
name := "mesh-controller" name := "node-tools"
start := time.Date(2026, 10, 7, 2, 0, 0, 0, time.UTC) start := time.Date(2026, 10, 7, 2, 0, 0, 0, time.UTC)
// Build A placed and running. // Build A placed and running.
p, err := Place(root, name, ByLease, buildA, "", "", start) p, err := Place(root, name, ByPing, buildA, "", "", start)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -99,7 +99,7 @@ func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T)
} }
// Build B placed while A still runs: A is moved aside, and reachable at its new path. // Build B placed while A still runs: A is moved aside, and reachable at its new path.
place(t, root, name, ByLease, buildB, "", "", start.Add(time.Minute)) place(t, root, name, ByPing, buildB, "", "", start.Add(time.Minute))
for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} { for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} {
info, err := os.Stat(dir) info, err := os.Stat(dir)
if err != nil { if err != nil {
+5 -20
View File
@@ -14,15 +14,14 @@ import (
// concluded (to-be 45 §8). // concluded (to-be 45 §8).
// //
// **Time counts only while it could ask.** A build's bound is spent only on looks that got an answer // **Time counts only while it could ask.** A build's bound is spent only on looks that got an answer
// from the bus — the lease read, or the PING delivered — so a machine whose own link is down, or a // from the bus — the PING delivered — so a machine whose own link is down, or a
// bus that refuses the question, never rolls a build back for the host's own trouble. A look that // bus that refuses the question, never rolls a build back for the host's own trouble. A look that
// could not ask counts towards GiveUp instead, and at GiveUp the build is said to be unwitnessed: // could not ask counts towards GiveUp instead, and at GiveUp the build is said to be unwitnessed:
// neither proved nor rolled back, and said. // neither proved nor rolled back, and said.
type Watcher struct { type Watcher struct {
Root string Root string
// Node is this machine's node name, the instance its runtime answers PING as; Host its hostname, // Node is this machine's node name, the instance its runtime answers PING as.
// as the controller's lease names its machine. Node string
Node, Host string
// Asker is the link open now, or nil. // Asker is the link open now, or nil.
Asker func() link.Asker Asker func() link.Asker
Run Runner Run Runner
@@ -69,7 +68,7 @@ func (w *Watcher) look(ctx context.Context, s State) {
every = Every every = Every
} }
now := w.now() now := w.now()
healthy, why, err := w.judge(ctx, s, now) healthy, why, err := w.judge(ctx, s)
w.hold(func() { w.hold(func() {
// Read again in turn: an apply may have placed another build meanwhile, which starts its own // Read again in turn: an apply may have placed another build meanwhile, which starts its own
@@ -123,7 +122,7 @@ func (w *Watcher) look(ctx context.Context, s State) {
} }
// judge asks once. An error is a look that could not ask; otherwise healthy, and why not when not. // judge asks once. An error is a look that could not ask; otherwise healthy, and why not when not.
func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, string, error) { func (w *Watcher) judge(ctx context.Context, s State) (bool, string, error) {
var asker link.Asker var asker link.Asker
if w.Asker != nil { if w.Asker != nil {
asker = w.Asker() asker = w.Asker()
@@ -134,20 +133,6 @@ func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, stri
asking, cancel := context.WithTimeout(ctx, PingWithin) asking, cancel := context.WithTimeout(ctx, PingWithin)
defer cancel() defer cancel()
switch s.Witness { switch s.Witness {
case ByLease:
value, found, err := asker.ReadKey(asking, LeaseBucket, LeaseKey)
if err != nil {
return false, "", err
}
if !found {
return false, "nobody holds the controller's lease", nil
}
lease, err := ParseLease(value)
if err != nil {
return false, err.Error(), nil
}
held, why := lease.HeldBySince(w.Host, s.Started, now)
return held, why, nil
case ByPing: case ByPing:
err := asker.Ping(asking, s.Process, w.Node) err := asker.Ping(asking, s.Process, w.Node)
switch { switch {
+47 -86
View File
@@ -13,8 +13,8 @@ import (
"github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/link"
) )
// Each rollback path, induced against a real directory: a controller that never takes the lease, a // Each rollback path, induced against a real directory: a runtime that never answers PING, a build
// runtime that never answers PING, a build declared not reversible; and a healthy update, which // declared not reversible; and a healthy update, which
// deletes nothing before it is proved and the build before it once it is (novox/hq to-be 45 §8). // deletes nothing before it is proved and the build before it once it is (novox/hq to-be 45 §8).
const ( const (
@@ -62,18 +62,10 @@ func kept(root, name string) bool {
// asker answers as the build it is told is running would. // asker answers as the build it is told is running would.
type asker struct { type asker struct {
mu sync.Mutex mu sync.Mutex
lease func() ([]byte, bool, error)
ping func() error ping func() error
asked int asked int
} }
func (a *asker) ReadKey(context.Context, string, string) ([]byte, bool, error) {
a.mu.Lock()
defer a.mu.Unlock()
a.asked++
return a.lease()
}
func (a *asker) Ping(context.Context, string, string) error { func (a *asker) Ping(context.Context, string, string) error {
a.mu.Lock() a.mu.Lock()
defer a.mu.Unlock() defer a.mu.Unlock()
@@ -95,7 +87,7 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
} }
func watcher(root string, a link.Asker, m *machine, clock *time.Time, verdicts *[]link.Rollback) *Watcher { func watcher(root string, a link.Asker, m *machine, clock *time.Time, verdicts *[]link.Rollback) *Watcher {
return &Watcher{Root: root, Node: "anchor", Host: "anchor", return &Watcher{Root: root, Node: "anchor",
Asker: func() link.Asker { Asker: func() link.Asker {
if a == nil { if a == nil {
return nil return nil
@@ -116,45 +108,37 @@ func look(w *Watcher, clock *time.Time, n int) {
} }
} }
func leaseHeldBy(instance string, taken time.Time, clock *time.Time) func() ([]byte, bool, error) { // A runtime that starts and never answers PING: restored to the build before, once, said once.
return func() ([]byte, bool, error) { func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) {
return []byte(fmt.Sprintf(`{"instance":%q,"host":"anchor","epoch":9,"taken":%q,"renewed":%q}`,
instance, taken.Format(time.RFC3339Nano), clock.Add(-time.Second).Format(time.RFC3339Nano))), true, nil
}
}
// A controller that starts and never takes the lease: restored to the build before, once, said once.
func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) {
root := t.TempDir() root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock) place(t, root, "node-tools", ByPing, buildA, "", "", clock)
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock) place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
if running(t, root, "mesh-controller") != buildB || !kept(root, "mesh-controller") { if running(t, root, "node-tools") != buildB || !kept(root, "node-tools") {
t.Fatal("the new controller is not running with the build before it kept beside it") t.Fatal("the new runtime is not running with the build before it kept beside it")
} }
// The old instance's lease, taken an hour ago, is all the bus ever shows. a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }}
a := &asker{lease: leaseHeldBy("controller@anchor pid 1 since earlier", clock.Add(-time.Hour), &clock)}
m := &machine{} m := &machine{}
var verdicts []link.Rollback var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts) w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, int(ControllerWithin/Every)-1) look(w, &clock, int(NodeToolsWithin/Every)-1)
if len(verdicts) != 0 || running(t, root, "mesh-controller") != buildB { if len(verdicts) != 0 || running(t, root, "node-tools") != buildB {
t.Fatalf("rolled back before its bound: %+v", verdicts) t.Fatalf("rolled back before its bound: %+v", verdicts)
} }
look(w, &clock, 1) look(w, &clock, 1)
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].From != buildB || verdicts[0].To != buildA || if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].From != buildB || verdicts[0].To != buildA ||
verdicts[0].Component != link.ComponentController { verdicts[0].Component != link.ComponentNodeTools {
t.Fatalf("the verdict is %+v, want controller rolled back from B to A", verdicts) t.Fatalf("the verdict is %+v, want node-tools rolled back from B to A", verdicts)
} }
if running(t, root, "mesh-controller") != buildA { if running(t, root, "node-tools") != buildA {
t.Fatalf("the controller running is %s, want the build before", running(t, root, "mesh-controller")) t.Fatalf("the runtime running is %s, want the build before", running(t, root, "node-tools"))
} }
if strings.Join(m.commands, "; ") != "systemctl stop mesh-controller.service; systemctl restart mesh-controller.service" { if strings.Join(m.commands, "; ") != "systemctl stop node-tools.service; systemctl restart node-tools.service" {
t.Fatalf("the machine was asked %v", m.commands) t.Fatalf("the machine was asked %v", m.commands)
} }
if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "before the new build started") { if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "is on the bus") {
t.Fatalf("the verdict does not say why: %s", verdicts[0].Why) t.Fatalf("the verdict does not say why: %s", verdicts[0].Why)
} }
@@ -168,81 +152,58 @@ func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) {
t.Fatalf("what every report says is %+v", s) t.Fatalf("what every report says is %+v", s)
} }
// The mesh still declares B: refused, A kept running, the verdict still standing. // The mesh still declares B: refused, A kept running, the verdict still standing.
p := place(t, root, "mesh-controller", ByLease, buildB, buildB, "", clock) p := place(t, root, "node-tools", ByPing, buildB, buildB, "", clock)
if p.Unpack || running(t, root, "mesh-controller") != buildA || !strings.Contains(p.Detail, "rolled back") { if p.Unpack || running(t, root, "node-tools") != buildA || !strings.Contains(p.Detail, "rolled back") {
t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "mesh-controller")) t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "node-tools"))
} }
if len(Standing(root)) != 1 || len(Trials(root)) != 0 { if len(Standing(root)) != 1 || len(Trials(root)) != 0 {
t.Fatal("refusing the rolled-back build changed what stands or started a trial") t.Fatal("refusing the rolled-back build changed what stands or started a trial")
} }
// A newer build is a new trial, and ends what was concluded once it is proved. // A newer build is a new trial, and ends what was concluded once it is proved.
place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock) place(t, root, "node-tools", ByPing, buildC, buildB, "", clock)
if running(t, root, "mesh-controller") != buildC || len(Trials(root)) != 1 { if running(t, root, "node-tools") != buildC || len(Trials(root)) != 1 {
t.Fatal("a newer build after a rollback was not placed on trial") t.Fatal("a newer build after a rollback was not placed on trial")
} }
if err := Proved(root, "mesh-controller"); err != nil { if err := Proved(root, "node-tools"); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if len(Standing(root)) != 0 || kept(root, "mesh-controller") { if len(Standing(root)) != 0 || kept(root, "node-tools") {
t.Fatal("a newer build proved and the rollback still stands, or the build before it is still kept") t.Fatal("a newer build proved and the rollback still stands, or the build before it is still kept")
} }
} }
// A runtime that never answers PING: restored, once.
func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) {
root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }}
m := &machine{}
var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, int(NodeToolsWithin/Every)+20)
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].Component != link.ComponentNodeTools {
t.Fatalf("the verdict is %+v, want node-tools rolled back once", verdicts)
}
if running(t, root, "node-tools") != buildA || kept(root, "node-tools") {
t.Fatal("the runtime running is not the build before, or a copy of it is still kept")
}
}
// A healthy update: nothing deleted before it is proved; the build before it deleted once it is. // A healthy update: nothing deleted before it is proved; the build before it deleted once it is.
func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) { func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
root := t.TempDir() root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
// The controller placed by a host from before any witness: a record, and no state. // The runtime placed by a host from before any witness: a record, and no state.
if err := os.MkdirAll(filepath.Join(root, "mesh-controller"), 0o755); err != nil { if err := os.MkdirAll(filepath.Join(root, "node-tools"), 0o755); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := os.WriteFile(filepath.Join(root, "mesh-controller", "build"), []byte(buildA), 0o644); err != nil { if err := os.WriteFile(filepath.Join(root, "node-tools", "build"), []byte(buildA), 0o644); err != nil {
t.Fatal(err) t.Fatal(err)
} }
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock) place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
started := clock
a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }} a := &asker{ping: func() error { return link.ErrNoAnswer }}
m := &machine{} m := &machine{}
var verdicts []link.Rollback var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts) w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, 5) look(w, &clock, 5)
if !kept(root, "mesh-controller") { if !kept(root, "node-tools") {
t.Fatal("the build before was deleted while the new one was still on trial") t.Fatal("the build before was deleted while the new one was still on trial")
} }
// A third build while the second is on trial: the one kept is still the one seen healthy. // A third build while the second is on trial: the one kept is still the one seen healthy.
place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock) place(t, root, "node-tools", ByPing, buildC, buildB, "", clock)
if s, _ := Load(root, "mesh-controller"); s.Previous != buildA { if s, _ := Load(root, "node-tools"); s.Previous != buildA {
t.Fatalf("the build kept to go back to is %s, want A — the last one seen healthy", s.Previous) t.Fatalf("the build kept to go back to is %s, want A — the last one seen healthy", s.Previous)
} }
started = clock a.ping = func() error { return nil }
a.lease = leaseHeldBy("controller@anchor pid 3 since now", started.Add(4*time.Second), &clock)
look(w, &clock, 2) look(w, &clock, 2)
if len(verdicts) != 0 || len(m.commands) != 0 { if len(verdicts) != 0 || len(m.commands) != 0 {
t.Fatalf("a healthy update was judged or acted on: %+v %v", verdicts, m.commands) t.Fatalf("a healthy update was judged or acted on: %+v %v", verdicts, m.commands)
} }
if kept(root, "mesh-controller") || len(Trials(root)) != 0 || running(t, root, "mesh-controller") != buildC { if kept(root, "node-tools") || len(Trials(root)) != 0 || running(t, root, "node-tools") != buildC {
t.Fatal("the build before was not deleted once the new one was proved") t.Fatal("the build before was not deleted once the new one was proved")
} }
} }
@@ -252,14 +213,14 @@ func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) { func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
root := t.TempDir() root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock) place(t, root, "node-tools", ByPing, buildA, "", "", clock)
place(t, root, "mesh-controller", ByLease, buildB, buildA, "migration 0073 cannot be undone", clock) place(t, root, "node-tools", ByPing, buildB, buildA, "migration 0073 cannot be undone", clock)
a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }} a := &asker{ping: func() error { return link.ErrNoAnswer }}
m := &machine{} m := &machine{}
var verdicts []link.Rollback var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts) w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, int(ControllerWithin/Every)+20) look(w, &clock, int(NodeToolsWithin/Every)+20)
if len(verdicts) != 1 || verdicts[0].Outcome != link.NotReversible || verdicts[0].To != "" { if len(verdicts) != 1 || verdicts[0].Outcome != link.NotReversible || verdicts[0].To != "" {
t.Fatalf("the verdict is %+v, want not-reversible with nothing restored", verdicts) t.Fatalf("the verdict is %+v, want not-reversible with nothing restored", verdicts)
@@ -267,9 +228,9 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
if !strings.Contains(verdicts[0].Why, "migration 0073 cannot be undone") { if !strings.Contains(verdicts[0].Why, "migration 0073 cannot be undone") {
t.Fatalf("the verdict does not say why it may not be rolled back: %s", verdicts[0].Why) t.Fatalf("the verdict does not say why it may not be rolled back: %s", verdicts[0].Why)
} }
if len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB { if len(m.commands) != 0 || running(t, root, "node-tools") != buildB {
t.Fatalf("the build before was started against the newer data: %v, running %s", m.commands, t.Fatalf("the build before was started against the newer data: %v, running %s", m.commands,
running(t, root, "mesh-controller")) running(t, root, "node-tools"))
} }
if len(Standing(root)) != 1 { if len(Standing(root)) != 1 {
t.Fatal("the not-reversible verdict does not stand") t.Fatal("the not-reversible verdict does not stand")
@@ -280,24 +241,24 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
func TestAWitnessThatCannotAskNeverRollsBack(t *testing.T) { func TestAWitnessThatCannotAskNeverRollsBack(t *testing.T) {
root := t.TempDir() root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock) place(t, root, "node-tools", ByPing, buildA, "", "", clock)
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock) place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
a := &asker{lease: func() ([]byte, bool, error) { a := &asker{ping: func() error {
return nil, false, fmt.Errorf("%w: this host's grant does not name the lease", link.ErrCannotAsk) return fmt.Errorf("%w: this host's grant does not name the PING", link.ErrCannotAsk)
}} }}
m := &machine{} m := &machine{}
var verdicts []link.Rollback var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts) w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, int(GiveUp/Every)-1) look(w, &clock, int(GiveUp/Every)-1)
if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB { if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "node-tools") != buildB {
t.Fatalf("a build was judged on questions that were never asked: %+v %v", verdicts, m.commands) t.Fatalf("a build was judged on questions that were never asked: %+v %v", verdicts, m.commands)
} }
look(w, &clock, 1) look(w, &clock, 1)
if len(verdicts) != 1 || verdicts[0].Outcome != link.Unwitnessed || len(m.commands) != 0 { if len(verdicts) != 1 || verdicts[0].Outcome != link.Unwitnessed || len(m.commands) != 0 {
t.Fatalf("the verdict at GiveUp is %+v, want unwitnessed with nothing done", verdicts) t.Fatalf("the verdict at GiveUp is %+v, want unwitnessed with nothing done", verdicts)
} }
if !kept(root, "mesh-controller") { if !kept(root, "node-tools") {
t.Fatal("the build before was deleted though the new one was never seen healthy") t.Fatal("the build before was deleted though the new one was never seen healthy")
} }