Witness the node tools alone: the controller's lease names no build any more
The controller ended by its unassignment (issue 1126, ADR 0590), so no build is judged by the lease: the lease witness, its contract and the direct get the witness asked it with are removed, and its tests now prove each rollback path on the node tools' PING. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
@@ -146,9 +146,8 @@ func launcherReplaced(pid int, proc, path string) bool {
|
||||
|
||||
// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
|
||||
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) {
|
||||
host, _ := os.Hostname()
|
||||
w := &witness.Watcher{
|
||||
Root: apply.DaemonRoot(), Node: node, Host: host,
|
||||
Root: apply.DaemonRoot(), Node: node,
|
||||
Asker: func() link.Asker { return queue.Asker() },
|
||||
Run: witness.Runner(apply.ExecRunner),
|
||||
Hold: func(f func()) {
|
||||
|
||||
@@ -488,9 +488,8 @@ const WitnessContract = 1
|
||||
|
||||
// The core components a witness judges (to-be 45 §8), as a rollback names them.
|
||||
const (
|
||||
ComponentEngine = "node-engine"
|
||||
ComponentController = "controller"
|
||||
ComponentNodeTools = "node-tools"
|
||||
ComponentEngine = "node-engine"
|
||||
ComponentNodeTools = "node-tools"
|
||||
)
|
||||
|
||||
// What a witness concluded, as a rollback says it.
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
|
||||
[]string{"protocol", "address", "port", "by", "published", "container-port"}},
|
||||
// novox/hq to-be 45 §8: a witness's verdicts, said on every report while they stand, and the
|
||||
// witness contract this host keeps.
|
||||
{Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentController}}, Witness: WitnessContract},
|
||||
{Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentNodeTools}}, Witness: WitnessContract},
|
||||
[]string{"node", "rollbacks", "witness"}},
|
||||
{Rollback{Component: ComponentNodeTools, From: "sha256:b", To: "sha256:a", Outcome: RolledBack, Why: "w"},
|
||||
[]string{"component", "from", "to", "outcome", "why", "at"}},
|
||||
|
||||
+3
-5
@@ -3,16 +3,15 @@
|
||||
# docker run -d --rm --name w -p 14224:4222 -v $PWD/internal/link/testdata:/c:ro nats:2.11-alpine -js -c /c/witness-grants.conf
|
||||
# MESH_TEST_NATS_GRANTS=nats://127.0.0.1:14224 go test ./internal/link/ -run TestNatsWitness
|
||||
#
|
||||
# `node.anchor` is a machine's host with exactly the two subjects the witness asks added to what a
|
||||
# host already has (its inbox); `node.bare` is a host without them. `runtime` stands in for the tool
|
||||
# runtime, `admin` for the controller writing the lease.
|
||||
# `node.anchor` is a machine's host with exactly the subject the witness asks added to what a host
|
||||
# already has (its inbox); `node.bare` is a host without it. `runtime` stands in for the tool runtime.
|
||||
jetstream: enabled
|
||||
accounts {
|
||||
MESH {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "node.anchor", password: "a", permissions: {
|
||||
publish: { allow: ["$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder", "$SRV.PING.node-tools.anchor"] }
|
||||
publish: { allow: ["$SRV.PING.node-tools.anchor"] }
|
||||
subscribe: { allow: ["_INBOX.node.anchor.>"] }
|
||||
} }
|
||||
{ user: "node.bare", password: "b", permissions: {
|
||||
@@ -23,7 +22,6 @@ accounts {
|
||||
subscribe: { allow: ["$SRV.PING.node-tools.>"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "admin", password: "x" }
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,14 +10,11 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the controller and the
|
||||
// node tools it placed on this machine, by what the bus says about them rather than by what they
|
||||
// say about themselves.
|
||||
// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the node tools it placed on this
|
||||
// machine, by what the bus says about them rather than by what they say about themselves.
|
||||
//
|
||||
// **Two questions, both read-only, both on the host's own link.** The controller's lease key, read
|
||||
// by JetStream's direct get — one subject, no stream information, nothing written; and this
|
||||
// machine's tool runtime, asked the NATS services protocol's PING by its name and this machine's —
|
||||
// so only this machine's runtime can answer it.
|
||||
// **One question, read-only, on the host's own link**: this machine's tool runtime, asked the NATS services protocol's
|
||||
// PING by its name and this machine's, so only this machine's runtime can answer it.
|
||||
|
||||
// ErrCannotAsk is a question the bus did not let this host put, or did not answer: no grant for it,
|
||||
// no such bucket, the bus slow. **It says nothing about the build being judged**, so a witness
|
||||
@@ -30,20 +27,11 @@ var ErrNoAnswer = errors.New("nothing answered")
|
||||
|
||||
// Asker is what a witness asks through. The link open now implements it; nil while there is none.
|
||||
type Asker interface {
|
||||
// ReadKey is a key-value bucket's current value for a key. Found false is nobody holding it —
|
||||
// absent, deleted or expired; an error wrapping ErrCannotAsk is no reading at all.
|
||||
ReadKey(ctx context.Context, bucket, key string) (value []byte, found bool, err error)
|
||||
// Ping asks the service `service`'s instance `id` whether it is there. Nil is an answer;
|
||||
// ErrNoAnswer is none in time; ErrCannotAsk is no asking.
|
||||
Ping(ctx context.Context, service, id string) error
|
||||
}
|
||||
|
||||
// DirectGetSubject is the one subject a host asks a bucket's key on: JetStream's direct get of the
|
||||
// bucket's stream, for the key's own subject. What a host's grant names exactly — no wildcard.
|
||||
func DirectGetSubject(bucket, key string) string {
|
||||
return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + key
|
||||
}
|
||||
|
||||
// PingSubject is the services protocol's PING of one instance of one service.
|
||||
func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id }
|
||||
|
||||
@@ -58,36 +46,6 @@ func (q *Session) Asker() Asker {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *natsLink) ReadKey(ctx context.Context, bucket, key string) ([]byte, bool, error) {
|
||||
subject := DirectGetSubject(bucket, key)
|
||||
before := l.conn.LastError()
|
||||
msg, err := l.conn.RequestWithContext(ctx, subject, nil)
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrNoResponders):
|
||||
return nil, false, fmt.Errorf("%w: the bus has no bucket %s that answers a direct get", ErrCannotAsk, bucket)
|
||||
case err != nil:
|
||||
return nil, false, fmt.Errorf("%w: reading %s from %s: %v%s", ErrCannotAsk, key, bucket, err, l.refusal(subject, before))
|
||||
}
|
||||
if msg.Header != nil {
|
||||
switch status := msg.Header.Get("Status"); status {
|
||||
case "":
|
||||
case "404":
|
||||
return nil, false, nil
|
||||
default:
|
||||
return nil, false, fmt.Errorf("%w: reading %s from %s: the bus answered %s %s", ErrCannotAsk, key,
|
||||
bucket, status, msg.Header.Get("Description"))
|
||||
}
|
||||
switch msg.Header.Get("KV-Operation") {
|
||||
case "DEL", "PURGE":
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
if len(msg.Data) == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
return msg.Data, true, nil
|
||||
}
|
||||
|
||||
func (l *natsLink) Ping(ctx context.Context, service, id string) error {
|
||||
subject := PingSubject(service, id)
|
||||
before := l.conn.LastError()
|
||||
|
||||
@@ -2,7 +2,6 @@ package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
@@ -12,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
// What a witness asks, asked of a real server with the grants the mesh composes (novox/hq to-be 45 §8):
|
||||
// whether the direct get and the PING work as the two subjects named, and whether a host without them
|
||||
// whether the PING works as the subject named, and whether a host without them
|
||||
// is told apart — "cannot ask", never "the build did not answer". See testdata/witness-grants.conf.
|
||||
func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS_GRANTS")
|
||||
@@ -28,63 +27,14 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
|
||||
t.Cleanup(conn.Close)
|
||||
return conn
|
||||
}
|
||||
admin := dial("admin", "x", "_INBOX.admin")
|
||||
js, err := admin.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = js.DeleteKeyValue("mesh-controller_lease")
|
||||
kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_lease", TTL: 15 * time.Second, History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
anchor := &natsLink{conn: dial("node.anchor", "a", "_INBOX.node.anchor")}
|
||||
bare := &natsLink{conn: dial("node.bare", "b", "_INBOX.node.bare")}
|
||||
ask := func() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), 2*time.Second)
|
||||
}
|
||||
|
||||
// Nobody holds it.
|
||||
ctx, cancel := ask()
|
||||
if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found {
|
||||
t.Fatalf("an empty lease read as %v, %v", found, err)
|
||||
}
|
||||
cancel()
|
||||
// Held.
|
||||
if _, err := kv.Put("holder", []byte(`{"instance":"controller@anchor pid 1 since now","epoch":3}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel = ask()
|
||||
value, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder")
|
||||
cancel()
|
||||
if err != nil || !found {
|
||||
t.Fatalf("a held lease read as %v, %v", found, err)
|
||||
}
|
||||
var holder struct {
|
||||
Instance string `json:"instance"`
|
||||
}
|
||||
if json.Unmarshal(value, &holder); holder.Instance == "" {
|
||||
t.Fatalf("the lease's value is %s", value)
|
||||
}
|
||||
// Given back.
|
||||
if err := kv.Delete("holder"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel = ask()
|
||||
if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found {
|
||||
t.Fatalf("a lease given back read as %v, %v", found, err)
|
||||
}
|
||||
cancel()
|
||||
// A host without the grant cannot ask — which is not "nobody holds it".
|
||||
ctx, cancel = ask()
|
||||
if _, _, err := bare.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) {
|
||||
t.Fatalf("a host without the grant read the lease as %v, want it unable to ask", err)
|
||||
}
|
||||
cancel()
|
||||
|
||||
// No runtime on the bus: not there to answer.
|
||||
ctx, cancel = ask()
|
||||
ctx, cancel := ask()
|
||||
if err := anchor.Ping(ctx, "node-tools", "anchor"); !errors.Is(err, ErrNoAnswer) {
|
||||
t.Fatalf("a PING nobody serves is %v, want no answer", err)
|
||||
}
|
||||
@@ -112,14 +62,4 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
|
||||
t.Fatalf("a host without the grant heard %v, want it unable to ask", err)
|
||||
}
|
||||
cancel()
|
||||
|
||||
// No lease bucket at all — a controller from before the lease: cannot ask, not "nobody holds it".
|
||||
if err := js.DeleteKeyValue("mesh-controller_lease"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel = ask()
|
||||
if _, _, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) {
|
||||
t.Fatalf("a bus with no lease bucket read as %v, want unable to ask", err)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
|
||||
+12
-113
@@ -1,155 +1,54 @@
|
||||
// Package witness is the node-engine watching a core build it placed — the controller on the control
|
||||
// node, the node tools on every machine — and restoring the build before it when the new one is not
|
||||
// healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the component being replaced is never the
|
||||
// only witness of its successor).
|
||||
// Package witness is the node-engine watching a core build it placed — the node tools on every machine — and
|
||||
// restoring the build before it when the new one is not healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the
|
||||
// component being replaced is never the only witness of its successor).
|
||||
//
|
||||
// **The contract is this file.** What the engine reads to call a build healthy, from where, in what
|
||||
// shape, within which bound — said once, here, and held by contract_test.go. The controller's side
|
||||
// writes what is read here (mesh-controller internal/lease Holder); a change on either side is a
|
||||
// change to this file and its test.
|
||||
// shape, within which bound — said once, here, and held by contract_test.go.
|
||||
package witness
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// The two witnesses, as a process declares which watches it (`witness`). A process that says none is
|
||||
// placed as ever; one that says nothing is judged by its name's default (Default).
|
||||
// The witnesses, as a process declares which watches it (`witness`). A process that says none is placed as ever; one
|
||||
// that says nothing is judged by its name's default (Default).
|
||||
const (
|
||||
// ByLease: healthy when the controller this machine started holds the controller's lease.
|
||||
ByLease = "lease"
|
||||
// ByPing: healthy when this machine's runtime answers the services protocol's PING.
|
||||
ByPing = "ping"
|
||||
// ByNone: not judged.
|
||||
ByNone = "none"
|
||||
)
|
||||
|
||||
// The controller's lease, as the host reads it (novox/hq ADR 0229): the key `holder` in the bucket
|
||||
// `mesh-controller_lease`, whose keys live fifteen seconds unless renewed, renewed every five.
|
||||
const (
|
||||
LeaseBucket = "mesh-controller_lease"
|
||||
LeaseKey = "holder"
|
||||
// LeaseAge is the bucket's age for its key. A holder whose last renewal is older than this is
|
||||
// not holding it, whatever the key still says.
|
||||
LeaseAge = 15 * time.Second
|
||||
// Skew is how far the controller's clock and this host's may disagree about when it took the
|
||||
// lease. One machine, one clock, in practice: a margin, not a tolerance.
|
||||
Skew = 2 * time.Second
|
||||
)
|
||||
|
||||
// Bounds (to-be 45 §8). A build is given Within of time the witness could ask in; asked every Every.
|
||||
const (
|
||||
// ControllerWithin: the controller holds the lease within sixty seconds of starting.
|
||||
ControllerWithin = 60 * time.Second
|
||||
// NodeToolsWithin: the runtime is announced and answering within sixty seconds of starting,
|
||||
// each PING answered within PingWithin.
|
||||
NodeToolsWithin = 60 * time.Second
|
||||
PingWithin = 5 * time.Second
|
||||
Every = 5 * time.Second
|
||||
// GiveUp is how long a witness goes on when it cannot ask at all before it says the build is
|
||||
// unwitnessed: the grant missing, the bucket missing, the bus away the whole time.
|
||||
// unwitnessed: the grant missing, the bus away the whole time.
|
||||
GiveUp = 30 * time.Minute
|
||||
)
|
||||
|
||||
// ControllerLease is the lease's value as the controller writes it — mesh-controller internal/lease
|
||||
// Holder, field for field by its JSON names. Only what the witness reads is required; a field the
|
||||
// controller adds later is ignored here.
|
||||
type ControllerLease struct {
|
||||
// Instance names one controller process: "controller@<machine> pid <pid> since <RFC 3339>".
|
||||
Instance string `json:"instance"`
|
||||
// Host is the machine it runs on, as its own hostname says.
|
||||
Host string `json:"host,omitempty"`
|
||||
// Build is the controller's build as it knows it; not read here — the toolchain stamps no
|
||||
// version, so it says "development build" — and a bundle's identity is its digest, which the
|
||||
// host already knows.
|
||||
Build string `json:"build,omitempty"`
|
||||
Epoch uint64 `json:"epoch,omitempty"`
|
||||
// Taken is when this instance took the key, Renewed when it last renewed it, by its own clock.
|
||||
Taken time.Time `json:"taken"`
|
||||
Renewed time.Time `json:"renewed"`
|
||||
}
|
||||
|
||||
// ParseLease reads the key's value.
|
||||
func ParseLease(value []byte) (ControllerLease, error) {
|
||||
var l ControllerLease
|
||||
if err := json.Unmarshal(value, &l); err != nil {
|
||||
return ControllerLease{}, fmt.Errorf("the lease's holder is not the controller's lease value: %w", err)
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// HeldBySince says whether the lease is held by a controller on machine `host` that took it at or
|
||||
// after `since` — the controller this machine started then, and not the one before it — and is
|
||||
// held now. Why says what it saw when it is not.
|
||||
func (l ControllerLease) HeldBySince(host string, since, now time.Time) (bool, string) {
|
||||
switch {
|
||||
case l.Instance == "":
|
||||
return false, "the lease names no holder"
|
||||
case host != "" && !sameMachine(l.Host, host):
|
||||
return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", l.Instance, l.Host)
|
||||
case l.Taken.Before(since.Add(-Skew)):
|
||||
return false, fmt.Sprintf("the lease is held by %s, taken %s — before the new build started at %s",
|
||||
l.Instance, l.Taken.UTC().Format(time.RFC3339), since.UTC().Format(time.RFC3339))
|
||||
case now.Sub(latest(l.Taken, l.Renewed)) > LeaseAge:
|
||||
return false, fmt.Sprintf("the lease names %s and was last renewed %s ago, past its %s",
|
||||
l.Instance, now.Sub(latest(l.Taken, l.Renewed)).Round(time.Second), LeaseAge)
|
||||
}
|
||||
return true, fmt.Sprintf("%s holds the lease, epoch %d", l.Instance, l.Epoch)
|
||||
}
|
||||
|
||||
// sameMachine compares two hostnames as names, so a short name and its fully qualified form agree.
|
||||
func sameMachine(a, b string) bool {
|
||||
short := func(s string) string {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
if i := strings.IndexByte(s, '.'); i > 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
return s
|
||||
}
|
||||
return short(a) == short(b)
|
||||
}
|
||||
|
||||
func latest(a, b time.Time) time.Time {
|
||||
if b.After(a) {
|
||||
return b
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// NodeToolsService is the runtime's name on the services protocol, and its instance is this
|
||||
// machine's node name: what `$SRV.PING.<service>.<node>` asks, so only this machine's runtime can
|
||||
// answer (mesh-tools node-tools internal/runtime, announce.Service{Name: module, ID: node}).
|
||||
const NodeToolsService = "node-tools"
|
||||
|
||||
// Default is the witness a process is judged by when it names none: the two core processes the mesh
|
||||
// composes, by the names it composes them under, and nothing else.
|
||||
// Default is the witness a process is judged by when it names none: the core process the mesh composes on every
|
||||
// machine, by the name it composes it under, and nothing else.
|
||||
func Default(process string) string {
|
||||
switch process {
|
||||
case "mesh-controller":
|
||||
return ByLease
|
||||
case NodeToolsService:
|
||||
if process == NodeToolsService {
|
||||
return ByPing
|
||||
}
|
||||
return ByNone
|
||||
}
|
||||
|
||||
// Within is the bound for a witness.
|
||||
func Within(by string) time.Duration {
|
||||
if by == ByLease {
|
||||
return ControllerWithin
|
||||
}
|
||||
return NodeToolsWithin
|
||||
}
|
||||
func Within(string) time.Duration { return NodeToolsWithin }
|
||||
|
||||
// Component names what a witness judges, as a rollback says it.
|
||||
func Component(by string) string {
|
||||
if by == ByLease {
|
||||
return link.ComponentController
|
||||
}
|
||||
return link.ComponentNodeTools
|
||||
}
|
||||
func Component(string) string { return link.ComponentNodeTools }
|
||||
|
||||
@@ -9,82 +9,7 @@ import (
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// The contract, held. Each of these is a line the controller's side relies on or writes; a change on
|
||||
// either side changes this test (novox/hq to-be 45 §8).
|
||||
|
||||
// The lease is read where the controller keeps it (ADR 0229), on the one subject a host's grant names.
|
||||
func TestTheLeaseIsReadWhereTheControllerKeepsIt(t *testing.T) {
|
||||
if LeaseBucket != "mesh-controller_lease" || LeaseKey != "holder" {
|
||||
t.Fatalf("the lease is read from %s/%s; the controller keeps it in mesh-controller_lease/holder",
|
||||
LeaseBucket, LeaseKey)
|
||||
}
|
||||
if got := link.DirectGetSubject(LeaseBucket, LeaseKey); got != "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder" {
|
||||
t.Fatalf("the host asks %s for the lease; its grant names exactly the direct get of the key", got)
|
||||
}
|
||||
if LeaseAge != 15*time.Second {
|
||||
t.Fatalf("the lease's age is %s; the controller's bucket keeps a key fifteen seconds", LeaseAge)
|
||||
}
|
||||
}
|
||||
|
||||
// What the controller writes — mesh-controller internal/lease Holder, by its JSON names — is what is
|
||||
// read. The value below is the shape that Holder marshals to.
|
||||
func TestTheLeaseValueIsTheControllersHolder(t *testing.T) {
|
||||
written := `{"instance":"controller@anchor pid 4242 since 2026-10-06T10:00:00Z","host":"anchor",` +
|
||||
`"build":"development build","epoch":57,"taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:31Z"}`
|
||||
l, err := ParseLease([]byte(written))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if l.Instance == "" || l.Host != "anchor" || l.Epoch != 57 || l.Taken.IsZero() || l.Renewed.IsZero() {
|
||||
t.Fatalf("the lease was not read whole: %+v", l)
|
||||
}
|
||||
// And a field the controller adds later does not stop it being read.
|
||||
if _, err := ParseLease([]byte(`{"instance":"i","taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:01Z","bundle":"sha256:x"}`)); err != nil {
|
||||
t.Fatalf("a lease value with a field this host does not know was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Healthy is: held now, on this machine, by an instance that took it after the new build started.
|
||||
func TestTheNewControllerHoldsTheLeaseOnlyWhenItTookItAfterItStarted(t *testing.T) {
|
||||
started := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||
now := started.Add(30 * time.Second)
|
||||
held := ControllerLease{Instance: "controller@anchor pid 2 since …", Host: "anchor.example",
|
||||
Taken: started.Add(3 * time.Second), Renewed: now.Add(-2 * time.Second), Epoch: 58}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
l ControllerLease
|
||||
want bool
|
||||
says string
|
||||
}{
|
||||
{"held by the new instance", held, true, "holds the lease"},
|
||||
{"nobody", ControllerLease{}, false, "no holder"},
|
||||
{"the old instance, taken before the restart", func() ControllerLease {
|
||||
l := held
|
||||
l.Taken = started.Add(-time.Hour)
|
||||
return l
|
||||
}(), false, "before the new build started"},
|
||||
{"another machine's controller", func() ControllerLease {
|
||||
l := held
|
||||
l.Host = "home-server"
|
||||
return l
|
||||
}(), false, "not this machine"},
|
||||
{"taken and no longer renewed", func() ControllerLease {
|
||||
l := held
|
||||
l.Renewed = now.Add(-20 * time.Second)
|
||||
return l
|
||||
}(), false, "past its"},
|
||||
{"taken within the clocks' skew of the start", func() ControllerLease {
|
||||
l := held
|
||||
l.Taken = started.Add(-time.Second)
|
||||
return l
|
||||
}(), true, "holds the lease"},
|
||||
} {
|
||||
got, why := c.l.HeldBySince("anchor", started, now)
|
||||
if got != c.want || !strings.Contains(why, c.says) {
|
||||
t.Errorf("%s: healthy %v (%s), want %v saying %q", c.name, got, why, c.want, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The contract, held (novox/hq to-be 45 §8).
|
||||
|
||||
// The runtime is asked by its own name and this machine's: only this machine's runtime can answer.
|
||||
func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) {
|
||||
@@ -93,23 +18,23 @@ func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// What is judged by default is the mesh's two core processes, by the names the controller composes
|
||||
// them under, and nothing else.
|
||||
func TestOnlyTheCoreProcessesAreJudgedByDefault(t *testing.T) {
|
||||
for name, want := range map[string]string{"mesh-controller": ByLease, "node-tools": ByPing, "greeter": ByNone} {
|
||||
// What is judged by default is the mesh's core process on every machine, by the name it is composed under, and
|
||||
// nothing else: the controller ended (issue 1126), so its name is judged as any other.
|
||||
func TestOnlyTheCoreProcessIsJudgedByDefault(t *testing.T) {
|
||||
for name, want := range map[string]string{"mesh-controller": ByNone, "node-tools": ByPing, "greeter": ByNone} {
|
||||
if got := Default(name); got != want {
|
||||
t.Errorf("%s is judged by %q, want %q", name, got, want)
|
||||
}
|
||||
}
|
||||
if ControllerWithin != 60*time.Second || NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second {
|
||||
t.Fatal("the bounds are to-be 45 §8's: the lease within sixty seconds, a PING answered within five")
|
||||
if NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second {
|
||||
t.Fatal("the bounds are to-be 45 §8's: the runtime within sixty seconds, a PING answered within five")
|
||||
}
|
||||
}
|
||||
|
||||
// A verdict on the wire names its component as the controller's condition does.
|
||||
func TestAVerdictNamesItsComponent(t *testing.T) {
|
||||
raw, _ := json.Marshal(link.Rollback{Component: Component(ByLease), Outcome: link.RolledBack})
|
||||
if !strings.Contains(string(raw), `"component":"controller"`) || Component(ByPing) != "node-tools" {
|
||||
raw, _ := json.Marshal(link.Rollback{Component: Component(ByPing), Outcome: link.RolledBack})
|
||||
if !strings.Contains(string(raw), `"component":"node-tools"`) {
|
||||
t.Fatalf("a verdict names its component as %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,11 +85,11 @@ func retiredBuilds(t *testing.T, root, name string) []string {
|
||||
// after its build was moved into a 0700 directory, and then deleted, could not run its own verbs.
|
||||
func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
name := "mesh-controller"
|
||||
name := "node-tools"
|
||||
start := time.Date(2026, 10, 7, 2, 0, 0, 0, time.UTC)
|
||||
|
||||
// Build A placed and running.
|
||||
p, err := Place(root, name, ByLease, buildA, "", "", start)
|
||||
p, err := Place(root, name, ByPing, buildA, "", "", start)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -99,7 +99,7 @@ func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T)
|
||||
}
|
||||
|
||||
// Build B placed while A still runs: A is moved aside, and reachable at its new path.
|
||||
place(t, root, name, ByLease, buildB, "", "", start.Add(time.Minute))
|
||||
place(t, root, name, ByPing, buildB, "", "", start.Add(time.Minute))
|
||||
for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} {
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
|
||||
@@ -14,15 +14,14 @@ import (
|
||||
// concluded (to-be 45 §8).
|
||||
//
|
||||
// **Time counts only while it could ask.** A build's bound is spent only on looks that got an answer
|
||||
// from the bus — the lease read, or the PING delivered — so a machine whose own link is down, or a
|
||||
// from the bus — the PING delivered — so a machine whose own link is down, or a
|
||||
// bus that refuses the question, never rolls a build back for the host's own trouble. A look that
|
||||
// could not ask counts towards GiveUp instead, and at GiveUp the build is said to be unwitnessed:
|
||||
// neither proved nor rolled back, and said.
|
||||
type Watcher struct {
|
||||
Root string
|
||||
// Node is this machine's node name, the instance its runtime answers PING as; Host its hostname,
|
||||
// as the controller's lease names its machine.
|
||||
Node, Host string
|
||||
// Node is this machine's node name, the instance its runtime answers PING as.
|
||||
Node string
|
||||
// Asker is the link open now, or nil.
|
||||
Asker func() link.Asker
|
||||
Run Runner
|
||||
@@ -69,7 +68,7 @@ func (w *Watcher) look(ctx context.Context, s State) {
|
||||
every = Every
|
||||
}
|
||||
now := w.now()
|
||||
healthy, why, err := w.judge(ctx, s, now)
|
||||
healthy, why, err := w.judge(ctx, s)
|
||||
|
||||
w.hold(func() {
|
||||
// Read again in turn: an apply may have placed another build meanwhile, which starts its own
|
||||
@@ -123,7 +122,7 @@ func (w *Watcher) look(ctx context.Context, s State) {
|
||||
}
|
||||
|
||||
// judge asks once. An error is a look that could not ask; otherwise healthy, and why not when not.
|
||||
func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, string, error) {
|
||||
func (w *Watcher) judge(ctx context.Context, s State) (bool, string, error) {
|
||||
var asker link.Asker
|
||||
if w.Asker != nil {
|
||||
asker = w.Asker()
|
||||
@@ -134,20 +133,6 @@ func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, stri
|
||||
asking, cancel := context.WithTimeout(ctx, PingWithin)
|
||||
defer cancel()
|
||||
switch s.Witness {
|
||||
case ByLease:
|
||||
value, found, err := asker.ReadKey(asking, LeaseBucket, LeaseKey)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if !found {
|
||||
return false, "nobody holds the controller's lease", nil
|
||||
}
|
||||
lease, err := ParseLease(value)
|
||||
if err != nil {
|
||||
return false, err.Error(), nil
|
||||
}
|
||||
held, why := lease.HeldBySince(w.Host, s.Started, now)
|
||||
return held, why, nil
|
||||
case ByPing:
|
||||
err := asker.Ping(asking, s.Process, w.Node)
|
||||
switch {
|
||||
|
||||
@@ -13,8 +13,8 @@ import (
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// Each rollback path, induced against a real directory: a controller that never takes the lease, a
|
||||
// runtime that never answers PING, a build declared not reversible; and a healthy update, which
|
||||
// Each rollback path, induced against a real directory: a runtime that never answers PING, a build
|
||||
// declared not reversible; and a healthy update, which
|
||||
// deletes nothing before it is proved and the build before it once it is (novox/hq to-be 45 §8).
|
||||
|
||||
const (
|
||||
@@ -62,18 +62,10 @@ func kept(root, name string) bool {
|
||||
// asker answers as the build it is told is running would.
|
||||
type asker struct {
|
||||
mu sync.Mutex
|
||||
lease func() ([]byte, bool, error)
|
||||
ping func() error
|
||||
asked int
|
||||
}
|
||||
|
||||
func (a *asker) ReadKey(context.Context, string, string) ([]byte, bool, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.asked++
|
||||
return a.lease()
|
||||
}
|
||||
|
||||
func (a *asker) Ping(context.Context, string, string) error {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
@@ -95,7 +87,7 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
|
||||
}
|
||||
|
||||
func watcher(root string, a link.Asker, m *machine, clock *time.Time, verdicts *[]link.Rollback) *Watcher {
|
||||
return &Watcher{Root: root, Node: "anchor", Host: "anchor",
|
||||
return &Watcher{Root: root, Node: "anchor",
|
||||
Asker: func() link.Asker {
|
||||
if a == nil {
|
||||
return nil
|
||||
@@ -116,45 +108,37 @@ func look(w *Watcher, clock *time.Time, n int) {
|
||||
}
|
||||
}
|
||||
|
||||
func leaseHeldBy(instance string, taken time.Time, clock *time.Time) func() ([]byte, bool, error) {
|
||||
return func() ([]byte, bool, error) {
|
||||
return []byte(fmt.Sprintf(`{"instance":%q,"host":"anchor","epoch":9,"taken":%q,"renewed":%q}`,
|
||||
instance, taken.Format(time.RFC3339Nano), clock.Add(-time.Second).Format(time.RFC3339Nano))), true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// A controller that starts and never takes the lease: restored to the build before, once, said once.
|
||||
func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) {
|
||||
// A runtime that starts and never answers PING: restored to the build before, once, said once.
|
||||
func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
|
||||
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
|
||||
if running(t, root, "mesh-controller") != buildB || !kept(root, "mesh-controller") {
|
||||
t.Fatal("the new controller is not running with the build before it kept beside it")
|
||||
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
|
||||
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
|
||||
if running(t, root, "node-tools") != buildB || !kept(root, "node-tools") {
|
||||
t.Fatal("the new runtime is not running with the build before it kept beside it")
|
||||
}
|
||||
|
||||
// The old instance's lease, taken an hour ago, is all the bus ever shows.
|
||||
a := &asker{lease: leaseHeldBy("controller@anchor pid 1 since earlier", clock.Add(-time.Hour), &clock)}
|
||||
a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }}
|
||||
m := &machine{}
|
||||
var verdicts []link.Rollback
|
||||
w := watcher(root, a, m, &clock, &verdicts)
|
||||
|
||||
look(w, &clock, int(ControllerWithin/Every)-1)
|
||||
if len(verdicts) != 0 || running(t, root, "mesh-controller") != buildB {
|
||||
look(w, &clock, int(NodeToolsWithin/Every)-1)
|
||||
if len(verdicts) != 0 || running(t, root, "node-tools") != buildB {
|
||||
t.Fatalf("rolled back before its bound: %+v", verdicts)
|
||||
}
|
||||
look(w, &clock, 1)
|
||||
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].From != buildB || verdicts[0].To != buildA ||
|
||||
verdicts[0].Component != link.ComponentController {
|
||||
t.Fatalf("the verdict is %+v, want controller rolled back from B to A", verdicts)
|
||||
verdicts[0].Component != link.ComponentNodeTools {
|
||||
t.Fatalf("the verdict is %+v, want node-tools rolled back from B to A", verdicts)
|
||||
}
|
||||
if running(t, root, "mesh-controller") != buildA {
|
||||
t.Fatalf("the controller running is %s, want the build before", running(t, root, "mesh-controller"))
|
||||
if running(t, root, "node-tools") != buildA {
|
||||
t.Fatalf("the runtime running is %s, want the build before", running(t, root, "node-tools"))
|
||||
}
|
||||
if strings.Join(m.commands, "; ") != "systemctl stop mesh-controller.service; systemctl restart mesh-controller.service" {
|
||||
if strings.Join(m.commands, "; ") != "systemctl stop node-tools.service; systemctl restart node-tools.service" {
|
||||
t.Fatalf("the machine was asked %v", m.commands)
|
||||
}
|
||||
if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "before the new build started") {
|
||||
if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "is on the bus") {
|
||||
t.Fatalf("the verdict does not say why: %s", verdicts[0].Why)
|
||||
}
|
||||
|
||||
@@ -168,81 +152,58 @@ func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) {
|
||||
t.Fatalf("what every report says is %+v", s)
|
||||
}
|
||||
// The mesh still declares B: refused, A kept running, the verdict still standing.
|
||||
p := place(t, root, "mesh-controller", ByLease, buildB, buildB, "", clock)
|
||||
if p.Unpack || running(t, root, "mesh-controller") != buildA || !strings.Contains(p.Detail, "rolled back") {
|
||||
t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "mesh-controller"))
|
||||
p := place(t, root, "node-tools", ByPing, buildB, buildB, "", clock)
|
||||
if p.Unpack || running(t, root, "node-tools") != buildA || !strings.Contains(p.Detail, "rolled back") {
|
||||
t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "node-tools"))
|
||||
}
|
||||
if len(Standing(root)) != 1 || len(Trials(root)) != 0 {
|
||||
t.Fatal("refusing the rolled-back build changed what stands or started a trial")
|
||||
}
|
||||
// A newer build is a new trial, and ends what was concluded once it is proved.
|
||||
place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock)
|
||||
if running(t, root, "mesh-controller") != buildC || len(Trials(root)) != 1 {
|
||||
place(t, root, "node-tools", ByPing, buildC, buildB, "", clock)
|
||||
if running(t, root, "node-tools") != buildC || len(Trials(root)) != 1 {
|
||||
t.Fatal("a newer build after a rollback was not placed on trial")
|
||||
}
|
||||
if err := Proved(root, "mesh-controller"); err != nil {
|
||||
if err := Proved(root, "node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(Standing(root)) != 0 || kept(root, "mesh-controller") {
|
||||
if len(Standing(root)) != 0 || kept(root, "node-tools") {
|
||||
t.Fatal("a newer build proved and the rollback still stands, or the build before it is still kept")
|
||||
}
|
||||
}
|
||||
|
||||
// A runtime that never answers PING: restored, once.
|
||||
func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
|
||||
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
|
||||
|
||||
a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }}
|
||||
m := &machine{}
|
||||
var verdicts []link.Rollback
|
||||
w := watcher(root, a, m, &clock, &verdicts)
|
||||
look(w, &clock, int(NodeToolsWithin/Every)+20)
|
||||
|
||||
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].Component != link.ComponentNodeTools {
|
||||
t.Fatalf("the verdict is %+v, want node-tools rolled back once", verdicts)
|
||||
}
|
||||
if running(t, root, "node-tools") != buildA || kept(root, "node-tools") {
|
||||
t.Fatal("the runtime running is not the build before, or a copy of it is still kept")
|
||||
}
|
||||
}
|
||||
|
||||
// A healthy update: nothing deleted before it is proved; the build before it deleted once it is.
|
||||
func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||
// The controller placed by a host from before any witness: a record, and no state.
|
||||
if err := os.MkdirAll(filepath.Join(root, "mesh-controller"), 0o755); err != nil {
|
||||
// The runtime placed by a host from before any witness: a record, and no state.
|
||||
if err := os.MkdirAll(filepath.Join(root, "node-tools"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "mesh-controller", "build"), []byte(buildA), 0o644); err != nil {
|
||||
if err := os.WriteFile(filepath.Join(root, "node-tools", "build"), []byte(buildA), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
|
||||
started := clock
|
||||
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
|
||||
|
||||
a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }}
|
||||
a := &asker{ping: func() error { return link.ErrNoAnswer }}
|
||||
m := &machine{}
|
||||
var verdicts []link.Rollback
|
||||
w := watcher(root, a, m, &clock, &verdicts)
|
||||
look(w, &clock, 5)
|
||||
if !kept(root, "mesh-controller") {
|
||||
if !kept(root, "node-tools") {
|
||||
t.Fatal("the build before was deleted while the new one was still on trial")
|
||||
}
|
||||
// A third build while the second is on trial: the one kept is still the one seen healthy.
|
||||
place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock)
|
||||
if s, _ := Load(root, "mesh-controller"); s.Previous != buildA {
|
||||
place(t, root, "node-tools", ByPing, buildC, buildB, "", clock)
|
||||
if s, _ := Load(root, "node-tools"); s.Previous != buildA {
|
||||
t.Fatalf("the build kept to go back to is %s, want A — the last one seen healthy", s.Previous)
|
||||
}
|
||||
started = clock
|
||||
a.lease = leaseHeldBy("controller@anchor pid 3 since now", started.Add(4*time.Second), &clock)
|
||||
a.ping = func() error { return nil }
|
||||
look(w, &clock, 2)
|
||||
if len(verdicts) != 0 || len(m.commands) != 0 {
|
||||
t.Fatalf("a healthy update was judged or acted on: %+v %v", verdicts, m.commands)
|
||||
}
|
||||
if kept(root, "mesh-controller") || len(Trials(root)) != 0 || running(t, root, "mesh-controller") != buildC {
|
||||
if kept(root, "node-tools") || len(Trials(root)) != 0 || running(t, root, "node-tools") != buildC {
|
||||
t.Fatal("the build before was not deleted once the new one was proved")
|
||||
}
|
||||
}
|
||||
@@ -252,14 +213,14 @@ func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
|
||||
func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
|
||||
place(t, root, "mesh-controller", ByLease, buildB, buildA, "migration 0073 cannot be undone", clock)
|
||||
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
|
||||
place(t, root, "node-tools", ByPing, buildB, buildA, "migration 0073 cannot be undone", clock)
|
||||
|
||||
a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }}
|
||||
a := &asker{ping: func() error { return link.ErrNoAnswer }}
|
||||
m := &machine{}
|
||||
var verdicts []link.Rollback
|
||||
w := watcher(root, a, m, &clock, &verdicts)
|
||||
look(w, &clock, int(ControllerWithin/Every)+20)
|
||||
look(w, &clock, int(NodeToolsWithin/Every)+20)
|
||||
|
||||
if len(verdicts) != 1 || verdicts[0].Outcome != link.NotReversible || verdicts[0].To != "" {
|
||||
t.Fatalf("the verdict is %+v, want not-reversible with nothing restored", verdicts)
|
||||
@@ -267,9 +228,9 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
|
||||
if !strings.Contains(verdicts[0].Why, "migration 0073 cannot be undone") {
|
||||
t.Fatalf("the verdict does not say why it may not be rolled back: %s", verdicts[0].Why)
|
||||
}
|
||||
if len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB {
|
||||
if len(m.commands) != 0 || running(t, root, "node-tools") != buildB {
|
||||
t.Fatalf("the build before was started against the newer data: %v, running %s", m.commands,
|
||||
running(t, root, "mesh-controller"))
|
||||
running(t, root, "node-tools"))
|
||||
}
|
||||
if len(Standing(root)) != 1 {
|
||||
t.Fatal("the not-reversible verdict does not stand")
|
||||
@@ -280,24 +241,24 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
|
||||
func TestAWitnessThatCannotAskNeverRollsBack(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
|
||||
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
|
||||
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
|
||||
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
|
||||
|
||||
a := &asker{lease: func() ([]byte, bool, error) {
|
||||
return nil, false, fmt.Errorf("%w: this host's grant does not name the lease", link.ErrCannotAsk)
|
||||
a := &asker{ping: func() error {
|
||||
return fmt.Errorf("%w: this host's grant does not name the PING", link.ErrCannotAsk)
|
||||
}}
|
||||
m := &machine{}
|
||||
var verdicts []link.Rollback
|
||||
w := watcher(root, a, m, &clock, &verdicts)
|
||||
look(w, &clock, int(GiveUp/Every)-1)
|
||||
if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB {
|
||||
if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "node-tools") != buildB {
|
||||
t.Fatalf("a build was judged on questions that were never asked: %+v %v", verdicts, m.commands)
|
||||
}
|
||||
look(w, &clock, 1)
|
||||
if len(verdicts) != 1 || verdicts[0].Outcome != link.Unwitnessed || len(m.commands) != 0 {
|
||||
t.Fatalf("the verdict at GiveUp is %+v, want unwitnessed with nothing done", verdicts)
|
||||
}
|
||||
if !kept(root, "mesh-controller") {
|
||||
if !kept(root, "node-tools") {
|
||||
t.Fatal("the build before was deleted though the new one was never seen healthy")
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user