From ff4dc71ce4d64074e1e3747ff3612c7e35bdcc08 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 11:22:21 +0100 Subject: [PATCH] Witness the node tools alone: the controller's lease names no build any more The controller ended by its unassignment (issue 1126, ADR 0590), so no build is judged by the lease: the lease witness, its contract and the direct get the witness asked it with are removed, and its tests now prove each rollback path on the node tools' PING. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- cmd/mesh-host/witnessing.go | 3 +- internal/link/messages.go | 5 +- internal/link/messages_test.go | 2 +- internal/link/testdata/witness-grants.conf | 8 +- internal/link/witnessing.go | 50 +------- internal/link/witnessing_nats_test.go | 64 +--------- internal/witness/contract.go | 125 ++----------------- internal/witness/contract_test.go | 93 ++------------ internal/witness/draining_test.go | 6 +- internal/witness/watch.go | 25 +--- internal/witness/watch_test.go | 133 ++++++++------------- 11 files changed, 89 insertions(+), 425 deletions(-) diff --git a/cmd/mesh-host/witnessing.go b/cmd/mesh-host/witnessing.go index d4aa7a0..2044f4e 100644 --- a/cmd/mesh-host/witnessing.go +++ b/cmd/mesh-host/witnessing.go @@ -146,9 +146,8 @@ func launcherReplaced(pid int, proc, path string) bool { // watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8). func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) { - host, _ := os.Hostname() w := &witness.Watcher{ - Root: apply.DaemonRoot(), Node: node, Host: host, + Root: apply.DaemonRoot(), Node: node, Asker: func() link.Asker { return queue.Asker() }, Run: witness.Runner(apply.ExecRunner), Hold: func(f func()) { diff --git a/internal/link/messages.go b/internal/link/messages.go index 7a64191..f0dd7a2 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -488,9 +488,8 @@ const WitnessContract = 1 // The core components a witness judges (to-be 45 §8), as a rollback names them. const ( - ComponentEngine = "node-engine" - ComponentController = "controller" - ComponentNodeTools = "node-tools" + ComponentEngine = "node-engine" + ComponentNodeTools = "node-tools" ) // What a witness concluded, as a rollback says it. diff --git a/internal/link/messages_test.go b/internal/link/messages_test.go index b27f39b..5bc7c29 100644 --- a/internal/link/messages_test.go +++ b/internal/link/messages_test.go @@ -27,7 +27,7 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { []string{"protocol", "address", "port", "by", "published", "container-port"}}, // novox/hq to-be 45 §8: a witness's verdicts, said on every report while they stand, and the // witness contract this host keeps. - {Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentController}}, Witness: WitnessContract}, + {Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentNodeTools}}, Witness: WitnessContract}, []string{"node", "rollbacks", "witness"}}, {Rollback{Component: ComponentNodeTools, From: "sha256:b", To: "sha256:a", Outcome: RolledBack, Why: "w"}, []string{"component", "from", "to", "outcome", "why", "at"}}, diff --git a/internal/link/testdata/witness-grants.conf b/internal/link/testdata/witness-grants.conf index ccc5311..78be114 100644 --- a/internal/link/testdata/witness-grants.conf +++ b/internal/link/testdata/witness-grants.conf @@ -3,16 +3,15 @@ # docker run -d --rm --name w -p 14224:4222 -v $PWD/internal/link/testdata:/c:ro nats:2.11-alpine -js -c /c/witness-grants.conf # MESH_TEST_NATS_GRANTS=nats://127.0.0.1:14224 go test ./internal/link/ -run TestNatsWitness # -# `node.anchor` is a machine's host with exactly the two subjects the witness asks added to what a -# host already has (its inbox); `node.bare` is a host without them. `runtime` stands in for the tool -# runtime, `admin` for the controller writing the lease. +# `node.anchor` is a machine's host with exactly the subject the witness asks added to what a host +# already has (its inbox); `node.bare` is a host without it. `runtime` stands in for the tool runtime. jetstream: enabled accounts { MESH { jetstream: enabled users = [ { user: "node.anchor", password: "a", permissions: { - publish: { allow: ["$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder", "$SRV.PING.node-tools.anchor"] } + publish: { allow: ["$SRV.PING.node-tools.anchor"] } subscribe: { allow: ["_INBOX.node.anchor.>"] } } } { user: "node.bare", password: "b", permissions: { @@ -23,7 +22,6 @@ accounts { subscribe: { allow: ["$SRV.PING.node-tools.>"] } allow_responses: { max: 1, ttl: "1m" } } } - { user: "admin", password: "x" } ] } } diff --git a/internal/link/witnessing.go b/internal/link/witnessing.go index 4b1f8ad..ed364d9 100644 --- a/internal/link/witnessing.go +++ b/internal/link/witnessing.go @@ -10,14 +10,11 @@ import ( "github.com/nats-io/nats.go" ) -// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the controller and the -// node tools it placed on this machine, by what the bus says about them rather than by what they -// say about themselves. +// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the node tools it placed on this +// machine, by what the bus says about them rather than by what they say about themselves. // -// **Two questions, both read-only, both on the host's own link.** The controller's lease key, read -// by JetStream's direct get — one subject, no stream information, nothing written; and this -// machine's tool runtime, asked the NATS services protocol's PING by its name and this machine's — -// so only this machine's runtime can answer it. +// **One question, read-only, on the host's own link**: this machine's tool runtime, asked the NATS services protocol's +// PING by its name and this machine's, so only this machine's runtime can answer it. // ErrCannotAsk is a question the bus did not let this host put, or did not answer: no grant for it, // no such bucket, the bus slow. **It says nothing about the build being judged**, so a witness @@ -30,20 +27,11 @@ var ErrNoAnswer = errors.New("nothing answered") // Asker is what a witness asks through. The link open now implements it; nil while there is none. type Asker interface { - // ReadKey is a key-value bucket's current value for a key. Found false is nobody holding it — - // absent, deleted or expired; an error wrapping ErrCannotAsk is no reading at all. - ReadKey(ctx context.Context, bucket, key string) (value []byte, found bool, err error) // Ping asks the service `service`'s instance `id` whether it is there. Nil is an answer; // ErrNoAnswer is none in time; ErrCannotAsk is no asking. Ping(ctx context.Context, service, id string) error } -// DirectGetSubject is the one subject a host asks a bucket's key on: JetStream's direct get of the -// bucket's stream, for the key's own subject. What a host's grant names exactly — no wildcard. -func DirectGetSubject(bucket, key string) string { - return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + key -} - // PingSubject is the services protocol's PING of one instance of one service. func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id } @@ -58,36 +46,6 @@ func (q *Session) Asker() Asker { return nil } -func (l *natsLink) ReadKey(ctx context.Context, bucket, key string) ([]byte, bool, error) { - subject := DirectGetSubject(bucket, key) - before := l.conn.LastError() - msg, err := l.conn.RequestWithContext(ctx, subject, nil) - switch { - case errors.Is(err, nats.ErrNoResponders): - return nil, false, fmt.Errorf("%w: the bus has no bucket %s that answers a direct get", ErrCannotAsk, bucket) - case err != nil: - return nil, false, fmt.Errorf("%w: reading %s from %s: %v%s", ErrCannotAsk, key, bucket, err, l.refusal(subject, before)) - } - if msg.Header != nil { - switch status := msg.Header.Get("Status"); status { - case "": - case "404": - return nil, false, nil - default: - return nil, false, fmt.Errorf("%w: reading %s from %s: the bus answered %s %s", ErrCannotAsk, key, - bucket, status, msg.Header.Get("Description")) - } - switch msg.Header.Get("KV-Operation") { - case "DEL", "PURGE": - return nil, false, nil - } - } - if len(msg.Data) == 0 { - return nil, false, nil - } - return msg.Data, true, nil -} - func (l *natsLink) Ping(ctx context.Context, service, id string) error { subject := PingSubject(service, id) before := l.conn.LastError() diff --git a/internal/link/witnessing_nats_test.go b/internal/link/witnessing_nats_test.go index 98c29e1..9c2c006 100644 --- a/internal/link/witnessing_nats_test.go +++ b/internal/link/witnessing_nats_test.go @@ -2,7 +2,6 @@ package link import ( "context" - "encoding/json" "errors" "os" "testing" @@ -12,7 +11,7 @@ import ( ) // What a witness asks, asked of a real server with the grants the mesh composes (novox/hq to-be 45 §8): -// whether the direct get and the PING work as the two subjects named, and whether a host without them +// whether the PING works as the subject named, and whether a host without them // is told apart — "cannot ask", never "the build did not answer". See testdata/witness-grants.conf. func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) { url := os.Getenv("MESH_TEST_NATS_GRANTS") @@ -28,63 +27,14 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) { t.Cleanup(conn.Close) return conn } - admin := dial("admin", "x", "_INBOX.admin") - js, err := admin.JetStream() - if err != nil { - t.Fatal(err) - } - _ = js.DeleteKeyValue("mesh-controller_lease") - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_lease", TTL: 15 * time.Second, History: 1}) - if err != nil { - t.Fatal(err) - } - anchor := &natsLink{conn: dial("node.anchor", "a", "_INBOX.node.anchor")} bare := &natsLink{conn: dial("node.bare", "b", "_INBOX.node.bare")} ask := func() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), 2*time.Second) } - // Nobody holds it. - ctx, cancel := ask() - if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found { - t.Fatalf("an empty lease read as %v, %v", found, err) - } - cancel() - // Held. - if _, err := kv.Put("holder", []byte(`{"instance":"controller@anchor pid 1 since now","epoch":3}`)); err != nil { - t.Fatal(err) - } - ctx, cancel = ask() - value, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder") - cancel() - if err != nil || !found { - t.Fatalf("a held lease read as %v, %v", found, err) - } - var holder struct { - Instance string `json:"instance"` - } - if json.Unmarshal(value, &holder); holder.Instance == "" { - t.Fatalf("the lease's value is %s", value) - } - // Given back. - if err := kv.Delete("holder"); err != nil { - t.Fatal(err) - } - ctx, cancel = ask() - if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found { - t.Fatalf("a lease given back read as %v, %v", found, err) - } - cancel() - // A host without the grant cannot ask — which is not "nobody holds it". - ctx, cancel = ask() - if _, _, err := bare.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) { - t.Fatalf("a host without the grant read the lease as %v, want it unable to ask", err) - } - cancel() - // No runtime on the bus: not there to answer. - ctx, cancel = ask() + ctx, cancel := ask() if err := anchor.Ping(ctx, "node-tools", "anchor"); !errors.Is(err, ErrNoAnswer) { t.Fatalf("a PING nobody serves is %v, want no answer", err) } @@ -112,14 +62,4 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) { t.Fatalf("a host without the grant heard %v, want it unable to ask", err) } cancel() - - // No lease bucket at all — a controller from before the lease: cannot ask, not "nobody holds it". - if err := js.DeleteKeyValue("mesh-controller_lease"); err != nil { - t.Fatal(err) - } - ctx, cancel = ask() - if _, _, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) { - t.Fatalf("a bus with no lease bucket read as %v, want unable to ask", err) - } - cancel() } diff --git a/internal/witness/contract.go b/internal/witness/contract.go index e3f2321..35e8b0b 100644 --- a/internal/witness/contract.go +++ b/internal/witness/contract.go @@ -1,155 +1,54 @@ -// Package witness is the node-engine watching a core build it placed — the controller on the control -// node, the node tools on every machine — and restoring the build before it when the new one is not -// healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the component being replaced is never the -// only witness of its successor). +// Package witness is the node-engine watching a core build it placed — the node tools on every machine — and +// restoring the build before it when the new one is not healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the +// component being replaced is never the only witness of its successor). // // **The contract is this file.** What the engine reads to call a build healthy, from where, in what -// shape, within which bound — said once, here, and held by contract_test.go. The controller's side -// writes what is read here (mesh-controller internal/lease Holder); a change on either side is a -// change to this file and its test. +// shape, within which bound — said once, here, and held by contract_test.go. package witness import ( - "encoding/json" - "fmt" - "strings" "time" "github.com/novox/mesh-host/internal/link" ) -// The two witnesses, as a process declares which watches it (`witness`). A process that says none is -// placed as ever; one that says nothing is judged by its name's default (Default). +// The witnesses, as a process declares which watches it (`witness`). A process that says none is placed as ever; one +// that says nothing is judged by its name's default (Default). const ( - // ByLease: healthy when the controller this machine started holds the controller's lease. - ByLease = "lease" // ByPing: healthy when this machine's runtime answers the services protocol's PING. ByPing = "ping" // ByNone: not judged. ByNone = "none" ) -// The controller's lease, as the host reads it (novox/hq ADR 0229): the key `holder` in the bucket -// `mesh-controller_lease`, whose keys live fifteen seconds unless renewed, renewed every five. -const ( - LeaseBucket = "mesh-controller_lease" - LeaseKey = "holder" - // LeaseAge is the bucket's age for its key. A holder whose last renewal is older than this is - // not holding it, whatever the key still says. - LeaseAge = 15 * time.Second - // Skew is how far the controller's clock and this host's may disagree about when it took the - // lease. One machine, one clock, in practice: a margin, not a tolerance. - Skew = 2 * time.Second -) - // Bounds (to-be 45 §8). A build is given Within of time the witness could ask in; asked every Every. const ( - // ControllerWithin: the controller holds the lease within sixty seconds of starting. - ControllerWithin = 60 * time.Second // NodeToolsWithin: the runtime is announced and answering within sixty seconds of starting, // each PING answered within PingWithin. NodeToolsWithin = 60 * time.Second PingWithin = 5 * time.Second Every = 5 * time.Second // GiveUp is how long a witness goes on when it cannot ask at all before it says the build is - // unwitnessed: the grant missing, the bucket missing, the bus away the whole time. + // unwitnessed: the grant missing, the bus away the whole time. GiveUp = 30 * time.Minute ) -// ControllerLease is the lease's value as the controller writes it — mesh-controller internal/lease -// Holder, field for field by its JSON names. Only what the witness reads is required; a field the -// controller adds later is ignored here. -type ControllerLease struct { - // Instance names one controller process: "controller@ pid since ". - Instance string `json:"instance"` - // Host is the machine it runs on, as its own hostname says. - Host string `json:"host,omitempty"` - // Build is the controller's build as it knows it; not read here — the toolchain stamps no - // version, so it says "development build" — and a bundle's identity is its digest, which the - // host already knows. - Build string `json:"build,omitempty"` - Epoch uint64 `json:"epoch,omitempty"` - // Taken is when this instance took the key, Renewed when it last renewed it, by its own clock. - Taken time.Time `json:"taken"` - Renewed time.Time `json:"renewed"` -} - -// ParseLease reads the key's value. -func ParseLease(value []byte) (ControllerLease, error) { - var l ControllerLease - if err := json.Unmarshal(value, &l); err != nil { - return ControllerLease{}, fmt.Errorf("the lease's holder is not the controller's lease value: %w", err) - } - return l, nil -} - -// HeldBySince says whether the lease is held by a controller on machine `host` that took it at or -// after `since` — the controller this machine started then, and not the one before it — and is -// held now. Why says what it saw when it is not. -func (l ControllerLease) HeldBySince(host string, since, now time.Time) (bool, string) { - switch { - case l.Instance == "": - return false, "the lease names no holder" - case host != "" && !sameMachine(l.Host, host): - return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", l.Instance, l.Host) - case l.Taken.Before(since.Add(-Skew)): - return false, fmt.Sprintf("the lease is held by %s, taken %s — before the new build started at %s", - l.Instance, l.Taken.UTC().Format(time.RFC3339), since.UTC().Format(time.RFC3339)) - case now.Sub(latest(l.Taken, l.Renewed)) > LeaseAge: - return false, fmt.Sprintf("the lease names %s and was last renewed %s ago, past its %s", - l.Instance, now.Sub(latest(l.Taken, l.Renewed)).Round(time.Second), LeaseAge) - } - return true, fmt.Sprintf("%s holds the lease, epoch %d", l.Instance, l.Epoch) -} - -// sameMachine compares two hostnames as names, so a short name and its fully qualified form agree. -func sameMachine(a, b string) bool { - short := func(s string) string { - s = strings.ToLower(strings.TrimSpace(s)) - if i := strings.IndexByte(s, '.'); i > 0 { - s = s[:i] - } - return s - } - return short(a) == short(b) -} - -func latest(a, b time.Time) time.Time { - if b.After(a) { - return b - } - return a -} - // NodeToolsService is the runtime's name on the services protocol, and its instance is this // machine's node name: what `$SRV.PING..` asks, so only this machine's runtime can // answer (mesh-tools node-tools internal/runtime, announce.Service{Name: module, ID: node}). const NodeToolsService = "node-tools" -// Default is the witness a process is judged by when it names none: the two core processes the mesh -// composes, by the names it composes them under, and nothing else. +// Default is the witness a process is judged by when it names none: the core process the mesh composes on every +// machine, by the name it composes it under, and nothing else. func Default(process string) string { - switch process { - case "mesh-controller": - return ByLease - case NodeToolsService: + if process == NodeToolsService { return ByPing } return ByNone } // Within is the bound for a witness. -func Within(by string) time.Duration { - if by == ByLease { - return ControllerWithin - } - return NodeToolsWithin -} +func Within(string) time.Duration { return NodeToolsWithin } // Component names what a witness judges, as a rollback says it. -func Component(by string) string { - if by == ByLease { - return link.ComponentController - } - return link.ComponentNodeTools -} +func Component(string) string { return link.ComponentNodeTools } diff --git a/internal/witness/contract_test.go b/internal/witness/contract_test.go index 5168968..2ea5c5c 100644 --- a/internal/witness/contract_test.go +++ b/internal/witness/contract_test.go @@ -9,82 +9,7 @@ import ( "github.com/novox/mesh-host/internal/link" ) -// The contract, held. Each of these is a line the controller's side relies on or writes; a change on -// either side changes this test (novox/hq to-be 45 §8). - -// The lease is read where the controller keeps it (ADR 0229), on the one subject a host's grant names. -func TestTheLeaseIsReadWhereTheControllerKeepsIt(t *testing.T) { - if LeaseBucket != "mesh-controller_lease" || LeaseKey != "holder" { - t.Fatalf("the lease is read from %s/%s; the controller keeps it in mesh-controller_lease/holder", - LeaseBucket, LeaseKey) - } - if got := link.DirectGetSubject(LeaseBucket, LeaseKey); got != "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder" { - t.Fatalf("the host asks %s for the lease; its grant names exactly the direct get of the key", got) - } - if LeaseAge != 15*time.Second { - t.Fatalf("the lease's age is %s; the controller's bucket keeps a key fifteen seconds", LeaseAge) - } -} - -// What the controller writes — mesh-controller internal/lease Holder, by its JSON names — is what is -// read. The value below is the shape that Holder marshals to. -func TestTheLeaseValueIsTheControllersHolder(t *testing.T) { - written := `{"instance":"controller@anchor pid 4242 since 2026-10-06T10:00:00Z","host":"anchor",` + - `"build":"development build","epoch":57,"taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:31Z"}` - l, err := ParseLease([]byte(written)) - if err != nil { - t.Fatal(err) - } - if l.Instance == "" || l.Host != "anchor" || l.Epoch != 57 || l.Taken.IsZero() || l.Renewed.IsZero() { - t.Fatalf("the lease was not read whole: %+v", l) - } - // And a field the controller adds later does not stop it being read. - if _, err := ParseLease([]byte(`{"instance":"i","taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:01Z","bundle":"sha256:x"}`)); err != nil { - t.Fatalf("a lease value with a field this host does not know was refused: %v", err) - } -} - -// Healthy is: held now, on this machine, by an instance that took it after the new build started. -func TestTheNewControllerHoldsTheLeaseOnlyWhenItTookItAfterItStarted(t *testing.T) { - started := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) - now := started.Add(30 * time.Second) - held := ControllerLease{Instance: "controller@anchor pid 2 since …", Host: "anchor.example", - Taken: started.Add(3 * time.Second), Renewed: now.Add(-2 * time.Second), Epoch: 58} - for _, c := range []struct { - name string - l ControllerLease - want bool - says string - }{ - {"held by the new instance", held, true, "holds the lease"}, - {"nobody", ControllerLease{}, false, "no holder"}, - {"the old instance, taken before the restart", func() ControllerLease { - l := held - l.Taken = started.Add(-time.Hour) - return l - }(), false, "before the new build started"}, - {"another machine's controller", func() ControllerLease { - l := held - l.Host = "home-server" - return l - }(), false, "not this machine"}, - {"taken and no longer renewed", func() ControllerLease { - l := held - l.Renewed = now.Add(-20 * time.Second) - return l - }(), false, "past its"}, - {"taken within the clocks' skew of the start", func() ControllerLease { - l := held - l.Taken = started.Add(-time.Second) - return l - }(), true, "holds the lease"}, - } { - got, why := c.l.HeldBySince("anchor", started, now) - if got != c.want || !strings.Contains(why, c.says) { - t.Errorf("%s: healthy %v (%s), want %v saying %q", c.name, got, why, c.want, c.says) - } - } -} +// The contract, held (novox/hq to-be 45 §8). // The runtime is asked by its own name and this machine's: only this machine's runtime can answer. func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) { @@ -93,23 +18,23 @@ func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) { } } -// What is judged by default is the mesh's two core processes, by the names the controller composes -// them under, and nothing else. -func TestOnlyTheCoreProcessesAreJudgedByDefault(t *testing.T) { - for name, want := range map[string]string{"mesh-controller": ByLease, "node-tools": ByPing, "greeter": ByNone} { +// What is judged by default is the mesh's core process on every machine, by the name it is composed under, and +// nothing else: the controller ended (issue 1126), so its name is judged as any other. +func TestOnlyTheCoreProcessIsJudgedByDefault(t *testing.T) { + for name, want := range map[string]string{"mesh-controller": ByNone, "node-tools": ByPing, "greeter": ByNone} { if got := Default(name); got != want { t.Errorf("%s is judged by %q, want %q", name, got, want) } } - if ControllerWithin != 60*time.Second || NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second { - t.Fatal("the bounds are to-be 45 §8's: the lease within sixty seconds, a PING answered within five") + if NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second { + t.Fatal("the bounds are to-be 45 §8's: the runtime within sixty seconds, a PING answered within five") } } // A verdict on the wire names its component as the controller's condition does. func TestAVerdictNamesItsComponent(t *testing.T) { - raw, _ := json.Marshal(link.Rollback{Component: Component(ByLease), Outcome: link.RolledBack}) - if !strings.Contains(string(raw), `"component":"controller"`) || Component(ByPing) != "node-tools" { + raw, _ := json.Marshal(link.Rollback{Component: Component(ByPing), Outcome: link.RolledBack}) + if !strings.Contains(string(raw), `"component":"node-tools"`) { t.Fatalf("a verdict names its component as %s", raw) } } diff --git a/internal/witness/draining_test.go b/internal/witness/draining_test.go index 4a7d32d..6429891 100644 --- a/internal/witness/draining_test.go +++ b/internal/witness/draining_test.go @@ -85,11 +85,11 @@ func retiredBuilds(t *testing.T, root, name string) []string { // after its build was moved into a 0700 directory, and then deleted, could not run its own verbs. func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T) { root := t.TempDir() - name := "mesh-controller" + name := "node-tools" start := time.Date(2026, 10, 7, 2, 0, 0, 0, time.UTC) // Build A placed and running. - p, err := Place(root, name, ByLease, buildA, "", "", start) + p, err := Place(root, name, ByPing, buildA, "", "", start) if err != nil { t.Fatal(err) } @@ -99,7 +99,7 @@ func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T) } // Build B placed while A still runs: A is moved aside, and reachable at its new path. - place(t, root, name, ByLease, buildB, "", "", start.Add(time.Minute)) + place(t, root, name, ByPing, buildB, "", "", start.Add(time.Minute)) for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} { info, err := os.Stat(dir) if err != nil { diff --git a/internal/witness/watch.go b/internal/witness/watch.go index 2fe5d2f..bb9d0a3 100644 --- a/internal/witness/watch.go +++ b/internal/witness/watch.go @@ -14,15 +14,14 @@ import ( // concluded (to-be 45 §8). // // **Time counts only while it could ask.** A build's bound is spent only on looks that got an answer -// from the bus — the lease read, or the PING delivered — so a machine whose own link is down, or a +// from the bus — the PING delivered — so a machine whose own link is down, or a // bus that refuses the question, never rolls a build back for the host's own trouble. A look that // could not ask counts towards GiveUp instead, and at GiveUp the build is said to be unwitnessed: // neither proved nor rolled back, and said. type Watcher struct { Root string - // Node is this machine's node name, the instance its runtime answers PING as; Host its hostname, - // as the controller's lease names its machine. - Node, Host string + // Node is this machine's node name, the instance its runtime answers PING as. + Node string // Asker is the link open now, or nil. Asker func() link.Asker Run Runner @@ -69,7 +68,7 @@ func (w *Watcher) look(ctx context.Context, s State) { every = Every } now := w.now() - healthy, why, err := w.judge(ctx, s, now) + healthy, why, err := w.judge(ctx, s) w.hold(func() { // Read again in turn: an apply may have placed another build meanwhile, which starts its own @@ -123,7 +122,7 @@ func (w *Watcher) look(ctx context.Context, s State) { } // judge asks once. An error is a look that could not ask; otherwise healthy, and why not when not. -func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, string, error) { +func (w *Watcher) judge(ctx context.Context, s State) (bool, string, error) { var asker link.Asker if w.Asker != nil { asker = w.Asker() @@ -134,20 +133,6 @@ func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, stri asking, cancel := context.WithTimeout(ctx, PingWithin) defer cancel() switch s.Witness { - case ByLease: - value, found, err := asker.ReadKey(asking, LeaseBucket, LeaseKey) - if err != nil { - return false, "", err - } - if !found { - return false, "nobody holds the controller's lease", nil - } - lease, err := ParseLease(value) - if err != nil { - return false, err.Error(), nil - } - held, why := lease.HeldBySince(w.Host, s.Started, now) - return held, why, nil case ByPing: err := asker.Ping(asking, s.Process, w.Node) switch { diff --git a/internal/witness/watch_test.go b/internal/witness/watch_test.go index 0f3d908..148b6a2 100644 --- a/internal/witness/watch_test.go +++ b/internal/witness/watch_test.go @@ -13,8 +13,8 @@ import ( "github.com/novox/mesh-host/internal/link" ) -// Each rollback path, induced against a real directory: a controller that never takes the lease, a -// runtime that never answers PING, a build declared not reversible; and a healthy update, which +// Each rollback path, induced against a real directory: a runtime that never answers PING, a build +// declared not reversible; and a healthy update, which // deletes nothing before it is proved and the build before it once it is (novox/hq to-be 45 §8). const ( @@ -62,18 +62,10 @@ func kept(root, name string) bool { // asker answers as the build it is told is running would. type asker struct { mu sync.Mutex - lease func() ([]byte, bool, error) ping func() error asked int } -func (a *asker) ReadKey(context.Context, string, string) ([]byte, bool, error) { - a.mu.Lock() - defer a.mu.Unlock() - a.asked++ - return a.lease() -} - func (a *asker) Ping(context.Context, string, string) error { a.mu.Lock() defer a.mu.Unlock() @@ -95,7 +87,7 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e } func watcher(root string, a link.Asker, m *machine, clock *time.Time, verdicts *[]link.Rollback) *Watcher { - return &Watcher{Root: root, Node: "anchor", Host: "anchor", + return &Watcher{Root: root, Node: "anchor", Asker: func() link.Asker { if a == nil { return nil @@ -116,45 +108,37 @@ func look(w *Watcher, clock *time.Time, n int) { } } -func leaseHeldBy(instance string, taken time.Time, clock *time.Time) func() ([]byte, bool, error) { - return func() ([]byte, bool, error) { - return []byte(fmt.Sprintf(`{"instance":%q,"host":"anchor","epoch":9,"taken":%q,"renewed":%q}`, - instance, taken.Format(time.RFC3339Nano), clock.Add(-time.Second).Format(time.RFC3339Nano))), true, nil - } -} - -// A controller that starts and never takes the lease: restored to the build before, once, said once. -func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) { +// A runtime that starts and never answers PING: restored to the build before, once, said once. +func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) - place(t, root, "mesh-controller", ByLease, buildA, "", "", clock) - place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock) - if running(t, root, "mesh-controller") != buildB || !kept(root, "mesh-controller") { - t.Fatal("the new controller is not running with the build before it kept beside it") + place(t, root, "node-tools", ByPing, buildA, "", "", clock) + place(t, root, "node-tools", ByPing, buildB, buildA, "", clock) + if running(t, root, "node-tools") != buildB || !kept(root, "node-tools") { + t.Fatal("the new runtime is not running with the build before it kept beside it") } - // The old instance's lease, taken an hour ago, is all the bus ever shows. - a := &asker{lease: leaseHeldBy("controller@anchor pid 1 since earlier", clock.Add(-time.Hour), &clock)} + a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) - look(w, &clock, int(ControllerWithin/Every)-1) - if len(verdicts) != 0 || running(t, root, "mesh-controller") != buildB { + look(w, &clock, int(NodeToolsWithin/Every)-1) + if len(verdicts) != 0 || running(t, root, "node-tools") != buildB { t.Fatalf("rolled back before its bound: %+v", verdicts) } look(w, &clock, 1) if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].From != buildB || verdicts[0].To != buildA || - verdicts[0].Component != link.ComponentController { - t.Fatalf("the verdict is %+v, want controller rolled back from B to A", verdicts) + verdicts[0].Component != link.ComponentNodeTools { + t.Fatalf("the verdict is %+v, want node-tools rolled back from B to A", verdicts) } - if running(t, root, "mesh-controller") != buildA { - t.Fatalf("the controller running is %s, want the build before", running(t, root, "mesh-controller")) + if running(t, root, "node-tools") != buildA { + t.Fatalf("the runtime running is %s, want the build before", running(t, root, "node-tools")) } - if strings.Join(m.commands, "; ") != "systemctl stop mesh-controller.service; systemctl restart mesh-controller.service" { + if strings.Join(m.commands, "; ") != "systemctl stop node-tools.service; systemctl restart node-tools.service" { t.Fatalf("the machine was asked %v", m.commands) } - if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "before the new build started") { + if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "is on the bus") { t.Fatalf("the verdict does not say why: %s", verdicts[0].Why) } @@ -168,81 +152,58 @@ func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) { t.Fatalf("what every report says is %+v", s) } // The mesh still declares B: refused, A kept running, the verdict still standing. - p := place(t, root, "mesh-controller", ByLease, buildB, buildB, "", clock) - if p.Unpack || running(t, root, "mesh-controller") != buildA || !strings.Contains(p.Detail, "rolled back") { - t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "mesh-controller")) + p := place(t, root, "node-tools", ByPing, buildB, buildB, "", clock) + if p.Unpack || running(t, root, "node-tools") != buildA || !strings.Contains(p.Detail, "rolled back") { + t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "node-tools")) } if len(Standing(root)) != 1 || len(Trials(root)) != 0 { t.Fatal("refusing the rolled-back build changed what stands or started a trial") } // A newer build is a new trial, and ends what was concluded once it is proved. - place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock) - if running(t, root, "mesh-controller") != buildC || len(Trials(root)) != 1 { + place(t, root, "node-tools", ByPing, buildC, buildB, "", clock) + if running(t, root, "node-tools") != buildC || len(Trials(root)) != 1 { t.Fatal("a newer build after a rollback was not placed on trial") } - if err := Proved(root, "mesh-controller"); err != nil { + if err := Proved(root, "node-tools"); err != nil { t.Fatal(err) } - if len(Standing(root)) != 0 || kept(root, "mesh-controller") { + if len(Standing(root)) != 0 || kept(root, "node-tools") { t.Fatal("a newer build proved and the rollback still stands, or the build before it is still kept") } } -// A runtime that never answers PING: restored, once. -func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) { - root := t.TempDir() - clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) - place(t, root, "node-tools", ByPing, buildA, "", "", clock) - place(t, root, "node-tools", ByPing, buildB, buildA, "", clock) - - a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }} - m := &machine{} - var verdicts []link.Rollback - w := watcher(root, a, m, &clock, &verdicts) - look(w, &clock, int(NodeToolsWithin/Every)+20) - - if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].Component != link.ComponentNodeTools { - t.Fatalf("the verdict is %+v, want node-tools rolled back once", verdicts) - } - if running(t, root, "node-tools") != buildA || kept(root, "node-tools") { - t.Fatal("the runtime running is not the build before, or a copy of it is still kept") - } -} - // A healthy update: nothing deleted before it is proved; the build before it deleted once it is. func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) - // The controller placed by a host from before any witness: a record, and no state. - if err := os.MkdirAll(filepath.Join(root, "mesh-controller"), 0o755); err != nil { + // The runtime placed by a host from before any witness: a record, and no state. + if err := os.MkdirAll(filepath.Join(root, "node-tools"), 0o755); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(root, "mesh-controller", "build"), []byte(buildA), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(root, "node-tools", "build"), []byte(buildA), 0o644); err != nil { t.Fatal(err) } - place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock) - started := clock + place(t, root, "node-tools", ByPing, buildB, buildA, "", clock) - a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }} + a := &asker{ping: func() error { return link.ErrNoAnswer }} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) look(w, &clock, 5) - if !kept(root, "mesh-controller") { + if !kept(root, "node-tools") { t.Fatal("the build before was deleted while the new one was still on trial") } // A third build while the second is on trial: the one kept is still the one seen healthy. - place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock) - if s, _ := Load(root, "mesh-controller"); s.Previous != buildA { + place(t, root, "node-tools", ByPing, buildC, buildB, "", clock) + if s, _ := Load(root, "node-tools"); s.Previous != buildA { t.Fatalf("the build kept to go back to is %s, want A — the last one seen healthy", s.Previous) } - started = clock - a.lease = leaseHeldBy("controller@anchor pid 3 since now", started.Add(4*time.Second), &clock) + a.ping = func() error { return nil } look(w, &clock, 2) if len(verdicts) != 0 || len(m.commands) != 0 { t.Fatalf("a healthy update was judged or acted on: %+v %v", verdicts, m.commands) } - if kept(root, "mesh-controller") || len(Trials(root)) != 0 || running(t, root, "mesh-controller") != buildC { + if kept(root, "node-tools") || len(Trials(root)) != 0 || running(t, root, "node-tools") != buildC { t.Fatal("the build before was not deleted once the new one was proved") } } @@ -252,14 +213,14 @@ func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) { func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) - place(t, root, "mesh-controller", ByLease, buildA, "", "", clock) - place(t, root, "mesh-controller", ByLease, buildB, buildA, "migration 0073 cannot be undone", clock) + place(t, root, "node-tools", ByPing, buildA, "", "", clock) + place(t, root, "node-tools", ByPing, buildB, buildA, "migration 0073 cannot be undone", clock) - a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }} + a := &asker{ping: func() error { return link.ErrNoAnswer }} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) - look(w, &clock, int(ControllerWithin/Every)+20) + look(w, &clock, int(NodeToolsWithin/Every)+20) if len(verdicts) != 1 || verdicts[0].Outcome != link.NotReversible || verdicts[0].To != "" { t.Fatalf("the verdict is %+v, want not-reversible with nothing restored", verdicts) @@ -267,9 +228,9 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) { if !strings.Contains(verdicts[0].Why, "migration 0073 cannot be undone") { t.Fatalf("the verdict does not say why it may not be rolled back: %s", verdicts[0].Why) } - if len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB { + if len(m.commands) != 0 || running(t, root, "node-tools") != buildB { t.Fatalf("the build before was started against the newer data: %v, running %s", m.commands, - running(t, root, "mesh-controller")) + running(t, root, "node-tools")) } if len(Standing(root)) != 1 { t.Fatal("the not-reversible verdict does not stand") @@ -280,24 +241,24 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) { func TestAWitnessThatCannotAskNeverRollsBack(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) - place(t, root, "mesh-controller", ByLease, buildA, "", "", clock) - place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock) + place(t, root, "node-tools", ByPing, buildA, "", "", clock) + place(t, root, "node-tools", ByPing, buildB, buildA, "", clock) - a := &asker{lease: func() ([]byte, bool, error) { - return nil, false, fmt.Errorf("%w: this host's grant does not name the lease", link.ErrCannotAsk) + a := &asker{ping: func() error { + return fmt.Errorf("%w: this host's grant does not name the PING", link.ErrCannotAsk) }} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) look(w, &clock, int(GiveUp/Every)-1) - if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB { + if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "node-tools") != buildB { t.Fatalf("a build was judged on questions that were never asked: %+v %v", verdicts, m.commands) } look(w, &clock, 1) if len(verdicts) != 1 || verdicts[0].Outcome != link.Unwitnessed || len(m.commands) != 0 { t.Fatalf("the verdict at GiveUp is %+v, want unwitnessed with nothing done", verdicts) } - if !kept(root, "mesh-controller") { + if !kept(root, "node-tools") { t.Fatal("the build before was deleted though the new one was never seen healthy") }