The launcher trusted a counter only a by-hand reconcile ever cleared and a known-good nothing in the daemon wrote, so no machine could roll its host back; the controller and the node tools were replaced in place with nothing kept. - The launcher runs a delivered host that is not known-good on trial: one that crashes, stops for nothing, or does not report within ten minutes goes back to known-good, once per version, recorded in rolled-back. The host proves itself when the mesh takes a report under its own build, says every standing verdict on its reports, never stands aside for a rolled-back version, and restarts its service once when its launcher was replaced on disk. - The engine keeps the controller's and the node tools' previous build beside the new one and judges the new one: the lease taken by the controller it started (read-only direct get of mesh-controller_lease/holder), or this machine's runtime answering $SRV.PING.node-tools.<node>, within sixty seconds of time it could ask. Not healthy: the previous restored, once, said. Proved: the previous deleted. A build declared not-reversible is never rolled back. - Retire never removes a version newer than the running one.
116 lines
4.8 KiB
Go
116 lines
4.8 KiB
Go
package witness
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/link"
|
|
)
|
|
|
|
// The contract, held. Each of these is a line the controller's side relies on or writes; a change on
|
|
// either side changes this test (novox/hq to-be 45 §8).
|
|
|
|
// The lease is read where the controller keeps it (ADR 0229), on the one subject a host's grant names.
|
|
func TestTheLeaseIsReadWhereTheControllerKeepsIt(t *testing.T) {
|
|
if LeaseBucket != "mesh-controller_lease" || LeaseKey != "holder" {
|
|
t.Fatalf("the lease is read from %s/%s; the controller keeps it in mesh-controller_lease/holder",
|
|
LeaseBucket, LeaseKey)
|
|
}
|
|
if got := link.DirectGetSubject(LeaseBucket, LeaseKey); got != "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder" {
|
|
t.Fatalf("the host asks %s for the lease; its grant names exactly the direct get of the key", got)
|
|
}
|
|
if LeaseAge != 15*time.Second {
|
|
t.Fatalf("the lease's age is %s; the controller's bucket keeps a key fifteen seconds", LeaseAge)
|
|
}
|
|
}
|
|
|
|
// What the controller writes — mesh-controller internal/lease Holder, by its JSON names — is what is
|
|
// read. The value below is the shape that Holder marshals to.
|
|
func TestTheLeaseValueIsTheControllersHolder(t *testing.T) {
|
|
written := `{"instance":"controller@anchor pid 4242 since 2026-10-06T10:00:00Z","host":"anchor",` +
|
|
`"build":"development build","epoch":57,"taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:31Z"}`
|
|
l, err := ParseLease([]byte(written))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if l.Instance == "" || l.Host != "anchor" || l.Epoch != 57 || l.Taken.IsZero() || l.Renewed.IsZero() {
|
|
t.Fatalf("the lease was not read whole: %+v", l)
|
|
}
|
|
// And a field the controller adds later does not stop it being read.
|
|
if _, err := ParseLease([]byte(`{"instance":"i","taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:01Z","bundle":"sha256:x"}`)); err != nil {
|
|
t.Fatalf("a lease value with a field this host does not know was refused: %v", err)
|
|
}
|
|
}
|
|
|
|
// Healthy is: held now, on this machine, by an instance that took it after the new build started.
|
|
func TestTheNewControllerHoldsTheLeaseOnlyWhenItTookItAfterItStarted(t *testing.T) {
|
|
started := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
|
now := started.Add(30 * time.Second)
|
|
held := ControllerLease{Instance: "controller@anchor pid 2 since …", Host: "anchor.example",
|
|
Taken: started.Add(3 * time.Second), Renewed: now.Add(-2 * time.Second), Epoch: 58}
|
|
for _, c := range []struct {
|
|
name string
|
|
l ControllerLease
|
|
want bool
|
|
says string
|
|
}{
|
|
{"held by the new instance", held, true, "holds the lease"},
|
|
{"nobody", ControllerLease{}, false, "no holder"},
|
|
{"the old instance, taken before the restart", func() ControllerLease {
|
|
l := held
|
|
l.Taken = started.Add(-time.Hour)
|
|
return l
|
|
}(), false, "before the new build started"},
|
|
{"another machine's controller", func() ControllerLease {
|
|
l := held
|
|
l.Host = "home-server"
|
|
return l
|
|
}(), false, "not this machine"},
|
|
{"taken and no longer renewed", func() ControllerLease {
|
|
l := held
|
|
l.Renewed = now.Add(-20 * time.Second)
|
|
return l
|
|
}(), false, "past its"},
|
|
{"taken within the clocks' skew of the start", func() ControllerLease {
|
|
l := held
|
|
l.Taken = started.Add(-time.Second)
|
|
return l
|
|
}(), true, "holds the lease"},
|
|
} {
|
|
got, why := c.l.HeldBySince("anchor", started, now)
|
|
if got != c.want || !strings.Contains(why, c.says) {
|
|
t.Errorf("%s: healthy %v (%s), want %v saying %q", c.name, got, why, c.want, c.says)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The runtime is asked by its own name and this machine's: only this machine's runtime can answer.
|
|
func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) {
|
|
if got := link.PingSubject(NodeToolsService, "anchor"); got != "$SRV.PING.node-tools.anchor" {
|
|
t.Fatalf("the host asks %s; the runtime answers $SRV.PING.node-tools.<node>", got)
|
|
}
|
|
}
|
|
|
|
// What is judged by default is the mesh's two core processes, by the names the controller composes
|
|
// them under, and nothing else.
|
|
func TestOnlyTheCoreProcessesAreJudgedByDefault(t *testing.T) {
|
|
for name, want := range map[string]string{"mesh-controller": ByLease, "node-tools": ByPing, "greeter": ByNone} {
|
|
if got := Default(name); got != want {
|
|
t.Errorf("%s is judged by %q, want %q", name, got, want)
|
|
}
|
|
}
|
|
if ControllerWithin != 60*time.Second || NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second {
|
|
t.Fatal("the bounds are to-be 45 §8's: the lease within sixty seconds, a PING answered within five")
|
|
}
|
|
}
|
|
|
|
// A verdict on the wire names its component as the controller's condition does.
|
|
func TestAVerdictNamesItsComponent(t *testing.T) {
|
|
raw, _ := json.Marshal(link.Rollback{Component: Component(ByLease), Outcome: link.RolledBack})
|
|
if !strings.Contains(string(raw), `"component":"controller"`) || Component(ByPing) != "node-tools" {
|
|
t.Fatalf("a verdict names its component as %s", raw)
|
|
}
|
|
}
|