novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mesh-healers act and artifact-store-tools sweeps without the controller's lease

Browse Source 
The controller ended by its unassignment, so its lease key is absent for good
and the healers stayed in shadow while nobody swept the artifact store (issue
1126, A4). Drop both gates and each manifest's read of mesh-controller.lease;
the would bucket stays readable until its keys expire.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
jschoubben
2026-11-02 10:38:58 +01:00
parent fe7d4504bf
commit 0e6c358623
15 changed files with 132 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/artifact-store-tools/cmd/artifact-store-tools/door.go
+1 -1
View File
@@ -88,7 +88,7 @@ func (runtimeBus) Publish(subject string, body any) error {
return stdio.SeatPublish(subject, body, "")
}
// follow watches the builder's done keys and the wants: a build's end sweeps (once the lease names no controller) and
// follow watches the builder's done keys and the wants: a build's end sweeps and
// observes again; a want changed observes again. Event-driven: nothing reads them on a timer.
func follow(h *Holder) {
<-stdio.Served()
modules/artifact-store-tools/cmd/artifact-store-tools/holder.go
+4 -20
View File
@@ -9,8 +9,8 @@ package main
// At each build's end (a done key written) it sweeps as the controller's take-in did: every kept archive held first
// (ADR 0251 §8), then what is eligible let go of, oldest first, within the same bound per sweep, an index left for a
// `collect` a person confirms (ADR 0251 §7), each deletion recorded in the hand-act log (seat mesh-hand-acts) and said
// as artifacts-gone. **Nothing is let go until the controller's lease names a holder that is no controller**: until
// then the controller's take-in sweeps, and two sweeps never run at once. It also observes S25, image-missing: a want's
// as artifacts-gone. **No lease turns it** (issue 1126): the controller ended by its unassignment, so its take-in sweeps
// nothing and this seat is the one sweep. It also observes S25, image-missing: a want's
// image the store does not hold, a finding in its own state `findings` the condition keeper reads.
import (
@@ -24,7 +24,6 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
@@ -40,7 +39,6 @@ const (
HavesRead = "node-engine.have"
// ManifestsRead is the indexer's manifests: the images a manifest pins by itself.
ManifestsRead = delivery.IndexerModule + ".manifests"
LeaseRead = buscompose.ControllerLease
// FindingsState is the holder's own findings (health.FindingsState): image-missing.
FindingsState = health.FindingsState
// Module is this module, the observer its findings name.
@@ -58,7 +56,7 @@ const (
ImageMissing = "image-missing"
)
// Bus is what the holder reads and writes on the bus: its read states (wants, builds, haves, the lease), its own
// Bus is what the holder reads and writes on the bus: its read states (wants, builds, haves), its own
// findings, and its seat's event. The runtime in main; a test's in memory.
type Bus interface {
Keys(state string) ([]string, error)
@@ -106,13 +104,6 @@ type Build struct {
Manifest string
}
// LeaseAway says the controller's lease names a holder that is no controller: then the seat sweeps.
func (h *Holder) LeaseAway() bool {
raw, found, err := h.Bus.Get(LeaseRead, buscompose.ControllerLeaseKey)
away, _ := buscompose.LeaseHandedAway(raw, found, err)
return away
}
// Builds is every done build the builder's keys hold, each with its end.
func (h *Holder) Builds() ([]Build, error) {
keys, err := h.Bus.Keys(BuildsRead)
@@ -495,10 +486,6 @@ func (h *Holder) Collect(why string, confirm bool, most int) (*CollectAnswer, er
if most > MostAtAll {
return nil, fmt.Errorf("collect lets go of at most %d at once, not %d", MostAtAll, most)
}
if confirm && !h.LeaseAway() {
return nil, errors.New("the controller holds its lease, and its take-in sweeps the store until it hands it away: " +
"this seat lets nothing go before then (novox/hq ADR 0571 clause 2); ask the controller's collect")
}
return h.Sweep(context.Background(), Sweeping{Confirmed: confirm, DryRun: !confirm, Most: most, Why: why, By: "collect"})
}
@@ -665,11 +652,8 @@ func (h *Holder) sayGone(j *judged, refs []string, how string) {
}
}
// AfterBuild is the sweep at a build's end: only once the lease names no controller.
// AfterBuild is the sweep at a build's end.
func (h *Holder) AfterBuild(key string) {
if !h.LeaseAway() {
return
}
a, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "after the build " + key})
if err != nil {
h.logf("the sweep after %s: %v", key, err)
modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go
+8 -17
View File
@@ -3,7 +3,7 @@
package main
// The seat mesh-artifact-sweep's holder whole against a real bus (novox/hq ADR 0571 clause 2, issue 1955): the builder's done
// keys, the wants and the controller's lease in a real nats-server's key-value buckets, a build's end heard by a watch
// keys and the wants in a real nats-server's key-value buckets, no lease key anywhere (issue 1126), a build's end heard by a watch
// of the builds as the runtime hands it over, each deletion recorded by a request to the seat mesh-hand-acts on the bus,
// artifacts-gone said on the seat's subject, and S25's finding written to the holder's own bucket. Only the store's
// door is in memory: its HTTP is the copied registry package's, tested there.
@@ -20,7 +20,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
@@ -117,7 +116,7 @@ func (b realBus) Publish(subject string, body any) error {
return b.nc.Publish(subject, raw)
}
func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
nc, err := nats.Connect(testbus.URL(t))
@@ -129,7 +128,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, s := range []string{BuildsRead, WantsRead, HavesRead, LeaseRead, ManifestsRead, "artifact-store-tools." + FindingsState} {
for _, s := range []string{BuildsRead, WantsRead, HavesRead, ManifestsRead, "artifact-store-tools." + FindingsState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(s), History: 5}); err != nil {
t.Fatal(err)
}
@@ -137,7 +136,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
bus := realBus{ctx: ctx, nc: nc, js: js}
// The mesh as aMesh lays it, copied onto the real buckets: letta's seven builds, a want, the store's listing.
mem, memb, door, _ := aMesh(t, &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3})
mem, memb, door, _ := aMesh(t)
// The builder's log as written today: aMesh dates it from 2026-10-01 for its fixed clock, and this holder runs on the
// real clock, so read as dated the entries fall out of the log's 30 days and every copy is kept as unrecorded.
shift := time.Since(memb.log[len(memb.log)-1].At)
@@ -145,7 +144,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
e.At = e.At.Add(shift)
bus.log = append(bus.log, e)
}
for _, s := range []string{BuildsRead, WantsRead, LeaseRead, ManifestsRead} {
for _, s := range []string{BuildsRead, WantsRead, ManifestsRead} {
keys, _ := memb.Keys(s)
for _, k := range keys[:min(len(keys), 6)] { // the seventh build ends later, on the bus
raw, _, _ := memb.Get(s, k)
@@ -175,7 +174,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
t.Fatal(err)
}
h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Now: time.Now, Logf: t.Logf}
h.Acts = handacts.Recorder{LeaseAway: h.LeaseAway, Ask: func(key string, body any) (json.RawMessage, error) {
h.Acts = handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
m, err := nc.Request("mesh.seat."+strings.TrimPrefix(strings.Replace(key, ".", ".tool.", 1), "seat:"), raw, 5*time.Second)
if err != nil {
@@ -204,16 +203,8 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
}
}
// While the lease names the controller, a build's end lets nothing go.
ended()
if len(door.asked) != 0 {
t.Fatalf("before the lease write the store was asked %v", door.asked)
}
// The lease handed away: the next build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved
// past the five newest and no want names (the want keeps the 1st), recording each deletion.
if err := bus.Put(LeaseRead, buscompose.ControllerLeaseKey, buscompose.LeaseHolder{HandedTo: "mesh-grants", Why: "the lease write"}); err != nil {
t.Fatal(err)
}
// No lease key: the build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved past the five
// newest and no want names (the want keeps the 1st), recording each deletion.
ended()
deadline := time.Now().Add(30 * time.Second)
for {
modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go
+48 -42
View File
@@ -11,7 +11,6 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/health"
@@ -139,8 +138,8 @@ func archiveRef(module string, n int) string {
}
// A mesh with seven done builds of letta, the n-th ending n hours after a day began, each with an image and an archive;
// a want names the first on ace; the store lists them all and one image no build names; the lease as given.
func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memDoor, *[]string) {
// a want names the first on ace; the store lists them all and one image no build names; no lease key (issue 1126).
func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
t.Helper()
day := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
b := &memBus{states: map[string]map[string][]byte{}, history: map[string][]stdio.StateRevision{}}
@@ -194,9 +193,6 @@ func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memD
_ = b.Put(ManifestsRead, mk, json.RawMessage(manifest))
want, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp(1), Self: true})
_ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want))
if lease != nil {
_ = b.Put(LeaseRead, buscompose.ControllerLeaseKey, lease)
}
var acts []string
h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil },
Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil },
@@ -206,18 +202,14 @@ func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memD
acts = append(acts, key+" "+string(raw))
return json.RawMessage(`{}`), nil
}}}
h.Acts.LeaseAway = h.LeaseAway
return h, b, door, &acts
}
// ADR 0571 clause 2: the keep set from the store's listing, the wants and the five newest builds; nothing let go of while
// the lease names a controller; once it names another holder, a build's end lets go of the eligible within the bound,
// leaves an index for a confirmed collect, records each deletion and says artifacts-gone.
func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t *testing.T) {
controller := &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3}
handed := &buscompose.LeaseHolder{HandedTo: "mesh-grants", HandedBy: "person.jochen", Why: "the lease write"}
h, _, _, _ := aMesh(t, controller)
// ADR 0571 clause 2, 1126/A4/AC2: the keep set from the store's listing, the wants and the five newest builds; with no
// lease key, a build's end and a confirmed collect let go of the eligible within the bound, leave an index for a
// confirmed collect, record each deletion as a hand act and say artifacts-gone.
func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T) {
h, _, _, _ := aMesh(t)
r, err := h.Artifacts("")
if err != nil {
t.Fatal(err)
@@ -244,9 +236,10 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t
t.Errorf("the references are %d, counted %v, at %q", len(r.References), r.Counts, r.Store)
}
all := []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}
for _, c := range []struct {
name string
lease *buscompose.LeaseHolder
how string // "build" (a build's end), "collect" (a confirmed collect), or a sweep asked with most
index bool
most int
letGo []string
@@ -254,45 +247,58 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t
skipped int
unlog int
}{
{name: "the lease names a controller", lease: controller},
{name: "no lease key", lease: nil},
{name: "handed away", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}},
{name: "an index waits", lease: handed, index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1},
{name: "bounded", lease: handed, most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2},
{name: "a build's end", how: "build", letGo: all},
{name: "a confirmed collect", how: "collect", letGo: all},
{name: "a sweep", most: MostPerSweep, letGo: all},
{name: "an index waits", index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1},
{name: "bounded", most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2},
// A kept build the log no longer holds: what it stood on is unknown, so no copy goes.
{name: "a kept build's bases unknown", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3},
{name: "a kept build's bases unknown", most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3},
} {
t.Run(c.name, func(t *testing.T) {
h, b, door, acts := aMesh(t, c.lease)
h, b, door, acts := aMesh(t)
door.indexes[imageRef("letta", 2)] = c.index
if c.unlog > 0 {
b.log = append(b.log[:c.unlog-1:c.unlog-1], b.log[c.unlog:]...)
}
if c.most == 0 {
switch c.how {
case "build":
h.AfterBuild("letta.x")
if _, err := h.Collect("tidy", true, 0); err == nil {
t.Error("a confirmed collect was let through while the controller sweeps")
var gone []string
for _, x := range door.asked {
if r, ok := strings.CutPrefix(x, "let go "); ok {
gone = append(gone, r)
}
}
if len(door.asked) != 0 || len(*acts) != 0 || len(b.published) != 0 {
t.Fatalf("before the lease write the store was asked %v, acts %v, said %v", door.asked, *acts, b.published)
if fmt.Sprint(gone) != fmt.Sprint(c.letGo) {
t.Fatalf("a build's end let go of %v, want %v", gone, c.letGo)
}
default:
var a *CollectAnswer
var err error
if c.how == "collect" {
a, err = h.Collect("tidy", true, 0)
} else {
a, err = h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"})
}
if err != nil {
t.Fatal(err)
}
if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 {
t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo)
}
return
}
a, err := h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"})
if err != nil {
t.Fatal(err)
}
if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 {
t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo)
}
// Every kept archive held before the first deletion.
first := -1
for i, x := range door.asked {
if strings.HasPrefix(x, "let go ") && first < 0 {
first = i
held := 0
for _, x := range door.asked {
if strings.HasPrefix(x, "let go ") {
break
}
if strings.HasPrefix(x, "hold ") {
held++
}
}
if first != 6 {
if held != 6 {
t.Errorf("the store was asked %v: the six kept archives are held first", door.asked)
}
if len(*acts) != len(c.letGo) || !strings.HasPrefix((*acts)[0], "seat:mesh-hand-acts.hand-act ") ||
@@ -310,7 +316,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t
// S25 (image-missing): a want whose build names an image the store does not hold is a finding in the seat's own state,
// written once and deleted once the store holds it again; images names what a machine's wants and haves name.
func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testing.T) {
h, b, _, _ := aMesh(t, nil)
h, b, _, _ := aMesh(t)
view, _ := h.Listed(context.Background())
repo, digest, _ := splitReference(imageRef("letta", 1))
delete(view.L.Repos[repo].Revisions, digest)
modules/artifact-store-tools/cmd/artifact-store-tools/main.go
+2 -3
View File
@@ -1,8 +1,7 @@
// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, ADR 0571 clause 2, to-be 51): a process the
// node's runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its repositories, its
// size, and what the mesh keeps of each manifest — read from the store's own files through the store's own container,
// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end once the controller's lease names
// no controller, and S25's image-missing.
// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end, and S25's image-missing.
//
// **A module beside the store, not the store's own.** A module that provides the artifact store cannot also build an
// artifact: building publishes to the store, so the store would be needed to create itself. So these tools are a
@@ -29,7 +28,7 @@ func main() {
}
h := &Holder{Bus: runtimeBus{}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View,
Now: time.Now, Logf: logf}
h.Acts = handacts.Recorder{Ask: stdio.Ask, LeaseAway: h.LeaseAway}
h.Acts = handacts.Recorder{Ask: stdio.Ask}
go follow(h)
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools.
if err := stdio.Serve("", append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...)); err != nil {
modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go
+2 -2
View File
@@ -436,7 +436,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
t.Fatalf("served %v, manifest %v", served, listed)
}
// ADR 0571 clause 2: it holds the seat mesh-artifact-sweep and serves its verbs, and calls no verb of the controller's;
// the lease is the one controller state it reads (clause 10's exemption).
// it reads no controller state, the lease neither (issue 1126).
want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule}
if !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes %v", m.Invokes)
@@ -455,7 +455,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
t.Errorf("claims %+v, serves %v, the seat's verbs %v", m.Claims, verbs, contract)
}
for _, r := range m.Reads {
if strings.HasPrefix(r, "mesh-controller.") && r != LeaseRead {
if strings.HasPrefix(r, "mesh-controller.") {
t.Errorf("reads the controller's %s", r)
}
}
modules/artifact-store-tools/module.json
+1 -2
View File
@@ -27,8 +27,7 @@
"mesh-delivery-builder.builds",
"mesh-delivery-builder.build-log",
"mesh-delivery-indexer.manifests",
"node-engine.have",
"mesh-controller.lease"
"node-engine.have"
],
"state": [
"findings"
modules/mesh-healers/README.md
+8 -8
View File
@@ -15,15 +15,15 @@ Each act is begun by a create of its key in `attempts` before it is made and fin
mid-act has spent it and a second instance cannot begin it again. The condition keeper writes the condition's
`tried`, resolver and escalation from `attempts`; a healer writes no condition.
**Only once the controller's lease names no controller.** Until the write that hands the lease away, each act it would
take is written to `would`, never to `attempts`: it counts for no budget, settle or brake and reaches no condition.
**They act, and no lease turns it** (issue 1126): the controller ended by its unassignment. While it ran, each act a
healer would take was written to `would`, never to `attempts`; nothing writes `would` any more.
Each key of `attempts` and `would` lives a week from its last write, its own time to live (the buckets are declared
`key-ttl`): nothing else deletes `would`, and once the lease is handed away nothing is written there. The healers' waits
`key-ttl`): nothing else deletes `would`. The healers' waits
are keys of `wakes` whose time to live is the wait: `settle.<condition key>` ends with a settle and judges the condition
again, `fortnight` ends when S15's oldest counted act leaves the fortnight and counts again. The watch of `wakes` hears
each expiry; no timer runs in the process. A wait still running is on the bus across a restart; one that expired while
no instance watched is judged at the start, once the lease is first read.
no instance watched is judged at the start.
S15: it hears the seat mesh-hand-acts' `act-recorded` and watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a
fortnight; a drill and a person's decision by design never count.
@@ -31,8 +31,8 @@ fortnight; a drill and a person's decision by design never count.
## What it keeps
- `mesh-healers_attempts`: every act and escalation, each key kept a week.
- `mesh-healers_would`: the newest act it would take per healer and condition while the controller holds the lease,
each key kept a week.
- `mesh-healers_would`: the newest act it would take per healer and condition while the controller ran, each key
kept a week; nothing writes it any more.
- `mesh-healers_findings`: the brake, its own blindness, and S15.
- `mesh-healers_wakes`: the waits, each key living as long as it waits.
@@ -41,6 +41,6 @@ fortnight; a drill and a person's decision by design never count.
- `mesh-healers.healers`: the registry, the acts of the last days, the would-acts, the brake, whether it acts and why;
`recount=true` counts the hand-act log again.
- `mesh-healers.heal`: one act now on an open condition, by key and why.
- `mesh-healers.drill`: records a drill in the hand-act log of the seat `mesh-hand-acts` as `hand-act drill` (the
controller's `drill` while that seat has no holder and the controller holds its lease). S15 never counts a drill.
- `mesh-healers.drill`: records a drill in the hand-act log of the seat `mesh-hand-acts` as `hand-act drill`; with
no holder of that seat it is refused. S15 never counts a drill.
- `healers_findings`, `healers_would`, `healers_wakes`: the reads of its buckets.
modules/mesh-healers/cmd/mesh-healers/healers.go
+11 -62
View File
@@ -9,10 +9,9 @@ package main
// healer's to say: the condition clears when its observer no longer sees it, and the condition keeper writes `tried`,
// the resolver and the escalation into the condition from `attempts`.
//
// **Only once the controller's lease names no controller** (ADR 0564 §8): until then each act it would take is
// written to the bucket `would`, never to `attempts`, so a would-act counts for no budget, settle or brake, reaches
// no `tried` and escalates nothing. `would` keeps each key a week (its bucket's age): nothing else deletes it, and once
// the lease is handed away nothing is written there again.
// **They act, and no lease turns it** (issue 1126): the controller ended by its unassignment, so the gate on its lease
// (ADR 0564 §8) is gone. The bucket `would` holds what they would have done while it ran, each key kept a week (its
// bucket's age); nothing writes there any more.
import (
"encoding/json"
@@ -42,8 +41,6 @@ const (
AttemptsState = health.AttemptsState
WouldState = health.WouldState
FindingsState = health.FindingsState
LeaseRead = buscompose.ControllerLease
LeaseKey = buscompose.ControllerLeaseKey
ConditionsRead = "mesh-conditions.conditions"
ActedEvent = health.EventHealerActed
// ActedSubject is where the seat says it, which its consumers hear (catalogue.SeatSays).
@@ -115,8 +112,8 @@ type Condition struct {
func (c Condition) Escalated() bool { return c.Resolver == "operator" && len(c.Tried) > 0 }
// Attempt, Would and Acted are mesh-sdk's health layouts: an act begun in `attempts` (its key
// `<healer>.<condition key>.<begun ns>` is the healers' own), an act written to `would` while the lease names a
// controller (one key per healer and condition, `<healer>.<condition key>`, its newest), and the body of healer-acted,
// `<healer>.<condition key>.<begun ns>` is the healers' own), an act written to `would` while the controller ran
// (one key per healer and condition, `<healer>.<condition key>`, its newest), and the body of healer-acted,
// epoch left out (ADR 0564 Limits 10).
type (
Attempt = health.Attempt
@@ -182,48 +179,14 @@ type Healers struct {
Registry []Healer
mu sync.Mutex
// lease is the controller's lease as last read: acting is on only while it is handed away.
acting bool
leaseWhy string
// declined is why each condition was last passed over, said once.
declined map[string]string
// acts are the hand-act log as watched, for S15.
acts map[string]HandAct
}
// LeaseHeard records what the lease says: acting is on only while it names a holder that is no controller (ADR 0564
// §8, Limits 8, as ADR 0561 §3's reader reads it).
func (h *Healers) LeaseHeard(raw []byte, found bool, readErr error) {
on, why := buscompose.LeaseHandedAway(raw, found, readErr)
h.mu.Lock()
flipped := on != h.acting || h.leaseWhy == ""
h.acting, h.leaseWhy = on, leaseWords(on, why)
h.mu.Unlock()
if flipped {
h.Logf("[%s] the healers %s: %s", Module, map[bool]string{true: "ACT", false: "act in shadow"}[on], h.leaseWhy)
}
}
// leaseWords says the lease's state as the healers read it; the SDK's words speak of minting.
func leaseWords(on bool, why string) string {
if on {
return "the controller's lease is handed away: the healers act"
}
if strings.Contains(why, "cannot be read") || strings.Contains(why, "not readable") {
return "the controller's lease cannot be read, so it is taken to name a controller: every act is written as would"
}
return "the controller's lease names a controller (or none is written, so one may take it): every act is written as would"
}
// Acting says whether the healers act, and why.
func (h *Healers) Acting() (bool, string) {
h.mu.Lock()
defer h.mu.Unlock()
if h.leaseWhy == "" {
return false, "the controller's lease has not been read yet: every act is written as would"
}
return h.acting, h.leaseWhy
}
// actingWhy is why the healers act: what the tool `healers` says.
const actingWhy = "the healers act: the controller ended by its unassignment, so no lease turns them (issue 1126)"
// healerFor is the healer that acts here on a kind.
func (h *Healers) healerFor(kind string) (Healer, bool) {
@@ -235,7 +198,7 @@ func (h *Healers) healerFor(kind string) (Healer, bool) {
return Healer{}, false
}
// Heard is a condition the keeper raised or changed: judged, and acted on, written as would, or passed over.
// Heard is a condition the keeper raised or changed: judged, and acted on or passed over.
func (h *Healers) Heard(body []byte) error {
var c Condition
if err := json.Unmarshal(body, &c); err != nil {
@@ -265,13 +228,6 @@ func (h *Healers) Judge(c Condition, by string) (string, error) {
}
delete(h.declined, c.Key)
now := h.Now()
if !h.acting {
w := Would{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: row.Repair, At: now.UTC(), Why: h.whyNot()}
if err := h.Store.PutTTL(WouldState, row.ID+"."+c.Key, w, KeptFor); err != nil {
return "", fmt.Errorf("the would-act of %s on %s is not written: %w", row.ID, c.Key, err)
}
return fmt.Sprintf("healer %s would act on %s: %s", row.ID, c.Key, w.Why), nil
}
all, err := h.attempts()
if err != nil {
h.blind(err)
@@ -299,13 +255,6 @@ func (h *Healers) Judge(c Condition, by string) (string, error) {
return h.act(row, c, budget, len(spent), by)
}
func (h *Healers) whyNot() string {
if h.leaseWhy == "" {
return "the controller's lease has not been read yet"
}
return h.leaseWhy
}
// act is one healer's act: begun by a create in attempts, made, finished, and said.
func (h *Healers) act(row Healer, c Condition, budget string, spent int, by string) (string, error) {
begun := h.Now().UTC()
@@ -379,7 +328,7 @@ func (h *Healers) tell(e Acted) {
}
}
// Rearm is the healers' start, once the lease is first read: a settle whose key expired while no instance watched has
// Rearm is the healers' start: a settle whose key expired while no instance watched has
// no expiry left to hear, so each newest act whose settle ended within its window is judged at once. A settle still
// running keeps its key on the bus and needs nothing.
func (h *Healers) Rearm() {
@@ -611,14 +560,14 @@ func (h *Healers) see(source string, found []Observation) {
// --- H3 --------------------------------------------------------------------------------------------------------
// moduleOf is the module whose objects a condition names: a consumer of a module's own (`<stream>.<module>`). The
// controller's own consumers are not a module's: the controller asserts them itself until its lease.
// controller's own consumers are not a module's: they go with the controller's leftovers (issue 1126).
func moduleOf(c Condition) (string, string) {
stream, consumer, ok := strings.Cut(c.Subject.ID, ".")
if c.Kind != "consumer-lost" || !ok || stream == "" || consumer == "" {
return "", "it names no consumer of a module whose objects the grants part asserts"
}
if consumer == buscompose.ControllerName || strings.HasPrefix(consumer, buscompose.ControllerName+"-") {
return "", "the consumer is the controller's own, which the controller asserts itself while it holds its lease"
return "", "the consumer is the controller's own, which the grants part does not assert"
}
return consumer, ""
}
modules/mesh-healers/cmd/mesh-healers/healers_test.go
+22 -70
View File
@@ -154,10 +154,6 @@ func (w *world) fresh() {
}
}
func (w *world) handAway() {
w.h.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil)
}
// lost is a module's consumer the bus does not hold, as D6 raises it.
func lost(module string) Condition {
return Condition{Key: "bus.EVENTS." + module + ".missing", Kind: "consumer-lost", Severity: "warning", Resolver: "self",
@@ -170,47 +166,21 @@ func body(c Condition) []byte {
return raw
}
// No act is made while the lease names a controller: each is written to would, never to attempts, so it counts for no
// budget or brake and nothing is asked or said; one is made from the write that hands the lease away (ADR 0564 §8).
func TestTheHealersActOnlyOnceTheLeaseNamesNoController(t *testing.T) {
cases := []struct {
name string
lease func(h *Healers)
acts bool
}{
{"never read", func(*Healers) {}, false},
{"absent", func(h *Healers) { h.LeaseHeard(nil, false, nil) }, false},
{"unreadable", func(h *Healers) { h.LeaseHeard(nil, false, errors.New("timeout")) }, false},
{"a controller holds it", func(h *Healers) {
h.LeaseHeard([]byte(`{"instance":"novox/1/2026-10-27","epoch":57}`), true, nil)
}, false},
{"handed away", func(h *Healers) { h.LeaseHeard([]byte(`{"handed-to":"the operator"}`), true, nil) }, true},
// 1126/A4/AC1: with no lease key read at all (the controller ended by its unassignment), a condition a healer covers is
// acted on: one attempt, finished, asked and said, and nothing written to would.
func TestTheHealersActWithNoLease(t *testing.T) {
w := newWorld()
if err := w.h.Heard(body(lost("records"))); err != nil {
t.Fatal(err)
}
for _, tc := range cases {
w := newWorld()
tc.lease(w.h)
if err := w.h.Heard(body(lost("records"))); err != nil {
t.Fatalf("%s: %v", tc.name, err)
}
attempts, would := w.store.keys(AttemptsState), w.store.keys(WouldState)
if tc.acts {
if len(attempts) != 1 || len(would) != 0 || len(w.asked) != 1 || !strings.Contains(w.asked[0], `"module":"records"`) ||
len(w.said) != 1 || w.said[0].Outcome != OutcomeActed {
t.Errorf("%s: attempts %v, would %v, asked %v, said %+v", tc.name, attempts, would, w.asked, w.said)
continue
}
a := w.store.attempt(t, attempts[0])
if a.Outcome != OutcomeActed || a.Finished == nil || a.Healer != "H3" || a.Condition != "bus.EVENTS.records.missing" {
t.Errorf("%s: the attempt is %+v", tc.name, a)
}
continue
}
if len(attempts) != 0 || len(w.asked) != 0 || len(w.said) != 0 || !slices.Equal(would, []string{"H3.bus.EVENTS.records.missing"}) {
t.Errorf("%s: attempts %v, would %v, asked %v, said %v", tc.name, attempts, would, w.asked, w.said)
}
if _, braked, _ := w.store.Get(FindingsState, brakeKey); braked {
t.Errorf("%s: a would-act moved the brake", tc.name)
}
attempts, would := w.store.keys(AttemptsState), w.store.keys(WouldState)
if len(attempts) != 1 || len(would) != 0 || len(w.asked) != 1 || !strings.Contains(w.asked[0], `"module":"records"`) ||
len(w.said) != 1 || w.said[0].Outcome != OutcomeActed {
t.Fatalf("attempts %v, would %v, asked %v, said %+v", attempts, would, w.asked, w.said)
}
a := w.store.attempt(t, attempts[0])
if a.Outcome != OutcomeActed || a.Finished == nil || a.Healer != "H3" || a.Condition != "bus.EVENTS.records.missing" {
t.Errorf("the attempt is %+v", a)
}
}
@@ -219,7 +189,6 @@ func TestTheHealersActOnlyOnceTheLeaseNamesNoController(t *testing.T) {
// in an hour and lets go an hour after the last (ADR 0231 §21, ADR 0564 §7).
func TestBudgetSettleEscalationAndBrake(t *testing.T) {
w := newWorld()
w.handAway()
c := lost("records")
_ = w.store.Put(ConditionsRead, c.Key, c)
settle := settlePrefix + c.Key
@@ -247,7 +216,6 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) {
switch s.how {
case "restart":
w.fresh()
w.handAway()
w.h.Rearm()
case "expire":
w.h.WakeHeard(settle, OpExpire)
@@ -277,28 +245,24 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) {
}
// A settle whose key expired while no instance watched is judged at the start.
r := newWorld()
r.handAway()
_ = r.store.Put(ConditionsRead, c.Key, c)
if _, err := r.h.Judge(c, ""); err != nil {
t.Fatal(err)
}
r.now = r.now.Add(7 * time.Minute)
r.fresh()
r.handAway()
r.h.Rearm()
if len(r.said) != 2 || r.said[1].Outcome != OutcomeEscalated {
t.Errorf("a settle that ended unheard is not judged at the start: %+v", r.said)
}
// A second instance cannot begin the act an attempt's key already holds.
x := newWorld()
x.handAway()
_ = x.store.CreateTTL(AttemptsState, attemptKey("H3", lost("mail").Key, x.now.UTC()), Attempt{}, KeptFor)
if said, _ := x.h.Judge(lost("mail"), ""); !strings.Contains(said, "another instance") || len(x.asked) != 0 {
t.Errorf("a second instance acted: %q, asked %v", said, x.asked)
}
// The brake: twelve acts in the hour stop every healer, said as an urgent finding.
b := newWorld()
b.handAway()
for i := 0; i < BrakeLimit; i++ {
b.now = b.now.Add(time.Minute)
_ = b.store.CreateTTL(AttemptsState, attemptKey("H3", "x", b.now), Attempt{Healer: "H3", Condition: "x", BudgetKey: "x",
@@ -325,7 +289,6 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) {
}
// Blind to what they did, they act on nothing and say so.
z := newWorld()
z.handAway()
z.store.keysErr[AttemptsState] = errors.New("the bus is away")
if _, err := z.h.Judge(lost("mail"), ""); err == nil || len(z.asked) != 0 {
t.Errorf("blind, the healers acted: %v", err)
@@ -350,7 +313,6 @@ func TestTheRegistryAndWhatH3Answers(t *testing.T) {
}
for _, tc := range cases {
w := newWorld()
w.handAway()
said, _ := w.h.Judge(tc.c, "")
if acted := len(w.asked) == 1; acted != tc.holds {
t.Errorf("%s: acted %v, want %v (%s)", tc.c.Key, acted, tc.holds, said)
@@ -420,7 +382,6 @@ func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) {
// answer every bucket.
func TestTheVerbsAndReads(t *testing.T) {
w := newWorld()
w.handAway()
c := lost("records")
_ = w.store.Put(ConditionsRead, c.Key, c)
if _, err := w.h.Heal(c.Key, "", "person.jochen"); err == nil {
@@ -444,28 +405,19 @@ func TestTheVerbsAndReads(t *testing.T) {
if a["acting"] != true || len(a["acts"].([]Attempt)) != 1 || len(a["healers"].([]map[string]any)) != 3 {
t.Errorf("healers answered %v", a)
}
// drill records into mesh-hand-acts as a drill, and never into the controller's log, whoever holds the lease
// (ADR 0565 §11, issue 1955).
// drill records into mesh-hand-acts as a drill, and never into the controller's log (ADR 0565 §11, issue 1955).
drills := []struct {
name string
seatGone bool
leaseAway bool
args map[string]any
log, err string
name string
seatGone bool
args map[string]any
log, err string
}{
{"into the seat", false, true, map[string]any{"what": "stopped records", "why": "does it raise"}, handacts.LogSeat, ""},
{"into the seat under a controller's lease", false, false, map[string]any{"what": "stopped records", "why": "x"}, handacts.LogSeat, ""},
{"no holder: refused, the controller's not asked", true, false, map[string]any{"what": "stopped records", "why": "x"}, "", "nothing was recorded"},
{"no holder, lease away", true, true, map[string]any{"what": "stopped records", "why": "x"}, "", "nothing was recorded"},
{"says nothing", false, true, map[string]any{}, "", "says what"},
{"into the seat", false, map[string]any{"what": "stopped records", "why": "does it raise"}, handacts.LogSeat, ""},
{"no holder: refused, the controller's not asked", true, map[string]any{"what": "stopped records", "why": "x"}, "", "nothing was recorded"},
{"says nothing", false, map[string]any{}, "", "says what"},
}
for _, d := range drills {
x := newWorld()
if d.leaseAway {
x.handAway()
} else {
x.h.LeaseHeard([]byte(`{"instance":"novox/1","epoch":5}`), true, nil)
}
var asked []string
x.h.Ask = func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
modules/mesh-healers/cmd/mesh-healers/main.go
+6 -12
View File
@@ -1,8 +1,8 @@
// mesh-healers: the holder of the seat mesh-healers (novox/hq ADR 0564 §7, §8). A Go bundle served by a runtime of its
// own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `attempts` (every act begun,
// made and finished, and every escalation, kept a week), `would` (what it would have done while the controller holds
// its lease, kept a week) and `findings` (the brake, its blindness, and S15). It hears the condition keeper's
// condition-raised and condition-changed on a durable consumer, watches the controller's lease, and counts the seat's hand acts
// made and finished, and every escalation, kept a week), `would` (what it would have done while the controller ran,
// kept a week; nothing writes it any more) and `findings` (the brake, its blindness, and S15). It hears the condition keeper's
// condition-raised and condition-changed on a durable consumer, and counts the seat's hand acts
// for S15. stdout is the MCP channel; what this module says, it says on stderr.
package main
@@ -11,7 +11,6 @@ import (
"errors"
"fmt"
"os"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
@@ -56,14 +55,9 @@ func main() {
h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf,
Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) },
Publish: stdio.SeatPublish}
// The lease first: until it is read the healers write would-acts only. Once it is first read, a settle that ended
// while no instance watched is judged (Rearm); the waits still running are keys on the bus, heard on their expiry.
var started sync.Once
go stdio.State(LeaseRead).Follow(LeaseKey, func(c stdio.StateChange) error {
h.LeaseHeard(c.Value, c.Op == "put", nil)
started.Do(func() { go h.Rearm() })
return nil
})
// At start, a settle that ended while no instance watched is judged (Rearm); the waits still running are keys on
// the bus, heard on their expiry.
go h.Rearm()
go stdio.State(WakesState).Follow("", func(c stdio.StateChange) error {
h.WakeHeard(c.Key, c.Op)
return nil
modules/mesh-healers/cmd/mesh-healers/manifest_test.go
+3 -3
View File
@@ -26,7 +26,7 @@ type manifest struct {
// The manifest says what the code does: it claims the seat mesh-healers once for the mesh with its verbs (ADR 0564 §1,
// §7), runs on its own account, says healer-acted, hears the keeper's raised and changed, asks only the grants part's
// assert, reads the lease, the keeper's conditions and the hand-act log, and keeps attempts and would a week each.
// assert, reads the keeper's conditions (never the controller's lease, issue 1126) and the hand-act log, and keeps attempts and would a week each.
// Every state it keeps has a tool that reads it, and every act it takes by itself a tool that starts it (ADR 0564 §12).
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
@@ -50,7 +50,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
{"emits (healer-acted is the seat's, said on its subject)", m.Emits, nil},
{"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent, RecordedEvent}},
{"invokes", m.Invokes, []string{AssertTool, "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts"}},
{"reads", m.Reads, []string{LeaseRead, ConditionsRead}},
{"reads", m.Reads, []string{ConditionsRead}},
} {
if !slices.Equal(c.got, c.want) {
t.Errorf("%s %v, want %v", c.name, c.got, c.want)
@@ -74,7 +74,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
}
readBy := map[string]string{AttemptsState: HealersTool, WouldState: WouldTool, FindingsState: FindingsTool,
WakesState: WakesTool, "a settle's end": HealTool,
"an act on an event": HealTool, "S15's count": HealersTool + " recount=true", "the lease as read": HealersTool}
"an act on an event": HealTool, "S15's count": HealersTool + " recount=true"}
tools := map[string]bool{}
for _, tool := range Tools(&Healers{}) {
tools[tool.Name] = true
modules/mesh-healers/cmd/mesh-healers/tools.go
+6 -8
View File
@@ -38,8 +38,8 @@ func Tools(h *Healers) []stdio.Tool {
return []stdio.Tool{
{Name: HealersTool,
Description: "The mesh's healers (novox/hq ADR 0564 §7, §8): the registry, every act and escalation of the last " +
"days (the bucket attempts), whether the mesh-wide brake holds, whether they act or only write what they " +
"would do while the controller holds its lease, and the would-acts. recount=true counts the hand-act log " +
"days (the bucket attempts), whether the mesh-wide brake holds, that they act (no lease turns them, " +
"issue 1126), and the would-acts kept from while the controller ran. recount=true counts the hand-act log " +
"again for S15 (a cause repaired by hand twice in a fortnight).",
Input: obj(map[string]any{
"days": map[string]any{"type": "number", "description": "how many days of acts back (default 7)"},
@@ -47,8 +47,7 @@ func Tools(h *Healers) []stdio.Tool {
Run: func(a map[string]any) (any, error) { return h.Answer(intArg(a, "days", 7), boolArg(a, "recount")) }},
{Name: HealTool,
Description: "One healer's act now on an open condition, by its key, within its budget and the brake, as an " +
"event would have it — or, while the controller holds its lease, the act it would take written as would. " +
"Says what was done.",
"event would have it. Says what was done.",
Input: obj(map[string]any{
"key": map[string]any{"type": "string", "description": "the condition's key"},
"why": map[string]any{"type": "string", "description": "why, in a sentence"}}),
@@ -76,8 +75,8 @@ func Tools(h *Healers) []stdio.Tool {
Input: obj(map[string]any{"key": map[string]any{"type": "string"}}),
Run: func(a map[string]any) (any, error) { return h.read(WakesState, strArg(a, "key")) }},
{Name: WouldTool,
Description: "What the healers would have done while the controller holds its lease (their bucket would, each " +
"key kept a week): one per healer and condition, its newest; or one whole by key.",
Description: "What the healers would have done while the controller ran (their bucket would, each " +
"key kept a week; nothing writes it any more): one per healer and condition, its newest; or one whole by key.",
Input: obj(map[string]any{"key": map[string]any{"type": "string"}}),
Run: func(a map[string]any) (any, error) { return h.read(WouldState, strArg(a, "key")) }},
}
@@ -147,7 +146,6 @@ func (h *Healers) Answer(days int, recount bool) (any, error) {
"budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle),
"then": r.Then, "acts-in": r.ActsIn, "from": r.From})
}
acting, why := h.Acting()
would, err := h.read(WouldState, "")
if err != nil {
would = map[string]any{"unread": err.Error()}
@@ -161,7 +159,7 @@ func (h *Healers) Answer(days int, recount bool) (any, error) {
if recent == nil {
recent = []Attempt{}
}
return map[string]any{"healers": rows, "acts": recent, "days": days, "acting": acting, "why": why, "would": would,
return map[string]any{"healers": rows, "acts": recent, "days": days, "acting": true, "why": actingWhy, "would": would,
"brake": map[string]any{"holds": braked, "said": said, "limit": BrakeLimit, "window": BrakeWindow.String()},
"declined": declined,
"note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"}, nil
modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go
+10 -24
View File
@@ -5,9 +5,9 @@ package main
// mesh-healers whole against a real bus (novox/hq ADR 0564 §7, §8; issue 1952 H3): the built binary, played over its
// real stdin and stdout as the node's tool runtime plays it (mesh/state.* and watches answered from a real nats-server's
// key-value buckets, the keeper's events handed over as the runtime hands them, the grants part's assert answered). An
// induced failure, a module's consumer gone, is only written as would while the controller holds its lease; from the
// write that hands the lease away a healer acts, its attempt on the bus and healer-acted said; past twelve acts in the
// hour the brake stops every healer, said as an urgent finding on the bus.
// induced failure, a module's consumer gone, is acted on with no lease key anywhere (issue 1126), its attempt on the bus
// and healer-acted said, nothing written as would; past twelve acts in the hour the brake stops every healer, said as an
// urgent finding on the bus.
import (
"bufio"
@@ -320,7 +320,7 @@ func raised(key, module string) []byte {
return raw
}
// 1952/H3: an induced failure; the healer acts only from the lease write, and brakes.
// 1952/H3, 1126/A4: an induced failure with no lease key; the healer acts, and brakes.
func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
@@ -341,7 +341,7 @@ func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) {
t.Fatal(err)
}
for _, b := range []string{Module + "_" + AttemptsState, Module + "_" + WouldState, Module + "_" + FindingsState, Module + "_" + WakesState,
"mesh-controller_lease", "mesh-conditions_conditions", "mesh-controller_hand-acts"} {
"mesh-conditions_conditions", "mesh-controller_hand-acts"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
t.Fatal(err)
}
@@ -357,10 +357,6 @@ func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) {
k, _ := bucket(name).Keys(ctx)
return k
}
lease := bucket("mesh-controller_lease")
if _, err := lease.Put(ctx, LeaseKey, []byte(`{"instance":"novox/1/2026-10-27","epoch":57}`)); err != nil {
t.Fatal(err)
}
acted, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + ActedEvent)
if err != nil {
t.Fatal(err)
@@ -385,24 +381,11 @@ func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) {
_ = json.Unmarshal(raw, &a)
return a
}
eventually(t, "the lease read as a controller's", func() bool {
a := healers()
return a != nil && strings.Contains(fmt.Sprint(a["why"]), "names a controller")
})
eventually(t, "the healers acting", func() bool { return healers()["acting"] == true })
// The runtime's consumer holds an event until the bundle subscribes: delivered once it hears them.
eventually(t, "the healers hearing the keeper's conditions", func() bool { return p.hears(RaisedEvent) && p.hears(ChangedEvent) })
// The controller holds the lease: the induced failure is written as would, never acted on.
p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records"))
eventually(t, "the would-act written", func() bool { return len(keys(Module+"_"+WouldState)) == 1 })
if k := keys(Module + "_" + AttemptsState); len(k) != 0 {
t.Fatalf("acted under the controller's lease: %v", k)
}
// The write that hands the lease away: the same failure is acted on.
if _, err := lease.Put(ctx, LeaseKey, []byte(`{"handed-to":"the operator's hand-over"}`)); err != nil {
t.Fatal(err)
}
eventually(t, "the healers acting", func() bool { return healers()["acting"] == true })
// No lease key: the induced failure is acted on.
p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records"))
eventually(t, "the act begun and finished on the bus", func() bool {
k := keys(Module + "_" + AttemptsState)
@@ -412,6 +395,9 @@ func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) {
e, err := bucket(Module+"_"+AttemptsState).Get(ctx, k[0])
return err == nil && strings.Contains(string(e.Value()), `"outcome":"acted"`)
})
if k := keys(Module + "_" + WouldState); len(k) != 0 {
t.Errorf("written as would: %v", k)
}
msg, err := acted.NextMsg(10 * time.Second)
if err != nil || !strings.Contains(string(msg.Data), `"healer":"H3"`) {
t.Fatalf("healer-acted was not said: %v", err)
modules/mesh-healers/module.json
-1
View File
@@ -24,7 +24,6 @@
"seat:mesh-hand-acts.hand-acts"
],
"reads": [
"mesh-controller.lease",
"mesh-conditions.conditions"
],
"state": [