novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'Without the controller: the vault retires and rotates, the healers act, the artifact store is swept, config and conditions stand alone (issue 1126)' (#592) from fix/1126-gates-catalog into main

Browse Source
This commit was merged in pull request #592.
This commit is contained in:
mesh-tools
2026-11-02 10:36:34 +00:00
parent db104bdb08 326897d877
commit d2de18fd6b
50 changed files with 384 additions and 759 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/artifact-store-tools/cmd/artifact-store-tools/door.go
+1 -1
View File
@@ -88,7 +88,7 @@ func (runtimeBus) Publish(subject string, body any) error {
return stdio.SeatPublish(subject, body, "")
}
// follow watches the builder's done keys and the wants: a build's end sweeps (once the lease names no controller) and
// follow watches the builder's done keys and the wants: a build's end sweeps and
// observes again; a want changed observes again. Event-driven: nothing reads them on a timer.
func follow(h *Holder) {
<-stdio.Served()
modules/artifact-store-tools/cmd/artifact-store-tools/holder.go
+4 -20
View File
@@ -9,8 +9,8 @@ package main
// At each build's end (a done key written) it sweeps as the controller's take-in did: every kept archive held first
// (ADR 0251 §8), then what is eligible let go of, oldest first, within the same bound per sweep, an index left for a
// `collect` a person confirms (ADR 0251 §7), each deletion recorded in the hand-act log (seat mesh-hand-acts) and said
// as artifacts-gone. **Nothing is let go until the controller's lease names a holder that is no controller**: until
// then the controller's take-in sweeps, and two sweeps never run at once. It also observes S25, image-missing: a want's
// as artifacts-gone. **No lease turns it** (issue 1126): the controller ended by its unassignment, so its take-in sweeps
// nothing and this seat is the one sweep. It also observes S25, image-missing: a want's
// image the store does not hold, a finding in its own state `findings` the condition keeper reads.
import (
@@ -24,7 +24,6 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
@@ -40,7 +39,6 @@ const (
HavesRead = "node-engine.have"
// ManifestsRead is the indexer's manifests: the images a manifest pins by itself.
ManifestsRead = delivery.IndexerModule + ".manifests"
LeaseRead = buscompose.ControllerLease
// FindingsState is the holder's own findings (health.FindingsState): image-missing.
FindingsState = health.FindingsState
// Module is this module, the observer its findings name.
@@ -58,7 +56,7 @@ const (
ImageMissing = "image-missing"
)
// Bus is what the holder reads and writes on the bus: its read states (wants, builds, haves, the lease), its own
// Bus is what the holder reads and writes on the bus: its read states (wants, builds, haves), its own
// findings, and its seat's event. The runtime in main; a test's in memory.
type Bus interface {
Keys(state string) ([]string, error)
@@ -106,13 +104,6 @@ type Build struct {
Manifest string
}
// LeaseAway says the controller's lease names a holder that is no controller: then the seat sweeps.
func (h *Holder) LeaseAway() bool {
raw, found, err := h.Bus.Get(LeaseRead, buscompose.ControllerLeaseKey)
away, _ := buscompose.LeaseHandedAway(raw, found, err)
return away
}
// Builds is every done build the builder's keys hold, each with its end.
func (h *Holder) Builds() ([]Build, error) {
keys, err := h.Bus.Keys(BuildsRead)
@@ -495,10 +486,6 @@ func (h *Holder) Collect(why string, confirm bool, most int) (*CollectAnswer, er
if most > MostAtAll {
return nil, fmt.Errorf("collect lets go of at most %d at once, not %d", MostAtAll, most)
}
if confirm && !h.LeaseAway() {
return nil, errors.New("the controller holds its lease, and its take-in sweeps the store until it hands it away: " +
"this seat lets nothing go before then (novox/hq ADR 0571 clause 2); ask the controller's collect")
}
return h.Sweep(context.Background(), Sweeping{Confirmed: confirm, DryRun: !confirm, Most: most, Why: why, By: "collect"})
}
@@ -665,11 +652,8 @@ func (h *Holder) sayGone(j *judged, refs []string, how string) {
}
}
// AfterBuild is the sweep at a build's end: only once the lease names no controller.
// AfterBuild is the sweep at a build's end.
func (h *Holder) AfterBuild(key string) {
if !h.LeaseAway() {
return
}
a, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "after the build " + key})
if err != nil {
h.logf("the sweep after %s: %v", key, err)
modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go
+8 -17
View File
@@ -3,7 +3,7 @@
package main
// The seat mesh-artifact-sweep's holder whole against a real bus (novox/hq ADR 0571 clause 2, issue 1955): the builder's done
// keys, the wants and the controller's lease in a real nats-server's key-value buckets, a build's end heard by a watch
// keys and the wants in a real nats-server's key-value buckets, no lease key anywhere (issue 1126), a build's end heard by a watch
// of the builds as the runtime hands it over, each deletion recorded by a request to the seat mesh-hand-acts on the bus,
// artifacts-gone said on the seat's subject, and S25's finding written to the holder's own bucket. Only the store's
// door is in memory: its HTTP is the copied registry package's, tested there.
@@ -20,7 +20,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
@@ -117,7 +116,7 @@ func (b realBus) Publish(subject string, body any) error {
return b.nc.Publish(subject, raw)
}
func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
nc, err := nats.Connect(testbus.URL(t))
@@ -129,7 +128,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, s := range []string{BuildsRead, WantsRead, HavesRead, LeaseRead, ManifestsRead, "artifact-store-tools." + FindingsState} {
for _, s := range []string{BuildsRead, WantsRead, HavesRead, ManifestsRead, "artifact-store-tools." + FindingsState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(s), History: 5}); err != nil {
t.Fatal(err)
}
@@ -137,7 +136,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
bus := realBus{ctx: ctx, nc: nc, js: js}
// The mesh as aMesh lays it, copied onto the real buckets: letta's seven builds, a want, the store's listing.
mem, memb, door, _ := aMesh(t, &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3})
mem, memb, door, _ := aMesh(t)
// The builder's log as written today: aMesh dates it from 2026-10-01 for its fixed clock, and this holder runs on the
// real clock, so read as dated the entries fall out of the log's 30 days and every copy is kept as unrecorded.
shift := time.Since(memb.log[len(memb.log)-1].At)
@@ -145,7 +144,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
e.At = e.At.Add(shift)
bus.log = append(bus.log, e)
}
for _, s := range []string{BuildsRead, WantsRead, LeaseRead, ManifestsRead} {
for _, s := range []string{BuildsRead, WantsRead, ManifestsRead} {
keys, _ := memb.Keys(s)
for _, k := range keys[:min(len(keys), 6)] { // the seventh build ends later, on the bus
raw, _, _ := memb.Get(s, k)
@@ -175,7 +174,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
t.Fatal(err)
}
h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Now: time.Now, Logf: t.Logf}
h.Acts = handacts.Recorder{LeaseAway: h.LeaseAway, Ask: func(key string, body any) (json.RawMessage, error) {
h.Acts = handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
m, err := nc.Request("mesh.seat."+strings.TrimPrefix(strings.Replace(key, ".", ".tool.", 1), "seat:"), raw, 5*time.Second)
if err != nil {
@@ -204,16 +203,8 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) {
}
}
// While the lease names the controller, a build's end lets nothing go.
ended()
if len(door.asked) != 0 {
t.Fatalf("before the lease write the store was asked %v", door.asked)
}
// The lease handed away: the next build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved
// past the five newest and no want names (the want keeps the 1st), recording each deletion.
if err := bus.Put(LeaseRead, buscompose.ControllerLeaseKey, buscompose.LeaseHolder{HandedTo: "mesh-grants", Why: "the lease write"}); err != nil {
t.Fatal(err)
}
// No lease key: the build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved past the five
// newest and no want names (the want keeps the 1st), recording each deletion.
ended()
deadline := time.Now().Add(30 * time.Second)
for {
modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go
+48 -42
View File
@@ -11,7 +11,6 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/health"
@@ -139,8 +138,8 @@ func archiveRef(module string, n int) string {
}
// A mesh with seven done builds of letta, the n-th ending n hours after a day began, each with an image and an archive;
// a want names the first on ace; the store lists them all and one image no build names; the lease as given.
func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memDoor, *[]string) {
// a want names the first on ace; the store lists them all and one image no build names; no lease key (issue 1126).
func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
t.Helper()
day := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
b := &memBus{states: map[string]map[string][]byte{}, history: map[string][]stdio.StateRevision{}}
@@ -194,9 +193,6 @@ func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memD
_ = b.Put(ManifestsRead, mk, json.RawMessage(manifest))
want, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp(1), Self: true})
_ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want))
if lease != nil {
_ = b.Put(LeaseRead, buscompose.ControllerLeaseKey, lease)
}
var acts []string
h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil },
Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil },
@@ -206,18 +202,14 @@ func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memD
acts = append(acts, key+" "+string(raw))
return json.RawMessage(`{}`), nil
}}}
h.Acts.LeaseAway = h.LeaseAway
return h, b, door, &acts
}
// ADR 0571 clause 2: the keep set from the store's listing, the wants and the five newest builds; nothing let go of while
// the lease names a controller; once it names another holder, a build's end lets go of the eligible within the bound,
// leaves an index for a confirmed collect, records each deletion and says artifacts-gone.
func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t *testing.T) {
controller := &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3}
handed := &buscompose.LeaseHolder{HandedTo: "mesh-grants", HandedBy: "person.jochen", Why: "the lease write"}
h, _, _, _ := aMesh(t, controller)
// ADR 0571 clause 2, 1126/A4/AC2: the keep set from the store's listing, the wants and the five newest builds; with no
// lease key, a build's end and a confirmed collect let go of the eligible within the bound, leave an index for a
// confirmed collect, record each deletion as a hand act and say artifacts-gone.
func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T) {
h, _, _, _ := aMesh(t)
r, err := h.Artifacts("")
if err != nil {
t.Fatal(err)
@@ -244,9 +236,10 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t
t.Errorf("the references are %d, counted %v, at %q", len(r.References), r.Counts, r.Store)
}
all := []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}
for _, c := range []struct {
name string
lease *buscompose.LeaseHolder
how string // "build" (a build's end), "collect" (a confirmed collect), or a sweep asked with most
index bool
most int
letGo []string
@@ -254,45 +247,58 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t
skipped int
unlog int
}{
{name: "the lease names a controller", lease: controller},
{name: "no lease key", lease: nil},
{name: "handed away", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}},
{name: "an index waits", lease: handed, index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1},
{name: "bounded", lease: handed, most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2},
{name: "a build's end", how: "build", letGo: all},
{name: "a confirmed collect", how: "collect", letGo: all},
{name: "a sweep", most: MostPerSweep, letGo: all},
{name: "an index waits", index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1},
{name: "bounded", most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2},
// A kept build the log no longer holds: what it stood on is unknown, so no copy goes.
{name: "a kept build's bases unknown", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3},
{name: "a kept build's bases unknown", most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3},
} {
t.Run(c.name, func(t *testing.T) {
h, b, door, acts := aMesh(t, c.lease)
h, b, door, acts := aMesh(t)
door.indexes[imageRef("letta", 2)] = c.index
if c.unlog > 0 {
b.log = append(b.log[:c.unlog-1:c.unlog-1], b.log[c.unlog:]...)
}
if c.most == 0 {
switch c.how {
case "build":
h.AfterBuild("letta.x")
if _, err := h.Collect("tidy", true, 0); err == nil {
t.Error("a confirmed collect was let through while the controller sweeps")
var gone []string
for _, x := range door.asked {
if r, ok := strings.CutPrefix(x, "let go "); ok {
gone = append(gone, r)
}
}
if len(door.asked) != 0 || len(*acts) != 0 || len(b.published) != 0 {
t.Fatalf("before the lease write the store was asked %v, acts %v, said %v", door.asked, *acts, b.published)
if fmt.Sprint(gone) != fmt.Sprint(c.letGo) {
t.Fatalf("a build's end let go of %v, want %v", gone, c.letGo)
}
default:
var a *CollectAnswer
var err error
if c.how == "collect" {
a, err = h.Collect("tidy", true, 0)
} else {
a, err = h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"})
}
if err != nil {
t.Fatal(err)
}
if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 {
t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo)
}
return
}
a, err := h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"})
if err != nil {
t.Fatal(err)
}
if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 {
t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo)
}
// Every kept archive held before the first deletion.
first := -1
for i, x := range door.asked {
if strings.HasPrefix(x, "let go ") && first < 0 {
first = i
held := 0
for _, x := range door.asked {
if strings.HasPrefix(x, "let go ") {
break
}
if strings.HasPrefix(x, "hold ") {
held++
}
}
if first != 6 {
if held != 6 {
t.Errorf("the store was asked %v: the six kept archives are held first", door.asked)
}
if len(*acts) != len(c.letGo) || !strings.HasPrefix((*acts)[0], "seat:mesh-hand-acts.hand-act ") ||
@@ -310,7 +316,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t
// S25 (image-missing): a want whose build names an image the store does not hold is a finding in the seat's own state,
// written once and deleted once the store holds it again; images names what a machine's wants and haves name.
func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testing.T) {
h, b, _, _ := aMesh(t, nil)
h, b, _, _ := aMesh(t)
view, _ := h.Listed(context.Background())
repo, digest, _ := splitReference(imageRef("letta", 1))
delete(view.L.Repos[repo].Revisions, digest)
modules/artifact-store-tools/cmd/artifact-store-tools/main.go
+2 -3
View File
@@ -1,8 +1,7 @@
// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, ADR 0571 clause 2, to-be 51): a process the
// node's runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its repositories, its
// size, and what the mesh keeps of each manifest — read from the store's own files through the store's own container,
// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end once the controller's lease names
// no controller, and S25's image-missing.
// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end, and S25's image-missing.
//
// **A module beside the store, not the store's own.** A module that provides the artifact store cannot also build an
// artifact: building publishes to the store, so the store would be needed to create itself. So these tools are a
@@ -29,7 +28,7 @@ func main() {
}
h := &Holder{Bus: runtimeBus{}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View,
Now: time.Now, Logf: logf}
h.Acts = handacts.Recorder{Ask: stdio.Ask, LeaseAway: h.LeaseAway}
h.Acts = handacts.Recorder{Ask: stdio.Ask}
go follow(h)
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools.
if err := stdio.Serve("", append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...)); err != nil {
modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go
+2 -2
View File
@@ -436,7 +436,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
t.Fatalf("served %v, manifest %v", served, listed)
}
// ADR 0571 clause 2: it holds the seat mesh-artifact-sweep and serves its verbs, and calls no verb of the controller's;
// the lease is the one controller state it reads (clause 10's exemption).
// it reads no controller state, the lease neither (issue 1126).
want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule}
if !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes %v", m.Invokes)
@@ -455,7 +455,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
t.Errorf("claims %+v, serves %v, the seat's verbs %v", m.Claims, verbs, contract)
}
for _, r := range m.Reads {
if strings.HasPrefix(r, "mesh-controller.") && r != LeaseRead {
if strings.HasPrefix(r, "mesh-controller.") {
t.Errorf("reads the controller's %s", r)
}
}
modules/artifact-store-tools/module.json
+1 -2
View File
@@ -27,8 +27,7 @@
"mesh-delivery-builder.builds",
"mesh-delivery-builder.build-log",
"mesh-delivery-indexer.manifests",
"node-engine.have",
"mesh-controller.lease"
"node-engine.have"
],
"state": [
"findings"
modules/mesh-conditions/cmd/mesh-conditions/compare.go
+5 -34
View File
@@ -3,8 +3,8 @@ package main
// **Built dark, compared, then switched by the operator** (novox/hq ADR 0564 clause 9; issue 1952 C5, C8): `compare`
// lists the open keys of the keeper and of the controller, each difference with its reason; the kinds that end with the
// controller, the kinds only the keeper raises and the keys held while their module is delivered are listed apart and
// not counted. `take-over` raises an approve ask under the keeper's own name (ADR 0259), refused while compare shows a
// difference, and on the warrant writes the take-over key naming the keeper. `hand-back` is gone (issue 774): nothing
// not counted. `take-over` is refused (issue 1126): the keeper is the one source since the controller ended by its
// unassignment; a yes to an ask raised before still writes the take-over key naming the keeper. `hand-back` is gone (issue 774): nothing
// reads the take-over key any more, so it is answered as gone and changes nothing.
import (
@@ -91,7 +91,7 @@ func (k *Keeper) Compare() (Comparison, error) {
}
switch {
case homed && (home.Carried || home.Replaced):
why := "ends with the controller: carried from its bucket until the lease write"
why := "ends with the controller: no longer carried (issue 1126)"
if home.Replaced {
why = "ends with the controller: its fault is observed otherwise"
}
@@ -125,41 +125,12 @@ type Asks interface {
Ask(a asks.Ask) error
}
// TakeOver raises the approve ask of a switch to the keeper, refused while compare differs or after the lease write.
// TakeOver is refused: the keeper is the one source of conditions since the controller ended (issue 1126).
func (k *Keeper) TakeOver(a Asks, by, why string) (map[string]any, error) {
if strings.TrimSpace(why) == "" {
return nil, errors.New("a take-over says why")
}
if k.LeaseAway() {
return nil, errors.New("the controller's lease is handed away: the keeper is the one source already")
}
cmp, err := k.Compare()
if err != nil {
return nil, err
}
if !cmp.Empty() {
return nil, fmt.Errorf("refused: compare shows %d difference(s), the first %s (%s); the take-over waits for none",
len(cmp.Differences), cmp.Differences[0].Key, cmp.Differences[0].Reason)
}
now := k.Now().UTC()
id := fmt.Sprintf("take-over-%d", now.UnixNano())
act := asks.Act{"verb": Seat + ".take-over", "module": Module, "ask": id}
bind, err := asks.ActDigest(act)
if err != nil {
return nil, err
}
ask := asks.Ask{ID: id, Headline: "Switch the mesh's alarms to the new keeper?", Who: asks.Operator,
Explanation: fmt.Sprintf("The new condition keeper holds the same %d open alarms as the controller. Your yes makes "+
"the alarm messages come from the keeper, with the same words. It can be handed back until the controller "+
"gives up its lease.", cmp.Same),
Options: []asks.Option{{ID: "yes", Label: "Yes, switch", Does: "Alarms come from the keeper.", Level: asks.Approve, Binds: bind},
{ID: "no", Label: "No, leave it", Does: "Alarms keep coming from the controller.", Level: asks.Acknowledge}},
Expires: now.Add(asks.ApproveLasts - time.Minute), OnExpiry: "Alarms keep coming from the controller.",
About: Module + "-take-over", Details: "Asked by " + by + ": " + why}
if err := a.Ask(ask); err != nil {
return nil, err
}
return map[string]any{"asked": id, "same": cmp.Same, "apart": len(cmp.Apart)}, nil
return nil, errors.New("the keeper is the one source already: the controller ended by its unassignment (issue 1126)")
}
// Warranted is the act on the operator's answer: a yes, compare still empty, writes the take-over key naming the keeper.
modules/mesh-conditions/cmd/mesh-conditions/hold.go
+4 -10
View File
@@ -3,7 +3,7 @@ package main
// **A module being delivered raises no condition until its delivery ends** (novox/hq ADR 0564 clause 13 and Limits 12;
// issue 1952 C7, C8). The keeper reads the assigner's wants and the nodes' haves, and judges a module's delivery on a
// node with mesh-sdk's delivery.MotionOf, as the view does. While it is delivered there, a finding about it there, or
// about what it serves, and a condition carried from the controller's bucket on those subjects, is kept in the state
// about what it serves, is kept in the state
// `held` and raises nothing. When the delivery ends, healthy at the want or failed, what still stands is raised once,
// saying it happened during the delivery; what went is dropped with a log line. The hold lasts at most the delivery's
// bound, a key of the state `bounds` whose time to live is the bound: its expiry raises what is held, saying the delivery
@@ -98,17 +98,11 @@ func (k *Keeper) motion(node, module string) (delivery.Motion, delivery.Want) {
}
// holdFor says whether key is held now, and as what. Called with mu held.
func (k *Keeper) holdFor(key string, f *health.Finding, carried *health.Condition, open bool) (held, bool) {
if open || (f == nil && carried == nil) {
func (k *Keeper) holdFor(key string, f *health.Finding, open bool) (held, bool) {
if open || f == nil {
return held{}, false
}
var kind, source string
var subject health.Subject
if f != nil {
kind, source, subject = f.Kind, f.Source, f.Subject
} else {
kind, source, subject = carried.Kind, carried.Source, carried.Subject
}
kind, source, subject := f.Kind, f.Source, f.Subject
if _, raisedOut := k.outOfHold[key]; raisedOut {
return held{}, false
}
modules/mesh-conditions/cmd/mesh-conditions/hold_test.go
+5 -11
View File
@@ -21,7 +21,7 @@ const (
)
// ADR 0564 clause 13 and Limits 12 (issue 1952 C7, C8): a delivery of mesh-controller on novox replayed want by have.
// Its own findings and the carried controller-deaf raise nothing during it and none after when they went; one still
// Its own findings raise nothing during it and none after when they went; one still
// standing at the end is raised once with its during-the-delivery line; one past the bound with its did-not-end line; a
// failed delivery raises what stands at once; another module's and the machine's own are raised at once; compare lists a
// held key as held.
@@ -34,8 +34,6 @@ func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) {
raw, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp, Composed: composed, Phase: phase, Health: h})
return raw
}
deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4",
Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self"})
for _, c := range []struct {
name string
// end is the have the delivery ends with, or nil for a delivery past its bound.
@@ -68,9 +66,6 @@ func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) {
t.Fatal(err)
}
}
if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil {
t.Fatal(err)
}
// The delivery's bound is its own key's time to live, the module's bound (the default where it names none).
if r.st.ttls["bounds/novox.mesh-controller"] != delivery.DefaultDeliveryBound {
t.Errorf("%s: the bound key lives %s", c.name, r.st.ttls["bounds/novox.mesh-controller"])
@@ -86,11 +81,10 @@ func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) {
held++
}
}
if held != 2 {
if held != 1 {
t.Errorf("%s: compare lists %d held, %+v", c.name, held, cmp)
}
// The carried condition about the controller goes within the delivery; its own finding goes unless it stands.
_ = r.k.Controller("core.controller.deaf", nil, true)
// Its own finding goes unless it stands.
if !c.standing {
_ = r.k.Finding(watchman, own, nil, true)
}
@@ -106,8 +100,8 @@ func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) {
if co, open := r.open(own); c.standing != open || (open && !strings.Contains(co.Explanation, c.line)) {
t.Errorf("%s: its own condition is %v: %q", c.name, open, co.Explanation)
}
if _, open := r.open("core.controller.deaf"); open || len(r.k.Held()) != 0 {
t.Errorf("%s: the carried deaf condition was raised, or something is still held: %+v", c.name, r.k.Held())
if len(r.k.Held()) != 0 {
t.Errorf("%s: something is still held: %+v", c.name, r.k.Held())
}
}
// A seat's finding is about the module the placement says holds the seat on that node (the lead, 2026-10-29): held
modules/mesh-conditions/cmd/mesh-conditions/homes.go
+4 -4
View File
@@ -2,8 +2,8 @@ package main
// **Where each condition kind lives after the take-over** (novox/hq ADR 0564 clauses 4, 6, 7 and 9, the Context's probe
// table; issue 1952 C3, C5): data, so a kind's home is a row and never code. `compare` reads it to tell a difference from
// a kind that ends with the controller or that the controller never raises, and the keeper reads it to carry, until the
// lease write, the controller's conditions of the kinds that end with it.
// a kind that ends with the controller or that the controller never raises, and the keeper reads it to clear, once, what
// it carried of the kinds that ended with the controller (EndCarried, issue 1126).
//
// The rows are ADR 0564's and the lead's table of 2026-10-29, by the controller's source of each kind. A kind with no
// row is named by `compare` as "no home decided".
@@ -17,8 +17,8 @@ type Home struct {
Controller bool
// Observer is the module whose findings raise it after the take-over; empty for a kind that ends.
Observer string
// Carried says it ends with the controller's lease and is carried from the controller's bucket until then
// (clause 9: S4, S7, S12, S13, S14, S25, D5, D9, H-controller).
// Carried says it ended with the controller: carried from its bucket while it ran (clause 9: S4, S7, S12, S13, S14,
// S25, D5, D9, H-controller), cleared once by EndCarried and never carried again (issue 1126).
Carried bool
// Replaced says the kind itself ends and another observes its fault (D1, D10, the stuck probe): listed apart in
// compare and not carried.
modules/mesh-conditions/cmd/mesh-conditions/keeper.go
+68 -88
View File
@@ -1,9 +1,10 @@
package main
// The condition keeper (novox/hq ADR 0564 clauses 2, 3, 7, 8, 9 and 13; issue 1952 C2, C3, C7): the one writer of the
// mesh's open conditions. It watches every observer's `findings`, the healers' `attempts`, the controller's lease
// alone (its `conditions` are no longer read, issue 774), and the assigner's wants and the nodes' haves for the hold. From
// all of them it derives, per key, the condition that should be open, and writes the difference by compare-and-set,
// mesh's open conditions. It watches every observer's `findings`, the healers' `attempts`, and the assigner's wants and
// the nodes' haves for the hold; nothing of the controller (its `conditions` are no longer read, issue 774, and its lease
// turns nothing, issue 1126: the controller ended by its unassignment). From all of them it derives, per key, the
// condition that should be open, and writes the difference by compare-and-set,
// saying each transition as the controller says it (go/health) and keeping it in `history`. Nothing clears a condition
// but its finding going (ADR 0231 §14).
@@ -46,10 +47,8 @@ var Observers = []string{health.WatchmanModule, health.SelfCheckModule, health.H
// The other states it reads, as `<module>.<name>`.
const (
AttemptsRead = health.HealersModule + "." + health.AttemptsState
ControllerLease = "mesh-controller.lease"
ControllerLeaseKey = "holder"
WantsRead = delivery.AssignerModule + ".wants"
AttemptsRead = health.HealersModule + "." + health.AttemptsState
WantsRead = delivery.AssignerModule + ".wants"
// PlacementRead is the mesh-placement seat's placement, read for which module holds a seat on a node (hold.go; the
// lead, 2026-10-29, for ADR 0564 clause 2's inputs).
PlacementRead = delivery.PlacementRead
@@ -68,7 +67,7 @@ type Store interface {
CreateTTL(state, key string, value any, ttl time.Duration) error
}
// held is a finding or a carried condition kept while its module is delivered (clause 13 (b)).
// held is a finding kept while its module is delivered (clause 13 (b)).
type held struct {
Key string `json:"key"`
Node string `json:"node"`
@@ -104,8 +103,9 @@ type Keeper struct {
findings map[string]map[string]health.Finding
blind map[string]string
attempts map[string]health.Attempt
// ctrl is the controller's conditions as Controller hands them: compare's other side alone, never a condition's
// input (issue 1126). Nothing feeds it since issue 774; it goes with compare.
ctrl map[string]health.Condition
leaseOut bool
takeover health.Takeover
wants map[string]delivery.Want
haves map[string]delivery.Have
@@ -209,7 +209,8 @@ func (k *Keeper) Attempt(key string, value []byte, deleted bool) error {
return k.Settle(condition)
}
// Controller takes one change of the controller's `conditions` bucket (clauses 8, 9): read until the lease write.
// Controller takes one change of the controller's `conditions` bucket, for compare alone: no condition is carried from
// it (issue 1126).
func (k *Keeper) Controller(key string, value []byte, deleted bool) error {
k.mu.Lock()
k.init()
@@ -219,38 +220,57 @@ func (k *Keeper) Controller(key string, value []byte, deleted bool) error {
var c health.Condition
if err := json.Unmarshal(value, &c); err != nil {
k.mu.Unlock()
k.Logf("the controller's condition %s is not readable, so nothing is carried from it: %v", key, err)
k.Logf("the controller's condition %s is not readable, so compare does not list it: %v", key, err)
return nil
}
k.ctrl[key] = c
}
k.mu.Unlock()
return k.Settle(key)
return nil
}
// Lease takes the controller's lease key as read (clause 8, Limits 8): once it is handed to a holder that is no
// controller, nothing more is carried from the controller's bucket, and the kinds that ended with it clear.
func (k *Keeper) Lease(away bool) error {
k.mu.Lock()
k.init()
was := k.leaseOut
k.leaseOut = away
keys := make([]string, 0, len(k.ctrl))
for key := range k.ctrl {
keys = append(keys, key)
}
k.mu.Unlock()
if was == away {
return nil
}
return k.settleAll(keys)
}
// endedWhy is why a condition of a kind that ended with the controller is cleared.
const endedWhy = "its kind ended with the controller, which was unassigned (issue 1126): no finding raises it again"
// LeaseAway says whether the controller's lease is handed away.
func (k *Keeper) LeaseAway() bool {
k.mu.Lock()
defer k.mu.Unlock()
return k.leaseOut
// EndCarried clears, once, every open condition of a kind that ended with the controller (Homes' Carried) that no
// observer's finding names: carried from the controller's bucket while it ran, nothing raises it again (issue 1126).
// It answers how many it cleared.
func (k *Keeper) EndCarried() (int, error) {
open, err := k.Open()
if err != nil {
return 0, err
}
n := 0
var problems []string
for _, c := range open {
if h, ok := k.homeOf(c.Kind); !ok || !h.Carried {
continue
}
k.mu.Lock()
k.init()
f := k.wanted(c.Key)
k.mu.Unlock()
if f != nil {
continue
}
raw, rev, found, err := k.Store.Get(ConditionsState, c.Key)
if err != nil || !found {
continue
}
var was health.Condition
if json.Unmarshal(raw, &was) != nil {
continue
}
if err := k.clear(c.Key, was, rev, k.Now().UTC(), endedWhy); err != nil {
problems = append(problems, fmt.Sprintf("%s: %v", c.Key, err))
continue
}
n++
}
if len(problems) > 0 {
return n, errors.New(strings.Join(problems, "; "))
}
return n, nil
}
// Takeover takes the keeper's own take-over key as read.
@@ -287,9 +307,9 @@ func (k *Keeper) settleAll(keys []string) error {
return nil
}
// wanted is what the inputs say key should be: a finding of an observer, or a condition carried from the controller's
// bucket of a kind that ends with the controller, until the lease write (clause 9). Nil when nothing says it.
func (k *Keeper) wanted(key string) (*health.Finding, *health.Condition) {
// wanted is what the inputs say key should be: a finding of an observer. Nil when nothing says it; nothing is carried
// from the controller's bucket (issue 1126).
func (k *Keeper) wanted(key string) *health.Finding {
observers := make([]string, 0, len(k.findings))
for o := range k.findings {
observers = append(observers, o)
@@ -297,18 +317,10 @@ func (k *Keeper) wanted(key string) (*health.Finding, *health.Condition) {
sort.Strings(observers)
for _, o := range observers {
if f, ok := k.findings[o][key]; ok {
return &f, nil
return &f
}
}
if c, ok := k.ctrl[key]; ok && !k.leaseOut && k.endsWithController(c) {
return nil, &c
}
return nil, nil
}
func (k *Keeper) endsWithController(c health.Condition) bool {
h, ok := k.homeOf(c.Kind)
return ok && h.Carried
return nil
}
// settleOnce is one try of Settle.
@@ -326,11 +338,11 @@ func (k *Keeper) settleOnce(key string) error {
}
k.mu.Lock()
k.init()
f, carried := k.wanted(key)
hold, holding := k.holdFor(key, f, carried, open)
f := k.wanted(key)
hold, holding := k.holdFor(key, f, open)
k.mu.Unlock()
if f == nil && carried == nil {
if f == nil {
k.dropHeld(key, "its finding went while its module was delivered")
if !open {
return nil
@@ -341,7 +353,7 @@ func (k *Keeper) settleOnce(key string) error {
return k.keepHeld(hold)
}
k.dropHeld(key, "")
next := k.derive(key, was, open, f, carried, now)
next := k.derive(key, was, open, f, now)
events := transitions(was, next, open)
if len(events) == 0 && open && sameBody(was, next) {
return nil
@@ -359,28 +371,16 @@ func (k *Keeper) settleOnce(key string) error {
k.mu.Unlock()
for _, e := range events {
e.Condition, e.At, e.Event = next, now, health.EventFor(e.Change)
e.Show = k.show(next, carried != nil)
e.Show = health.ShowKeeper(next.Key)
k.tell(e)
}
return nil
}
func (k *Keeper) show(c health.Condition, carried bool) string {
if carried {
return health.ShowController(c.Key)
}
return health.ShowKeeper(c.Key)
}
// derive is the condition key should be now, from what was open and what the inputs say.
func (k *Keeper) derive(key string, was health.Condition, open bool, f *health.Finding, carried *health.Condition, now time.Time) health.Condition {
func (k *Keeper) derive(key string, was health.Condition, open bool, f *health.Finding, now time.Time) health.Condition {
var c health.Condition
switch {
case carried != nil:
// Raised and cleared as the controller's bucket says, with its show and actions (Limits 9).
c = *carried
c.Epoch = 0
c.LooksUnread = ""
case open:
c = was
k.fromFinding(&c, *f, now)
@@ -402,9 +402,7 @@ func (k *Keeper) derive(key string, was health.Condition, open bool, f *health.F
}
k.mu.Unlock()
}
if carried == nil {
k.overlay(&c, now)
}
k.overlay(&c, now)
if !open {
k.mu.Lock()
if line := k.outOfHold[key]; line != "" {
@@ -447,9 +445,7 @@ func (k *Keeper) fromFinding(c *health.Condition, f health.Finding, now time.Tim
_ = now
}
// overlay writes the healers' attempts into the condition (clause 7) and, until the lease write, what the controller's
// bucket holds for the same key (clause 8): its tried, its resolver and escalation, and its silence where the keeper has
// none of its own.
// overlay writes the healers' attempts into the condition (clause 7); nothing of the controller's bucket (issue 1126).
func (k *Keeper) overlay(c *health.Condition, now time.Time) {
k.mu.Lock()
var mine []health.Attempt
@@ -458,18 +454,8 @@ func (k *Keeper) overlay(c *health.Condition, now time.Time) {
mine = append(mine, a)
}
}
ctrl, carry := k.ctrl[c.Key]
carry = carry && !k.leaseOut
k.mu.Unlock()
sort.Slice(mine, func(i, j int) bool { return mine[i].Begun.Before(mine[j].Begun) })
if carry && len(ctrl.Tried) > 0 {
c.Tried = append([]health.Tried(nil), ctrl.Tried...)
if ctrl.Escalated() {
c.Resolver, c.Severity = health.ResolverOperator, health.Urgent
} else if ctrl.Resolver != "" && ctrl.Resolver != health.ResolverSelf {
c.Resolver = ctrl.Resolver
}
}
for _, a := range mine {
t := a.Tried()
if !containsTried(c.Tried, t) {
@@ -487,9 +473,6 @@ func (k *Keeper) overlay(c *health.Condition, now time.Time) {
if len(c.Tried) > health.KeptTried {
c.Tried = c.Tried[len(c.Tried)-health.KeptTried:]
}
if carry && c.Silenced == nil && ctrl.Silenced != nil && now.Before(ctrl.Silenced.Until) {
c.Silenced = ctrl.Silenced
}
}
func containsTried(list []health.Tried, t health.Tried) bool {
@@ -556,7 +539,7 @@ func (k *Keeper) clear(key string, was health.Condition, rev uint64, now time.Ti
k.mu.Unlock()
cleared := now
k.tell(health.Event{Condition: was, Event: health.EventCleared, At: now, Change: health.ChangeCleared, Why: why,
Cleared: &cleared, Show: k.show(was, false)})
Cleared: &cleared, Show: health.ShowKeeper(was.Key)})
return nil
}
@@ -623,9 +606,6 @@ func (k *Keeper) Resync() (int, error) {
seen[key] = true
}
}
for key := range k.ctrl {
seen[key] = true
}
k.mu.Unlock()
keys := make([]string, 0, len(seen))
for key := range seen {
modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go
+60 -40
View File
@@ -141,7 +141,7 @@ func finding(t *testing.T, observer, scope, id, kind, source string, sev health.
}
// ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer
// clears nothing; the healers' attempts and the controller's escalation reach the condition; a reopening within ten
// clears nothing; the healers' attempts reach the condition; a reopening within ten
// minutes is the same fault; a silence is a transition in history.
func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) {
r := newRig()
@@ -206,25 +206,12 @@ func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) {
}
return ""
}},
// Until the lease write the controller's escalation for the same key reaches the condition (clause 8).
{name: "the controller's escalation is carried", do: func() error {
ctrl, _ := json.Marshal(health.Condition{Key: key, Kind: "silent", Severity: health.Urgent, Resolver: health.ResolverOperator,
Tried: []health.Tried{{At: r.now, What: "x", Outcome: "escalated", By: "healer H1"}}})
_ = r.k.Attempt("h3-1", nil, true)
_ = r.k.Attempt("h3-2", nil, true)
return r.k.Controller(key, ctrl, false)
}, check: func(c health.Condition, _ bool) string {
if c.Resolver != health.ResolverOperator || c.Severity != health.Urgent {
return fmt.Sprintf("%+v", c)
}
return ""
}},
}
for _, s := range steps {
if err := s.do(); err != nil {
t.Fatalf("%s: %v", s.name, err)
}
if got := r.changes(); s.said != "" && got != s.said || s.said == "" && got != "" && s.name != "the controller's escalation is carried" {
if got := r.changes(); s.said != "" && got != s.said || s.said == "" && got != "" {
t.Errorf("%s: said %q, want %q", s.name, got, s.said)
}
if s.check != nil {
@@ -243,28 +230,64 @@ func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) {
if !strings.Contains(string(raw0), `"show":"mesh-conditions.conditions key=machine.g14.silent"`) {
t.Errorf("the first transition is %s", raw0)
}
// A carried kind (it ends with the controller) is raised and cleared as the controller's bucket says, until the
// lease is handed away, with the controller's show (clause 9, Limits 9).
deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4",
Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self", Epoch: 9})
if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil {
}
// 1126/A5/AC2: the keeper carries no controller condition: the controller's bucket, as Controller hands it, opens no
// condition and changes none (no escalation, no silence). What it carried while the controller ran, still open, is cleared
// once by EndCarried, said as cleared; a condition an observer's finding holds stays; a second EndCarried clears nothing.
func TestTheKeeperCarriesNoControllerConditionAndClearsWhatItCarriedOnce(t *testing.T) {
r := newRig()
key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now)
if err := r.k.Finding(watchman, key, raw, false); err != nil {
t.Fatal(err)
}
if c, open := r.open("core.controller.deaf"); !open || c.Epoch != 0 || r.said[0].Show != "mesh-controller.conditions key=core.controller.deaf" {
t.Errorf("a carried kind is %+v, said %+v", c, r.said)
}
r.said = nil
if err := r.k.Lease(true); err != nil {
t.Fatal(err)
until := r.now.Add(time.Hour)
for _, c := range []health.Condition{
{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4", Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"},
Severity: health.Urgent, Summary: "deaf", Resolver: "self"},
{Key: key, Kind: "silent", Severity: health.Urgent, Resolver: health.ResolverOperator,
Tried: []health.Tried{{At: r.now, What: "x", Outcome: "escalated", By: "healer H1"}},
Silenced: &health.Silence{Until: until}},
} {
raw, _ := json.Marshal(c)
if err := r.k.Controller(c.Key, raw, false); err != nil {
t.Fatal(err)
}
if _, err := r.k.Resync(); err != nil {
t.Fatal(err)
}
}
if _, open := r.open("core.controller.deaf"); open || r.changes() != "cleared core.controller.deaf" {
t.Error("a carried kind outlived the lease write")
if _, open := r.open("core.controller.deaf"); open || r.changes() != "" {
t.Error("the controller's condition was carried")
}
if c, _ := r.open(key); c.Severity != health.Warning || c.Resolver != health.ResolverSelf || len(c.Tried) != 0 || c.Silenced != nil {
t.Errorf("the controller's escalation reached the keeper's condition: %+v", c)
}
// Carried while the controller ran: two of kinds that ended with it, still open in conditions.
for _, c := range []health.Condition{
{Key: "core.controller.deaf", Kind: "controller-deaf", Severity: health.Urgent, Resolver: "self", Count: 1},
{Key: "seat.vault.ace.holder-silent", Kind: "holder-silent", Severity: health.Warning, Resolver: "self", Count: 1},
} {
if _, err := r.st.Create(ConditionsState, c.Key, c); err != nil {
t.Fatal(err)
}
}
n, err := r.k.EndCarried()
if err != nil || n != 2 || r.changes() != "cleared core.controller.deaf, cleared seat.vault.ace.holder-silent" {
t.Fatalf("EndCarried cleared %d (%v)", n, err)
}
if _, open := r.open(key); !open {
t.Error("a condition a finding holds was cleared")
}
if n, err := r.k.EndCarried(); err != nil || n != 0 || r.changes() != "" {
t.Errorf("a second EndCarried cleared %d (%v)", n, err)
}
}
// ADR 0564 clause 9 (issue 1952 C5, C8; the lead 2026-10-29): compare counts only what both sides can raise. A key held
// while its module is delivered, a kind that ends with the controller and a kind the controller never raises are listed
// apart; a kind with no home decided is a difference naming the gap; take-over is refused while compare differs, and
// apart; a kind with no home decided is a difference naming the gap; take-over is refused (issue 1126), and
// hand-back is gone (issue 774, review g1): answered as gone, naming its record, and writing nothing.
func TestCompareCountsOnlyWhatBothSidesRaise(t *testing.T) {
r := newRig()
@@ -296,7 +319,7 @@ func TestCompareCountsOnlyWhatBothSidesRaise(t *testing.T) {
"delivery.x.mystery": "no home is decided for the kind mystery",
"bus.x.consumer-wrong": "mesh-self-check has not written its finding",
// A kind of a probe the operator dropped (2026-10-30) is carried, never missing.
"seat.vault.ace.holder-silent": "carried from its bucket until the lease write (dropped 2026-10-30 by the operator",
"seat.vault.ace.holder-silent": "no longer carried (issue 1126) (dropped 2026-10-30 by the operator",
// The controller's own bus upgrade ends with its lease (the operator, 2026-10-31; issue 1952).
"bus.nats.upgrade-failed": "ends with the controller",
} {
@@ -332,17 +355,14 @@ func TestCompareCountsOnlyWhatBothSidesRaise(t *testing.T) {
if len(Homes) != 84 || carried != 34 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled
t.Errorf("the table holds %d kinds, %d carried", len(Homes), carried)
}
if _, err := r.k.TakeOver(nil, "jochen", "now"); err == nil || !strings.Contains(err.Error(), "2 difference") {
t.Errorf("take-over while compare differs: %v", err)
if _, err := r.k.TakeOver(nil, "jochen", "now"); err == nil || !strings.Contains(err.Error(), "one source already") {
t.Errorf("take-over: %v", err)
}
for _, away := range []bool{false, true} {
_ = r.k.Lease(away)
if _, err := r.k.HandBack("jochen", "back"); err == nil || !strings.Contains(err.Error(), "hand-back is gone") ||
!strings.Contains(err.Error(), "ADR 0564 clause 9") {
t.Errorf("hand-back with the lease away %v: %v, want it answered as gone naming its record", away, err)
}
if _, _, found, _ := r.k.Store.Get(TakeoverState, health.TakeoverKey); found {
t.Errorf("hand-back with the lease away %v wrote the take-over key", away)
}
if _, err := r.k.HandBack("jochen", "back"); err == nil || !strings.Contains(err.Error(), "hand-back is gone") ||
!strings.Contains(err.Error(), "ADR 0564 clause 9") {
t.Errorf("hand-back: %v, want it answered as gone naming its record", err)
}
if _, _, found, _ := r.k.Store.Get(TakeoverState, health.TakeoverKey); found {
t.Error("hand-back or take-over wrote the take-over key")
}
}
modules/mesh-conditions/cmd/mesh-conditions/main.go
+5 -9
View File
@@ -15,7 +15,6 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/health"
)
@@ -106,14 +105,11 @@ func Keep(k *Keeper) {
}
return nil
}
watch(k, ControllerLease, "", func(c stdio.StateChange) error {
if c.Key != ControllerLeaseKey {
return nil
}
away, why := buscompose.LeaseHandedAway(c.Value, c.Op == "put", nil)
logf("the controller's lease: %s", why)
return said(k.Lease(away))
})
// Once at start: what was carried from the controller's bucket while it ran is cleared (issue 1126); nothing raises
// those kinds again.
if n, err := k.EndCarried(); n > 0 || err != nil {
logf("cleared %d condition(s) of kinds that ended with the controller: %v", n, err)
}
watch(k, WantsRead, "", func(c stdio.StateChange) error { return said(k.Want(c.Key, c.Value, c.Op != "put")) })
watch(k, HavesRead, "", func(c stdio.StateChange) error { return said(k.Have(c.Key, c.Value, c.Op != "put")) })
watch(k, AttemptsRead, health.HealersModule+" attempts", func(c stdio.StateChange) error {
modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go
+2 -2
View File
@@ -29,14 +29,14 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
!slices.Equal(Verbs, catalogue.VerbNames(seat.Serves)) {
t.Errorf("module %s runs as %s and claims %+v; the seat serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(seat.Serves))
}
reads := []string{ControllerLease, WantsRead, HavesRead, AttemptsRead, PlacementRead}
reads := []string{WantsRead, HavesRead, AttemptsRead, PlacementRead}
for _, o := range Observers {
reads = append(reads, o+"."+health.FindingsState)
}
slices.Sort(reads)
got := slices.Clone(m.Reads)
slices.Sort(got)
if slices.Contains(m.Reads, "mesh-controller.conditions") || !slices.Equal(got, reads) || slices.Contains(m.Reads, health.HealersModule+"."+health.WouldState) {
if slices.Contains(m.Reads, "mesh-controller.conditions") || slices.Contains(m.Reads, "mesh-controller.lease") || !slices.Equal(got, reads) || slices.Contains(m.Reads, health.HealersModule+"."+health.WouldState) {
t.Errorf("reads %v, want %v and never the healers' would", got, reads)
}
ttl, keyTTL := map[string]int{}, map[string]bool{}
modules/mesh-conditions/cmd/mesh-conditions/tools.go
+3 -3
View File
@@ -73,8 +73,8 @@ func Tools(k *Keeper, a Asks) []stdio.Tool {
{Name: Seat + ".compare", Description: "The keeper's open conditions against the controller's, each difference with " +
"its reason; what ends with the controller, what only the keeper raises and what is held listed apart.",
Run: func(map[string]any) (any, error) { return k.Compare() }},
{Name: Seat + ".take-over", Description: "Ask the operator's Approve to switch the readers of conditions to the " +
"keeper; refused while compare differs.",
{Name: Seat + ".take-over", Description: "Refused: the keeper is the one source of conditions since the " +
"controller ended by its unassignment (issue 1126); kept as the seat's verb until the seat drops it.",
Input: obj(map[string]any{"why": str("why now")}, "why"),
Run: func(args map[string]any) (any, error) {
return k.TakeOver(a, stdio.CallerOf(args), strArg(args, "why"))
@@ -212,7 +212,7 @@ func (k *Keeper) Inputs() map[string]any {
findings[o] = len(fs)
}
return map[string]any{"findings": findings, "blind": k.blind, "attempts": len(k.attempts),
"controller-conditions": len(k.ctrl), "lease-handed-away": k.leaseOut, "follow": k.takeover.Follow,
"controller-conditions": len(k.ctrl), "follow": k.takeover.Follow,
"wants": len(k.wants), "haves": len(k.haves)}
}
modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go
+1 -1
View File
@@ -261,7 +261,7 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) {
if err != nil {
t.Fatal(err)
}
buckets := []string{"mesh-controller_conditions", "mesh-controller_lease", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_placement",
buckets := []string{"mesh-controller_conditions", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_placement",
"mesh-healers_attempts"}
for _, o := range Observers {
buckets = append(buckets, o+"_findings")
modules/mesh-conditions/module.json
-1
View File
@@ -28,7 +28,6 @@
"mesh-provisioner.findings",
"artifact-store-tools.findings",
"mesh-healers.attempts",
"mesh-controller.lease",
"mesh-delivery-assigner.wants",
"node-engine.have",
"mesh-delivery-assigner.placement"
modules/mesh-config/README.md
+2 -3
View File
@@ -75,6 +75,5 @@ Each directory or archive this module declares says how it treats a file it did
controller told by `pull-declares`; the controller's mirrors are no longer read (ADR 0565 §4, issue 774).
`config_take_over` asks the operator's Approve, refused while any difference stands and switched on the Approve only
if none stands then. `config_hand_back` is gone: it answers so and changes nothing. `config_declared` and `config_resync` read and write again what it keeps.
- From the controller's lease write a trusted value is set only through propose and the operator's Approve.
- Its sets, clears and warrants are recorded in the hand-act log of the seat `mesh-hand-acts`, falling back to the
controller's while that seat has no holder and the controller holds its lease.
- A trusted value is set only through propose and the operator's Approve: no controller relay is trusted (issue 1126).
- Its sets, clears and warrants are recorded in the hand-act log of the seat `mesh-hand-acts` alone.
modules/mesh-config/cmd/mesh-config/config.go
+10 -16
View File
@@ -16,9 +16,10 @@ import (
// values it moves; records the change in the history and the hand-act log; raises `config-changed`; and answers
// with each module on each node whose value moved. A config change opens no walk and waits at no first-node gate.
//
// **Trusted values at the terminal or on the operator's Approve.** A trusted value (ADR 0348 §8) is set only by a
// call the controller carries from its terminal, or on the operator's warrant for a proposal (proposals.go, design
// 57 K4); a set through the verb is refused and names the proposal as the way.
// **Trusted values on the operator's Approve alone.** A trusted value (ADR 0348 §8) is set only on the operator's
// warrant for a proposal (proposals.go, design 57 K4); a set through the verb is refused and names the proposal as the
// way. No controller relay is trusted (ADR 0565 §13, issue 1126): the controller ended by its unassignment, so no lease
// turns its terminal on.
// ControllerCaller is the controller's bus user: the one caller that carries a value given at its terminal.
const ControllerCaller = "controller"
@@ -275,9 +276,9 @@ func (c *Config) change(call Call) (*SetAnswer, error) {
return nil, fmt.Errorf("%s is refused for %s at %s: %s", show(value), call.Key, level,
strings.Join(problems, "; "))
}
if trusted && call.approval == nil && !c.atTheTerminal(call) {
return nil, fmt.Errorf("%s is trusted (it decides what runs or who can reach what), so it is set only at "+
"the controller's terminal or on the operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: "+
if trusted && call.approval == nil {
return nil, fmt.Errorf("%s is trusted (it decides what runs or who can reach what), so it is set only on "+
"the operator's Approve (novox/hq ADR 0348 §8; no controller relay is trusted, ADR 0565 §13), not by %s: "+
"propose it instead, with config's action propose", call.Key, call.Caller)
}
v := Value{Value: compact(raw), By: by, Via: call.Caller, Why: strings.TrimSpace(call.Why), At: now}
@@ -293,9 +294,9 @@ func (c *Config) change(call Call) (*SetAnswer, error) {
if !had {
return nil, fmt.Errorf("nothing is set for %s at %s, so there is nothing to clear", call.Key, level)
}
if trusted && call.approval == nil && !c.atTheTerminal(call) {
return nil, fmt.Errorf("%s is trusted, so it is cleared only at the controller's terminal or on the "+
"operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: propose the clear instead, with "+
if trusted && call.approval == nil {
return nil, fmt.Errorf("%s is trusted, so it is cleared only on the operator's Approve (novox/hq ADR 0348 "+
"§8; no controller relay is trusted, ADR 0565 §13), not by %s: propose the clear instead, with "+
"config's action propose and clear", call.Key, call.Caller)
}
if err := c.Store.Delete(ValuesState, vk); err != nil {
@@ -326,13 +327,6 @@ func (c *Config) change(call Call) (*SetAnswer, error) {
return answer, nil
}
// atTheTerminal says a call is the controller's relay of its terminal, which holds only until the controller's lease is
// handed away: from the lease write a trusted value is set only through propose and the operator's Approve, whoever
// calls (novox/hq ADR 0565 §13).
func (c *Config) atTheTerminal(call Call) bool {
return call.Caller == ControllerCaller && !c.leaseAway()
}
// declared is the rule a value at a level is held to, and whether it is trusted; refused, naming the declared
// keys, for a key the module or the list of predefined keys does not declare (ADR 0348 §10).
func (c *Config) declared(s Sources, level Level, key string) (rule, bool, error) {
modules/mesh-config/cmd/mesh-config/config_test.go
+25 -21
View File
@@ -404,10 +404,10 @@ func TestTheVerbRefusesWhatADeclarationDoesNotAllow(t *testing.T) {
{"a value not in its enum", Call{Key: "position", Value: "middle", Node: "g14", Module: "dunst"}, "is not one of"},
{"a value not matching its pattern", Call{Key: "origin", Value: "Home 1", Node: "g14", Module: "dunst"},
"does not match the pattern"},
{"a trusted value at the mesh, not from the terminal", Call{Key: "agent-managed-settings",
Value: map[string]any{"a": 1.0}}, "set only at the controller's terminal"},
{"a trusted module value, not from the terminal", Call{Key: "managed", Value: map[string]any{},
Node: "g14", Module: "claude-code"}, "set only at the controller's terminal"},
{"a trusted value at the mesh, not on an Approve", Call{Key: "agent-managed-settings",
Value: map[string]any{"a": 1.0}}, "set only on the operator's Approve"},
{"a trusted module value, not on an Approve", Call{Key: "managed", Value: map[string]any{},
Node: "g14", Module: "claude-code"}, "set only on the operator's Approve"},
{"a change without why", Call{Key: "font-size", Value: 12.0, Why: " "}, "say why"},
{"an unproven caller", Call{Key: "font-size", Value: 12.0, Caller: Unproven}, "no proven caller"},
{"by from anyone but the controller", Call{Key: "font-size", Value: 12.0, By: "jochen"}, "only the controller says"},
@@ -438,23 +438,27 @@ func TestTheVerbRefusesWhatADeclarationDoesNotAllow(t *testing.T) {
}
}
// **A trusted value from the controller's terminal is set**, with who gave it there (ADR 0348 §8).
func TestATrustedValueFromTheTerminalIsSetWithWhoGaveIt(t *testing.T) {
c, m := aMesh(t)
a := set(t, c, Call{Key: "agent-managed-settings", Value: map[string]any{"b": 2.0, "a": 1.0},
Caller: ControllerCaller, By: "jochen"})
if string(a.Value) != `{"a":1,"b":2}` {
t.Errorf("the value is written as %s", a.Value)
}
r := m.resolved(t, "g14", "claude-code", "managed")
if r.Source != SourceMesh || !r.Trusted || string(r.Value) != `{"a":1,"b":2}` {
t.Errorf("claude-code's managed settings resolve as %+v", r)
}
e, _ := m.Get(ValuesState, "mesh.agent-managed-settings")
var v Value
_ = json.Unmarshal(e.Value, &v)
if v.By != "jochen" || v.Via != ControllerCaller {
t.Errorf("the value says it was set by %s via %s", v.By, v.Via)
// 1126/A5/AC1: **no controller relay is trusted.** With no lease key (the controller ended by its unassignment), a trusted
// value set or cleared by the controller's account is refused, naming propose; nothing is written (ADR 0565 §13).
func TestNoControllerRelayIsTrusted(t *testing.T) {
for _, call := range []Call{
{Action: "set", Key: "agent-managed-settings", Value: map[string]any{"a": 1.0}},
{Action: "set", Key: "managed", Node: "g14", Module: "claude-code", Value: map[string]any{"a": 1.0}},
{Action: "clear", Key: "agent-managed-settings"},
} {
c, m := aMesh(t)
m.mirror(t, ValuesState, "mesh.agent-managed-settings", `{"value":{"b":2},"by":"person.jochen","via":"person.jochen"}`)
call.Caller, call.By, call.Why = ControllerCaller, "jochen", "at the terminal"
_, err := c.Change(call)
if err == nil || !strings.Contains(err.Error(), "propose") {
t.Errorf("%s %s from the controller: %v", call.Action, call.Key, err)
}
if v, _ := storedValue(t, m, "mesh.agent-managed-settings"); string(v.Value) != `{"b":2}` {
t.Errorf("%s %s from the controller left %s", call.Action, call.Key, v.Value)
}
if _, held := storedValue(t, m, "module.g14.claude-code.managed"); held {
t.Errorf("%s %s from the controller wrote claude-code's own value", call.Action, call.Key)
}
}
}
modules/mesh-config/cmd/mesh-config/main.go
+2 -11
View File
@@ -19,7 +19,6 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/handacts"
)
@@ -77,15 +76,7 @@ func (runtimeStore) Emit(event string, body any) error { return stdio.Emit(event
// recorder records into the hand-act log of the seat mesh-hand-acts alone (novox/hq ADR 0565 §11, issue 1953): the manifest
// invokes no verb of the controller and the SDK recorder has no fall back to its log.
var recorder = handacts.Recorder{Ask: stdio.Ask, LeaseAway: func() bool {
e, err := stdio.State(LeaseRead).Get(buscompose.ControllerLeaseKey)
var raw []byte
if e != nil {
raw = e.Value
}
away, _ := buscompose.LeaseHandedAway(raw, e != nil, err)
return away
}}
var recorder = handacts.Recorder{Ask: stdio.Ask}
// HandAct records a change in the hand-act log (to-be 45 §7).
func (runtimeStore) HandAct(what, why, cause string) error {
@@ -238,7 +229,7 @@ func Tools(c *Config) []stdio.Tool {
"one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " +
"module's own value on one node (module and node). Each set or clear needs why, is validated against " +
"its declaration, names each module on each node whose value moves, and is kept in the history with " +
"who and why. A trusted value is set only at the controller's terminal or on the operator's Approve: " +
"who and why. A trusted value is set only on the operator's Approve: " +
"propose asks the operator, under mesh-config's own name on a channel that proves who answered, and sets " +
"the value on Approve if what it was shown against still stands; proposals lists them, or one with " +
"proposal. A proposal naming the open pull request that introduces its key (pull) is judged against what " +
modules/mesh-config/cmd/mesh-config/manifest_test.go
+1 -1
View File
@@ -33,7 +33,7 @@ func TestEveryStateAndActHasItsTool(t *testing.T) {
ProposalsState: "config proposals", PullsState: "config list", KeptState: "config list", HistoryLog: "config history",
DeclaredState: DeclaredTool, TakeoverState: DeclaredTool,
PlacementRead: CompareTool, WantsRead: CompareTool, BuildsRead: CompareTool, ModulesRead: DeclaredTool,
ManifestsRead: DeclaredTool, IndexerPredefinedRead: CompareTool, CandidatesRead: CompareTool, LeaseRead: HandBackTool,
ManifestsRead: DeclaredTool, IndexerPredefinedRead: CompareTool, CandidatesRead: CompareTool,
"a resolution": ResyncTool, "the indexer's candidates taken": ResyncTool, "a take-over decided": TakeOverTool,
}
served := map[string]bool{}
modules/mesh-config/cmd/mesh-config/proposals_test.go
+3 -4
View File
@@ -266,8 +266,8 @@ func TestATrustedValueIsProposedAskedUnderItsOwnNameAndSetOnTheOperatorsApprove(
func TestAnApproveIsRefusedWhenTheStoredValueChangedSince(t *testing.T) {
c, m, r := aProposingMesh(t)
a := propose(t, c, Call{Key: "agent-managed-settings", Value: map[string]any{"a": 1.0}})
// Set at the terminal meanwhile.
set(t, c, Call{Key: "agent-managed-settings", Value: map[string]any{"b": 2.0}, Caller: ControllerCaller, By: "jochen"})
// Changed meanwhile.
m.mirror(t, ValuesState, "mesh.agent-managed-settings", `{"value":{"b":2},"by":"person.jochen","via":"person.jochen"}`)
m.events = nil
r.choose(t, a.Proposal, OptionApprove, nil)
said, _ := c.Decide()
@@ -501,8 +501,7 @@ func TestAClearIsProposedAndATrustedSetNamesTheProposal(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "propose it instead") {
t.Errorf("a trusted set through the verb: %v", err)
}
set(t, c, Call{Key: "managed", Value: map[string]any{"a": 1.0}, Node: "g14", Module: "claude-code",
Caller: ControllerCaller, By: "jochen"})
m.mirror(t, ValuesState, "module.g14.claude-code.managed", `{"value":{"a":1},"by":"person.jochen","via":"person.jochen"}`)
a := propose(t, c, Call{Key: "managed", Clear: true, Node: "g14", Module: "claude-code"})
ask := r.asks[a.Proposal]
if !strings.HasPrefix(ask.Headline, "Clear managed for claude-code on g14") ||
modules/mesh-config/cmd/mesh-config/takeover.go
-14
View File
@@ -26,7 +26,6 @@ import (
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -41,8 +40,6 @@ const (
// IndexerPredefinedRead and CandidatesRead are the indexer's states (ADR 0565 §2).
IndexerPredefinedRead = delivery.IndexerModule + "." + delivery.IndexerPredefinedState
CandidatesRead = delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState
// LeaseRead is the controller's lease: after its write hand-back is gone and the terminal sets nothing (§13).
LeaseRead = buscompose.ControllerLease
)
// The sources the switch names.
@@ -95,17 +92,6 @@ func (c *Config) source() (Switch, error) {
return s, nil
}
// leaseAway says the controller's lease is handed away (ADR 0561 §3's reading).
func (c *Config) leaseAway() bool {
e, err := c.Store.Get(LeaseRead, buscompose.ControllerLeaseKey)
var raw []byte
if e != nil {
raw = e.Value
}
away, _ := buscompose.LeaseHandedAway(raw, e != nil, err)
return away
}
// ---- declared --------------------------------------------------------------------------------------------
// DeclaredKept is one key of `declared`: a module's declarations as the manifest the indexer's `modules.<module>`
modules/mesh-config/cmd/mesh-config/takeover_test.go
+1 -11
View File
@@ -118,8 +118,7 @@ func TestCompareListsEveryDifferenceAndTheTakeOverWaitsForNone(t *testing.T) {
}
// The operator's Approve switches config to the indexer: pull-declares is refused, the list and the pull requests'
// config come from the indexer; hand-back is gone, and a switch naming the controller is read as the indexer; from the
// lease write a trusted value is set only on an Approve; declared is written once per manifest (ADR 0565 §3, §4, §13).
// config come from the indexer; hand-back is gone, and a switch naming the controller is read as the indexer; declared is written once per manifest (ADR 0565 §3, §4, §13).
func TestTheTakeOverAndTheHandBack(t *testing.T) {
c, m, r := anAgreeingMesh(t)
steps := []struct {
@@ -174,15 +173,6 @@ func TestTheTakeOverAndTheHandBack(t *testing.T) {
res, err := c.Reconcile()
return res.SourceGone + "|" + string(m.resolved(t, "g14", "dunst", "width").Value), err
}, "is gone: config no longer reads its records"},
{"the lease is written", func() (string, error) {
m.mirror(t, LeaseRead, "holder", `{"handed-to":"the operator's hand-over"}`)
return "lease", nil
}, "lease"},
{"the terminal sets no trusted value after the lease write", func() (string, error) {
_, err := c.Change(Call{Action: "set", Key: "managed", Module: "claude-code", Node: "g14", Value: map[string]any{},
Why: "x", Caller: ControllerCaller, By: "person.jochen"})
return "", err
}, "propose"},
}
for _, s := range steps {
got, err := s.do()
modules/mesh-config/module.json
+1 -2
View File
@@ -37,8 +37,7 @@
"mesh-delivery-indexer.modules",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.predefined",
"mesh-delivery-indexer.config-candidates",
"mesh-controller.lease"
"mesh-delivery-indexer.config-candidates"
],
"tools": [
"config_compare",
modules/mesh-healers/README.md
+8 -8
View File
@@ -15,15 +15,15 @@ Each act is begun by a create of its key in `attempts` before it is made and fin
mid-act has spent it and a second instance cannot begin it again. The condition keeper writes the condition's
`tried`, resolver and escalation from `attempts`; a healer writes no condition.
**Only once the controller's lease names no controller.** Until the write that hands the lease away, each act it would
take is written to `would`, never to `attempts`: it counts for no budget, settle or brake and reaches no condition.
**They act, and no lease turns it** (issue 1126): the controller ended by its unassignment. While it ran, each act a
healer would take was written to `would`, never to `attempts`; nothing writes `would` any more.
Each key of `attempts` and `would` lives a week from its last write, its own time to live (the buckets are declared
`key-ttl`): nothing else deletes `would`, and once the lease is handed away nothing is written there. The healers' waits
`key-ttl`): nothing else deletes `would`. The healers' waits
are keys of `wakes` whose time to live is the wait: `settle.<condition key>` ends with a settle and judges the condition
again, `fortnight` ends when S15's oldest counted act leaves the fortnight and counts again. The watch of `wakes` hears
each expiry; no timer runs in the process. A wait still running is on the bus across a restart; one that expired while
no instance watched is judged at the start, once the lease is first read.
no instance watched is judged at the start.
S15: it hears the seat mesh-hand-acts' `act-recorded` and watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a
fortnight; a drill and a person's decision by design never count.
@@ -31,8 +31,8 @@ fortnight; a drill and a person's decision by design never count.
## What it keeps
- `mesh-healers_attempts`: every act and escalation, each key kept a week.
- `mesh-healers_would`: the newest act it would take per healer and condition while the controller holds the lease,
each key kept a week.
- `mesh-healers_would`: the newest act it would take per healer and condition while the controller ran, each key
kept a week; nothing writes it any more.
- `mesh-healers_findings`: the brake, its own blindness, and S15.
- `mesh-healers_wakes`: the waits, each key living as long as it waits.
@@ -41,6 +41,6 @@ fortnight; a drill and a person's decision by design never count.
- `mesh-healers.healers`: the registry, the acts of the last days, the would-acts, the brake, whether it acts and why;
`recount=true` counts the hand-act log again.
- `mesh-healers.heal`: one act now on an open condition, by key and why.
- `mesh-healers.drill`: records a drill in the hand-act log of the seat `mesh-hand-acts` as `hand-act drill` (the
controller's `drill` while that seat has no holder and the controller holds its lease). S15 never counts a drill.
- `mesh-healers.drill`: records a drill in the hand-act log of the seat `mesh-hand-acts` as `hand-act drill`; with
no holder of that seat it is refused. S15 never counts a drill.
- `healers_findings`, `healers_would`, `healers_wakes`: the reads of its buckets.
modules/mesh-healers/cmd/mesh-healers/healers.go
+11 -62
View File
@@ -9,10 +9,9 @@ package main
// healer's to say: the condition clears when its observer no longer sees it, and the condition keeper writes `tried`,
// the resolver and the escalation into the condition from `attempts`.
//
// **Only once the controller's lease names no controller** (ADR 0564 §8): until then each act it would take is
// written to the bucket `would`, never to `attempts`, so a would-act counts for no budget, settle or brake, reaches
// no `tried` and escalates nothing. `would` keeps each key a week (its bucket's age): nothing else deletes it, and once
// the lease is handed away nothing is written there again.
// **They act, and no lease turns it** (issue 1126): the controller ended by its unassignment, so the gate on its lease
// (ADR 0564 §8) is gone. The bucket `would` holds what they would have done while it ran, each key kept a week (its
// bucket's age); nothing writes there any more.
import (
"encoding/json"
@@ -42,8 +41,6 @@ const (
AttemptsState = health.AttemptsState
WouldState = health.WouldState
FindingsState = health.FindingsState
LeaseRead = buscompose.ControllerLease
LeaseKey = buscompose.ControllerLeaseKey
ConditionsRead = "mesh-conditions.conditions"
ActedEvent = health.EventHealerActed
// ActedSubject is where the seat says it, which its consumers hear (catalogue.SeatSays).
@@ -115,8 +112,8 @@ type Condition struct {
func (c Condition) Escalated() bool { return c.Resolver == "operator" && len(c.Tried) > 0 }
// Attempt, Would and Acted are mesh-sdk's health layouts: an act begun in `attempts` (its key
// `<healer>.<condition key>.<begun ns>` is the healers' own), an act written to `would` while the lease names a
// controller (one key per healer and condition, `<healer>.<condition key>`, its newest), and the body of healer-acted,
// `<healer>.<condition key>.<begun ns>` is the healers' own), an act written to `would` while the controller ran
// (one key per healer and condition, `<healer>.<condition key>`, its newest), and the body of healer-acted,
// epoch left out (ADR 0564 Limits 10).
type (
Attempt = health.Attempt
@@ -182,48 +179,14 @@ type Healers struct {
Registry []Healer
mu sync.Mutex
// lease is the controller's lease as last read: acting is on only while it is handed away.
acting bool
leaseWhy string
// declined is why each condition was last passed over, said once.
declined map[string]string
// acts are the hand-act log as watched, for S15.
acts map[string]HandAct
}
// LeaseHeard records what the lease says: acting is on only while it names a holder that is no controller (ADR 0564
// §8, Limits 8, as ADR 0561 §3's reader reads it).
func (h *Healers) LeaseHeard(raw []byte, found bool, readErr error) {
on, why := buscompose.LeaseHandedAway(raw, found, readErr)
h.mu.Lock()
flipped := on != h.acting || h.leaseWhy == ""
h.acting, h.leaseWhy = on, leaseWords(on, why)
h.mu.Unlock()
if flipped {
h.Logf("[%s] the healers %s: %s", Module, map[bool]string{true: "ACT", false: "act in shadow"}[on], h.leaseWhy)
}
}
// leaseWords says the lease's state as the healers read it; the SDK's words speak of minting.
func leaseWords(on bool, why string) string {
if on {
return "the controller's lease is handed away: the healers act"
}
if strings.Contains(why, "cannot be read") || strings.Contains(why, "not readable") {
return "the controller's lease cannot be read, so it is taken to name a controller: every act is written as would"
}
return "the controller's lease names a controller (or none is written, so one may take it): every act is written as would"
}
// Acting says whether the healers act, and why.
func (h *Healers) Acting() (bool, string) {
h.mu.Lock()
defer h.mu.Unlock()
if h.leaseWhy == "" {
return false, "the controller's lease has not been read yet: every act is written as would"
}
return h.acting, h.leaseWhy
}
// actingWhy is why the healers act: what the tool `healers` says.
const actingWhy = "the healers act: the controller ended by its unassignment, so no lease turns them (issue 1126)"
// healerFor is the healer that acts here on a kind.
func (h *Healers) healerFor(kind string) (Healer, bool) {
@@ -235,7 +198,7 @@ func (h *Healers) healerFor(kind string) (Healer, bool) {
return Healer{}, false
}
// Heard is a condition the keeper raised or changed: judged, and acted on, written as would, or passed over.
// Heard is a condition the keeper raised or changed: judged, and acted on or passed over.
func (h *Healers) Heard(body []byte) error {
var c Condition
if err := json.Unmarshal(body, &c); err != nil {
@@ -265,13 +228,6 @@ func (h *Healers) Judge(c Condition, by string) (string, error) {
}
delete(h.declined, c.Key)
now := h.Now()
if !h.acting {
w := Would{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: row.Repair, At: now.UTC(), Why: h.whyNot()}
if err := h.Store.PutTTL(WouldState, row.ID+"."+c.Key, w, KeptFor); err != nil {
return "", fmt.Errorf("the would-act of %s on %s is not written: %w", row.ID, c.Key, err)
}
return fmt.Sprintf("healer %s would act on %s: %s", row.ID, c.Key, w.Why), nil
}
all, err := h.attempts()
if err != nil {
h.blind(err)
@@ -299,13 +255,6 @@ func (h *Healers) Judge(c Condition, by string) (string, error) {
return h.act(row, c, budget, len(spent), by)
}
func (h *Healers) whyNot() string {
if h.leaseWhy == "" {
return "the controller's lease has not been read yet"
}
return h.leaseWhy
}
// act is one healer's act: begun by a create in attempts, made, finished, and said.
func (h *Healers) act(row Healer, c Condition, budget string, spent int, by string) (string, error) {
begun := h.Now().UTC()
@@ -379,7 +328,7 @@ func (h *Healers) tell(e Acted) {
}
}
// Rearm is the healers' start, once the lease is first read: a settle whose key expired while no instance watched has
// Rearm is the healers' start: a settle whose key expired while no instance watched has
// no expiry left to hear, so each newest act whose settle ended within its window is judged at once. A settle still
// running keeps its key on the bus and needs nothing.
func (h *Healers) Rearm() {
@@ -611,14 +560,14 @@ func (h *Healers) see(source string, found []Observation) {
// --- H3 --------------------------------------------------------------------------------------------------------
// moduleOf is the module whose objects a condition names: a consumer of a module's own (`<stream>.<module>`). The
// controller's own consumers are not a module's: the controller asserts them itself until its lease.
// controller's own consumers are not a module's: they go with the controller's leftovers (issue 1126).
func moduleOf(c Condition) (string, string) {
stream, consumer, ok := strings.Cut(c.Subject.ID, ".")
if c.Kind != "consumer-lost" || !ok || stream == "" || consumer == "" {
return "", "it names no consumer of a module whose objects the grants part asserts"
}
if consumer == buscompose.ControllerName || strings.HasPrefix(consumer, buscompose.ControllerName+"-") {
return "", "the consumer is the controller's own, which the controller asserts itself while it holds its lease"
return "", "the consumer is the controller's own, which the grants part does not assert"
}
return consumer, ""
}
modules/mesh-healers/cmd/mesh-healers/healers_test.go
+22 -70
View File
@@ -154,10 +154,6 @@ func (w *world) fresh() {
}
}
func (w *world) handAway() {
w.h.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil)
}
// lost is a module's consumer the bus does not hold, as D6 raises it.
func lost(module string) Condition {
return Condition{Key: "bus.EVENTS." + module + ".missing", Kind: "consumer-lost", Severity: "warning", Resolver: "self",
@@ -170,47 +166,21 @@ func body(c Condition) []byte {
return raw
}
// No act is made while the lease names a controller: each is written to would, never to attempts, so it counts for no
// budget or brake and nothing is asked or said; one is made from the write that hands the lease away (ADR 0564 §8).
func TestTheHealersActOnlyOnceTheLeaseNamesNoController(t *testing.T) {
cases := []struct {
name string
lease func(h *Healers)
acts bool
}{
{"never read", func(*Healers) {}, false},
{"absent", func(h *Healers) { h.LeaseHeard(nil, false, nil) }, false},
{"unreadable", func(h *Healers) { h.LeaseHeard(nil, false, errors.New("timeout")) }, false},
{"a controller holds it", func(h *Healers) {
h.LeaseHeard([]byte(`{"instance":"novox/1/2026-10-27","epoch":57}`), true, nil)
}, false},
{"handed away", func(h *Healers) { h.LeaseHeard([]byte(`{"handed-to":"the operator"}`), true, nil) }, true},
// 1126/A4/AC1: with no lease key read at all (the controller ended by its unassignment), a condition a healer covers is
// acted on: one attempt, finished, asked and said, and nothing written to would.
func TestTheHealersActWithNoLease(t *testing.T) {
w := newWorld()
if err := w.h.Heard(body(lost("records"))); err != nil {
t.Fatal(err)
}
for _, tc := range cases {
w := newWorld()
tc.lease(w.h)
if err := w.h.Heard(body(lost("records"))); err != nil {
t.Fatalf("%s: %v", tc.name, err)
}
attempts, would := w.store.keys(AttemptsState), w.store.keys(WouldState)
if tc.acts {
if len(attempts) != 1 || len(would) != 0 || len(w.asked) != 1 || !strings.Contains(w.asked[0], `"module":"records"`) ||
len(w.said) != 1 || w.said[0].Outcome != OutcomeActed {
t.Errorf("%s: attempts %v, would %v, asked %v, said %+v", tc.name, attempts, would, w.asked, w.said)
continue
}
a := w.store.attempt(t, attempts[0])
if a.Outcome != OutcomeActed || a.Finished == nil || a.Healer != "H3" || a.Condition != "bus.EVENTS.records.missing" {
t.Errorf("%s: the attempt is %+v", tc.name, a)
}
continue
}
if len(attempts) != 0 || len(w.asked) != 0 || len(w.said) != 0 || !slices.Equal(would, []string{"H3.bus.EVENTS.records.missing"}) {
t.Errorf("%s: attempts %v, would %v, asked %v, said %v", tc.name, attempts, would, w.asked, w.said)
}
if _, braked, _ := w.store.Get(FindingsState, brakeKey); braked {
t.Errorf("%s: a would-act moved the brake", tc.name)
}
attempts, would := w.store.keys(AttemptsState), w.store.keys(WouldState)
if len(attempts) != 1 || len(would) != 0 || len(w.asked) != 1 || !strings.Contains(w.asked[0], `"module":"records"`) ||
len(w.said) != 1 || w.said[0].Outcome != OutcomeActed {
t.Fatalf("attempts %v, would %v, asked %v, said %+v", attempts, would, w.asked, w.said)
}
a := w.store.attempt(t, attempts[0])
if a.Outcome != OutcomeActed || a.Finished == nil || a.Healer != "H3" || a.Condition != "bus.EVENTS.records.missing" {
t.Errorf("the attempt is %+v", a)
}
}
@@ -219,7 +189,6 @@ func TestTheHealersActOnlyOnceTheLeaseNamesNoController(t *testing.T) {
// in an hour and lets go an hour after the last (ADR 0231 §21, ADR 0564 §7).
func TestBudgetSettleEscalationAndBrake(t *testing.T) {
w := newWorld()
w.handAway()
c := lost("records")
_ = w.store.Put(ConditionsRead, c.Key, c)
settle := settlePrefix + c.Key
@@ -247,7 +216,6 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) {
switch s.how {
case "restart":
w.fresh()
w.handAway()
w.h.Rearm()
case "expire":
w.h.WakeHeard(settle, OpExpire)
@@ -277,28 +245,24 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) {
}
// A settle whose key expired while no instance watched is judged at the start.
r := newWorld()
r.handAway()
_ = r.store.Put(ConditionsRead, c.Key, c)
if _, err := r.h.Judge(c, ""); err != nil {
t.Fatal(err)
}
r.now = r.now.Add(7 * time.Minute)
r.fresh()
r.handAway()
r.h.Rearm()
if len(r.said) != 2 || r.said[1].Outcome != OutcomeEscalated {
t.Errorf("a settle that ended unheard is not judged at the start: %+v", r.said)
}
// A second instance cannot begin the act an attempt's key already holds.
x := newWorld()
x.handAway()
_ = x.store.CreateTTL(AttemptsState, attemptKey("H3", lost("mail").Key, x.now.UTC()), Attempt{}, KeptFor)
if said, _ := x.h.Judge(lost("mail"), ""); !strings.Contains(said, "another instance") || len(x.asked) != 0 {
t.Errorf("a second instance acted: %q, asked %v", said, x.asked)
}
// The brake: twelve acts in the hour stop every healer, said as an urgent finding.
b := newWorld()
b.handAway()
for i := 0; i < BrakeLimit; i++ {
b.now = b.now.Add(time.Minute)
_ = b.store.CreateTTL(AttemptsState, attemptKey("H3", "x", b.now), Attempt{Healer: "H3", Condition: "x", BudgetKey: "x",
@@ -325,7 +289,6 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) {
}
// Blind to what they did, they act on nothing and say so.
z := newWorld()
z.handAway()
z.store.keysErr[AttemptsState] = errors.New("the bus is away")
if _, err := z.h.Judge(lost("mail"), ""); err == nil || len(z.asked) != 0 {
t.Errorf("blind, the healers acted: %v", err)
@@ -350,7 +313,6 @@ func TestTheRegistryAndWhatH3Answers(t *testing.T) {
}
for _, tc := range cases {
w := newWorld()
w.handAway()
said, _ := w.h.Judge(tc.c, "")
if acted := len(w.asked) == 1; acted != tc.holds {
t.Errorf("%s: acted %v, want %v (%s)", tc.c.Key, acted, tc.holds, said)
@@ -420,7 +382,6 @@ func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) {
// answer every bucket.
func TestTheVerbsAndReads(t *testing.T) {
w := newWorld()
w.handAway()
c := lost("records")
_ = w.store.Put(ConditionsRead, c.Key, c)
if _, err := w.h.Heal(c.Key, "", "person.jochen"); err == nil {
@@ -444,28 +405,19 @@ func TestTheVerbsAndReads(t *testing.T) {
if a["acting"] != true || len(a["acts"].([]Attempt)) != 1 || len(a["healers"].([]map[string]any)) != 3 {
t.Errorf("healers answered %v", a)
}
// drill records into mesh-hand-acts as a drill, and never into the controller's log, whoever holds the lease
// (ADR 0565 §11, issue 1955).
// drill records into mesh-hand-acts as a drill, and never into the controller's log (ADR 0565 §11, issue 1955).
drills := []struct {
name string
seatGone bool
leaseAway bool
args map[string]any
log, err string
name string
seatGone bool
args map[string]any
log, err string
}{
{"into the seat", false, true, map[string]any{"what": "stopped records", "why": "does it raise"}, handacts.LogSeat, ""},
{"into the seat under a controller's lease", false, false, map[string]any{"what": "stopped records", "why": "x"}, handacts.LogSeat, ""},
{"no holder: refused, the controller's not asked", true, false, map[string]any{"what": "stopped records", "why": "x"}, "", "nothing was recorded"},
{"no holder, lease away", true, true, map[string]any{"what": "stopped records", "why": "x"}, "", "nothing was recorded"},
{"says nothing", false, true, map[string]any{}, "", "says what"},
{"into the seat", false, map[string]any{"what": "stopped records", "why": "does it raise"}, handacts.LogSeat, ""},
{"no holder: refused, the controller's not asked", true, map[string]any{"what": "stopped records", "why": "x"}, "", "nothing was recorded"},
{"says nothing", false, map[string]any{}, "", "says what"},
}
for _, d := range drills {
x := newWorld()
if d.leaseAway {
x.handAway()
} else {
x.h.LeaseHeard([]byte(`{"instance":"novox/1","epoch":5}`), true, nil)
}
var asked []string
x.h.Ask = func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
modules/mesh-healers/cmd/mesh-healers/main.go
+6 -12
View File
@@ -1,8 +1,8 @@
// mesh-healers: the holder of the seat mesh-healers (novox/hq ADR 0564 §7, §8). A Go bundle served by a runtime of its
// own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `attempts` (every act begun,
// made and finished, and every escalation, kept a week), `would` (what it would have done while the controller holds
// its lease, kept a week) and `findings` (the brake, its blindness, and S15). It hears the condition keeper's
// condition-raised and condition-changed on a durable consumer, watches the controller's lease, and counts the seat's hand acts
// made and finished, and every escalation, kept a week), `would` (what it would have done while the controller ran,
// kept a week; nothing writes it any more) and `findings` (the brake, its blindness, and S15). It hears the condition keeper's
// condition-raised and condition-changed on a durable consumer, and counts the seat's hand acts
// for S15. stdout is the MCP channel; what this module says, it says on stderr.
package main
@@ -11,7 +11,6 @@ import (
"errors"
"fmt"
"os"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
@@ -56,14 +55,9 @@ func main() {
h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf,
Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) },
Publish: stdio.SeatPublish}
// The lease first: until it is read the healers write would-acts only. Once it is first read, a settle that ended
// while no instance watched is judged (Rearm); the waits still running are keys on the bus, heard on their expiry.
var started sync.Once
go stdio.State(LeaseRead).Follow(LeaseKey, func(c stdio.StateChange) error {
h.LeaseHeard(c.Value, c.Op == "put", nil)
started.Do(func() { go h.Rearm() })
return nil
})
// At start, a settle that ended while no instance watched is judged (Rearm); the waits still running are keys on
// the bus, heard on their expiry.
go h.Rearm()
go stdio.State(WakesState).Follow("", func(c stdio.StateChange) error {
h.WakeHeard(c.Key, c.Op)
return nil
modules/mesh-healers/cmd/mesh-healers/manifest_test.go
+3 -3
View File
@@ -26,7 +26,7 @@ type manifest struct {
// The manifest says what the code does: it claims the seat mesh-healers once for the mesh with its verbs (ADR 0564 §1,
// §7), runs on its own account, says healer-acted, hears the keeper's raised and changed, asks only the grants part's
// assert, reads the lease, the keeper's conditions and the hand-act log, and keeps attempts and would a week each.
// assert, reads the keeper's conditions (never the controller's lease, issue 1126) and the hand-act log, and keeps attempts and would a week each.
// Every state it keeps has a tool that reads it, and every act it takes by itself a tool that starts it (ADR 0564 §12).
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
@@ -50,7 +50,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
{"emits (healer-acted is the seat's, said on its subject)", m.Emits, nil},
{"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent, RecordedEvent}},
{"invokes", m.Invokes, []string{AssertTool, "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts"}},
{"reads", m.Reads, []string{LeaseRead, ConditionsRead}},
{"reads", m.Reads, []string{ConditionsRead}},
} {
if !slices.Equal(c.got, c.want) {
t.Errorf("%s %v, want %v", c.name, c.got, c.want)
@@ -74,7 +74,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
}
readBy := map[string]string{AttemptsState: HealersTool, WouldState: WouldTool, FindingsState: FindingsTool,
WakesState: WakesTool, "a settle's end": HealTool,
"an act on an event": HealTool, "S15's count": HealersTool + " recount=true", "the lease as read": HealersTool}
"an act on an event": HealTool, "S15's count": HealersTool + " recount=true"}
tools := map[string]bool{}
for _, tool := range Tools(&Healers{}) {
tools[tool.Name] = true
modules/mesh-healers/cmd/mesh-healers/tools.go
+6 -8
View File
@@ -38,8 +38,8 @@ func Tools(h *Healers) []stdio.Tool {
return []stdio.Tool{
{Name: HealersTool,
Description: "The mesh's healers (novox/hq ADR 0564 §7, §8): the registry, every act and escalation of the last " +
"days (the bucket attempts), whether the mesh-wide brake holds, whether they act or only write what they " +
"would do while the controller holds its lease, and the would-acts. recount=true counts the hand-act log " +
"days (the bucket attempts), whether the mesh-wide brake holds, that they act (no lease turns them, " +
"issue 1126), and the would-acts kept from while the controller ran. recount=true counts the hand-act log " +
"again for S15 (a cause repaired by hand twice in a fortnight).",
Input: obj(map[string]any{
"days": map[string]any{"type": "number", "description": "how many days of acts back (default 7)"},
@@ -47,8 +47,7 @@ func Tools(h *Healers) []stdio.Tool {
Run: func(a map[string]any) (any, error) { return h.Answer(intArg(a, "days", 7), boolArg(a, "recount")) }},
{Name: HealTool,
Description: "One healer's act now on an open condition, by its key, within its budget and the brake, as an " +
"event would have it — or, while the controller holds its lease, the act it would take written as would. " +
"Says what was done.",
"event would have it. Says what was done.",
Input: obj(map[string]any{
"key": map[string]any{"type": "string", "description": "the condition's key"},
"why": map[string]any{"type": "string", "description": "why, in a sentence"}}),
@@ -76,8 +75,8 @@ func Tools(h *Healers) []stdio.Tool {
Input: obj(map[string]any{"key": map[string]any{"type": "string"}}),
Run: func(a map[string]any) (any, error) { return h.read(WakesState, strArg(a, "key")) }},
{Name: WouldTool,
Description: "What the healers would have done while the controller holds its lease (their bucket would, each " +
"key kept a week): one per healer and condition, its newest; or one whole by key.",
Description: "What the healers would have done while the controller ran (their bucket would, each " +
"key kept a week; nothing writes it any more): one per healer and condition, its newest; or one whole by key.",
Input: obj(map[string]any{"key": map[string]any{"type": "string"}}),
Run: func(a map[string]any) (any, error) { return h.read(WouldState, strArg(a, "key")) }},
}
@@ -147,7 +146,6 @@ func (h *Healers) Answer(days int, recount bool) (any, error) {
"budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle),
"then": r.Then, "acts-in": r.ActsIn, "from": r.From})
}
acting, why := h.Acting()
would, err := h.read(WouldState, "")
if err != nil {
would = map[string]any{"unread": err.Error()}
@@ -161,7 +159,7 @@ func (h *Healers) Answer(days int, recount bool) (any, error) {
if recent == nil {
recent = []Attempt{}
}
return map[string]any{"healers": rows, "acts": recent, "days": days, "acting": acting, "why": why, "would": would,
return map[string]any{"healers": rows, "acts": recent, "days": days, "acting": true, "why": actingWhy, "would": would,
"brake": map[string]any{"holds": braked, "said": said, "limit": BrakeLimit, "window": BrakeWindow.String()},
"declined": declined,
"note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"}, nil
modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go
+10 -24
View File
@@ -5,9 +5,9 @@ package main
// mesh-healers whole against a real bus (novox/hq ADR 0564 §7, §8; issue 1952 H3): the built binary, played over its
// real stdin and stdout as the node's tool runtime plays it (mesh/state.* and watches answered from a real nats-server's
// key-value buckets, the keeper's events handed over as the runtime hands them, the grants part's assert answered). An
// induced failure, a module's consumer gone, is only written as would while the controller holds its lease; from the
// write that hands the lease away a healer acts, its attempt on the bus and healer-acted said; past twelve acts in the
// hour the brake stops every healer, said as an urgent finding on the bus.
// induced failure, a module's consumer gone, is acted on with no lease key anywhere (issue 1126), its attempt on the bus
// and healer-acted said, nothing written as would; past twelve acts in the hour the brake stops every healer, said as an
// urgent finding on the bus.
import (
"bufio"
@@ -320,7 +320,7 @@ func raised(key, module string) []byte {
return raw
}
// 1952/H3: an induced failure; the healer acts only from the lease write, and brakes.
// 1952/H3, 1126/A4: an induced failure with no lease key; the healer acts, and brakes.
func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
@@ -341,7 +341,7 @@ func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) {
t.Fatal(err)
}
for _, b := range []string{Module + "_" + AttemptsState, Module + "_" + WouldState, Module + "_" + FindingsState, Module + "_" + WakesState,
"mesh-controller_lease", "mesh-conditions_conditions", "mesh-controller_hand-acts"} {
"mesh-conditions_conditions", "mesh-controller_hand-acts"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
t.Fatal(err)
}
@@ -357,10 +357,6 @@ func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) {
k, _ := bucket(name).Keys(ctx)
return k
}
lease := bucket("mesh-controller_lease")
if _, err := lease.Put(ctx, LeaseKey, []byte(`{"instance":"novox/1/2026-10-27","epoch":57}`)); err != nil {
t.Fatal(err)
}
acted, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + ActedEvent)
if err != nil {
t.Fatal(err)
@@ -385,24 +381,11 @@ func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) {
_ = json.Unmarshal(raw, &a)
return a
}
eventually(t, "the lease read as a controller's", func() bool {
a := healers()
return a != nil && strings.Contains(fmt.Sprint(a["why"]), "names a controller")
})
eventually(t, "the healers acting", func() bool { return healers()["acting"] == true })
// The runtime's consumer holds an event until the bundle subscribes: delivered once it hears them.
eventually(t, "the healers hearing the keeper's conditions", func() bool { return p.hears(RaisedEvent) && p.hears(ChangedEvent) })
// The controller holds the lease: the induced failure is written as would, never acted on.
p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records"))
eventually(t, "the would-act written", func() bool { return len(keys(Module+"_"+WouldState)) == 1 })
if k := keys(Module + "_" + AttemptsState); len(k) != 0 {
t.Fatalf("acted under the controller's lease: %v", k)
}
// The write that hands the lease away: the same failure is acted on.
if _, err := lease.Put(ctx, LeaseKey, []byte(`{"handed-to":"the operator's hand-over"}`)); err != nil {
t.Fatal(err)
}
eventually(t, "the healers acting", func() bool { return healers()["acting"] == true })
// No lease key: the induced failure is acted on.
p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records"))
eventually(t, "the act begun and finished on the bus", func() bool {
k := keys(Module + "_" + AttemptsState)
@@ -412,6 +395,9 @@ func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) {
e, err := bucket(Module+"_"+AttemptsState).Get(ctx, k[0])
return err == nil && strings.Contains(string(e.Value()), `"outcome":"acted"`)
})
if k := keys(Module + "_" + WouldState); len(k) != 0 {
t.Errorf("written as would: %v", k)
}
msg, err := acted.NextMsg(10 * time.Second)
if err != nil || !strings.Contains(string(msg.Data), `"healer":"H3"`) {
t.Fatalf("healer-acted was not said: %v", err)
modules/mesh-healers/module.json
-1
View File
@@ -24,7 +24,6 @@
"seat:mesh-hand-acts.hand-acts"
],
"reads": [
"mesh-controller.lease",
"mesh-conditions.conditions"
],
"state": [
modules/mesh-vault/cmd/mesh-vault/enrolment_test.go
+1 -2
View File
@@ -72,7 +72,7 @@ func (l *loginsState) Keys() ([]string, error) {
return out, nil
}
// enrolling is a vault that mints, with the lease handed away, a loaded user list at revision 570, the bus on novox, the
// enrolling is a vault that mints, with no lease heard (issue 1126), a loaded user list at revision 570, the bus on novox, the
// nodes g14, ace and novox placed, their existing logins adopted, and hand acts of person.jochen's.
type enrolling struct {
testVault
@@ -87,7 +87,6 @@ func newEnrolling(t *testing.T) *enrolling {
e := &enrolling{testVault: v, accounts: &fakeAccounts{keys: map[string]AccountHash{}}, logins: &loginsState{keys: map[string]NodeLogin{}}, acts: map[string]HandActSeen{}}
v.Accounts, v.Logins = e.accounts, e.logins
v.Placement = placementsOf(t, map[string][]string{"novox": {"nats"}, "g14": {"web"}, "ace": {"shop"}})
v.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil)
v.LoadedListHeard(570)
v.TokenBase = func() (TokenBase, error) {
return TokenBase{Broker: "tls://10.77.0.1:4222", Fingerprint: "sha256:bus"}, nil
modules/mesh-vault/cmd/mesh-vault/lease.go
-41
View File
@@ -1,41 +0,0 @@
package main
// The controller's lease as the vault reads it (novox/hq ADR 0561 §3): handed away only to a holder that is no
// controller, and not for a controller's value, a key deleted or expired, or one that cannot be read. It turns retirement
// (retire.go) and rotation (rotation.go), never minting: the vault mints every wanted account it holds no value for,
// whoever holds the lease (mint.go, issue 774). No clock and no setting turns it. A key's age passing is heard by no
// watch, so the lease is read afresh before a retirement or a rotation (LeaseRead).
import "git.novox.be/novox/mesh-sdk/go/buscompose"
// LeaseHeard is the watch of the controller's lease: it records whether it is handed away, and says when that turned.
func (v *Vault) LeaseHeard(raw []byte, found bool, readErr error) {
if !v.setLease(raw, found, readErr) {
return
}
on, why := v.leaseHanded()
v.log("the controller's lease is %s: %s", map[bool]string{true: "HANDED AWAY", false: "not handed away"}[on], why)
}
// setLease records what the lease says, and answers whether minting turned on or off by it.
func (v *Vault) setLease(raw []byte, found bool, readErr error) bool {
on, why := buscompose.LeaseHandedAway(raw, found, readErr)
m := &v.minting
m.mu.Lock()
defer m.mu.Unlock()
flipped := on != m.on || m.why == ""
if flipped {
m.since = v.Now().UTC()
}
m.on, m.why = on, why
return flipped
}
// leaseAfresh reads the lease again before a retirement or a rotation, where the vault can: a key that expired since is heard by no watch.
func (v *Vault) leaseAfresh() {
if v.LeaseRead == nil {
return
}
raw, found, err := v.LeaseRead()
v.setLease(raw, found, err)
}
modules/mesh-vault/cmd/mesh-vault/main.go
+1 -33
View File
@@ -58,7 +58,7 @@ func build() *Env {
watchedPlan := func(key string) ([]byte, error) { raw, err := assigned.Read(key); return asAssignment(key, raw, err) }
env.Vault = &Vault{Store: store, Own: own, Askers: Declared{Read: watchedPlan, Confirm: readAssignment, Store: store},
Assignments: Mirror{Keys: assignmentKeys, Read: readAssignment}, Emit: emit, Ask: stdio.Ask,
Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, LeaseRead: readLease, WantRead: readWant,
Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, WantRead: readWant,
HaveKeys: func() ([]string, error) { return stdio.State(HaveRead).Keys() }, Placement: placement, Signers: busSigners{},
AskCallers: askCallers(os.Getenv("MESH_VAULT_SECRET_ASK_CALLERS")), EnrolCallers: askCallers(os.Getenv("MESH_VAULT_ENROLMENT_CALLERS")),
Logins: busNodeLogins{}, ReadAct: readHandAct, TokenBase: TokenBaseFromFile(os.Getenv("MESH_VAULT_STATE")),
@@ -229,34 +229,6 @@ func readWant(user string) ([]byte, bool, error) {
return e.Value, true, nil
}
// readLease is the controller's lease key, read through the runtime under the vault's own account (its manifest reads it).
func readLease() ([]byte, bool, error) {
e, err := stdio.State(LeaseRead).Get(LeaseKey)
if err != nil || e == nil {
return nil, false, err
}
return e.Value, true, nil
}
// watchLease turns minting on and off as the controller's lease is written (lease.go, ADR 0561 §3): a watch, never a
// poll; and the lease is read afresh before every mint. It answers whether the watch began.
func watchLease(env *Env) bool {
if env.Vault == nil {
return true
}
env.Vault.LeaseHeard(readLease())
err := stdio.State(LeaseRead).Watch(LeaseKey, func(c stdio.StateChange) error {
deleted := c.Op == "delete" || c.Op == "purge" || len(c.Value) == 0
env.Vault.LeaseHeard(c.Value, !deleted, nil)
return nil
})
if err != nil {
say("the controller's lease is not watched, so minting stays as last read; begun again at the next pass: %v", err)
return false
}
return true
}
// adoptOperatorKey takes the operator key from the export the mesh writes to the vault's root (what
// `operator key set` recorded; the controller keeps only a public half there), and seals again any
// operator copy not yet sealed to the current key. Run every minute, so a failure partway is finished. It
@@ -319,7 +291,6 @@ func main() {
// The operator key first: nothing is kept without it, so a want heard before it would not be minted.
env.pass()
watched, watching := watchHandings(env), watchAssignments()
lease := watchLease(env)
wants := watchWants(env)
leavers := watchLeavers(env)
for {
@@ -332,9 +303,6 @@ func main() {
if !watching {
watching = watchAssignments()
}
if !lease {
lease = watchLease(env)
}
if !wants {
wants = watchWants(env)
}
modules/mesh-vault/cmd/mesh-vault/mint.go
+13 -45
View File
@@ -11,10 +11,10 @@ package main
// on `secret-changed` and composes.
//
// **The vault is the one issuer** (issue 774, research 072's gap "vault mints after the lease"): it mints every wanted
// account it holds no value for, whoever holds the controller's lease, and the controller mints none once a vault is
// installed (mesh-controller forwards `module issue` and its assignment's issue to issue-account). No account depends
// on a node reporting to the controller. The lease still turns retirement and rotation (retire.go, rotation.go), and
// vault_status names it. **Never two issuers of one value** (§3, §4): an account whose value the vault holds, or whose
// account it holds no value for, and the controller mints none once a vault is installed (mesh-controller forwards
// `module issue` and its assignment's issue to issue-account). No account depends on a node reporting to the
// controller. No lease turns minting, retirement or rotation: the controller ended by its unassignment (issue 1126).
// **Never two issuers of one value** (§3, §4): an account whose value the vault holds, or whose
// hash it adopted, is never minted; the node accounts and the controller's own are never wanted; a value the controller
// moves in never replaces one held (move.go), and one it hands with origin issued never replaces an account the vault
// minted (AcceptTake): during the delivery an older controller may still mint, and the vault's value wins.
@@ -38,23 +38,13 @@ import (
// WantedRead is the grants part's state of wanted accounts, as the vault's manifest reads it.
const WantedRead = buscompose.GrantsAccounts
// LeaseRead is the controller's lease, as the vault's manifest reads it, and LeaseKey its one key.
const (
LeaseRead = buscompose.ControllerLease
LeaseKey = buscompose.ControllerLeaseKey
)
// AccountIssuedEvent is what the vault says when it minted an account (mesh-sdk buscompose.AccountIssued).
const AccountIssuedEvent = buscompose.AccountIssuedEvent
// minting is what the vault knows of the wanted accounts and of the controller's lease, kept in memory and read again
// from the bus at every start: the grants part's state and the lease are the record, never this. on, why and since are
// the lease (handed away or not), which turns retirement and rotation, never minting.
// minting is what the vault knows of the wanted accounts, kept in memory and read again from the bus at every start:
// the grants part's state is the record, never this.
type minting struct {
mu sync.Mutex
on bool
why string
since time.Time
mu sync.Mutex
// wants are the wanted accounts, by user, as last heard.
wants map[string]buscompose.WantedAccount
// heldBack are the wanted users the vault did not mint, with why (vault_status).
@@ -81,34 +71,25 @@ type Minted struct {
Hashed bool `json:"hashed"`
}
// MintState is what vault_status says of minting: always on (the vault is the one issuer, issue 774), and the lease,
// which turns retirement and rotation.
// MintState is what vault_status says of minting: always on (the vault is the one issuer, issue 774).
type MintState struct {
On bool `json:"on"`
Why string `json:"why"`
// Lease is whether the controller's lease is handed away, LeaseWhy why, and Since when that last turned.
Lease bool `json:"lease_handed_away"`
LeaseWhy string `json:"lease_why"`
Since time.Time `json:"since,omitzero"`
On bool `json:"on"`
Why string `json:"why"`
Wanted int `json:"wanted"`
HeldBack map[string]string `json:"held_back"`
Minted []Minted `json:"minted"`
}
// mintingWhy is why the vault mints, whoever holds the lease.
// mintingWhy is why the vault mints.
const mintingWhy = "the vault is the one issuer of bus accounts: it mints every wanted account it holds no value for " +
"and adopted no hash of, whoever holds the controller's lease (issue 774)"
"and adopted no hash of (issue 774)"
// MintState answers how minting stands. A user held back whose account the vault holds a value for meanwhile (moved in,
// handed, or minted by another path) is not said held back: it is not (issue 774, 2026-10-31).
func (v *Vault) MintState() MintState {
m := &v.minting
m.mu.Lock()
out := MintState{On: true, Why: mintingWhy, Lease: m.on, LeaseWhy: m.why, Since: m.since, Wanted: len(m.wants),
HeldBack: map[string]string{}, Minted: []Minted{}}
if out.LeaseWhy == "" {
out.LeaseWhy = leaseUnread
}
out := MintState{On: true, Why: mintingWhy, Wanted: len(m.wants), HeldBack: map[string]string{}, Minted: []Minted{}}
wants := map[string]buscompose.WantedAccount{}
for u, why := range m.heldBack {
out.HeldBack[u] = why
@@ -143,19 +124,6 @@ func (v *Vault) holdsAccount(w buscompose.WantedAccount) bool {
return err == nil && BusAccountValue(secret, value)
}
// leaseUnread is why nothing is retired or rotated before the lease is read.
const leaseUnread = "the controller's lease has not been read yet, so nothing is retired or rotated"
// leaseHanded says whether the controller's lease is handed away, and why: what retirement and rotation wait for.
func (v *Vault) leaseHanded() (bool, string) {
v.minting.mu.Lock()
defer v.minting.mu.Unlock()
if v.minting.why == "" {
return false, leaseUnread
}
return v.minting.on, v.minting.why
}
func (v *Vault) holdBack(user, why string) {
m := &v.minting
m.mu.Lock()
modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go
+5 -6
View File
@@ -5,7 +5,7 @@ package main
// The vault whole against a real bus (novox/hq ADR 0561 §1 to §4, issue 774's acceptance 12): the built binary, played
// over its real stdin and stdout as the node's tool runtime plays it (tools/call with the caller in _meta; mesh/state.*
// and mesh/publish answered from a real nats-server's key-value buckets and subjects), its store in a directory of its
// own. A want written to the grants part's state is minted whoever holds the controller's lease (issue 774: the vault is
// own. A want written to the grants part's state is minted with no lease asked (issue 774: the vault is
// the one issuer): the hash is in bus-accounts, own answers the value sealed to the node, the value is the controller's
// shape and its hash's password, account-issued is said; an adopted user is never minted.
@@ -330,7 +330,7 @@ func TestTheVaultWholeMintsOnTheBusWhoeverHoldsTheLease(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, b := range []string{"mesh-vault_bus-accounts", "mesh-grants_accounts", "mesh-controller_lease",
for _, b := range []string{"mesh-vault_bus-accounts", "mesh-grants_accounts",
"mesh-provisioner_plans", "node-engine_have", "mesh-delivery-assigner_placement"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 2}); err != nil {
t.Fatal(err)
@@ -372,14 +372,13 @@ func TestTheVaultWholeMintsOnTheBusWhoeverHoldsTheLease(t *testing.T) {
_ = json.Unmarshal(raw, &s)
return s.Minting
}
// The controller holds the lease, and the want is minted all the same.
p.put(t, "mesh-controller.lease", LeaseKey, map[string]any{"instance": "novox/1/2026-10-27", "epoch": 57})
// No lease key: the want is minted all the same (issue 1126).
x, y := aWant("mod-x"), aWant("mod-y")
p.put(t, "mesh-vault.bus-accounts", y.User, AccountHash{User: y.User, Hash: "$2a$04$adoptedadoptedadoptedadoptedadoptedadoptedadopte", Adopted: true})
p.put(t, "mesh-grants.accounts", x.User, x)
p.put(t, "mesh-grants.accounts", y.User, y)
eventually(t, "the account minted under the controller's lease", func() bool { return p.hashOf(t, x.User) != nil })
if st := status(); !st.On || st.Lease || len(st.HeldBack) != 0 {
eventually(t, "the account minted", func() bool { return p.hashOf(t, x.User) != nil })
if st := status(); !st.On || len(st.HeldBack) != 0 {
t.Errorf("vault_status says %+v", st)
}
nodePub, nodePriv := aSealingKey(t)
modules/mesh-vault/cmd/mesh-vault/mint_test.go
+14 -19
View File
@@ -24,9 +24,9 @@ func wantRaw(t *testing.T, w buscompose.WantedAccount) []byte {
return raw
}
// mintingVault is a vault with a state of bus accounts, ace's sealing key stated, ace's modules assigned, and minting
// turned on or off as on says.
func mintingVault(t *testing.T, on bool) (testVault, *fakeAccounts, string) {
// mintingVault is a vault with a state of bus accounts, ace's sealing key stated and ace's modules assigned. No lease is
// heard: none turns minting, retirement or rotation (issue 1126).
func mintingVault(t *testing.T) (testVault, *fakeAccounts, string) {
t.Helper()
plans := assignments{"ace.mod-x": `{"node":"ace","module":"mod-x"}`, "ace.mod-y": `{"node":"ace","module":"mod-y"}`}
v := newVault(t, nil)
@@ -37,11 +37,6 @@ func mintingVault(t *testing.T, on bool) (testVault, *fakeAccounts, string) {
if err := v.recordNodeKey("ace", pub, "node.ace"); err != nil {
t.Fatal(err)
}
if on {
v.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil)
} else {
v.LeaseHeard([]byte(`{"instance":"novox/1/2026-10-27","epoch":57}`), true, nil)
}
return v, accounts, priv
}
@@ -49,7 +44,7 @@ func mintingVault(t *testing.T, on bool) (testVault, *fakeAccounts, string) {
// shape as the module's broker, its hash in bus-accounts, said account-issued before secret-changed, and answered by own
// sealed to the node; a value held or a hash adopted is never minted.
func TestTheVaultMintsAWantedAccountOnceAndNothingItHolds(t *testing.T) {
v, accounts, nodePriv := mintingVault(t, true)
v, accounts, nodePriv := mintingVault(t)
w := aWant("mod-x")
before := len(v.bus.events)
v.WantHeard(w.User, wantRaw(t, w), false)
@@ -107,11 +102,11 @@ func TestTheVaultMintsAWantedAccountOnceAndNothingItHolds(t *testing.T) {
}
// 774/AC8, after research 072's gap "vault mints after the lease": the vault is the one issuer, so a wanted account is
// minted while the controller still holds the lease (never an adopted one), vault_status says minting is on and the
// lease apart, and a want held back is not said held back once the vault holds its value. During the delivery an older
// minted with no lease asked (never an adopted one), vault_status says minting is on and names no lease (issue 1126),
// and a want held back is not said held back once the vault holds its value. During the delivery an older
// controller may still mint: its value handed with origin issued never replaces the account the vault minted.
func TestTheVaultMintsWhoeverHoldsTheLease(t *testing.T) {
v, accounts, _ := mintingVault(t, false)
v, accounts, _ := mintingVault(t)
accounts.keys = map[string]AccountHash{"ace.mod-y": {User: "ace.mod-y", Hash: "$2a$04$adopted", Adopted: true}}
for _, m := range []string{"mod-x", "mod-y"} {
w := aWant(m)
@@ -119,14 +114,14 @@ func TestTheVaultMintsWhoeverHoldsTheLease(t *testing.T) {
}
minted, held, _ := v.Store.Read("own/ace/mod-x/broker")
if !held {
t.Fatalf("not minted under the controller's lease: %+v", v.MintState())
t.Fatalf("not minted: %+v", v.MintState())
}
if _, held, _ := v.Store.Read("own/ace/mod-y/broker"); held {
t.Error("an adopted account was minted")
}
st := v.MintState()
if !st.On || st.Lease || !strings.Contains(st.LeaseWhy, "controller holds the lease") || len(st.HeldBack) != 0 {
t.Errorf("under the controller's lease vault_status says %+v", st)
if !st.On || strings.Contains(st.Why, "lease") || len(st.HeldBack) != 0 {
t.Errorf("vault_status says %+v", st)
}
// An older controller's covered account, handed with origin issued: refused, the vault's value kept.
x := aWant("mod-x")
@@ -156,7 +151,7 @@ func TestTheVaultMintsWhoeverHoldsTheLease(t *testing.T) {
// 774/AC10 issue-account (novox/hq ADR 0561 §9): the verb mints one wanted user now, read afresh, whoever holds the lease;
// it is refused for a node's account and the controller's, and for a user nobody wants; a user held already is said so.
func TestIssueAccountMintsOneWantedUserNowAndRefusesTheRest(t *testing.T) {
v, _, _ := mintingVault(t, false)
v, _, _ := mintingVault(t)
w := aWant("mod-x")
raw := wantRaw(t, w)
v.WantRead = func(user string) ([]byte, bool, error) {
@@ -191,7 +186,7 @@ func TestIssueAccountMintsOneWantedUserNowAndRefusesTheRest(t *testing.T) {
// over the second login `<user>--b`, never of a node's account, the controller's or an adopted hash, and `rotate` of a
// covered account stays refused, so nothing rotates in place.
func TestOneRotationBeginsPerRevisionOfTheLoadedList(t *testing.T) {
v, accounts, _ := mintingVault(t, true)
v, accounts, _ := mintingVault(t)
for _, m := range []string{"mod-x", "mod-y"} {
w := aWant(m)
v.WantHeard(w.User, wantRaw(t, w), false)
@@ -227,7 +222,7 @@ func TestOneRotationBeginsPerRevisionOfTheLoadedList(t *testing.T) {
// A mint that failed (here: no operator key yet, so nothing can be kept) is tried again at the pass, never lost.
func TestAFailedMintIsTriedAgainAtThePass(t *testing.T) {
v, accounts, _ := mintingVault(t, true)
v, accounts, _ := mintingVault(t)
key, _ := v.Store.OperatorKey()
if err := v.Store.setOperatorKey("", "the test", v.Now()); err != nil {
t.Fatal(err)
@@ -250,7 +245,7 @@ func TestAFailedMintIsTriedAgainAtThePass(t *testing.T) {
// broker-management on 2026-10-30, issue 774) is replaced by a minted account when the grants part wants its user;
// a held account still never is.
func TestAHeldValueThatIsNoAccountIsReplacedByAMintedOne(t *testing.T) {
v, accounts, _ := mintingVault(t, true)
v, accounts, _ := mintingVault(t)
w := aWant("mod-x")
if _, err := v.Put("own/ace/mod-x/broker", `{"password":"pw"}`, "the test"); err != nil {
t.Fatal(err)
modules/mesh-vault/cmd/mesh-vault/noderotation.go
-8
View File
@@ -268,10 +268,6 @@ func (v *Vault) RotateNode(caller string, a RotateNodeArgs) (RotateNodeAnswer, e
if v.Accounts == nil {
return RotateNodeAnswer{}, errors.New("the vault has no state of bus accounts to write to")
}
v.leaseAfresh()
if on, why := v.leaseHanded(); !on {
return RotateNodeAnswer{}, fmt.Errorf("no rotation begins: %s", why)
}
v.minting.mu.Lock()
revision := v.minting.loadedRevision
v.minting.mu.Unlock()
@@ -596,10 +592,6 @@ func (v *Vault) RotationPass() {
if err != nil || len(running) > 0 {
return
}
v.leaseAfresh()
if on, _ := v.leaseHanded(); !on {
return
}
bus, err := v.busNode()
if err != nil {
return
modules/mesh-vault/cmd/mesh-vault/retire.go
+3 -13
View File
@@ -7,8 +7,8 @@ package main
// retired in its store, keeping it (a value is deleted only by a person, ADR 0230). As ADR 0230 §15 and §18 have it, the
// same set of accounts gone must stand for five passes and ten minutes first, and a set of more than three, or more than
// half of the accounts held, waits for a person, who retires them with `account_retire`. Wanted again, an account is
// re-enabled at once: the mark taken off and its hash written again. Nothing is retired while the controller holds the
// lease (it forgets the accounts it mints), or before the wanted accounts are read whole.
// re-enabled at once: the mark taken off and its hash written again. Nothing is retired before the wanted accounts are read
// whole. No lease turns it: the controller ended by its unassignment (issue 1126).
import (
"errors"
@@ -135,8 +135,6 @@ func (v *Vault) RetirePass() {
if v.Accounts == nil {
return
}
v.leaseAfresh()
on, leaseWhy := v.leaseHanded()
m := &v.minting
m.mu.Lock()
read := m.retiring.wantsRead
@@ -150,9 +148,6 @@ func (v *Vault) RetirePass() {
case !read:
reset("the wanted accounts are not read whole yet, so none is known to be gone")
return
case !on:
reset("nothing is retired: " + leaseWhy)
return
}
gone, held, err := v.goneAccounts()
if err != nil {
@@ -239,12 +234,7 @@ func (v *Vault) RetireAccounts(caller string, users []string, why string) (any,
if len(users) == 0 {
return nil, errors.New("account_retire names the accounts (`users`), as vault_status lists them waiting")
}
// As the passes: never while the controller holds the lease (it forgets the accounts it mints), nor before the wanted
// accounts are read whole (an account not read yet is not known to be gone).
v.leaseAfresh()
if on, why := v.leaseHanded(); !on {
return nil, fmt.Errorf("nothing was retired: %s", why)
}
// As the passes: never before the wanted accounts are read whole (an account not read yet is not known to be gone).
v.minting.mu.Lock()
read := v.minting.retiring.wantsRead
v.minting.mu.Unlock()
modules/mesh-vault/cmd/mesh-vault/retire_test.go
+2 -9
View File
@@ -11,7 +11,7 @@ import (
// more than three, or more than half of those held, at once: past that it waits for a person (account_retire). Wanted
// again, it is re-enabled at once; and a retired account is not hashed again at a start.
func TestAWantGoneRetiresItsAccountOnlyWithinTheBounds(t *testing.T) {
v, accounts, _ := mintingVault(t, true)
v, accounts, _ := mintingVault(t)
clock := time.Date(2026, 10, 28, 9, 0, 0, 0, time.UTC)
v.Now = func() time.Time { return clock }
modules := []string{"m1", "m2", "m3", "m4", "m5", "m6", "m7", "m8"}
@@ -65,14 +65,7 @@ func TestAWantGoneRetiresItsAccountOnlyWithinTheBounds(t *testing.T) {
if _, err := v.RetireAccounts("person.jochen", []string{"ace.m2"}, "test"); err == nil {
t.Error("retired by hand an account still wanted")
}
// Never while the controller holds the lease, nor before the wants are read whole.
held := v.LeaseRead
v.LeaseRead = func() ([]byte, bool, error) { return []byte(`{"instance":"novox/1","epoch":57}`), true, nil }
if _, err := v.RetireAccounts("person.jochen", []string{"ace.m5"}, "the modules left"); err == nil || !strings.Contains(err.Error(), "controller holds the lease") {
t.Errorf("retired by hand under the controller's lease: %v", err)
}
v.LeaseRead = held
v.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil)
// Never before the wants are read whole; no lease is asked (issue 1126).
v.minting.mu.Lock()
v.minting.retiring.wantsRead = false
v.minting.mu.Unlock()
modules/mesh-vault/cmd/mesh-vault/rotation.go
+2 -6
View File
@@ -4,9 +4,8 @@ package main
// 2026-10-27). A module's, a process's or a provisioner client's account rotates over two credentials (ADR 0114 §10):
// a second login, `<user>--b`, carrying the first login's permissions verbatim, which the bus composer composes only while
// the account's key in `bus-accounts` names it in `rotating`. The vault starts at most one rotation per revision of the
// user list the bus loaded. What composes and delivers the second login is a subtask of its own, before the lease write;
// until it is built nothing begins a rotation, and `rotate` of a covered account stays refused, so nothing rotates in
// place.
// user list the bus loaded, and no lease turns it (issue 1126). `rotate` of a covered account stays refused, so nothing
// rotates in place.
import (
"fmt"
@@ -42,9 +41,6 @@ func (v *Vault) BeginRotation(user string) (Rotating, error) {
if nodeOrController(user) {
return Rotating{}, fmt.Errorf("%s does not rotate here: a node's account and the controller's keep the hash they have (ADR 0561 §4)", user)
}
if on, why := v.leaseHanded(); !on {
return Rotating{}, fmt.Errorf("no rotation begins: %s", why)
}
v.minting.mu.Lock()
revision := v.minting.loadedRevision
v.minting.mu.Unlock()
modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go
+3 -3
View File
@@ -53,7 +53,7 @@ func TestTheVaultWholeRotatesANodesLoginWithAStandInEngine(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, b := range []string{"mesh-vault_bus-accounts", "mesh-vault_node-logins", "mesh-grants_accounts", "mesh-controller_lease",
for _, b := range []string{"mesh-vault_bus-accounts", "mesh-vault_node-logins", "mesh-grants_accounts",
"mesh-provisioner_plans", "node-engine_have", "mesh-delivery-assigner_placement"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 2}); err != nil {
t.Fatal(err)
@@ -90,7 +90,8 @@ func TestTheVaultWholeRotatesANodesLoginWithAStandInEngine(t *testing.T) {
}
go p.serve(stdout)
// The bus node (novox, running nats) and the node that rotates (g14); the lease handed away; g14's login adopted from
// The bus node (novox, running nats) and the node that rotates (g14); no lease key anywhere (issue 1126: the controller
// ended by its unassignment); g14's login adopted from
// the loaded list, its password known to the engine.
for node, modules := range map[string][]string{"novox": {"nats"}, "g14": {"web"}} {
pl := delivery.Placement{V: delivery.Version, Node: node}
@@ -108,7 +109,6 @@ func TestTheVaultWholeRotatesANodesLoginWithAStandInEngine(t *testing.T) {
}
}
}
p.put(t, "mesh-controller.lease", LeaseKey, map[string]any{"handed-to": "the operator's hand-over"})
oldPassword := "the-old-password-of-g14"
oldHash, _ := bcrypt.GenerateFromPassword([]byte(oldPassword), bcrypt.MinCost)
p.put(t, "mesh-vault.bus-accounts", "node.g14", AccountHash{User: "node.g14", Hash: string(oldHash), Adopted: true})
modules/mesh-vault/cmd/mesh-vault/tools.go
+1 -2
View File
@@ -307,7 +307,7 @@ func definitions(env *Env) []toolDef {
{Name: Seat + ".issue-account",
Description: "Mint one bus account the grants part wants, now (novox/hq ADR 0561 §9): the recovery when the vault did not " +
"hear a want, and what the controller's `module issue` forwards to (issue 774). Reads the wanted key afresh from " +
"mesh-grants.accounts and mints it as a want would, whoever holds the controller's lease: never for a node's " +
"mesh-grants.accounts and mints it as a want would: never for a node's " +
"account or the controller's, never a user nobody wants, and never anew for one whose value the vault holds or whose hash it adopted (it says so). Answers " +
"whether it minted, the secret and generation, never the value or the hash.",
Input: map[string]any{"user": str("the bus user, e.g. ace.mesh-build-go")},
@@ -645,7 +645,6 @@ func definitions(env *Env) []toolDef {
Description: "What needs a person's eye in the vault: operator copies still sealed to an earlier key, the operator " +
"key the mesh's export holds if the vault refused it, how often the record of hand-outs was rolled, and " +
"minting (novox/hq ADR 0561 §9, issue 774): the vault mints every wanted bus account it holds no value for, " +
"whoever holds the controller's lease; whether that lease is handed away (it turns retirement and rotation), " +
"every wanted account it holds back with why (never one whose value it holds), and those it minted.",
run: func(string, map[string]any) (any, error) {
if env.Vault == nil {
modules/mesh-vault/cmd/mesh-vault/vault.go
+1 -3
View File
@@ -69,9 +69,7 @@ type Vault struct {
WantRead func(user string) ([]byte, bool, error)
// Placement reads the seat's placement (placed.go): where modules stand once the assignments are gone.
Placement PlacementReader
// LeaseRead reads the controller's lease afresh before a mint (lease.go): nil reads it only as the watch hands it.
LeaseRead func() ([]byte, bool, error)
// minting is the switch on the lease and what the vault knows of the wanted bus accounts (mint.go).
// minting is what the vault knows of the wanted bus accounts (mint.go).
minting minting
// Signers is the vault's state `signers`, where it publishes its signing halves (signing.go, ADR 0550 §2): nil writes
// none.
modules/mesh-vault/cmd/mesh-vault/vault_test.go
+4 -7
View File
@@ -135,7 +135,7 @@ func filesUnder(t *testing.T, dir string) map[string]string {
// (ADR 0547 clause 9), opened in accounts.go alone. It reads five buckets of others', and only reads them: the
// seat mesh-provisioner's plans, for what each node's plan names (ADR 0570 clause 2), (askers.go, design 57 V4), the node-engines' haves, where
// the bus node hands the loaded list's hashes sealed to the vault (adopt.go, ADR 0547 clause 9), the grants part's wanted
// accounts and the controller's lease (mint.go, lease.go, ADR 0561 §2, §3), and the seat's placement (placed.go), in
// accounts (mint.go, ADR 0561 §2), and the seat's placement (placed.go), in
// main.go alone.
func TestNoBucketHoldsASecret(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
@@ -151,9 +151,9 @@ func TestNoBucketHoldsASecret(t *testing.T) {
"buckets are the accounts' hashes (ADR 0547 clause 9), the signing keys' public halves (ADR 0550 §2) and the "+
"credentials' outcomes, names and generations alone (ADR 0570 clauses 3, 6) and each node's login in use (ADR 0583)", s)
}
if r, _ := json.Marshal(manifest["reads"]); string(r) != `["node-engine.have","mesh-grants.accounts","mesh-controller.lease","mesh-delivery-assigner.placement","mesh-provisioner.plans","mesh-provisioner.provisions"]` {
if r, _ := json.Marshal(manifest["reads"]); string(r) != `["node-engine.have","mesh-grants.accounts","mesh-delivery-assigner.placement","mesh-provisioner.plans","mesh-provisioner.provisions"]` {
t.Errorf("the manifest reads %s: the vault reads the seat mesh-provisioner's plans and provisions, the haves, the "+
"grants part's wanted accounts, the lease and the placement alone", r)
"grants part's wanted accounts and the placement alone, never the controller's lease (issue 1126)", r)
}
sources, _ := filepath.Glob("*.go")
bucket := regexp.MustCompile(`stdio\.State\(|nats-io|jetstream|KeyValue`)
@@ -195,9 +195,6 @@ func TestNoBucketHoldsASecret(t *testing.T) {
code = []byte(strings.Replace(string(code), "stdio.State(HaveRead).Keys(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(WantedRead).Watch(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(WantedRead).Get(", "", 1))
// And the controller's lease, one read and one watch (ADR 0561 §3).
code = []byte(strings.Replace(string(code), "stdio.State(LeaseRead).Get(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(LeaseRead).Watch(", "", 1))
// And the seat's placement, where modules stand (placed.go): its keys and one key, read.
code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Keys(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Get(", "", 1))
@@ -922,7 +919,7 @@ func TestTheRecordOfHandOutsIsBounded(t *testing.T) {
// broker-nats-tools, wanted as a process account, has the random value the controller moved in replaced by a minted
// account, and a value that is no account is refused for it from then on.
func TestABusAccountIsKeptOnlyAsAnAccount(t *testing.T) {
v, accounts, _ := mintingVault(t, true)
v, accounts, _ := mintingVault(t)
const tools, address = "own/novox/nats/broker-nats-tools", "own/novox/mesh-controller/broker-address"
for _, row := range []struct {
name, value string
modules/mesh-vault/module.json
-1
View File
@@ -32,7 +32,6 @@
"reads": [
"node-engine.have",
"mesh-grants.accounts",
"mesh-controller.lease",
"mesh-delivery-assigner.placement",
"mesh-provisioner.plans",
"mesh-provisioner.provisions"