Design 47's last milestones: the keeper, self-check, healers, hand-acts, provisioner, vault, artifact sweep, disk readings, credential forms (issues 774, 1952, 1953, 1954, 1955)
mesh/delivery-group group feat/774-l3-reland rejected: composed together: the checker's verdict at the members' heads (b48c90fea31f): integration: mesh-self-ch…
mesh/delivery closed: a newer head of the same pull request
mesh/check failed: integration: mesh-self-check/cmd/mesh-self-check.TestTheSelfCheckWholeRunsOnTheBus: the run answered {Run:doctor-1793351737-1 At:20…

One commit with exactly the tree of the reviewed head 35b4509c0a, on main 12c2574c9d, so two compiled programs added and removed on the branch leave no trace in history (issue 1577).
This commit is contained in:
2026-10-30 02:21:51 +01:00
parent 12c2574c9d
commit 35ff67bda0
331 changed files with 24352 additions and 1792 deletions
+1 -1
View File
@@ -133,7 +133,7 @@ func pruneGrants(grants map[string]Grant, kind string, now time.Time, keep int)
// stands, closes the ask with a warrant for its asker (asks.go); Details are shown where they were asked.
// Anything else is refused, and said there. The same answer is acted on once.
func (h *Holder) Chosen(kind string, c asks.Choice) (string, error) {
// The controller's root-free word is asked before the work is taken (the confirmation review of 2026-10-09):
// The machines' engines' root-free word is asked before the work is taken (the confirmation review of 2026-10-09):
// up to RootFreeWithin, which no ask, answer or link should wait behind. Asked fresh here, for this answer,
// and judged with in it.
root := h.rootWordFor(kind)
@@ -600,6 +600,16 @@ func TestAnAskIsRefusedInWordsToItsAsker(t *testing.T) {
if _, kept := store.asks["mesh-delivery.d-1"]; kept {
t.Error("a refused ask was recorded")
}
// A node's engine is an asker under node-engine-<node> (novox/hq ADR 0563 §3, changing ADR 0259 §2): its ask is
// recorded under that name, with its own bound of open asks; a name that is no asker's is dropped.
submit(t, h, "node-engine-ace", heldAsk(c, "hand-over-1"))
submit(t, h, "node.ace", heldAsk(c, "hand-over-2"))
if rec, kept := store.asks["node-engine-ace.hand-over-1"]; !kept || rec.Asker != "node-engine-ace" {
t.Errorf("a node's engine's ask was not recorded under its name: %+v", rec)
}
if _, kept := store.asks["node.ace.hand-over-2"]; kept {
t.Error("an ask under a name that is no asker's was recorded")
}
// No channel can carry anything: held with the reason (ADR 0397), never queued in silence.
h2, _, _, c2, store2, w2 := newRouter(t)
h2.Bench.Stand("telegram", Standing{Ready: false, Why: "no bot token", At: c2.now()})
+51 -37
View File
@@ -242,8 +242,9 @@ func (h *Holder) carriers(level asks.Level) []string {
return out
}
// RootVerdict is the controller's word on one machine (its verb root-free, novox/hq ADR 0259 §8): free only
// when it measured, recently, that no agent can become root there without a person.
// RootVerdict is a machine's engine's word on that machine (the node seat node-engine's root-free, novox/hq ADR 0259
// §8 as ADR 0563 §6 changes it): free only when it measured, recently, that no agent can become root there without a
// person.
type RootVerdict struct {
Machine string `json:"machine"`
Free bool `json:"free"`
@@ -251,51 +252,64 @@ type RootVerdict struct {
Judged string `json:"judged,omitempty"`
}
// RootFreeVerb is the controller's verb the router asks, as this module, which machines are root-free.
const RootFreeVerb = "seat:" + ControllerSeat + ".root-free"
// EngineSeat is the node seat each machine's engine holds, and RootFreeVerb its verb the router asks of a machine, as
// this module: `seat:node-engine.root-free@<machine>`, which that machine's engine alone answers (ADR 0563 §6).
const (
EngineSeat = "node-engine"
RootFreeVerb = "seat:" + EngineSeat + ".root-free"
)
// RootFreeWithin is how long the router waits for the controller's word; RootFreeKept how long a word is
// used to render a message's buttons. An answer is always judged on a word asked then.
// RootFreeWithin is how long the router waits for the machines' words; RootFreeKept how long a word is used to render a
// message's buttons. An answer is always judged on a word asked then.
const (
RootFreeWithin = 10 * time.Second
RootFreeKept = 30 * time.Second
)
// askRootFree asks the controller about machines, failing on no answer in time.
// askRootFree asks each machine's own engine about that machine, at once, failing on no answer in time. A machine's
// word is the verdict its engine gives on it; a verdict an engine gives on another machine is not taken.
func (h *Holder) askRootFree(machines []string) (map[string]RootVerdict, error) {
if h.RootFree != nil {
return h.RootFree(machines)
}
if h.Ask == nil {
return nil, errors.New("this router cannot ask the controller")
return nil, errors.New("this router cannot ask a machine's engine")
}
type answer struct {
raw json.RawMessage
err error
machine string
raw json.RawMessage
err error
}
done := make(chan answer, 1)
go func() {
raw, err := h.Ask(RootFreeVerb, map[string]any{"machines": machines})
done <- answer{raw, err}
}()
var a answer
select {
case a = <-done:
case <-time.After(RootFreeWithin):
return nil, fmt.Errorf("the controller did not answer within %s", RootFreeWithin)
}
if a.err != nil {
return nil, a.err
}
var body struct {
Machines []RootVerdict `json:"machines"`
}
if err := json.Unmarshal(a.raw, &body); err != nil {
return nil, fmt.Errorf("the controller's answer could not be read: %v", err)
done := make(chan answer, len(machines))
for _, m := range machines {
go func(m string) {
raw, err := h.Ask(RootFreeVerb+"@"+m, map[string]any{})
done <- answer{m, raw, err}
}(m)
}
out := map[string]RootVerdict{}
for _, v := range body.Machines {
out[v.Machine] = v
deadline := time.After(RootFreeWithin)
for range machines {
var a answer
select {
case a = <-done:
case <-deadline:
return nil, fmt.Errorf("a machine's engine did not answer within %s", RootFreeWithin)
}
if a.err != nil {
return nil, fmt.Errorf("%s's engine: %w", a.machine, a.err)
}
var body struct {
Machines []RootVerdict `json:"machines"`
}
if err := json.Unmarshal(a.raw, &body); err != nil {
return nil, fmt.Errorf("%s's engine's answer could not be read: %v", a.machine, err)
}
for _, v := range body.Machines {
if v.Machine == a.machine {
out[v.Machine] = v
}
}
}
return out, nil
}
@@ -329,7 +343,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string {
case cached:
// Read inside the work, which never waits on the controller: no recent word is no pass. warmRootFree
// asks outside it.
return "the controller's word on whether an agent can become root there is not recent"
return "the machines' engines' word on whether an agent can become root there is not recent"
default:
got, err := h.askRootFree(machines)
if err != nil {
@@ -351,7 +365,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string {
v, said := verdicts[m]
switch {
case !said:
return "the controller said nothing of whether an agent can become root on " + m
return "its engine said nothing of whether an agent can become root on " + m
case !v.Free:
why := v.Why
if why == "" {
@@ -364,7 +378,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string {
return ""
}
// warmRootFree asks the controller's root-free word for every kind that proves its sender, outside the work,
// warmRootFree asks the machines' engines' root-free word for every kind that proves its sender, outside the work,
// so what is rendered and offered inside it reads a recent word.
// force asks even a recent word again: taking an ask, whose refusal its asker keeps until something changes.
func (h *Holder) warmRootFree(force bool) {
@@ -390,7 +404,7 @@ func (h *Holder) warmRootFree(force bool) {
}
}
// rootWordFor asks the controller's root-free word for a kind fresh, before the work is taken, and answers
// rootWordFor asks the machines' engines' root-free word for a kind fresh, before the work is taken, and answers
// what judging an answer of that kind then reads: the word for that very kind's placement, or why there is none.
// A kind that does not prove its sender is asked nothing: its answers never approve.
func (h *Holder) rootWordFor(kind string) func(KindInfo) string {
@@ -453,7 +467,7 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error {
if verb == ConditionVerb {
// Before the work lock: Condition takes it, and decides as for the controller's conditions.
if !asks.UsableID(strings.ReplaceAll(asker, "-", "_")) {
h.Logf("[messenger] a condition from %q was taken and dropped: not a module's name", asker)
h.Logf("[messenger] a condition from %q was taken and dropped: not an asker's name (a module's, or a node's engine's)", asker)
return nil
}
h.takeCondition(asker, body)
@@ -466,7 +480,7 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error {
h.init()
h.mu.Unlock()
if !asks.UsableID(strings.ReplaceAll(asker, "-", "_")) {
h.Logf("[messenger] an ask from %q was taken and dropped: not a module's name", asker)
h.Logf("[messenger] an ask from %q was taken and dropped: not an asker's name (a module's, or a node's engine's)", asker)
return nil
}
if verb == "cancel" {
+2 -10
View File
@@ -256,16 +256,8 @@ func (r *busRuntime) runtime(id json.RawMessage, method string, params json.RawM
case "mesh/seat.publish":
return map[string]any{}, r.nc.Publish(p.Subject, p.Body)
case "mesh/ask":
if p.Key == RootFreeVerb {
var a struct {
Machines []string `json:"machines"`
}
_ = json.Unmarshal(p.Body, &a)
var out []RootVerdict
for _, m := range a.Machines {
out = append(out, RootVerdict{Machine: m, Free: true})
}
return map[string]any{"machines": out}, nil
if m, ok := strings.CutPrefix(p.Key, RootFreeVerb+"@"); ok {
return map[string]any{"machines": []RootVerdict{{Machine: m, Free: true}}}, nil
}
return nil, fmt.Errorf("the test runtime answers nothing for %s", p.Key)
}
+5 -2
View File
@@ -122,9 +122,12 @@ func (c Condition) SilencedAt(now time.Time) bool {
// eventOf is the local event name of an envelope's key: `mesh-controller.condition-raised` is
// `condition-raised`. Anything not of the controller's seat is not a condition event.
func eventOf(key string) (string, bool) {
func eventOf(key string) (string, bool) { return eventOfSeat(key, ControllerSeat) }
// eventOfSeat is eventOf for the seat the take-over key says is followed: the controller's, or the keeper's (ADR 0564 clause 9).
func eventOfSeat(key, seat string) (string, bool) {
emitter, event, ok := strings.Cut(key, ".")
if !ok || emitter != ControllerSeat {
if !ok || emitter != seat {
return "", false
}
switch event {
+2 -1
View File
@@ -315,7 +315,8 @@ type Holder struct {
Emit func(event string, body any) error
// Ask calls a seat verb through the mesh, as this module: the answers the operator chooses.
Ask Asker
// RootFree asks the controller's verb root-free about machines; nil asks it through Ask (askRootFree).
// RootFree asks each machine's engine's root-free (the node seat node-engine's, ADR 0563 §6); nil asks it through Ask
// (askRootFree).
RootFree func(machines []string) (map[string]RootVerdict, error)
// Zone is the operator's time zone, which every time in a message is said in; nil is this machine's.
Zone func() *time.Location
+13 -5
View File
@@ -18,6 +18,7 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/health"
)
func errorf(format string, a ...any) error { return fmt.Errorf(format, a...) }
@@ -415,8 +416,9 @@ func run(h *Holder, l *listening) {
// The state first: what is open now is read from the controller before any event is taken, so
// what the stream replays is history (novox/hq issue 271). Not reachable yet, it is asked for
// again in the loop below, and old events meanwhile are kept as state, never said.
follow := newTakeover()
h.Start()
syncFromController(h)
syncFromController(h, follow)
go func() {
tick := time.NewTicker(5 * time.Second)
defer tick.Stop()
@@ -429,10 +431,11 @@ func run(h *Holder, l *listening) {
}
if h.WantsSync() && now.Sub(lastAsk) >= time.Minute {
lastAsk = now
syncFromController(h)
syncFromController(h, follow)
}
}
}()
go followTakeover(follow, h)
handle := func(e stdio.Envelope) error {
if event, ok := strings.CutPrefix(e.Key, ViewModule+"."); ok {
// A module in motion on a node, settled or stuck: a walk's start or end now that plans are gone (ADR 0308 §1, ADR 0539 §17).
@@ -446,7 +449,9 @@ func run(h *Holder, l *listening) {
}
return nil
}
event, ok := eventOf(e.Key)
// Only the seat the take-over key names is heard: the other's events are dropped, so the keeper's conditions
// are never held a second time beside the controller's (ADR 0564 clause 9).
event, ok := eventOfSeat(e.Key, follow.seat())
if !ok {
return nil
}
@@ -476,6 +481,9 @@ func run(h *Holder, l *listening) {
}()
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
err := stdio.Subscribe(ControllerSeat+".*", handle)
if err == nil {
err = stdio.Subscribe(health.KeeperModule+".*", handle)
}
if err == nil {
l.set("listening")
logf("[messenger] listening for the controller's condition events")
@@ -625,7 +633,7 @@ func followKinds(h *Holder) {
}
// syncFromController reads the controller's open conditions and gives them to the holder.
func syncFromController(h *Holder) {
func syncFromController(h *Holder, follow *takeover) {
asked := h.Now()
type answer struct {
raw json.RawMessage
@@ -633,7 +641,7 @@ func syncFromController(h *Holder) {
}
done := make(chan answer, 1)
go func() {
raw, err := stdio.Ask("seat:"+ControllerSeat+".conditions", map[string]any{})
raw, err := stdio.Ask("seat:"+follow.seat()+".conditions", map[string]any{})
done <- answer{raw, err}
}()
var a answer
@@ -81,13 +81,14 @@ func TestItDeclaresTheSeatsOfAnAskAndHoldsTheOperatorChannel(t *testing.T) { //
}
if !reflect.DeepEqual(m.Consumes, []string{
ControllerSeat + "." + EventRaised, ControllerSeat + "." + EventChanged, ControllerSeat + "." + EventCleared,
"mesh-conditions." + EventRaised, "mesh-conditions." + EventChanged, "mesh-conditions." + EventCleared,
ViewModule + "." + EventMoving, ViewModule + "." + EventSettled, ViewModule + "." + EventStuck, "intake.choice", "intake.link", "intake.failed", "intake.standing"}) {
t.Fatalf("consumes %v", m.Consumes)
}
// It performs nothing: the verbs it calls read the open conditions, whether root is free, and which issues
// track which conditions.
if !reflect.DeepEqual(m.Emits, []string{"refused"}) || !reflect.DeepEqual(m.Invokes, []string{"seat:" + ControllerSeat + ".conditions",
"seat:" + ControllerSeat + ".root-free", TrackingVerb}) {
"seat:mesh-conditions.conditions", "seat:" + EngineSeat + ".root-free", TrackingVerb}) {
t.Fatalf("emits %v invokes %v", m.Emits, m.Invokes)
}
var state []string
+19 -7
View File
@@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"sync"
"testing"
@@ -66,26 +67,37 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) {
}
}
// H3: the word is asked of the controller's verb, as this module, and read from its answer.
func TestTheRootWordIsAskedOfTheControllersVerb(t *testing.T) {
// H3, as ADR 0563 §6 changes it: the word on each machine is asked of that machine's own engine, the node seat
// node-engine's root-free there, as this module, and read from its answer; a machine's word is that machine's alone.
func TestTheRootWordIsAskedOfEachMachinesEngine(t *testing.T) {
h, _, _, _, _, _ := newRouter(t)
h.RootFree = nil
var asked []string
h.Ask = func(key string, body any) (json.RawMessage, error) {
asked = append(asked, key)
raw, _ := json.Marshal(body)
if !strings.Contains(string(raw), `"machines":["anchor"]`) {
t.Errorf("asked about %s", raw)
machine := strings.TrimPrefix(key, "seat:node-engine.root-free@")
if machine == "elsewhere" {
machine = "anchor" // an engine that answers for another machine is not believed
}
return json.RawMessage(`{"machines":[{"machine":"anchor","free":true,"why":"judged","judged":"2026-10-09T10:00:00Z"}]}`), nil
return json.RawMessage(`{"machines":[{"machine":"` + machine + `","free":true,"why":"judged","judged":"2026-10-09T10:00:00Z"}]}`), nil
}
info, _ := h.Bench.Kind("telegram")
if why := h.rootFree(info, false); why != "" {
t.Fatalf("free not read: %s", why)
}
if len(asked) != 1 || asked[0] != "seat:mesh-controller.root-free" {
if len(asked) != 1 || asked[0] != "seat:node-engine.root-free@anchor" {
t.Errorf("asked %v", asked)
}
asked = nil
h.Node = "elsewhere"
if why := h.rootFree(info, false); !strings.Contains(why, "said nothing of whether an agent can become root on elsewhere") {
t.Errorf("one machine's word was taken for another's: %q", why)
}
sort.Strings(asked)
if len(asked) != 2 || asked[0] != "seat:node-engine.root-free@anchor" || asked[1] != "seat:node-engine.root-free@elsewhere" {
t.Errorf("the router's machine and the channel's were asked as %v", asked)
}
h.Node = "anchor"
h.Ask = func(string, any) (json.RawMessage, error) { return json.RawMessage(`not json`), nil }
if why := h.rootFree(info, false); !strings.Contains(why, "could not be read") {
t.Errorf("an unreadable answer was a pass: %q", why)
+8
View File
@@ -180,3 +180,11 @@ func (h *Holder) WantsSync() bool {
h.init()
return h.wantSync || h.syncedAt.IsZero() || h.Now().Sub(h.syncedAt) >= ResyncEvery
}
// AskSync has the next pass of the loop read the open conditions again (the take-over key named another seat).
func (h *Holder) AskSync() {
h.mu.Lock()
h.init()
h.wantSync = true
h.mu.Unlock()
}
@@ -0,0 +1,59 @@
package main
import (
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
)
// takeover is which seat's condition events the messenger hears: the controller's until the keeper's take-over key
// says otherwise (novox/hq ADR 0564 clause 9, issue 1952 R1). An absent key follows the controller.
type takeover struct {
mu sync.Mutex
t health.Takeover
}
func newTakeover() *takeover {
return &takeover{t: health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController}}
}
func (f *takeover) seat() string { f.mu.Lock(); defer f.mu.Unlock(); return f.t.Seat() }
// set takes the key as a watch handed it and says whether the followed seat changed.
func (f *takeover) set(value []byte, found bool) (changed bool, err error) {
t, err := health.DecodeTakeover(value, found)
if err != nil {
return false, err
}
f.mu.Lock()
defer f.mu.Unlock()
changed = f.t.Seat() != t.Seat()
f.t = t
return changed, nil
}
// followTakeover watches the take-over key for the life of the process, and asks the holder to read the followed
// seat's open conditions again when it changes. A key it cannot read leaves the seat it followed.
func followTakeover(f *takeover, h *Holder) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
err := stdio.State(health.KeeperModule+"."+health.TakeoverState).Watch(health.TakeoverKey, func(c stdio.StateChange) error {
changed, err := f.set(c.Value, c.Op != "delete" && len(c.Value) > 0)
if err != nil {
logf("[messenger] the take-over key is unreadable (%v); still hearing %s", err, f.seat())
return nil
}
if changed {
logf("[messenger] the take-over key now names %s", f.seat())
h.AskSync()
}
return nil
})
if err == nil {
return
}
logf("[messenger] not watching the take-over key yet (%v); hearing %s", err, f.seat())
time.Sleep(wait)
}
}