Ask the root-free word of the controller's verb, which answers, and say every failed ask, so Telegram carries asks that offer an approval again (issue 774)

The messenger asked seat:node-engine.root-free@<machine>, which nothing on the mesh answers (the engine's acts are
left out of its declared state, mesh-host #178; ADR 0571 §11's self-check probe is not held either), and deleted
its kept word on each failure without a log line: every ask from an agent or a merge was held, silently.

Quick fix: point RootFreeVerb back at mesh-controller.root-free. Thorough fix: the same, plus every failed ask is
logged with its error, the failure is kept beside the word and becomes the held ask's reason, a fresh word is no
longer thrown away by one failed ask (an answer is still judged only on a word asked then), and a held ask whose
reason changes is logged. Chose thorough: the silence is what hid the fault for a day.
This commit is contained in:
2026-10-30 18:01:54 +01:00
parent 6d67c22bf3
commit bc27189627
7 changed files with 138 additions and 71 deletions
+1 -1
View File
@@ -133,7 +133,7 @@ func pruneGrants(grants map[string]Grant, kind string, now time.Time, keep int)
// stands, closes the ask with a warrant for its asker (asks.go); Details are shown where they were asked.
// Anything else is refused, and said there. The same answer is acted on once.
func (h *Holder) Chosen(kind string, c asks.Choice) (string, error) {
// The machines' engines' root-free word is asked before the work is taken (the confirmation review of 2026-10-09):
// The root-free word is asked before the work is taken (the confirmation review of 2026-10-09):
// up to RootFreeWithin, which no ask, answer or link should wait behind. Asked fresh here, for this answer,
// and judged with in it.
root := h.rootWordFor(kind)
+73 -51
View File
@@ -242,9 +242,8 @@ func (h *Holder) carriers(level asks.Level) []string {
return out
}
// RootVerdict is a machine's engine's word on that machine (the node seat node-engine's root-free, novox/hq ADR 0259
// §8 as ADR 0563 §6 changes it): free only when it measured, recently, that no agent can become root there without a
// person.
// RootVerdict is the word on one machine (novox/hq ADR 0259 §8): free only when it was measured, recently, that no
// agent can become root there without a person.
type RootVerdict struct {
Machine string `json:"machine"`
Free bool `json:"free"`
@@ -252,63 +251,61 @@ type RootVerdict struct {
Judged string `json:"judged,omitempty"`
}
// EngineSeat is the node seat each machine's engine holds, and RootFreeVerb its verb the router asks of a machine, as
// this module: `seat:node-engine.root-free@<machine>`, which that machine's engine alone answers (ADR 0563 §6).
const (
EngineSeat = "node-engine"
RootFreeVerb = "seat:" + EngineSeat + ".root-free"
)
// RootFreeVerb is the verb the router asks, as this module, which machines are root-free: the controller's, which
// judges each machine from its node-engine's own verdict (ADR 0259 §8). ADR 0563 §6 moves the word to the node seat
// node-engine and ADR 0571 §11 to the self-check's probe; neither answers on the mesh yet (the engine's acts are left
// out of its declared state, mesh-host #178, and no seat mesh-self-check is held), so asking them held every ask
// that offers an approval (issue 774, the sweep of 2026-10-30). The controller's verb is the one that answers.
const RootFreeVerb = "seat:" + ControllerSeat + ".root-free"
// RootFreeWithin is how long the router waits for the machines' words; RootFreeKept how long a word is used to render a
// RootFreeWithin is how long the router waits for the word; RootFreeKept how long a word is used to render a
// message's buttons. An answer is always judged on a word asked then.
const (
RootFreeWithin = 10 * time.Second
RootFreeKept = 30 * time.Second
)
// askRootFree asks each machine's own engine about that machine, at once, failing on no answer in time. A machine's
// word is the verdict its engine gives on it; a verdict an engine gives on another machine is not taken.
// askRootFree asks the word on machines at once, failing on no answer in time. Only a verdict on a machine asked is
// taken.
func (h *Holder) askRootFree(machines []string) (map[string]RootVerdict, error) {
if h.RootFree != nil {
return h.RootFree(machines)
}
if h.Ask == nil {
return nil, errors.New("this router cannot ask a machine's engine")
return nil, errors.New("this router cannot ask the controller")
}
type answer struct {
machine string
raw json.RawMessage
err error
raw json.RawMessage
err error
}
done := make(chan answer, len(machines))
done := make(chan answer, 1)
go func() {
raw, err := h.Ask(RootFreeVerb, map[string]any{"machines": machines})
done <- answer{raw, err}
}()
var a answer
select {
case a = <-done:
case <-time.After(RootFreeWithin):
return nil, fmt.Errorf("%s did not answer within %s", RootFreeVerb, RootFreeWithin)
}
if a.err != nil {
return nil, fmt.Errorf("%s: %w", RootFreeVerb, a.err)
}
var body struct {
Machines []RootVerdict `json:"machines"`
}
if err := json.Unmarshal(a.raw, &body); err != nil {
return nil, fmt.Errorf("%s's answer could not be read: %v", RootFreeVerb, err)
}
asked := map[string]bool{}
for _, m := range machines {
go func(m string) {
raw, err := h.Ask(RootFreeVerb+"@"+m, map[string]any{})
done <- answer{m, raw, err}
}(m)
asked[m] = true
}
out := map[string]RootVerdict{}
deadline := time.After(RootFreeWithin)
for range machines {
var a answer
select {
case a = <-done:
case <-deadline:
return nil, fmt.Errorf("a machine's engine did not answer within %s", RootFreeWithin)
}
if a.err != nil {
return nil, fmt.Errorf("%s's engine: %w", a.machine, a.err)
}
var body struct {
Machines []RootVerdict `json:"machines"`
}
if err := json.Unmarshal(a.raw, &body); err != nil {
return nil, fmt.Errorf("%s's engine's answer could not be read: %v", a.machine, err)
}
for _, v := range body.Machines {
if v.Machine == a.machine {
out[v.Machine] = v
}
for _, v := range body.Machines {
if asked[v.Machine] {
out[v.Machine] = v
}
}
return out, nil
@@ -337,20 +334,36 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string {
kept, ok := h.rootWord[key]
h.mu.Unlock()
var verdicts map[string]RootVerdict
fresh := ok && kept.verdicts != nil && now.Sub(kept.at) < RootFreeKept && now.Sub(kept.at) >= 0
switch {
case cached && ok && now.Sub(kept.at) < RootFreeKept && now.Sub(kept.at) >= 0:
case cached && fresh:
verdicts = kept.verdicts
case cached:
// Read inside the work, which never waits on the controller: no recent word is no pass. warmRootFree
// asks outside it.
return "the machines' engines' word on whether an agent can become root there is not recent"
// asks outside it; when its last ask failed, that failure is the reason, so a held ask says it.
if ok && kept.failed != "" {
return "the word on whether an agent can become root there is not recent: asking it at " +
kept.failedAt.UTC().Format("15:04:05") + " UTC failed: " + kept.failed
}
return "the word on whether an agent can become root there is not recent"
default:
got, err := h.askRootFree(machines)
if err != nil {
// A word that could not be asked again is not kept from before: no recent word, no pass.
// The failure is said and kept beside the last word. That word is not thrown away: it still renders
// until it is RootFreeKept old, and an answer is never judged on it (this path is not a pass).
h.mu.Lock()
delete(h.rootWord, key)
if h.rootWord == nil {
h.rootWord = map[string]rootWord{}
}
w := h.rootWord[key]
w.failed, w.failedAt = err.Error(), now
h.rootWord[key] = w
h.mu.Unlock()
age := "no word was kept"
if fresh {
age = "the word of " + now.Sub(kept.at).Round(time.Second).String() + " ago is kept for rendering"
}
h.Logf("[messenger] could not ask whether an agent can become root on %s: %v (%s)", key, err, age)
return "whether an agent can become root where it runs could not be asked: " + err.Error()
}
verdicts = got
@@ -358,6 +371,10 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string {
if h.rootWord == nil {
h.rootWord = map[string]rootWord{}
}
if w := h.rootWord[key]; w.failed != "" {
h.Logf("[messenger] the word on whether an agent can become root on %s was asked again after it failed at %s",
key, w.failedAt.UTC().Format(time.RFC3339))
}
h.rootWord[key] = rootWord{at: now, verdicts: got}
h.mu.Unlock()
}
@@ -365,7 +382,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string {
v, said := verdicts[m]
switch {
case !said:
return "its engine said nothing of whether an agent can become root on " + m
return "the controller said nothing of whether an agent can become root on " + m
case !v.Free:
why := v.Why
if why == "" {
@@ -378,7 +395,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string {
return ""
}
// warmRootFree asks the machines' engines' root-free word for every kind that proves its sender, outside the work,
// warmRootFree asks the root-free word for every kind that proves its sender, outside the work,
// so what is rendered and offered inside it reads a recent word.
// force asks even a recent word again: taking an ask, whose refusal its asker keeps until something changes.
func (h *Holder) warmRootFree(force bool) {
@@ -404,7 +421,7 @@ func (h *Holder) warmRootFree(force bool) {
}
}
// rootWordFor asks the machines' engines' root-free word for a kind fresh, before the work is taken, and answers
// rootWordFor asks the root-free word for a kind fresh, before the work is taken, and answers
// what judging an answer of that kind then reads: the word for that very kind's placement, or why there is none.
// A kind that does not prove its sender is asked nothing: its answers never approve.
func (h *Holder) rootWordFor(kind string) func(KindInfo) string {
@@ -424,10 +441,13 @@ func (h *Holder) rootWordFor(kind string) func(KindInfo) string {
}
}
// rootWord is the controller's last word on a set of machines, and when it was asked.
// rootWord is the controller's last word on a set of machines, and when it was asked; failed and failedAt the last
// ask that failed since, and why.
type rootWord struct {
at time.Time
verdicts map[string]RootVerdict
failed string
failedAt time.Time
}
// approveHolds is why an answer on a kind could not approve now, or "": the kind verifies its sender, an
@@ -704,6 +724,8 @@ func (h *Holder) releaseHeld() {
rec.Tickets = nil
if why := h.holdWhy(rec.Ask); why != rec.HeldWhy {
rec.HeldWhy = why
h.Logf("[messenger] %s's ask %q (%s) is still held, since %s: %s", rec.Asker, rec.Ask.Headline, rec.Ask.ID,
rec.HeldSince.UTC().Format(time.RFC3339), why)
h.saveHeld(key, rec, rev)
}
continue
+8 -2
View File
@@ -256,8 +256,14 @@ func (r *busRuntime) runtime(id json.RawMessage, method string, params json.RawM
case "mesh/seat.publish":
return map[string]any{}, r.nc.Publish(p.Subject, p.Body)
case "mesh/ask":
if m, ok := strings.CutPrefix(p.Key, RootFreeVerb+"@"); ok {
return map[string]any{"machines": []RootVerdict{{Machine: m, Free: true}}}, nil
if p.Key == RootFreeVerb {
var args struct{ Machines []string }
_ = json.Unmarshal(p.Body, &args)
var out []RootVerdict
for _, m := range args.Machines {
out = append(out, RootVerdict{Machine: m, Free: true})
}
return map[string]any{"machines": out}, nil
}
return nil, fmt.Errorf("the test runtime answers nothing for %s", p.Key)
}
+1 -1
View File
@@ -315,7 +315,7 @@ type Holder struct {
Emit func(event string, body any) error
// Ask calls a seat verb through the mesh, as this module: the answers the operator chooses.
Ask Asker
// RootFree asks each machine's engine's root-free (the node seat node-engine's, ADR 0563 §6); nil asks it through Ask
// RootFree asks the root-free word on machines (the controller's verb, RootFreeVerb); nil asks it through Ask
// (askRootFree).
RootFree func(machines []string) (map[string]RootVerdict, error)
// Zone is the operator's time zone, which every time in a message is said in; nil is this machine's.
@@ -88,7 +88,7 @@ func TestItDeclaresTheSeatsOfAnAskAndHoldsTheOperatorChannel(t *testing.T) { //
// It performs nothing: the verbs it calls read the open conditions, whether root is free, and which issues
// track which conditions.
if !reflect.DeepEqual(m.Emits, []string{"refused"}) || !reflect.DeepEqual(m.Invokes, []string{"seat:" + ControllerSeat + ".conditions",
"seat:mesh-conditions.conditions", "seat:" + EngineSeat + ".root-free", TrackingVerb}) {
"seat:mesh-conditions.conditions", RootFreeVerb, TrackingVerb}) {
t.Fatalf("emits %v invokes %v", m.Emits, m.Invokes)
}
var state []string
+53 -14
View File
@@ -5,10 +5,10 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"sync"
"testing"
@@ -30,6 +30,8 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) {
k.Node = ""
h.Bench.SetKinds([]KindInfo{k})
}, "has not said which machine"},
// The word asked before it, still fresh, renders until it is RootFreeKept old (a failed ask throws no fresh
// word away; the sweep of 2026-10-30); the answer itself is refused all the same.
"a controller that does not answer": {func(h *Holder) {
h.RootFree = func([]string) (map[string]RootVerdict, error) { return nil, errors.New("no responders") }
}, "could not be asked: no responders"},
@@ -60,6 +62,10 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) {
if len(w.of("mesh-delivery")) != 0 {
t.Error("a warrant was issued")
}
if strings.Contains(tc.want, "could not be asked") {
c.pass(RootFreeKept)
stand(h, c)
}
if got := h.carriers(asks.Approve); len(got) != 0 {
t.Errorf("approval still carried on %v", got)
}
@@ -67,34 +73,32 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) {
}
}
// H3, as ADR 0563 §6 changes it: the word on each machine is asked of that machine's own engine, the node seat
// node-engine's root-free there, as this module, and read from its answer; a machine's word is that machine's alone.
func TestTheRootWordIsAskedOfEachMachinesEngine(t *testing.T) {
// H3: the word on the router's machine and the channel's is asked of the verb that answers it on the mesh, the
// controller's root-free, once with both machines, as this module; a verdict on a machine not asked is not taken.
// The sweep of 2026-10-30 (issue 774): the node seat node-engine's root-free answered nowhere, so every ask that
// offers an approval was held, silently.
func TestTheRootWordIsAskedOfTheVerbThatAnswers(t *testing.T) {
h, _, _, _, _, _ := newRouter(t)
h.RootFree = nil
var asked []string
h.Ask = func(key string, body any) (json.RawMessage, error) {
asked = append(asked, key)
machine := strings.TrimPrefix(key, "seat:node-engine.root-free@")
if machine == "elsewhere" {
machine = "anchor" // an engine that answers for another machine is not believed
}
return json.RawMessage(`{"machines":[{"machine":"` + machine + `","free":true,"why":"judged","judged":"2026-10-09T10:00:00Z"}]}`), nil
ms, _ := body.(map[string]any)["machines"].([]string)
asked = append(asked, key+" "+strings.Join(ms, ","))
return json.RawMessage(`{"machines":[{"machine":"anchor","free":true,"why":"judged","judged":"2026-10-09T10:00:00Z"}]}`), nil
}
info, _ := h.Bench.Kind("telegram")
if why := h.rootFree(info, false); why != "" {
t.Fatalf("free not read: %s", why)
}
if len(asked) != 1 || asked[0] != "seat:node-engine.root-free@anchor" {
if len(asked) != 1 || asked[0] != "seat:mesh-controller.root-free anchor" {
t.Errorf("asked %v", asked)
}
asked = nil
h.Node = "elsewhere"
if why := h.rootFree(info, false); !strings.Contains(why, "said nothing of whether an agent can become root on elsewhere") {
t.Errorf("one machine's word was taken for another's: %q", why)
t.Errorf("a machine left out of the answer was a pass: %q", why)
}
sort.Strings(asked)
if len(asked) != 2 || asked[0] != "seat:node-engine.root-free@anchor" || asked[1] != "seat:node-engine.root-free@elsewhere" {
if len(asked) != 1 || asked[0] != "seat:mesh-controller.root-free elsewhere,anchor" {
t.Errorf("the router's machine and the channel's were asked as %v", asked)
}
h.Node = "anchor"
@@ -108,6 +112,41 @@ func TestTheRootWordIsAskedOfEachMachinesEngine(t *testing.T) {
}
}
// A failed ask of the root word is said with its error, keeps the word still fresh for rendering, never passes an
// answer, and is the reason a held ask carries (the sweep of 2026-10-30: the word was deleted silently and the held
// reason said only "not recent").
func TestAFailedRootAskIsSaidAndKeepsAFreshWord(t *testing.T) {
h, _, _, c, store, _ := newRouter(t)
link(h, store, c, "42", c.now().Add(-2*time.Hour))
var logs []string
h.Logf = func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }
info, _ := h.Bench.Kind("telegram")
if why := h.rootFree(info, false); why != "" {
t.Fatalf("free not read: %s", why)
}
h.RootFree = func([]string) (map[string]RootVerdict, error) {
return nil, errors.New("nothing in the mesh is called node-engine")
}
c.pass(RootFreeKept / 3)
if why := h.rootFree(info, false); !strings.Contains(why, "could not be asked: nothing in the mesh is called node-engine") {
t.Errorf("a failed ask judged an answer: %q", why)
}
if len(logs) != 1 || !strings.Contains(logs[0], "nothing in the mesh is called node-engine") || !strings.Contains(logs[0], "kept for rendering") {
t.Errorf("the failure was not said with its error: %v", logs)
}
if why := h.rootFree(info, true); why != "" {
t.Errorf("one failed ask threw away a word still fresh: %q", why)
}
c.pass(RootFreeKept)
stand(h, c)
if why := h.rootFree(info, true); !strings.Contains(why, "failed: nothing in the mesh is called node-engine") {
t.Errorf("a stale word passed, or its reason hid the failure: %q", why)
}
if why := h.holdWhy(heldAsk(c, "d-9")); !strings.Contains(why, "nothing in the mesh is called node-engine") {
t.Errorf("a held ask does not say why: %q", why)
}
}
// H3: a message rendered while root is not free offers no approval; a word kept for rendering is never the
// word an answer is judged on.
func TestAMessageRenderedWhileRootIsNotFreeOffersNoApproval(t *testing.T) {
+1 -1
View File
@@ -94,7 +94,7 @@
"invokes": [
"seat:mesh-controller.conditions",
"seat:mesh-conditions.conditions",
"seat:node-engine.root-free",
"seat:mesh-controller.root-free",
"seat:issue-tracker.tracking"
],
"reads": [