Ask the root-free word of the controller's verb, which answers, and say every failed ask, so Telegram carries asks that offer an approval again (issue 774)
The messenger asked seat:node-engine.root-free@<machine>, which nothing on the mesh answers (the engine's acts are left out of its declared state, mesh-host #178; ADR 0571 §11's self-check probe is not held either), and deleted its kept word on each failure without a log line: every ask from an agent or a merge was held, silently. Quick fix: point RootFreeVerb back at mesh-controller.root-free. Thorough fix: the same, plus every failed ask is logged with its error, the failure is kept beside the word and becomes the held ask's reason, a fresh word is no longer thrown away by one failed ask (an answer is still judged only on a word asked then), and a held ask whose reason changes is logged. Chose thorough: the silence is what hid the fault for a day.
This commit is contained in:
@@ -133,7 +133,7 @@ func pruneGrants(grants map[string]Grant, kind string, now time.Time, keep int)
|
||||
// stands, closes the ask with a warrant for its asker (asks.go); Details are shown where they were asked.
|
||||
// Anything else is refused, and said there. The same answer is acted on once.
|
||||
func (h *Holder) Chosen(kind string, c asks.Choice) (string, error) {
|
||||
// The machines' engines' root-free word is asked before the work is taken (the confirmation review of 2026-10-09):
|
||||
// The root-free word is asked before the work is taken (the confirmation review of 2026-10-09):
|
||||
// up to RootFreeWithin, which no ask, answer or link should wait behind. Asked fresh here, for this answer,
|
||||
// and judged with in it.
|
||||
root := h.rootWordFor(kind)
|
||||
|
||||
@@ -242,9 +242,8 @@ func (h *Holder) carriers(level asks.Level) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// RootVerdict is a machine's engine's word on that machine (the node seat node-engine's root-free, novox/hq ADR 0259
|
||||
// §8 as ADR 0563 §6 changes it): free only when it measured, recently, that no agent can become root there without a
|
||||
// person.
|
||||
// RootVerdict is the word on one machine (novox/hq ADR 0259 §8): free only when it was measured, recently, that no
|
||||
// agent can become root there without a person.
|
||||
type RootVerdict struct {
|
||||
Machine string `json:"machine"`
|
||||
Free bool `json:"free"`
|
||||
@@ -252,63 +251,61 @@ type RootVerdict struct {
|
||||
Judged string `json:"judged,omitempty"`
|
||||
}
|
||||
|
||||
// EngineSeat is the node seat each machine's engine holds, and RootFreeVerb its verb the router asks of a machine, as
|
||||
// this module: `seat:node-engine.root-free@<machine>`, which that machine's engine alone answers (ADR 0563 §6).
|
||||
const (
|
||||
EngineSeat = "node-engine"
|
||||
RootFreeVerb = "seat:" + EngineSeat + ".root-free"
|
||||
)
|
||||
// RootFreeVerb is the verb the router asks, as this module, which machines are root-free: the controller's, which
|
||||
// judges each machine from its node-engine's own verdict (ADR 0259 §8). ADR 0563 §6 moves the word to the node seat
|
||||
// node-engine and ADR 0571 §11 to the self-check's probe; neither answers on the mesh yet (the engine's acts are left
|
||||
// out of its declared state, mesh-host #178, and no seat mesh-self-check is held), so asking them held every ask
|
||||
// that offers an approval (issue 774, the sweep of 2026-10-30). The controller's verb is the one that answers.
|
||||
const RootFreeVerb = "seat:" + ControllerSeat + ".root-free"
|
||||
|
||||
// RootFreeWithin is how long the router waits for the machines' words; RootFreeKept how long a word is used to render a
|
||||
// RootFreeWithin is how long the router waits for the word; RootFreeKept how long a word is used to render a
|
||||
// message's buttons. An answer is always judged on a word asked then.
|
||||
const (
|
||||
RootFreeWithin = 10 * time.Second
|
||||
RootFreeKept = 30 * time.Second
|
||||
)
|
||||
|
||||
// askRootFree asks each machine's own engine about that machine, at once, failing on no answer in time. A machine's
|
||||
// word is the verdict its engine gives on it; a verdict an engine gives on another machine is not taken.
|
||||
// askRootFree asks the word on machines at once, failing on no answer in time. Only a verdict on a machine asked is
|
||||
// taken.
|
||||
func (h *Holder) askRootFree(machines []string) (map[string]RootVerdict, error) {
|
||||
if h.RootFree != nil {
|
||||
return h.RootFree(machines)
|
||||
}
|
||||
if h.Ask == nil {
|
||||
return nil, errors.New("this router cannot ask a machine's engine")
|
||||
return nil, errors.New("this router cannot ask the controller")
|
||||
}
|
||||
type answer struct {
|
||||
machine string
|
||||
raw json.RawMessage
|
||||
err error
|
||||
raw json.RawMessage
|
||||
err error
|
||||
}
|
||||
done := make(chan answer, len(machines))
|
||||
done := make(chan answer, 1)
|
||||
go func() {
|
||||
raw, err := h.Ask(RootFreeVerb, map[string]any{"machines": machines})
|
||||
done <- answer{raw, err}
|
||||
}()
|
||||
var a answer
|
||||
select {
|
||||
case a = <-done:
|
||||
case <-time.After(RootFreeWithin):
|
||||
return nil, fmt.Errorf("%s did not answer within %s", RootFreeVerb, RootFreeWithin)
|
||||
}
|
||||
if a.err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", RootFreeVerb, a.err)
|
||||
}
|
||||
var body struct {
|
||||
Machines []RootVerdict `json:"machines"`
|
||||
}
|
||||
if err := json.Unmarshal(a.raw, &body); err != nil {
|
||||
return nil, fmt.Errorf("%s's answer could not be read: %v", RootFreeVerb, err)
|
||||
}
|
||||
asked := map[string]bool{}
|
||||
for _, m := range machines {
|
||||
go func(m string) {
|
||||
raw, err := h.Ask(RootFreeVerb+"@"+m, map[string]any{})
|
||||
done <- answer{m, raw, err}
|
||||
}(m)
|
||||
asked[m] = true
|
||||
}
|
||||
out := map[string]RootVerdict{}
|
||||
deadline := time.After(RootFreeWithin)
|
||||
for range machines {
|
||||
var a answer
|
||||
select {
|
||||
case a = <-done:
|
||||
case <-deadline:
|
||||
return nil, fmt.Errorf("a machine's engine did not answer within %s", RootFreeWithin)
|
||||
}
|
||||
if a.err != nil {
|
||||
return nil, fmt.Errorf("%s's engine: %w", a.machine, a.err)
|
||||
}
|
||||
var body struct {
|
||||
Machines []RootVerdict `json:"machines"`
|
||||
}
|
||||
if err := json.Unmarshal(a.raw, &body); err != nil {
|
||||
return nil, fmt.Errorf("%s's engine's answer could not be read: %v", a.machine, err)
|
||||
}
|
||||
for _, v := range body.Machines {
|
||||
if v.Machine == a.machine {
|
||||
out[v.Machine] = v
|
||||
}
|
||||
for _, v := range body.Machines {
|
||||
if asked[v.Machine] {
|
||||
out[v.Machine] = v
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
@@ -337,20 +334,36 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string {
|
||||
kept, ok := h.rootWord[key]
|
||||
h.mu.Unlock()
|
||||
var verdicts map[string]RootVerdict
|
||||
fresh := ok && kept.verdicts != nil && now.Sub(kept.at) < RootFreeKept && now.Sub(kept.at) >= 0
|
||||
switch {
|
||||
case cached && ok && now.Sub(kept.at) < RootFreeKept && now.Sub(kept.at) >= 0:
|
||||
case cached && fresh:
|
||||
verdicts = kept.verdicts
|
||||
case cached:
|
||||
// Read inside the work, which never waits on the controller: no recent word is no pass. warmRootFree
|
||||
// asks outside it.
|
||||
return "the machines' engines' word on whether an agent can become root there is not recent"
|
||||
// asks outside it; when its last ask failed, that failure is the reason, so a held ask says it.
|
||||
if ok && kept.failed != "" {
|
||||
return "the word on whether an agent can become root there is not recent: asking it at " +
|
||||
kept.failedAt.UTC().Format("15:04:05") + " UTC failed: " + kept.failed
|
||||
}
|
||||
return "the word on whether an agent can become root there is not recent"
|
||||
default:
|
||||
got, err := h.askRootFree(machines)
|
||||
if err != nil {
|
||||
// A word that could not be asked again is not kept from before: no recent word, no pass.
|
||||
// The failure is said and kept beside the last word. That word is not thrown away: it still renders
|
||||
// until it is RootFreeKept old, and an answer is never judged on it (this path is not a pass).
|
||||
h.mu.Lock()
|
||||
delete(h.rootWord, key)
|
||||
if h.rootWord == nil {
|
||||
h.rootWord = map[string]rootWord{}
|
||||
}
|
||||
w := h.rootWord[key]
|
||||
w.failed, w.failedAt = err.Error(), now
|
||||
h.rootWord[key] = w
|
||||
h.mu.Unlock()
|
||||
age := "no word was kept"
|
||||
if fresh {
|
||||
age = "the word of " + now.Sub(kept.at).Round(time.Second).String() + " ago is kept for rendering"
|
||||
}
|
||||
h.Logf("[messenger] could not ask whether an agent can become root on %s: %v (%s)", key, err, age)
|
||||
return "whether an agent can become root where it runs could not be asked: " + err.Error()
|
||||
}
|
||||
verdicts = got
|
||||
@@ -358,6 +371,10 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string {
|
||||
if h.rootWord == nil {
|
||||
h.rootWord = map[string]rootWord{}
|
||||
}
|
||||
if w := h.rootWord[key]; w.failed != "" {
|
||||
h.Logf("[messenger] the word on whether an agent can become root on %s was asked again after it failed at %s",
|
||||
key, w.failedAt.UTC().Format(time.RFC3339))
|
||||
}
|
||||
h.rootWord[key] = rootWord{at: now, verdicts: got}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
@@ -365,7 +382,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string {
|
||||
v, said := verdicts[m]
|
||||
switch {
|
||||
case !said:
|
||||
return "its engine said nothing of whether an agent can become root on " + m
|
||||
return "the controller said nothing of whether an agent can become root on " + m
|
||||
case !v.Free:
|
||||
why := v.Why
|
||||
if why == "" {
|
||||
@@ -378,7 +395,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// warmRootFree asks the machines' engines' root-free word for every kind that proves its sender, outside the work,
|
||||
// warmRootFree asks the root-free word for every kind that proves its sender, outside the work,
|
||||
// so what is rendered and offered inside it reads a recent word.
|
||||
// force asks even a recent word again: taking an ask, whose refusal its asker keeps until something changes.
|
||||
func (h *Holder) warmRootFree(force bool) {
|
||||
@@ -404,7 +421,7 @@ func (h *Holder) warmRootFree(force bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// rootWordFor asks the machines' engines' root-free word for a kind fresh, before the work is taken, and answers
|
||||
// rootWordFor asks the root-free word for a kind fresh, before the work is taken, and answers
|
||||
// what judging an answer of that kind then reads: the word for that very kind's placement, or why there is none.
|
||||
// A kind that does not prove its sender is asked nothing: its answers never approve.
|
||||
func (h *Holder) rootWordFor(kind string) func(KindInfo) string {
|
||||
@@ -424,10 +441,13 @@ func (h *Holder) rootWordFor(kind string) func(KindInfo) string {
|
||||
}
|
||||
}
|
||||
|
||||
// rootWord is the controller's last word on a set of machines, and when it was asked.
|
||||
// rootWord is the controller's last word on a set of machines, and when it was asked; failed and failedAt the last
|
||||
// ask that failed since, and why.
|
||||
type rootWord struct {
|
||||
at time.Time
|
||||
verdicts map[string]RootVerdict
|
||||
failed string
|
||||
failedAt time.Time
|
||||
}
|
||||
|
||||
// approveHolds is why an answer on a kind could not approve now, or "": the kind verifies its sender, an
|
||||
@@ -704,6 +724,8 @@ func (h *Holder) releaseHeld() {
|
||||
rec.Tickets = nil
|
||||
if why := h.holdWhy(rec.Ask); why != rec.HeldWhy {
|
||||
rec.HeldWhy = why
|
||||
h.Logf("[messenger] %s's ask %q (%s) is still held, since %s: %s", rec.Asker, rec.Ask.Headline, rec.Ask.ID,
|
||||
rec.HeldSince.UTC().Format(time.RFC3339), why)
|
||||
h.saveHeld(key, rec, rev)
|
||||
}
|
||||
continue
|
||||
|
||||
@@ -256,8 +256,14 @@ func (r *busRuntime) runtime(id json.RawMessage, method string, params json.RawM
|
||||
case "mesh/seat.publish":
|
||||
return map[string]any{}, r.nc.Publish(p.Subject, p.Body)
|
||||
case "mesh/ask":
|
||||
if m, ok := strings.CutPrefix(p.Key, RootFreeVerb+"@"); ok {
|
||||
return map[string]any{"machines": []RootVerdict{{Machine: m, Free: true}}}, nil
|
||||
if p.Key == RootFreeVerb {
|
||||
var args struct{ Machines []string }
|
||||
_ = json.Unmarshal(p.Body, &args)
|
||||
var out []RootVerdict
|
||||
for _, m := range args.Machines {
|
||||
out = append(out, RootVerdict{Machine: m, Free: true})
|
||||
}
|
||||
return map[string]any{"machines": out}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("the test runtime answers nothing for %s", p.Key)
|
||||
}
|
||||
|
||||
@@ -315,7 +315,7 @@ type Holder struct {
|
||||
Emit func(event string, body any) error
|
||||
// Ask calls a seat verb through the mesh, as this module: the answers the operator chooses.
|
||||
Ask Asker
|
||||
// RootFree asks each machine's engine's root-free (the node seat node-engine's, ADR 0563 §6); nil asks it through Ask
|
||||
// RootFree asks the root-free word on machines (the controller's verb, RootFreeVerb); nil asks it through Ask
|
||||
// (askRootFree).
|
||||
RootFree func(machines []string) (map[string]RootVerdict, error)
|
||||
// Zone is the operator's time zone, which every time in a message is said in; nil is this machine's.
|
||||
|
||||
@@ -88,7 +88,7 @@ func TestItDeclaresTheSeatsOfAnAskAndHoldsTheOperatorChannel(t *testing.T) { //
|
||||
// It performs nothing: the verbs it calls read the open conditions, whether root is free, and which issues
|
||||
// track which conditions.
|
||||
if !reflect.DeepEqual(m.Emits, []string{"refused"}) || !reflect.DeepEqual(m.Invokes, []string{"seat:" + ControllerSeat + ".conditions",
|
||||
"seat:mesh-conditions.conditions", "seat:" + EngineSeat + ".root-free", TrackingVerb}) {
|
||||
"seat:mesh-conditions.conditions", RootFreeVerb, TrackingVerb}) {
|
||||
t.Fatalf("emits %v invokes %v", m.Emits, m.Invokes)
|
||||
}
|
||||
var state []string
|
||||
|
||||
@@ -5,10 +5,10 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -30,6 +30,8 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) {
|
||||
k.Node = ""
|
||||
h.Bench.SetKinds([]KindInfo{k})
|
||||
}, "has not said which machine"},
|
||||
// The word asked before it, still fresh, renders until it is RootFreeKept old (a failed ask throws no fresh
|
||||
// word away; the sweep of 2026-10-30); the answer itself is refused all the same.
|
||||
"a controller that does not answer": {func(h *Holder) {
|
||||
h.RootFree = func([]string) (map[string]RootVerdict, error) { return nil, errors.New("no responders") }
|
||||
}, "could not be asked: no responders"},
|
||||
@@ -60,6 +62,10 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) {
|
||||
if len(w.of("mesh-delivery")) != 0 {
|
||||
t.Error("a warrant was issued")
|
||||
}
|
||||
if strings.Contains(tc.want, "could not be asked") {
|
||||
c.pass(RootFreeKept)
|
||||
stand(h, c)
|
||||
}
|
||||
if got := h.carriers(asks.Approve); len(got) != 0 {
|
||||
t.Errorf("approval still carried on %v", got)
|
||||
}
|
||||
@@ -67,34 +73,32 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// H3, as ADR 0563 §6 changes it: the word on each machine is asked of that machine's own engine, the node seat
|
||||
// node-engine's root-free there, as this module, and read from its answer; a machine's word is that machine's alone.
|
||||
func TestTheRootWordIsAskedOfEachMachinesEngine(t *testing.T) {
|
||||
// H3: the word on the router's machine and the channel's is asked of the verb that answers it on the mesh, the
|
||||
// controller's root-free, once with both machines, as this module; a verdict on a machine not asked is not taken.
|
||||
// The sweep of 2026-10-30 (issue 774): the node seat node-engine's root-free answered nowhere, so every ask that
|
||||
// offers an approval was held, silently.
|
||||
func TestTheRootWordIsAskedOfTheVerbThatAnswers(t *testing.T) {
|
||||
h, _, _, _, _, _ := newRouter(t)
|
||||
h.RootFree = nil
|
||||
var asked []string
|
||||
h.Ask = func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
machine := strings.TrimPrefix(key, "seat:node-engine.root-free@")
|
||||
if machine == "elsewhere" {
|
||||
machine = "anchor" // an engine that answers for another machine is not believed
|
||||
}
|
||||
return json.RawMessage(`{"machines":[{"machine":"` + machine + `","free":true,"why":"judged","judged":"2026-10-09T10:00:00Z"}]}`), nil
|
||||
ms, _ := body.(map[string]any)["machines"].([]string)
|
||||
asked = append(asked, key+" "+strings.Join(ms, ","))
|
||||
return json.RawMessage(`{"machines":[{"machine":"anchor","free":true,"why":"judged","judged":"2026-10-09T10:00:00Z"}]}`), nil
|
||||
}
|
||||
info, _ := h.Bench.Kind("telegram")
|
||||
if why := h.rootFree(info, false); why != "" {
|
||||
t.Fatalf("free not read: %s", why)
|
||||
}
|
||||
if len(asked) != 1 || asked[0] != "seat:node-engine.root-free@anchor" {
|
||||
if len(asked) != 1 || asked[0] != "seat:mesh-controller.root-free anchor" {
|
||||
t.Errorf("asked %v", asked)
|
||||
}
|
||||
asked = nil
|
||||
h.Node = "elsewhere"
|
||||
if why := h.rootFree(info, false); !strings.Contains(why, "said nothing of whether an agent can become root on elsewhere") {
|
||||
t.Errorf("one machine's word was taken for another's: %q", why)
|
||||
t.Errorf("a machine left out of the answer was a pass: %q", why)
|
||||
}
|
||||
sort.Strings(asked)
|
||||
if len(asked) != 2 || asked[0] != "seat:node-engine.root-free@anchor" || asked[1] != "seat:node-engine.root-free@elsewhere" {
|
||||
if len(asked) != 1 || asked[0] != "seat:mesh-controller.root-free elsewhere,anchor" {
|
||||
t.Errorf("the router's machine and the channel's were asked as %v", asked)
|
||||
}
|
||||
h.Node = "anchor"
|
||||
@@ -108,6 +112,41 @@ func TestTheRootWordIsAskedOfEachMachinesEngine(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A failed ask of the root word is said with its error, keeps the word still fresh for rendering, never passes an
|
||||
// answer, and is the reason a held ask carries (the sweep of 2026-10-30: the word was deleted silently and the held
|
||||
// reason said only "not recent").
|
||||
func TestAFailedRootAskIsSaidAndKeepsAFreshWord(t *testing.T) {
|
||||
h, _, _, c, store, _ := newRouter(t)
|
||||
link(h, store, c, "42", c.now().Add(-2*time.Hour))
|
||||
var logs []string
|
||||
h.Logf = func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }
|
||||
info, _ := h.Bench.Kind("telegram")
|
||||
if why := h.rootFree(info, false); why != "" {
|
||||
t.Fatalf("free not read: %s", why)
|
||||
}
|
||||
h.RootFree = func([]string) (map[string]RootVerdict, error) {
|
||||
return nil, errors.New("nothing in the mesh is called node-engine")
|
||||
}
|
||||
c.pass(RootFreeKept / 3)
|
||||
if why := h.rootFree(info, false); !strings.Contains(why, "could not be asked: nothing in the mesh is called node-engine") {
|
||||
t.Errorf("a failed ask judged an answer: %q", why)
|
||||
}
|
||||
if len(logs) != 1 || !strings.Contains(logs[0], "nothing in the mesh is called node-engine") || !strings.Contains(logs[0], "kept for rendering") {
|
||||
t.Errorf("the failure was not said with its error: %v", logs)
|
||||
}
|
||||
if why := h.rootFree(info, true); why != "" {
|
||||
t.Errorf("one failed ask threw away a word still fresh: %q", why)
|
||||
}
|
||||
c.pass(RootFreeKept)
|
||||
stand(h, c)
|
||||
if why := h.rootFree(info, true); !strings.Contains(why, "failed: nothing in the mesh is called node-engine") {
|
||||
t.Errorf("a stale word passed, or its reason hid the failure: %q", why)
|
||||
}
|
||||
if why := h.holdWhy(heldAsk(c, "d-9")); !strings.Contains(why, "nothing in the mesh is called node-engine") {
|
||||
t.Errorf("a held ask does not say why: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// H3: a message rendered while root is not free offers no approval; a word kept for rendering is never the
|
||||
// word an answer is judged on.
|
||||
func TestAMessageRenderedWhileRootIsNotFreeOffersNoApproval(t *testing.T) {
|
||||
|
||||
@@ -94,7 +94,7 @@
|
||||
"invokes": [
|
||||
"seat:mesh-controller.conditions",
|
||||
"seat:mesh-conditions.conditions",
|
||||
"seat:node-engine.root-free",
|
||||
"seat:mesh-controller.root-free",
|
||||
"seat:issue-tracker.tracking"
|
||||
],
|
||||
"reads": [
|
||||
|
||||
Reference in New Issue
Block a user