From bc271896277419620eb5e92f624ccc3c7b9021ca Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 18:01:54 +0100 Subject: [PATCH] Ask the root-free word of the controller's verb, which answers, and say every failed ask, so Telegram carries asks that offer an approval again (issue 774) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The messenger asked seat:node-engine.root-free@, which nothing on the mesh answers (the engine's acts are left out of its declared state, mesh-host #178; ADR 0571 §11's self-check probe is not held either), and deleted its kept word on each failure without a log line: every ask from an agent or a merge was held, silently. Quick fix: point RootFreeVerb back at mesh-controller.root-free. Thorough fix: the same, plus every failed ask is logged with its error, the failure is kept beside the word and becomes the held ask's reason, a fresh word is no longer thrown away by one failed ask (an answer is still judged only on a word asked then), and a held ask whose reason changes is logged. Chose thorough: the silence is what hid the fault for a day. --- modules/messenger/cmd/messenger/answers.go | 2 +- modules/messenger/cmd/messenger/asks.go | 124 +++++++++++------- modules/messenger/cmd/messenger/bus_test.go | 10 +- modules/messenger/cmd/messenger/holder.go | 2 +- .../messenger/cmd/messenger/manifest_test.go | 2 +- .../messenger/cmd/messenger/review_test.go | 67 ++++++++-- modules/messenger/module.json | 2 +- 7 files changed, 138 insertions(+), 71 deletions(-) diff --git a/modules/messenger/cmd/messenger/answers.go b/modules/messenger/cmd/messenger/answers.go index 6a3fbb5a..ab0159f2 100644 --- a/modules/messenger/cmd/messenger/answers.go +++ b/modules/messenger/cmd/messenger/answers.go @@ -133,7 +133,7 @@ func pruneGrants(grants map[string]Grant, kind string, now time.Time, keep int) // stands, closes the ask with a warrant for its asker (asks.go); Details are shown where they were asked. // Anything else is refused, and said there. The same answer is acted on once. func (h *Holder) Chosen(kind string, c asks.Choice) (string, error) { - // The machines' engines' root-free word is asked before the work is taken (the confirmation review of 2026-10-09): + // The root-free word is asked before the work is taken (the confirmation review of 2026-10-09): // up to RootFreeWithin, which no ask, answer or link should wait behind. Asked fresh here, for this answer, // and judged with in it. root := h.rootWordFor(kind) diff --git a/modules/messenger/cmd/messenger/asks.go b/modules/messenger/cmd/messenger/asks.go index b719e68f..301429f6 100644 --- a/modules/messenger/cmd/messenger/asks.go +++ b/modules/messenger/cmd/messenger/asks.go @@ -242,9 +242,8 @@ func (h *Holder) carriers(level asks.Level) []string { return out } -// RootVerdict is a machine's engine's word on that machine (the node seat node-engine's root-free, novox/hq ADR 0259 -// §8 as ADR 0563 §6 changes it): free only when it measured, recently, that no agent can become root there without a -// person. +// RootVerdict is the word on one machine (novox/hq ADR 0259 §8): free only when it was measured, recently, that no +// agent can become root there without a person. type RootVerdict struct { Machine string `json:"machine"` Free bool `json:"free"` @@ -252,63 +251,61 @@ type RootVerdict struct { Judged string `json:"judged,omitempty"` } -// EngineSeat is the node seat each machine's engine holds, and RootFreeVerb its verb the router asks of a machine, as -// this module: `seat:node-engine.root-free@`, which that machine's engine alone answers (ADR 0563 §6). -const ( - EngineSeat = "node-engine" - RootFreeVerb = "seat:" + EngineSeat + ".root-free" -) +// RootFreeVerb is the verb the router asks, as this module, which machines are root-free: the controller's, which +// judges each machine from its node-engine's own verdict (ADR 0259 §8). ADR 0563 §6 moves the word to the node seat +// node-engine and ADR 0571 §11 to the self-check's probe; neither answers on the mesh yet (the engine's acts are left +// out of its declared state, mesh-host #178, and no seat mesh-self-check is held), so asking them held every ask +// that offers an approval (issue 774, the sweep of 2026-10-30). The controller's verb is the one that answers. +const RootFreeVerb = "seat:" + ControllerSeat + ".root-free" -// RootFreeWithin is how long the router waits for the machines' words; RootFreeKept how long a word is used to render a +// RootFreeWithin is how long the router waits for the word; RootFreeKept how long a word is used to render a // message's buttons. An answer is always judged on a word asked then. const ( RootFreeWithin = 10 * time.Second RootFreeKept = 30 * time.Second ) -// askRootFree asks each machine's own engine about that machine, at once, failing on no answer in time. A machine's -// word is the verdict its engine gives on it; a verdict an engine gives on another machine is not taken. +// askRootFree asks the word on machines at once, failing on no answer in time. Only a verdict on a machine asked is +// taken. func (h *Holder) askRootFree(machines []string) (map[string]RootVerdict, error) { if h.RootFree != nil { return h.RootFree(machines) } if h.Ask == nil { - return nil, errors.New("this router cannot ask a machine's engine") + return nil, errors.New("this router cannot ask the controller") } type answer struct { - machine string - raw json.RawMessage - err error + raw json.RawMessage + err error } - done := make(chan answer, len(machines)) + done := make(chan answer, 1) + go func() { + raw, err := h.Ask(RootFreeVerb, map[string]any{"machines": machines}) + done <- answer{raw, err} + }() + var a answer + select { + case a = <-done: + case <-time.After(RootFreeWithin): + return nil, fmt.Errorf("%s did not answer within %s", RootFreeVerb, RootFreeWithin) + } + if a.err != nil { + return nil, fmt.Errorf("%s: %w", RootFreeVerb, a.err) + } + var body struct { + Machines []RootVerdict `json:"machines"` + } + if err := json.Unmarshal(a.raw, &body); err != nil { + return nil, fmt.Errorf("%s's answer could not be read: %v", RootFreeVerb, err) + } + asked := map[string]bool{} for _, m := range machines { - go func(m string) { - raw, err := h.Ask(RootFreeVerb+"@"+m, map[string]any{}) - done <- answer{m, raw, err} - }(m) + asked[m] = true } out := map[string]RootVerdict{} - deadline := time.After(RootFreeWithin) - for range machines { - var a answer - select { - case a = <-done: - case <-deadline: - return nil, fmt.Errorf("a machine's engine did not answer within %s", RootFreeWithin) - } - if a.err != nil { - return nil, fmt.Errorf("%s's engine: %w", a.machine, a.err) - } - var body struct { - Machines []RootVerdict `json:"machines"` - } - if err := json.Unmarshal(a.raw, &body); err != nil { - return nil, fmt.Errorf("%s's engine's answer could not be read: %v", a.machine, err) - } - for _, v := range body.Machines { - if v.Machine == a.machine { - out[v.Machine] = v - } + for _, v := range body.Machines { + if asked[v.Machine] { + out[v.Machine] = v } } return out, nil @@ -337,20 +334,36 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { kept, ok := h.rootWord[key] h.mu.Unlock() var verdicts map[string]RootVerdict + fresh := ok && kept.verdicts != nil && now.Sub(kept.at) < RootFreeKept && now.Sub(kept.at) >= 0 switch { - case cached && ok && now.Sub(kept.at) < RootFreeKept && now.Sub(kept.at) >= 0: + case cached && fresh: verdicts = kept.verdicts case cached: // Read inside the work, which never waits on the controller: no recent word is no pass. warmRootFree - // asks outside it. - return "the machines' engines' word on whether an agent can become root there is not recent" + // asks outside it; when its last ask failed, that failure is the reason, so a held ask says it. + if ok && kept.failed != "" { + return "the word on whether an agent can become root there is not recent: asking it at " + + kept.failedAt.UTC().Format("15:04:05") + " UTC failed: " + kept.failed + } + return "the word on whether an agent can become root there is not recent" default: got, err := h.askRootFree(machines) if err != nil { - // A word that could not be asked again is not kept from before: no recent word, no pass. + // The failure is said and kept beside the last word. That word is not thrown away: it still renders + // until it is RootFreeKept old, and an answer is never judged on it (this path is not a pass). h.mu.Lock() - delete(h.rootWord, key) + if h.rootWord == nil { + h.rootWord = map[string]rootWord{} + } + w := h.rootWord[key] + w.failed, w.failedAt = err.Error(), now + h.rootWord[key] = w h.mu.Unlock() + age := "no word was kept" + if fresh { + age = "the word of " + now.Sub(kept.at).Round(time.Second).String() + " ago is kept for rendering" + } + h.Logf("[messenger] could not ask whether an agent can become root on %s: %v (%s)", key, err, age) return "whether an agent can become root where it runs could not be asked: " + err.Error() } verdicts = got @@ -358,6 +371,10 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { if h.rootWord == nil { h.rootWord = map[string]rootWord{} } + if w := h.rootWord[key]; w.failed != "" { + h.Logf("[messenger] the word on whether an agent can become root on %s was asked again after it failed at %s", + key, w.failedAt.UTC().Format(time.RFC3339)) + } h.rootWord[key] = rootWord{at: now, verdicts: got} h.mu.Unlock() } @@ -365,7 +382,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { v, said := verdicts[m] switch { case !said: - return "its engine said nothing of whether an agent can become root on " + m + return "the controller said nothing of whether an agent can become root on " + m case !v.Free: why := v.Why if why == "" { @@ -378,7 +395,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { return "" } -// warmRootFree asks the machines' engines' root-free word for every kind that proves its sender, outside the work, +// warmRootFree asks the root-free word for every kind that proves its sender, outside the work, // so what is rendered and offered inside it reads a recent word. // force asks even a recent word again: taking an ask, whose refusal its asker keeps until something changes. func (h *Holder) warmRootFree(force bool) { @@ -404,7 +421,7 @@ func (h *Holder) warmRootFree(force bool) { } } -// rootWordFor asks the machines' engines' root-free word for a kind fresh, before the work is taken, and answers +// rootWordFor asks the root-free word for a kind fresh, before the work is taken, and answers // what judging an answer of that kind then reads: the word for that very kind's placement, or why there is none. // A kind that does not prove its sender is asked nothing: its answers never approve. func (h *Holder) rootWordFor(kind string) func(KindInfo) string { @@ -424,10 +441,13 @@ func (h *Holder) rootWordFor(kind string) func(KindInfo) string { } } -// rootWord is the controller's last word on a set of machines, and when it was asked. +// rootWord is the controller's last word on a set of machines, and when it was asked; failed and failedAt the last +// ask that failed since, and why. type rootWord struct { at time.Time verdicts map[string]RootVerdict + failed string + failedAt time.Time } // approveHolds is why an answer on a kind could not approve now, or "": the kind verifies its sender, an @@ -704,6 +724,8 @@ func (h *Holder) releaseHeld() { rec.Tickets = nil if why := h.holdWhy(rec.Ask); why != rec.HeldWhy { rec.HeldWhy = why + h.Logf("[messenger] %s's ask %q (%s) is still held, since %s: %s", rec.Asker, rec.Ask.Headline, rec.Ask.ID, + rec.HeldSince.UTC().Format(time.RFC3339), why) h.saveHeld(key, rec, rev) } continue diff --git a/modules/messenger/cmd/messenger/bus_test.go b/modules/messenger/cmd/messenger/bus_test.go index c42fb278..c42a9ade 100644 --- a/modules/messenger/cmd/messenger/bus_test.go +++ b/modules/messenger/cmd/messenger/bus_test.go @@ -256,8 +256,14 @@ func (r *busRuntime) runtime(id json.RawMessage, method string, params json.RawM case "mesh/seat.publish": return map[string]any{}, r.nc.Publish(p.Subject, p.Body) case "mesh/ask": - if m, ok := strings.CutPrefix(p.Key, RootFreeVerb+"@"); ok { - return map[string]any{"machines": []RootVerdict{{Machine: m, Free: true}}}, nil + if p.Key == RootFreeVerb { + var args struct{ Machines []string } + _ = json.Unmarshal(p.Body, &args) + var out []RootVerdict + for _, m := range args.Machines { + out = append(out, RootVerdict{Machine: m, Free: true}) + } + return map[string]any{"machines": out}, nil } return nil, fmt.Errorf("the test runtime answers nothing for %s", p.Key) } diff --git a/modules/messenger/cmd/messenger/holder.go b/modules/messenger/cmd/messenger/holder.go index b3dff885..4917f08d 100644 --- a/modules/messenger/cmd/messenger/holder.go +++ b/modules/messenger/cmd/messenger/holder.go @@ -315,7 +315,7 @@ type Holder struct { Emit func(event string, body any) error // Ask calls a seat verb through the mesh, as this module: the answers the operator chooses. Ask Asker - // RootFree asks each machine's engine's root-free (the node seat node-engine's, ADR 0563 §6); nil asks it through Ask + // RootFree asks the root-free word on machines (the controller's verb, RootFreeVerb); nil asks it through Ask // (askRootFree). RootFree func(machines []string) (map[string]RootVerdict, error) // Zone is the operator's time zone, which every time in a message is said in; nil is this machine's. diff --git a/modules/messenger/cmd/messenger/manifest_test.go b/modules/messenger/cmd/messenger/manifest_test.go index 6e61b0de..89278493 100644 --- a/modules/messenger/cmd/messenger/manifest_test.go +++ b/modules/messenger/cmd/messenger/manifest_test.go @@ -88,7 +88,7 @@ func TestItDeclaresTheSeatsOfAnAskAndHoldsTheOperatorChannel(t *testing.T) { // // It performs nothing: the verbs it calls read the open conditions, whether root is free, and which issues // track which conditions. if !reflect.DeepEqual(m.Emits, []string{"refused"}) || !reflect.DeepEqual(m.Invokes, []string{"seat:" + ControllerSeat + ".conditions", - "seat:mesh-conditions.conditions", "seat:" + EngineSeat + ".root-free", TrackingVerb}) { + "seat:mesh-conditions.conditions", RootFreeVerb, TrackingVerb}) { t.Fatalf("emits %v invokes %v", m.Emits, m.Invokes) } var state []string diff --git a/modules/messenger/cmd/messenger/review_test.go b/modules/messenger/cmd/messenger/review_test.go index 9c042edc..384a7afb 100644 --- a/modules/messenger/cmd/messenger/review_test.go +++ b/modules/messenger/cmd/messenger/review_test.go @@ -5,10 +5,10 @@ package main import ( "encoding/json" "errors" + "fmt" "os" "path/filepath" "regexp" - "sort" "strings" "sync" "testing" @@ -30,6 +30,8 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) { k.Node = "" h.Bench.SetKinds([]KindInfo{k}) }, "has not said which machine"}, + // The word asked before it, still fresh, renders until it is RootFreeKept old (a failed ask throws no fresh + // word away; the sweep of 2026-10-30); the answer itself is refused all the same. "a controller that does not answer": {func(h *Holder) { h.RootFree = func([]string) (map[string]RootVerdict, error) { return nil, errors.New("no responders") } }, "could not be asked: no responders"}, @@ -60,6 +62,10 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) { if len(w.of("mesh-delivery")) != 0 { t.Error("a warrant was issued") } + if strings.Contains(tc.want, "could not be asked") { + c.pass(RootFreeKept) + stand(h, c) + } if got := h.carriers(asks.Approve); len(got) != 0 { t.Errorf("approval still carried on %v", got) } @@ -67,34 +73,32 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) { } } -// H3, as ADR 0563 §6 changes it: the word on each machine is asked of that machine's own engine, the node seat -// node-engine's root-free there, as this module, and read from its answer; a machine's word is that machine's alone. -func TestTheRootWordIsAskedOfEachMachinesEngine(t *testing.T) { +// H3: the word on the router's machine and the channel's is asked of the verb that answers it on the mesh, the +// controller's root-free, once with both machines, as this module; a verdict on a machine not asked is not taken. +// The sweep of 2026-10-30 (issue 774): the node seat node-engine's root-free answered nowhere, so every ask that +// offers an approval was held, silently. +func TestTheRootWordIsAskedOfTheVerbThatAnswers(t *testing.T) { h, _, _, _, _, _ := newRouter(t) h.RootFree = nil var asked []string h.Ask = func(key string, body any) (json.RawMessage, error) { - asked = append(asked, key) - machine := strings.TrimPrefix(key, "seat:node-engine.root-free@") - if machine == "elsewhere" { - machine = "anchor" // an engine that answers for another machine is not believed - } - return json.RawMessage(`{"machines":[{"machine":"` + machine + `","free":true,"why":"judged","judged":"2026-10-09T10:00:00Z"}]}`), nil + ms, _ := body.(map[string]any)["machines"].([]string) + asked = append(asked, key+" "+strings.Join(ms, ",")) + return json.RawMessage(`{"machines":[{"machine":"anchor","free":true,"why":"judged","judged":"2026-10-09T10:00:00Z"}]}`), nil } info, _ := h.Bench.Kind("telegram") if why := h.rootFree(info, false); why != "" { t.Fatalf("free not read: %s", why) } - if len(asked) != 1 || asked[0] != "seat:node-engine.root-free@anchor" { + if len(asked) != 1 || asked[0] != "seat:mesh-controller.root-free anchor" { t.Errorf("asked %v", asked) } asked = nil h.Node = "elsewhere" if why := h.rootFree(info, false); !strings.Contains(why, "said nothing of whether an agent can become root on elsewhere") { - t.Errorf("one machine's word was taken for another's: %q", why) + t.Errorf("a machine left out of the answer was a pass: %q", why) } - sort.Strings(asked) - if len(asked) != 2 || asked[0] != "seat:node-engine.root-free@anchor" || asked[1] != "seat:node-engine.root-free@elsewhere" { + if len(asked) != 1 || asked[0] != "seat:mesh-controller.root-free elsewhere,anchor" { t.Errorf("the router's machine and the channel's were asked as %v", asked) } h.Node = "anchor" @@ -108,6 +112,41 @@ func TestTheRootWordIsAskedOfEachMachinesEngine(t *testing.T) { } } +// A failed ask of the root word is said with its error, keeps the word still fresh for rendering, never passes an +// answer, and is the reason a held ask carries (the sweep of 2026-10-30: the word was deleted silently and the held +// reason said only "not recent"). +func TestAFailedRootAskIsSaidAndKeepsAFreshWord(t *testing.T) { + h, _, _, c, store, _ := newRouter(t) + link(h, store, c, "42", c.now().Add(-2*time.Hour)) + var logs []string + h.Logf = func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) } + info, _ := h.Bench.Kind("telegram") + if why := h.rootFree(info, false); why != "" { + t.Fatalf("free not read: %s", why) + } + h.RootFree = func([]string) (map[string]RootVerdict, error) { + return nil, errors.New("nothing in the mesh is called node-engine") + } + c.pass(RootFreeKept / 3) + if why := h.rootFree(info, false); !strings.Contains(why, "could not be asked: nothing in the mesh is called node-engine") { + t.Errorf("a failed ask judged an answer: %q", why) + } + if len(logs) != 1 || !strings.Contains(logs[0], "nothing in the mesh is called node-engine") || !strings.Contains(logs[0], "kept for rendering") { + t.Errorf("the failure was not said with its error: %v", logs) + } + if why := h.rootFree(info, true); why != "" { + t.Errorf("one failed ask threw away a word still fresh: %q", why) + } + c.pass(RootFreeKept) + stand(h, c) + if why := h.rootFree(info, true); !strings.Contains(why, "failed: nothing in the mesh is called node-engine") { + t.Errorf("a stale word passed, or its reason hid the failure: %q", why) + } + if why := h.holdWhy(heldAsk(c, "d-9")); !strings.Contains(why, "nothing in the mesh is called node-engine") { + t.Errorf("a held ask does not say why: %q", why) + } +} + // H3: a message rendered while root is not free offers no approval; a word kept for rendering is never the // word an answer is judged on. func TestAMessageRenderedWhileRootIsNotFreeOffersNoApproval(t *testing.T) { diff --git a/modules/messenger/module.json b/modules/messenger/module.json index eec860f0..ca694b40 100644 --- a/modules/messenger/module.json +++ b/modules/messenger/module.json @@ -94,7 +94,7 @@ "invokes": [ "seat:mesh-controller.conditions", "seat:mesh-conditions.conditions", - "seat:node-engine.root-free", + "seat:mesh-controller.root-free", "seat:issue-tracker.tracking" ], "reads": [