COND1: the keeper hears findings as events and reads no observer's state

The keeper takes finding-written and finding-cleared from each observer's stream
(`*.finding-written`, `*.finding-cleared`, `*.findings-stated` in consumes); its reads of
every `findings`, the views.findings consumes and the read_findings invokes go.
node-engine's name is a local constant since mesh-sdk's buscompose lost it.

Builds: design 47 G §383, §386

Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
2026-11-11 12:05:00 +01:00
parent e6aab55008
commit b8c3f5f09a
7 changed files with 246 additions and 70 deletions
@@ -0,0 +1,136 @@
package main
// The keeper hears each observer's findings as events on the observer's own stream (design 47 G §383): it reads no
// observer's state. `finding-written` raises or changes the condition of its key, `finding-cleared` clears it. The
// observer and the node are the event's subject (its envelope's key and node); the bus's grants keep another account
// from publishing there, and the keeper refuses a body that names another observer.
import (
"encoding/json"
"fmt"
"slices"
"strings"
"time"
cc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The events an observer says on its stream (design 47 G §383).
const (
EventFindingWritten = "finding-written"
EventFindingCleared = "finding-cleared"
)
// emitterOf is the module an event's envelope key `<module>.<event>` names.
func emitterOf(key, event string) (string, bool) {
m, ok := strings.CutSuffix(key, "."+event)
return m, ok && m != "" && !strings.Contains(m, ".")
}
// FindingEvent is one `finding-written` or `finding-cleared` heard from an observer on a node.
func (k *Keeper) FindingEvent(e stdio.Envelope) error {
if observer, ok := emitterOf(e.Key, EventFindingWritten); ok {
return k.FindingWritten(observer, e.Node, e.Body)
}
if observer, ok := emitterOf(e.Key, EventFindingCleared); ok {
return k.FindingCleared(observer, e.Node, e.Body)
}
return nil
}
// isSource says whether the keeper takes findings from the module.
func (k *Keeper) isSource(observer string) bool { return slices.Contains(Observers, observer) }
// FindingWritten raises or changes the condition of the finding an observer says on a node. A body that is not a finding
// the keeper can raise, or names another observer, is said in the journal and changes nothing.
func (k *Keeper) FindingWritten(observer, node string, body []byte) error {
k.mu.Lock()
k.init()
k.mu.Unlock()
if !k.isSource(observer) {
k.Logf("%s says finding-written on %s, and is no source of findings: left out", observer, node)
return nil
}
var f seat.Finding
if err := json.Unmarshal(body, &f); err != nil {
k.Logf("%s's finding-written on %s is not readable: %v", observer, node, err)
return nil
}
if f.Observer != observer {
k.Logf("%s's finding-written on %s names the observer %q: refused", observer, node, f.Observer)
return nil
}
if f.Scope != f.Subject.Scope || f.ID != f.Subject.ID {
k.Logf("%s's finding-written on %s names the scope %q and id %q, and its subject %q and %q: refused", observer, node,
f.Scope, f.ID, f.Subject.Scope, f.Subject.ID)
return nil
}
v := storedFinding(f)
value, err := json.Marshal(v)
if err != nil {
return err
}
return k.Finding(observer, v.Key, value, false)
}
// FindingCleared clears the condition of the finding an observer no longer observes.
func (k *Keeper) FindingCleared(observer, node string, body []byte) error {
if !k.isSource(observer) {
k.Logf("%s says finding-cleared on %s, and is no source of findings: left out", observer, node)
return nil
}
var c seat.FindingCleared
if err := json.Unmarshal(body, &c); err != nil {
k.Logf("%s's finding-cleared on %s is not readable: %v", observer, node, err)
return nil
}
k.mu.Lock()
k.init()
key := cc.Key(c.Scope, c.ID, c.Kind)
if _, ok := k.findings[observer][key]; !ok {
// A finding whose key ends in its ref rather than its kind.
for other, f := range k.findings[observer] {
if f.Subject.Scope == c.Scope && f.Subject.ID == c.ID && f.Kind == c.Kind {
key = other
}
}
}
k.mu.Unlock()
return k.Finding(observer, key, nil, true)
}
// storedFinding is the event's finding as the keeper holds one.
func storedFinding(f seat.Finding) seat.FindingsByKeyValue {
token := f.Ref
if token == "" {
token = f.Kind
}
evidence := f.Evidence
if evidence == nil {
evidence = []seat.Evidence{}
}
return seat.FindingsByKeyValue{V: cc.FindingVersion, Key: cc.Key(f.Scope, f.ID, token), Kind: f.Kind, Subject: f.Subject,
Severity: f.Severity, Summary: f.Summary, Headline: f.Headline, Explanation: f.Explanation, Needs: f.Needs,
Actions: f.Actions, Evidence: evidence, Source: f.Source, Resolver: f.Resolver, Observer: f.Observer, Owner: f.Owner,
Ref: f.Ref, Run: f.Run, WaitingOn: f.WaitingOn, At: f.At}
}
// hear subscribes to the observers' events once the runtime serves, asked again until it takes each.
func hear(k *Keeper) {
for _, event := range []string{EventFindingWritten, EventFindingCleared} {
pattern := "*." + event
go func() {
<-stdio.Served()
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
err := stdio.Subscribe(pattern, k.FindingEvent)
if err == nil {
return
}
logf("%s", fmt.Sprintf("%s is not heard yet (%v); asked again in %s", pattern, err, wait))
time.Sleep(wait)
}
}()
}
}
@@ -20,7 +20,6 @@ import (
assignerclient "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mesh "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/motion"
cc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
@@ -36,6 +35,8 @@ const (
healersModule = "mesh-healers"
// The holders of the assignments and the catalog, whose views and findings the keeper follows (the manifest test holds
// these equal to the manifest's `consumes` and `invokes`).
// engineModule is the node-engine, an observer of §387 (design 47 G §387 names mesh-host).
engineModule = "mesh-host"
assignerModule = "mesh-delivery-assigner"
indexerModule = "mesh-delivery-indexer"
)
@@ -69,7 +70,7 @@ var Observers = []string{watchmanModule, selfCheckModule, healersModule, assigne
"node-tools", "claude-code",
// The node-engine's: a node that has read no seat list five minutes after its start, seat-list-unread (ADR 0636 §1;
// issue 1966: written on ace and raised by nobody).
buscompose.EngineModule}
engineModule}
// The other states it reads, as `<module>.<name>`.
const (
@@ -163,6 +163,22 @@ func finding(t *testing.T, observer, scope, id, kind, source string, sev string,
return f.Key, raw
}
func mustJSON(t *testing.T, v any) []byte {
t.Helper()
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return raw
}
// eventFinding is the machine g14's silent finding as an observer says it in finding-written.
func eventFinding(observer string, at time.Time) seat.Finding {
return seat.Finding{Scope: cc.ScopeMachine, ID: "g14", Kind: "silent", Subject: seat.Subject{Scope: cc.ScopeMachine, ID: "g14"},
Severity: cc.Warning, Summary: "silent of g14", Headline: "silent", Explanation: "It is silent.", Source: "S1",
Observer: observer, At: at, Since: at, Evidence: []seat.Evidence{{At: at, Said: "silent"}}}
}
// ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer
// clears nothing; the healers' attempts reach the condition; a reopening within ten
// minutes is the same fault; a silence is a transition in history.
@@ -229,6 +245,29 @@ func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) {
}
return ""
}},
// Design 47 G §383: the same findings heard as events on the observer's stream.
{name: "finding-cleared clears", do: func() error {
return r.k.FindingCleared(watchman, "ace", mustJSON(t, seat.FindingCleared{At: r.now, ID: "g14", Kind: "silent", Scope: cc.ScopeMachine, Why: "back"}))
}, said: "cleared machine.g14.silent", check: func(_ Condition, open bool) string {
if open {
return "still open"
}
return ""
}},
{name: "finding-written raises", do: func() error {
return r.k.FindingWritten(watchman, "ace", mustJSON(t, eventFinding(watchman, r.now)))
}, said: "reopened machine.g14.silent"},
{name: "a body naming another observer changes nothing", do: func() error {
return r.k.FindingWritten(watchman, "ace", mustJSON(t, eventFinding(selfCheck, r.now)))
}},
{name: "a module that is no source changes nothing", do: func() error {
return r.k.FindingWritten("stranger", "ace", mustJSON(t, eventFinding("stranger", r.now)))
}},
{name: "changed words change the condition", do: func() error {
f := eventFinding(watchman, r.now)
f.Needs = "a cable"
return r.k.FindingWritten(watchman, "ace", mustJSON(t, f))
}, said: "words machine.g14.silent"},
}
for _, s := range steps {
if err := s.do(); err != nil {
@@ -1,8 +1,8 @@
// mesh-conditions: the condition keeper, holder of the seat mesh-conditions (novox/hq ADR 0564 clauses 1-3, 9, 13). A
// Go bundle served by a runtime of its own, on its own bus account (`runs-as`), so that its account alone writes its
// states: `conditions` (one key per open condition), `history` (every transition, ninety days), `takeover` (the key the
// readers follow), `held` and `bounds` (what a delivery holds, and for how long). It watches every observer's
// `findings`, the healers' `attempts`, the assigner's wants, the nodes' haves and the controller's lease and, until the
// readers follow), `held` and `bounds` (what a delivery holds, and for how long). It hears every observer's
// findings as events (design 47 G §383), watches the healers' `attempts`, the assigner's wants, the nodes' haves and the controller's lease and, until the
// lease write, its `conditions`; it says every transition on its seat with the controller's body. stdout is the MCP
// channel; what this module says, it says on stderr.
package main
@@ -18,7 +18,6 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/views"
)
@@ -122,11 +121,7 @@ func Keep(k *Keeper, v *views.Reader) {
watch(k, v, AttemptsRead, healersModule+" attempts", func(c stdio.StateChange) error {
return said(k.Attempt(c.Key, c.Value, c.Op != "put"))
})
for _, o := range Observers {
watch(k, v, o+"."+seat.StateFindings, o, func(c stdio.StateChange) error {
return said(k.Finding(o, c.Key, c.Value, c.Op != "put"))
})
}
hear(k)
watch(k, v, BoundsState, "", func(c stdio.StateChange) error {
if c.Op == stdio.OpExpire {
return said(k.BoundPassed(c.Key))
@@ -143,19 +138,14 @@ func Keep(k *Keeper, v *views.Reader) {
const (
WantsTool = assignerModule + ".assigner_read_wants"
AssignmentsTool = assignerModule + ".assigner_read_assignments"
FindingsTool = assignerModule + ".assigner_read_findings"
// IndexerFindingsTool is the indexer's read tool of its findings: the indexer's state, too, is asked of its owner.
IndexerFindingsTool = indexerModule + ".indexer_read_findings"
)
// NewViews is the keeper's pictures of the assigner's wants, assignments and findings and of the indexer's findings. A first view it cannot get past its
// NewViews is the keeper's pictures of the assigner's wants and assignments (the findings come as events, design 47 G §383). A first view it cannot get past its
// bound is a finding of its own, taken in as any observer's: the keeper raises the condition itself.
func NewViews(k *Keeper) *views.Reader {
v := &views.Reader{Module: Module, Logf: logf, Findings: ownFindings{k}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}}
v.Follow(assignerModule, "wants", WantsTool)
v.Follow(assignerModule, "assignments", AssignmentsTool)
v.Follow(assignerModule, seat.StateFindings, FindingsTool)
v.Follow(indexerModule, seat.StateFindings, IndexerFindingsTool)
return v
}
@@ -4,6 +4,7 @@ import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
@@ -32,19 +33,19 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
!slices.Equal(Verbs, catalogue.VerbNames(def.Serves)) {
t.Errorf("module %s runs as %s and claims %+v; the def serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(def.Serves))
}
// The assigner's wants, assignments and findings, and the indexer's findings, are asked once of their read tools and
// followed by their events, not read (ADR 0615 §1).
// The assigner's wants and assignments are asked once of their read tools and followed by their events (ADR 0615 §1);
// the observers' findings are heard as events and no `findings` is read (design 47 G §383).
reads := []string{HavesRead, AttemptsRead}
for _, o := range Observers {
if o != assignerModule && o != indexerModule {
reads = append(reads, o+"."+seat.StateFindings)
}
}
if !slices.Equal(m.Invokes, []string{WantsTool, AssignmentsTool, FindingsTool, IndexerFindingsTool, "seat:mesh-hand-acts.hand-act"}) ||
if !slices.Equal(m.Invokes, []string{WantsTool, AssignmentsTool, "seat:mesh-hand-acts.hand-act"}) ||
!slices.Equal(m.Consumes, []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-assigner.views.assignments",
"mesh-delivery-assigner.views.findings", "mesh-delivery-indexer.views.findings"}) {
"*.finding-written", "*.finding-cleared", "*.findings-stated"}) {
t.Errorf("invokes %v, consumes %v", m.Invokes, m.Consumes)
}
for _, r := range m.Reads {
if strings.HasSuffix(r, "."+seat.StateFindings) {
t.Errorf("the keeper reads %s, and hears findings as events", r)
}
}
slices.Sort(reads)
got := slices.Clone(m.Reads)
slices.Sort(got)
@@ -51,6 +51,28 @@ type player struct {
in io.Writer
next int
pending map[string]chan wire
// subscribed is each pattern the keeper asked its events for.
subscribed chan string
}
// event hands the keeper one event as the runtime does, answered once its handlers took it.
func (p *player) event(key, node string, body any) error {
p.mu.Lock()
p.next++
id := fmt.Sprintf("e-%d", p.next)
ch := make(chan wire, 1)
p.pending[id] = ch
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": map[string]any{"key": key, "node": node, "body": body}}})
select {
case m := <-ch:
if m.Error != nil {
return errors.New(m.Error.Message)
}
return nil
case <-time.After(30 * time.Second):
return fmt.Errorf("%s was not taken in 30s", key)
}
}
func (p *player) send(m map[string]any) {
@@ -197,7 +219,15 @@ func (p *player) handle(m wire) {
case "mesh/seat.publish":
p.answer(m.ID, map[string]any{"sequence": 1}, p.nc.Publish(a.Subject, a.Body))
case "mesh/subscribe":
var sub struct {
Pattern string `json:"pattern"`
}
_ = json.Unmarshal(m.Params, &sub)
p.answer(m.ID, map[string]any{}, nil)
select {
case p.subscribed <- sub.Pattern:
default:
}
default:
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
}
@@ -242,8 +272,8 @@ func (p *player) call(name string, args map[string]any) (json.RawMessage, error)
}
}
// 1952/C6: an observer's finding raises a condition on the bus, said with the controller's body, and its deletion clears
// it.
// 1952/C6, design 47 G §383: an observer's finding-written raises a condition on the bus, said with the controller's body,
// and its finding-cleared clears it.
func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
@@ -264,9 +294,6 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) {
}
buckets := []string{"mesh-controller_conditions", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_assignments",
"mesh-healers_attempts"}
for _, o := range Observers {
buckets = append(buckets, o+"_findings")
}
for _, s := range []string{ConditionsState, HistoryState, HeldState, BoundsState} {
buckets = append(buckets, Module+"_"+s)
}
@@ -288,25 +315,29 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) {
t.Fatal(err)
}
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}, subscribed: make(chan string, 32)}
go p.serve(stdout)
// The first call serves; the watches start after it.
if _, err := p.call(InputsTool, map[string]any{}); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
at := time.Now().UTC()
f := seat.FindingsByKeyValue{Key: "machine.g14.silent", Kind: "silent", Subject: seat.Subject{Scope: cc.ScopeMachine, ID: "g14",
Machine: "g14"}, Severity: cc.Warning, Summary: "g14 is silent", Headline: "g14 is offline",
Explanation: "g14 said nothing for three minutes.", Resolved: "g14 is back", Source: "S1",
Observer: watchmanModule, At: at, Evidence: []seat.Evidence{{At: at, Said: "no presence for 3m"}}}
raw, err := encodeFinding(f)
if err != nil {
t.Fatal(err)
heard := map[string]bool{}
for deadline := time.After(20 * time.Second); !(heard["*."+EventFindingWritten] && heard["*."+EventFindingCleared]); {
select {
case got := <-p.subscribed:
heard[got] = true
case <-deadline:
t.Fatalf("the keeper asked for %v, not both finding events", heard)
}
}
findings, _ := js.KeyValue(ctx, watchmanModule+"_findings")
if _, err := findings.Put(ctx, f.Key, raw); err != nil {
at := time.Now().UTC()
f := seat.Finding{Scope: cc.ScopeMachine, ID: "g14", Kind: "silent", Subject: seat.Subject{Scope: cc.ScopeMachine, ID: "g14",
Machine: "g14"}, Severity: cc.Warning, Summary: "g14 is silent", Headline: "g14 is offline",
Explanation: "g14 said nothing for three minutes.", Source: "S1", Observer: watchmanModule, At: at, Since: at,
Evidence: []seat.Evidence{{At: at, Said: "no presence for 3m"}}}
key := cc.Key(f.Scope, f.ID, f.Kind)
if err := p.event(watchmanModule+"."+EventFindingWritten, "ace", f); err != nil {
t.Fatal(err)
}
msg, err := events.NextMsg(10 * time.Second)
@@ -315,14 +346,14 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) {
}
var e Event
if err := json.Unmarshal(msg.Data, &e); err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-raised" ||
e.Key != f.Key || e.Change != cc.ChangeRaised || e.Show != cc.ShowKeeper(f.Key) || e.Headline != f.Headline {
e.Key != key || e.Change != cc.ChangeRaised || e.Show != cc.ShowKeeper(key) || e.Headline != f.Headline {
t.Fatalf("said %s on %s", msg.Data, msg.Subject)
}
listed, err := p.call(Seat+".conditions", map[string]any{})
if err != nil || !strings.Contains(string(listed), `"key":"machine.g14.silent"`) {
t.Fatalf("conditions answered %s, %v", listed, err)
}
if err := findings.Delete(ctx, f.Key); err != nil {
if err := p.event(watchmanModule+"."+EventFindingCleared, "ace", seat.FindingCleared{At: at, ID: f.ID, Kind: f.Kind, Scope: f.Scope, Why: "back"}); err != nil {
t.Fatal(err)
}
msg, err = events.NextMsg(10 * time.Second)
@@ -330,7 +361,7 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) {
t.Fatalf("no condition-cleared was said: %v", err)
}
conditions, _ := js.KeyValue(ctx, Module+"_"+ConditionsState)
if _, err := conditions.Get(ctx, f.Key); !errors.Is(err, jetstream.ErrKeyNotFound) {
if _, err := conditions.Get(ctx, key); !errors.Is(err, jetstream.ErrKeyNotFound) {
t.Errorf("the cleared condition is still kept: %v", err)
}
}
+3 -25
View File
@@ -14,42 +14,20 @@
}
],
"reads": [
"mesh-delivery-watchman.findings",
"mesh-self-check.findings",
"mesh-healers.findings",
"messenger.findings",
"mesh-provisioner.findings",
"artifact-store-tools.findings",
"mesh-config.findings",
"mesh-vault.findings",
"mesh-watcher.findings",
"minio.findings",
"nfs-server.findings",
"gitea.findings",
"mesh-build-go.findings",
"mesh-delivery-builder.findings",
"mesh-delivery-checker.findings",
"mesh-delivery-teller.findings",
"mesh-delivery-view.findings",
"mesh-enrolment.findings",
"node-tools.findings",
"claude-code.findings",
"mesh-host.findings",
"mesh-healers.attempts",
"node-engine.have"
],
"invokes": [
"mesh-delivery-assigner.assigner_read_wants",
"mesh-delivery-assigner.assigner_read_assignments",
"mesh-delivery-assigner.assigner_read_findings",
"mesh-delivery-indexer.indexer_read_findings",
"seat:mesh-hand-acts.hand-act"
],
"consumes": [
"mesh-delivery-assigner.views.wants",
"mesh-delivery-assigner.views.assignments",
"mesh-delivery-assigner.views.findings",
"mesh-delivery-indexer.views.findings"
"*.finding-written",
"*.finding-cleared",
"*.findings-stated"
],
"state": [
"conditions",