diff --git a/node-tools/cmd/node-tools/approvals.go b/node-tools/cmd/node-tools/approvals.go index 3b3b4ed..d048b7c 100644 --- a/node-tools/cmd/node-tools/approvals.go +++ b/node-tools/cmd/node-tools/approvals.go @@ -3,6 +3,11 @@ package main import ( "encoding/json" "errors" + "log" + "time" + + handacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts" + "git.novox.be/novox/mesh-sdk/go/busretry" stdio "git.novox.be/novox/mesh-sdk/go" @@ -99,3 +104,46 @@ func routerRecord(conn *bus.Conn) func(module, id string) (*stdio.StateEntry, er return &stdio.StateEntry{Key: e.Key, Value: e.Value, Revision: e.Revision, At: e.At}, nil } } + +// boundState is the runner's key-ttl state bound (lead answers A-157, A-158). +type boundState struct { + conn *bus.Conn + module string +} + +// BoundState is the runner's state its bound keys live in. +const BoundState = "bound" + +func (b boundState) CreateTTL(key string, value any, ttl time.Duration) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + _, err = b.conn.StateCreateTTL(b.module, BoundState, key, raw, ttl) + return err +} + +func (b boundState) Delete(key string) error { return b.conn.StateDelete(b.module, BoundState, key) } + +func (b boundState) Watch(pattern string, expired func(key string)) (func(), error) { + return b.conn.StateWatch(b.module, BoundState, pattern, func(c bus.StateChange) error { + if c.Op == "expire" { + expired(c.Key) + } + return nil + }) +} + +// warranted asks mesh-hand-acts.warranted for one acted ask, each try a kept call open while open says so (design 46 +// 7-approval-mark §16; lead answer A-157). +func warranted(conn *bus.Conn) func(asker, ask string, open func() (bool, error)) error { + return func(asker, ask string, open func() (bool, error)) error { + _, err := busretry.DoKept("mesh-hand-acts.warranted of "+asker+"."+ask, + busretry.Open{Key: asker + "." + ask, Open: func(string) (bool, error) { return open() }, Logf: log.Printf}, + func() (bus.Answered, error) { + return conn.Ask("seat:"+handacts.Seat+"."+handacts.VerbWarranted, handacts.WarrantedInput{Asker: asker, Ask: ask}, + "mesh.seat."+handacts.Seat+".tool."+handacts.VerbWarranted) + }, nil) + return err + } +} diff --git a/node-tools/cmd/node-tools/main.go b/node-tools/cmd/node-tools/main.go index 78e15cd..168250b 100644 --- a/node-tools/cmd/node-tools/main.go +++ b/node-tools/cmd/node-tools/main.go @@ -25,7 +25,9 @@ import ( "syscall" "time" + handacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts" "github.com/novox/mesh-tools/node-tools/internal/alive" + "github.com/novox/mesh-tools/node-tools/internal/approval" "github.com/novox/mesh-tools/node-tools/internal/bus" "github.com/novox/mesh-tools/node-tools/internal/console" @@ -77,8 +79,16 @@ func main() { defer stopViews() gate := approval.New(approval.Deps{Node: conn.Node(), Runner: cred.Module, Catalog: source.ApprovalCatalog(definitionsOf(seatList)), Wait: source.WaitCatalog, Findings: source.Findings(), Logf: log.Printf, State: approvalsState{conn: conn, module: cred.Module}, - Publish: publishAsk(conn), Record: routerRecord(conn)}) + Publish: publishAsk(conn), Record: routerRecord(conn), Warranted: warranted(conn), Bounds: boundState{conn: conn, module: cred.Module}}) opts = append(opts, runtime.WithGate(gate)) + // A run still unrecorded is asked again at each start and when the hand-act seat's holder is heard again (lead answer A-157). + if stopBounds, err := gate.WatchBounds(); err != nil { + log.Printf("[mesh-tools] the gate's bound keys are not watched (%v): an unanswered hand-act is said in the log only", err) + } else { + defer stopBounds() + } + source.OnSeatHolder(handacts.Seat, gate.Recover) + go gate.Recover() stop, err := runtime.Run(conn, served, envs, log.Printf, opts...) if err != nil { log.Fatalf("node-tools: %v", err) diff --git a/node-tools/go.mod b/node-tools/go.mod index e9db0f0..0eda7d2 100644 --- a/node-tools/go.mod +++ b/node-tools/go.mod @@ -7,6 +7,7 @@ require ( git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000 diff --git a/node-tools/internal/approval/approval_test.go b/node-tools/internal/approval/approval_test.go index 95ed780..2baddfb 100644 --- a/node-tools/internal/approval/approval_test.go +++ b/node-tools/internal/approval/approval_test.go @@ -2,6 +2,7 @@ package approval import ( "encoding/json" + "errors" "sort" "strings" "sync" @@ -12,6 +13,7 @@ import ( seat "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" catalogseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/seatdef" ) @@ -103,17 +105,49 @@ type world struct { now time.Time runs int findings *findings + bounds *bounds + // warranted is what mesh-hand-acts.warranted answers, and recorded each ask it answered for. + warranted error + recorded []string +} + +// bounds is the runner's key-ttl state bound as a fixture: expire hands a key's expiry to the watch. +type bounds struct { + keys map[string]time.Duration + expired func(string) +} + +func (b *bounds) CreateTTL(key string, _ any, ttl time.Duration) error { b.keys[key] = ttl; return nil } +func (b *bounds) Delete(key string) error { delete(b.keys, key); return nil } +func (b *bounds) Watch(_ string, expired func(string)) (func(), error) { + b.expired = expired + return func() {}, nil +} +func (b *bounds) expire(key string) { + delete(b.keys, key) + b.expired(key) } func newWorld() *world { shell := entry("src2:1", []string{"node-login-shell.execute"}, map[string]any{"needs-approval": []any{"wipe"}}) w := &world{state: newMem(), records: map[string]seat.AsksByAskerIDValue{}, takes: true, findings: &findings{}, - now: time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC), + bounds: &bounds{keys: map[string]time.Duration{}}, + now: time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC), cat: &fakeCatalog{revision: "7", byFp: map[string]catalogseat.CatalogEntry{"src2:1": shell}, newest: map[string]catalogseat.CatalogEntry{"shell": shell}, defs: map[string]seatdef.Definition{"node-login-shell": {Seat: "node-login-shell", Verbs: []seatdef.Verb{{Name: "execute", NeedsApproval: true}}}}}} - w.g = New(Deps{Node: "n1", Runner: "node-tools", Catalog: w.cat, State: w.state, Findings: w.findings, + w.g = New(Deps{Node: "n1", Runner: "node-tools", Catalog: w.cat, State: w.state, Findings: w.findings, Bounds: w.bounds, + Go: func(f func()) { f() }, + Warranted: func(asker, ask string, open func() (bool, error)) error { + if still, _ := open(); !still { + return nil + } + if w.warranted == nil { + w.recorded = append(w.recorded, asker+"."+ask) + } + return w.warranted + }, Now: func() time.Time { return w.now }, After: func(d time.Duration) <-chan time.Time { w.now = w.now.Add(d) @@ -309,7 +343,8 @@ func TestTheRefusalsAndAnswers(t *testing.T) { _, _ = w.do(call("wipe", `{"dir":"scratch"}`)) w.decide(OptionApprove, w.now.Add(time.Minute)) w.now = w.now.Add(2 * time.Minute) - if got, err := w.do(call("wipe", `{"dir":"scratch"}`)); err != nil || got != "done" || w.runs != 1 { + if got, err := w.do(call("wipe", `{"dir":"scratch"}`)); err != nil || got.(Answered).Answer != "done" || w.runs != 1 || + !strings.HasPrefix(got.(Answered).Approval, "run on the operator's approval via telegram at 2026-11-12T10:01:00Z, ask ") { t.Fatalf("%v %v, runs %d", got, err, w.runs) } _, err := w.do(call("wipe", `{"dir":"scratch"}`)) @@ -432,6 +467,34 @@ func TestTheRefusalsAndAnswers(t *testing.T) { t.Fatalf("%v, asked %d", err, len(w.asked)) } }}, + {"§16: the run is recorded once; unanswered within five minutes it is a finding, cleared when warranted answers", func(t *testing.T, w *world) { + _, _ = w.do(call("wipe", `{}`)) + a := w.decide(OptionApprove, w.now) + stop, _ := w.g.WatchBounds() + defer stop() + w.warranted = errors.New("no responders") + if _, err := w.do(call("wipe", `{}`)); err != nil || w.runs != 1 || len(w.recorded) != 0 { + t.Fatalf("%v, runs %d, recorded %v", err, w.runs, w.recorded) + } + if w.bounds.keys["n1.unanswered."+a.ID] != UnansweredAfter || w.bounds.keys["n1.unrecorded."+a.ID] != KeptFor-SaidBefore { + t.Fatalf("bounds %v", w.bounds.keys) + } + w.bounds.expire("n1.unanswered." + a.ID) + if w.findings.raised[keys.EscapeName("n1."+a.ID)] != KindCallUnanswered { + t.Fatalf("findings %v", w.findings.raised) + } + w.warranted = nil + w.g.Recover() // a start, or the hand-act seat's holder heard again + w.g.Recover() + if len(w.recorded) != 1 || w.recorded[0] != "shell."+a.ID || len(w.findings.raised) != 0 || len(w.bounds.keys) != 0 { + t.Fatalf("recorded %v, findings %v, bounds %v", w.recorded, w.findings.raised, w.bounds.keys) + } + var k operatorchannelclient.Kept + _ = json.Unmarshal(w.state.vals["n1.ask."+a.ID], &k) + if !k.Recorded || !k.Acted { + t.Fatalf("kept %+v", k) + } + }}, } { t.Run(tc.name, func(t *testing.T) { tc.run(t, newWorld()) }) } diff --git a/node-tools/internal/approval/gate.go b/node-tools/internal/approval/gate.go index a8c8498..5381ec9 100644 --- a/node-tools/internal/approval/gate.go +++ b/node-tools/internal/approval/gate.go @@ -49,6 +49,12 @@ type Deps struct { Publish func(module, subject string, body any, id string) error // Record reads the router's record of an ask, `.` in the seat's state asks, on the runner's own account (§14). Record func(module, id string) (*stdio.StateEntry, error) + // Warranted asks `mesh-hand-acts.warranted` {asker, ask}, each try a kept call open while open says so (§16, A-157). + Warranted func(asker, ask string, open func() (bool, error)) error + // Bounds is the runner's key-ttl state bound (A-157, A-158); nil keeps no bound. + Bounds Bounds + // Go runs what goes on beside the call (a run's recording); nil is a goroutine. + Go func(func()) } // Gate is the runner's approval gate: every call of a module it serves goes through it before the bundle (§20). @@ -61,6 +67,8 @@ type Gate struct { unserved map[string][]string // unmarkedAt is each caller's address it last called unmarked, so a mark added since is named (§12). unmarkedAt map[string]bool + // inFlight is each ask whose recording runs now. + inFlight map[string]bool } // New is a gate over d. @@ -74,6 +82,9 @@ func New(d Deps) *Gate { if d.After == nil { d.After = time.After } + if d.Go == nil { + d.Go = func(f func()) { go f() } + } return &Gate{d: d, registered: map[string]map[string]bool{}, unserved: map[string][]string{}} } diff --git a/node-tools/internal/approval/record.go b/node-tools/internal/approval/record.go new file mode 100644 index 0000000..a01a038 --- /dev/null +++ b/node-tools/internal/approval/record.go @@ -0,0 +1,220 @@ +package approval + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + seat "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The bounds of a run's recording (§13, §16; lead answers A-157, A-158). +const ( + UnansweredAfter = 5 * time.Minute + KeptFor = 48 * time.Hour + SaidBefore = time.Hour +) + +// Bounds is the runner's key-ttl state `bound` (lead answers A-157, A-158): a key created with its own time to live, which +// the bus removes when it passes, every watch hearing the removal as an expiry, so the gate's waits need no timer. +type Bounds interface { + CreateTTL(key string, value any, ttl time.Duration) error + Delete(key string) error + // Watch hands expired every key under the pattern whose time to live passed, until the stop is called. + Watch(pattern string, expired func(key string)) (stop func(), err error) +} + +// Answered is a call run on the operator's approval: the tool's own answer and the line naming the approval (§16), in the +// form a process holder's `gated-ended` says it (design 47 G §677; lead answer A-158). +type Answered struct { + Answer any `json:"answer"` + Approval string `json:"approval"` +} + +// The bound keys of a run (lead answer A-158; §13). +func (g *Gate) unansweredKey(id string) string { return g.d.Node + ".unanswered." + id } +func (g *Gate) unrecordedKey(id string) string { return g.d.Node + ".unrecorded." + id } + +// ran is a call run on the operator's approval: answered with the tool's answer and the approval's line, and its warrant +// recorded in the hand-act log (§16). +func (g *Gate) ran(c Call, id string, rec *seat.AsksByAskerIDValue, answer any, err error) (any, error) { + channel := "" + at := "" + if w := rec.Warrant; w != nil { + channel, at = w.Channel, w.At + } + if channel == "" { + channel = strings.Join(rec.OfferedOn, ", ") + } + line := fmt.Sprintf("run on the operator's approval via %s at %s, ask %s", orNone(channel), at, id) + g.d.Logf("[mesh-tools] %s on %s %s", c.Address, g.d.Node, line) + g.bindRecording(c.Module, id) + g.d.Go(func() { g.recordRun(c.Module, id) }) + if err != nil { + return nil, fmt.Errorf("%w (%s)", err, line) + } + return Answered{Answer: answer, Approval: line}, nil +} + +// bindRecording keeps the run's two bound keys: one that raises `call-unanswered` when `warranted` has not answered within +// five minutes, one that says an unrecorded run an hour before its key's 48 hours end (§13, §16). +func (g *Gate) bindRecording(module, id string) { + if g.d.Bounds == nil { + return + } + for key, ttl := range map[string]time.Duration{g.unansweredKey(id): UnansweredAfter, g.unrecordedKey(id): KeptFor - SaidBefore} { + if err := g.d.Bounds.CreateTTL(key, map[string]string{"asker": module, "ask": id}, ttl); err != nil { + g.d.Logf("[mesh-tools] the bound %s of the run of ask %s is not kept: %v", key, id, err) + } + } +} + +// recordRun calls `mesh-hand-acts.warranted` for one acted ask, each try a kept call open while the ask lacks `recorded` +// (§16, A-157), and on its answer writes `recorded` and clears what waited on it. One recording per ask at a time. +func (g *Gate) recordRun(module, id string) { + if !g.recording(id, true) { + return + } + defer g.recording(id, false) + if g.d.Warranted == nil { + g.d.Logf("[mesh-tools] the run of ask %s is not recorded: the tool runner reaches no hand-act log", id) + return + } + if err := g.d.Warranted(module, id, func() (bool, error) { return g.unrecorded(id) }); err != nil { + g.d.Logf("[mesh-tools] mesh-hand-acts.warranted has not answered for ask %s (%v): asked again when the hand-act seat's "+ + "holder is heard again and at the next start", id, err) + return + } + if err := g.markRecorded(id); err != nil { + g.d.Logf("[mesh-tools] ask %s is recorded in the hand-act log and its mark recorded is not kept: %v", id, err) + return + } + if g.d.Bounds != nil { + _ = g.d.Bounds.Delete(g.unansweredKey(id)) + _ = g.d.Bounds.Delete(g.unrecordedKey(id)) + } + g.finding(id, KindCallUnanswered, false, "mesh-hand-acts.warranted answered for ask "+id, "") +} + +// recording claims or releases an ask's recording: false when one runs already. +func (g *Gate) recording(id string, start bool) bool { + g.mu.Lock() + defer g.mu.Unlock() + if g.inFlight == nil { + g.inFlight = map[string]bool{} + } + if start { + if g.inFlight[id] { + return false + } + g.inFlight[id] = true + return true + } + delete(g.inFlight, id) + return true +} + +// kept is the ask the client keeps under its id, with its revision; nil when there is none. +func (g *Gate) kept(id string) (*operatorchannelclient.Kept, uint64, error) { + e, err := g.d.State.Get(g.askKey(id)) + if err != nil || e == nil { + return nil, 0, err + } + var k operatorchannelclient.Kept + if err := json.Unmarshal(e.Value, &k); err != nil { + return nil, 0, err + } + return &k, e.Revision, nil +} + +// unrecorded says the ask is acted on and its run not yet recorded: the key that keeps `warranted` open (§16). +func (g *Gate) unrecorded(id string) (bool, error) { + k, _, err := g.kept(id) + if err != nil || k == nil { + return false, err + } + return k.Acted && !k.Recorded, nil +} + +// markRecorded writes `recorded` on the kept ask by compare-and-set, again when another write came first. +func (g *Gate) markRecorded(id string) error { + for { + k, rev, err := g.kept(id) + if err != nil || k == nil { + return fmt.Errorf("the kept ask cannot be read: %v", err) + } + if k.Recorded { + return nil + } + k.Recorded = true + if _, err := g.d.State.Update(g.askKey(id), k, rev); err == nil { + return nil + } else if !isChanged(err) { + return err + } + } +} + +// Recover calls `warranted` again for every acted, unrecorded ask this machine's keys hold within their 48 hours (§16): at +// each start, and when the hand-act seat's holder is heard again (A-157). +func (g *Gate) Recover() { + if g.d.State == nil { + return + } + keys, err := g.d.State.Keys() + if err != nil { + g.d.Logf("[mesh-tools] the acted, unrecorded asks cannot be read (%v): read again at the next start or holder heard", err) + return + } + prefix := g.d.Node + ".ask." + for _, key := range keys { + if !strings.HasPrefix(key, prefix) { + continue + } + id := strings.TrimPrefix(key, prefix) + k, _, err := g.kept(id) + if err != nil || k == nil || !k.Acted || k.Recorded || g.d.Now().Sub(k.Asked) > KeptFor { + continue + } + module := moduleOf(k) + g.d.Go(func() { g.recordRun(module, id) }) + } +} + +// moduleOf is the module an ask was asked under: the warrant's asker. +func moduleOf(k *operatorchannelclient.Kept) string { + if k.Answer != nil { + return k.Answer.Asker + } + return "" +} + +// WatchBounds hears the run's bound keys expire (A-157): `call-unanswered` raised when `warranted` has not answered within +// five minutes; an unrecorded run said in the log an hour before its key's 48 hours end (§13). +func (g *Gate) WatchBounds() (func(), error) { + if g.d.Bounds == nil { + return func() {}, nil + } + return g.d.Bounds.Watch(g.d.Node+".**", func(key string) { + rest := strings.TrimPrefix(key, g.d.Node+".") + kind, id, _ := strings.Cut(rest, ".") + if open, err := g.unrecorded(id); err != nil || !open { + return + } + switch kind { + case "unanswered": + g.finding(id, KindCallUnanswered, true, fmt.Sprintf("mesh-hand-acts.warranted has not answered for ask %s, run on the "+ + "operator's approval on %s, within %s: the run is not yet in the hand-act log", id, g.d.Node, UnansweredAfter), + "a run's hand-act is unrecorded on "+g.d.Node) + case "unrecorded": + g.d.Logf("[mesh-tools] ERROR the run of ask %s on %s is still unrecorded an hour before its key's 48 hours end", id, g.d.Node) + } + }) +} + +// isChanged says a compare-and-set lost to another write. +func isChanged(err error) bool { return errors.Is(err, stdio.ErrChanged) } diff --git a/node-tools/internal/approval/run.go b/node-tools/internal/approval/run.go index b1d3bef..c828e6f 100644 --- a/node-tools/internal/approval/run.go +++ b/node-tools/internal/approval/run.go @@ -170,11 +170,6 @@ func (g *Gate) judge(c Call, act operatorchannelclient.Act, digest, id, since st return g.ran(c, id, rec, answer, ran) } -// ran is a call run on the operator's approval: the tool's own answer. -func (g *Gate) ran(_ Call, _ string, _ *seat.AsksByAskerIDValue, answer any, err error) (any, error) { - return answer, err -} - func errText(err error) string { if err == nil { return "no reason given" diff --git a/node-tools/internal/firstview/approval.go b/node-tools/internal/firstview/approval.go index 587612e..45e1b6c 100644 --- a/node-tools/internal/firstview/approval.go +++ b/node-tools/internal/firstview/approval.go @@ -3,6 +3,7 @@ package firstview import ( "context" "strconv" + "strings" "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" @@ -155,3 +156,27 @@ func waitAny(next <-chan struct{}, bound <-chan time.Time, ready []<-chan struct } return true } + +// OnSeatHolder calls f whenever the assigner's seat-holders view changes for the seat (lead answer A-157: the gate asks +// mesh-hand-acts.warranted again when the hand-act seat's holder is heard again). +func (v *Views) OnSeatHolder(seat string, f func()) { + v.mu.Lock() + defer v.mu.Unlock() + if v.onKey == nil { + v.onKey = map[string][]func(){} + } + k := "seat-holders " + seat + v.onKey[k] = append(v.onKey[k], f) +} + +// heardKey calls the hearers of a key of an owner's state just changed: a seat's holders, under the seat's own key and +// its `.` keys. +func (v *Views) heardKey(state, key string) { + seat, _, _ := strings.Cut(key, ".") + v.mu.Lock() + fs := append([]func(){}, v.onKey[state+" "+seat]...) + v.mu.Unlock() + for _, f := range fs { + go f() + } +} diff --git a/node-tools/internal/firstview/firstview.go b/node-tools/internal/firstview/firstview.go index 1251cac..8cc6eaa 100644 --- a/node-tools/internal/firstview/firstview.go +++ b/node-tools/internal/firstview/firstview.go @@ -69,6 +69,8 @@ type Views struct { deps Deps // heard is closed and made again at every event heard, so a wait for the picture wakes on events, never on a timer. heard chan struct{} + // onKey are the hearers of one key of an owner's state, by ` ` (OnSeatHolder). + onKey map[string][]func() } // FindingsState is the runner's own state the findings are written in. @@ -230,6 +232,7 @@ func (v *Views) hear(env bus.Envelope) error { if f == nil || f.Owner != ev.Owner { return nil } + defer v.heardKey(ev.State, ev.Key) if gap, why := f.Apply(ev); gap { v.logf("[mesh-tools] a gap in %s's %s (%s): its first view %s is asked once more", f.Owner, f.State, why, f.Tool) go func() { _ = f.Again(v.ctx, "a gap: "+why) }()