Merge the release branch; checks/health takes its newer sdk pin (issue 1952)

This commit is contained in:
2026-10-29 22:45:43 +01:00
88 changed files with 11880 additions and 96 deletions
+8
View File
@@ -0,0 +1,8 @@
# The calls of the controller's hand-act log a manifest of this catalogue may still invoke, each `<module> <verb> <why>`:
# the fallback of a recorder (mesh-sdk handacts.Recorder) while the seat mesh-hand-acts has no holder that answers, until
# the controller's lease is written to a holder that is no controller. The check of this directory (handacts) refuses any
# other, and an entry whose module no longer invokes it: the list only shrinks, and it is empty before the lease write
# (novox/hq ADR 0565 §11; issue 1953, issue 774).
mesh-config hand-act fallback until the lease write: its sets, clears and held values record through mesh-hand-acts first
mesh-config warranted fallback until the lease write: its warrants record through mesh-hand-acts first
mesh-healers drill fallback until the lease write: a drill records through mesh-hand-acts first
+3
View File
@@ -0,0 +1,3 @@
module git.novox.be/novox/mesh-catalog/checks/handacts
go 1.22
+109
View File
@@ -0,0 +1,109 @@
// Command handacts refuses a manifest that records into the controller's hand-act log (novox/hq ADR 0565 §11): every
// recorder records through the seat mesh-hand-acts from its delivery, so no manifest invokes
// `seat:mesh-controller.hand-act`, `.warranted` or `.drill`. The recorders' fallbacks to the controller's log while the
// seat has no holder are named in checks/handacts/fallbacks, each module and verb with why; that list only shrinks, and
// an entry whose module no longer invokes the verb is refused too.
//
// go run . <catalog root>
//
// Exits 1 on a finding.
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
)
// Verbs are the controller's hand-act verbs, as a manifest invokes them, and Allowed the list of named fallbacks.
var Verbs = []string{"hand-act", "warranted", "drill"}
const (
invokePrefix = "seat:mesh-controller."
Allowed = "checks/handacts/fallbacks"
)
// Findings is every refusal in the catalog at root, sorted.
func Findings(root string) ([]string, error) {
allowed := map[string]bool{}
raw, err := os.ReadFile(filepath.Join(root, Allowed))
if err != nil && !os.IsNotExist(err) {
return nil, err
}
for _, line := range strings.Split(string(raw), "\n") {
if f := strings.Fields(line); len(f) >= 3 && !strings.HasPrefix(f[0], "#") {
allowed[f[0]+" "+f[1]] = true
} else if len(f) > 0 && !strings.HasPrefix(f[0], "#") {
return nil, fmt.Errorf("%s: %q is not `<module> <verb> <why>`", Allowed, line)
}
}
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil {
return nil, err
}
var out []string
invoked := map[string]bool{}
for _, path := range manifests {
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var m struct {
Module string `json:"module"`
Invokes []string `json:"invokes"`
}
if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" {
continue
}
for _, inv := range m.Invokes {
verb, ok := strings.CutPrefix(strings.TrimSpace(inv), invokePrefix)
if !ok || !contains(Verbs, verb) {
continue
}
invoked[m.Module+" "+verb] = true
if !allowed[m.Module+" "+verb] {
out = append(out, fmt.Sprintf("%s invokes %s%s: record through seat:mesh-hand-acts.%s (novox/hq ADR 0565 §11), "+
"or name the fallback in %s", m.Module, invokePrefix, verb, verb, Allowed))
}
}
}
for entry := range allowed {
if !invoked[entry] {
out = append(out, fmt.Sprintf("%s names %q, which is no longer invoked: take it off the list, which only shrinks",
Allowed, entry))
}
}
sort.Strings(out)
return out, nil
}
func contains(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
func main() {
if len(os.Args) != 2 {
fmt.Fprintln(os.Stderr, "usage: handacts <catalog root>")
os.Exit(2)
}
found, err := Findings(os.Args[1])
if err != nil {
fmt.Println(err)
os.Exit(1)
}
for _, f := range found {
fmt.Println(f)
}
if len(found) > 0 {
os.Exit(1)
}
fmt.Println("no manifest records into the controller's hand-act log but the fallbacks named")
}
+59
View File
@@ -0,0 +1,59 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
// No manifest invokes the controller's hand-act verbs but a named fallback; a named fallback no longer invoked is
// refused (novox/hq ADR 0565 §11's catalog test).
func TestFindings(t *testing.T) {
for _, row := range []struct {
name string
files map[string]string
want string // empty: clean
}{
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""},
{"an unnamed call", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.warranted"]}`},
"a invokes seat:mesh-controller.warranted"},
{"a named fallback, and the controller's own", map[string]string{
"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.drill"]}`,
"modules/mesh-controller/module.json": `{"module":"mesh-controller","invokes":["seat:mesh-controller.hand-act"]}`,
Allowed: "# why\na drill fallback until the lease write\n"}, ""},
{"a fallback no longer invoked", map[string]string{Allowed: "gone hand-act fallback\n"}, `names "gone hand-act"`},
{"another verb of the controller", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.conditions"]}`}, ""},
} {
t.Run(row.name, func(t *testing.T) {
root := t.TempDir()
for path, text := range row.files {
p := filepath.Join(root, path)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := Findings(root)
if err != nil {
t.Fatal(err)
}
if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) {
t.Fatalf("findings = %q, want one containing %q", all, row.want)
}
})
}
}
// The run of this check over the real tree.
func TestTheCatalogRecordsOnlyThroughTheSeat(t *testing.T) {
got, err := Findings(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
for _, f := range got {
t.Error(f)
}
}
+3
View File
@@ -0,0 +1,3 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
# every manifest, for what it invokes.
modules/*/module.json
+1 -1
View File
@@ -2,4 +2,4 @@ module checks/health
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791
+2 -2
View File
@@ -1,2 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268 h1:+jH77IJmxUiFDdbJljhynQRY0Cc5/5jtwmMV7UUqRI4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
@@ -0,0 +1,218 @@
package main
// **Built dark, compared, then switched by the operator** (novox/hq ADR 0564 clause 9; issue 1952 C5, C8): `compare`
// lists the open keys of the keeper and of the controller, each difference with its reason; the kinds that end with the
// controller, the kinds only the keeper raises and the keys held while their module is delivered are listed apart and
// not counted. `take-over` raises an approve ask under the keeper's own name (ADR 0259), refused while compare shows a
// difference, and on the warrant writes the take-over key naming the keeper. `hand-back` writes it back to the
// controller until the lease write, and is refused after it.
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Difference is one key the two sides do not agree on, or one listed apart.
type Difference struct {
Key string `json:"key"`
Kind string `json:"kind"`
Reason string `json:"reason"`
}
// Comparison is compare's answer.
type Comparison struct {
Differences []Difference `json:"differences"`
// Apart are the keys not counted: of kinds that end with the controller, raised only by the keeper, or held.
Apart []Difference `json:"apart"`
Same int `json:"same"`
At time.Time `json:"at"`
}
// Empty says nothing differs.
func (c Comparison) Empty() bool { return len(c.Differences) == 0 }
// Compare is the keeper's open conditions against the controller's.
func (k *Keeper) Compare() (Comparison, error) {
mine, err := k.Open()
if err != nil {
return Comparison{}, err
}
k.mu.Lock()
k.init()
theirs := map[string]health.Condition{}
for key, c := range k.ctrl {
theirs[key] = c
}
heldNow := map[string]held{}
for key, h := range k.held {
heldNow[key] = h
}
k.mu.Unlock()
out := Comparison{Differences: []Difference{}, Apart: []Difference{}, At: k.Now().UTC()}
open := map[string]health.Condition{}
for _, c := range mine {
open[c.Key] = c
}
keys := map[string]bool{}
for key := range open {
keys[key] = true
}
for key := range theirs {
keys[key] = true
}
for key := range heldNow {
keys[key] = true
}
sorted := make([]string, 0, len(keys))
for key := range keys {
sorted = append(sorted, key)
}
sort.Strings(sorted)
for _, key := range sorted {
c, inMine := open[key]
t, inTheirs := theirs[key]
kind := c.Kind
if !inMine {
kind = t.Kind
}
home, homed := k.homeOf(kind)
if h, ok := heldNow[key]; ok {
out.Apart = append(out.Apart, Difference{Key: key, Kind: kind,
Reason: fmt.Sprintf("held while %s is delivered on %s (version %s)", h.Module, h.Node, short(h.Version))})
continue
}
switch {
case homed && (home.Carried || home.Replaced):
why := "ends with the controller: carried from its bucket until the lease write"
if home.Replaced {
why = "ends with the controller: its fault is observed otherwise"
}
out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: why})
case inMine && !inTheirs && (!homed || !home.Controller):
out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: "keeper only: the controller never raises this kind"})
case inMine && !inTheirs:
out.Differences = append(out.Differences, Difference{Key: key, Kind: kind,
Reason: "open at the keeper, not at the controller"})
case !inMine && !homed:
out.Differences = append(out.Differences, Difference{Key: key, Kind: kind,
Reason: fmt.Sprintf("open at the controller; no home is decided for the kind %s (ADR 0564's gap)", kind)})
case !inMine:
out.Differences = append(out.Differences, Difference{Key: key, Kind: kind,
Reason: fmt.Sprintf("open at the controller; %s has not written its finding", home.Observer)})
case c.Severity != t.Severity:
out.Differences = append(out.Differences, Difference{Key: key, Kind: kind,
Reason: fmt.Sprintf("%s at the keeper, %s at the controller", c.Severity, t.Severity)})
default:
out.Same++
}
}
return out, nil
}
// Asks is the operator channel as the keeper asks it: an approve ask under its own name, and where one stands.
type Asks interface {
Ask(a asks.Ask) error
}
// TakeOver raises the approve ask of a switch to the keeper, refused while compare differs or after the lease write.
func (k *Keeper) TakeOver(a Asks, by, why string) (map[string]any, error) {
if strings.TrimSpace(why) == "" {
return nil, errors.New("a take-over says why")
}
if k.LeaseAway() {
return nil, errors.New("the controller's lease is handed away: the keeper is the one source already")
}
cmp, err := k.Compare()
if err != nil {
return nil, err
}
if !cmp.Empty() {
return nil, fmt.Errorf("refused: compare shows %d difference(s), the first %s (%s); the take-over waits for none",
len(cmp.Differences), cmp.Differences[0].Key, cmp.Differences[0].Reason)
}
now := k.Now().UTC()
id := fmt.Sprintf("take-over-%d", now.UnixNano())
act := asks.Act{"verb": Seat + ".take-over", "module": Module, "ask": id}
bind, err := asks.ActDigest(act)
if err != nil {
return nil, err
}
ask := asks.Ask{ID: id, Headline: "Switch the mesh's alarms to the new keeper?", Who: asks.Operator,
Explanation: fmt.Sprintf("The new condition keeper holds the same %d open alarms as the controller. Your yes makes "+
"the alarm messages come from the keeper, with the same words. It can be handed back until the controller "+
"gives up its lease.", cmp.Same),
Options: []asks.Option{{ID: "yes", Label: "Yes, switch", Does: "Alarms come from the keeper.", Level: asks.Approve, Binds: bind},
{ID: "no", Label: "No, leave it", Does: "Alarms keep coming from the controller.", Level: asks.Acknowledge}},
Expires: now.Add(asks.ApproveLasts - time.Minute), OnExpiry: "Alarms keep coming from the controller.",
About: Module + "-take-over", Details: "Asked by " + by + ": " + why}
if err := a.Ask(ask); err != nil {
return nil, err
}
return map[string]any{"asked": id, "same": cmp.Same, "apart": len(cmp.Apart)}, nil
}
// Warranted is the act on the operator's answer: a yes, compare still empty, writes the take-over key naming the keeper.
func (k *Keeper) Warranted(o asks.Option, w asks.Warrant) error {
if o.ID != "yes" {
k.Logf("the take-over %s was answered %s: alarms stay with the controller", w.Ask, o.ID)
return nil
}
cmp, err := k.Compare()
if err != nil {
return err
}
if !cmp.Empty() {
return fmt.Errorf("the take-over %s was approved, and compare now shows %d difference(s): not switched", w.Ask,
len(cmp.Differences))
}
return k.writeTakeover(health.Takeover{Follow: health.FollowKeeper, At: k.Now().UTC(), By: w.Says(), Ask: w.Ask})
}
// HandBack switches the readers back to the controller, until the lease write.
func (k *Keeper) HandBack(by, why string) (map[string]any, error) {
if strings.TrimSpace(why) == "" {
return nil, errors.New("a hand-back says why")
}
if k.LeaseAway() {
return nil, errors.New("refused: the controller's lease is handed away, so there is no controller to hand back to")
}
t := health.Takeover{Follow: health.FollowController, At: k.Now().UTC(), By: by, Why: why}
if err := k.writeTakeover(t); err != nil {
return nil, err
}
return map[string]any{"follow": t.Follow, "at": t.At}, nil
}
// writeTakeover writes the take-over key, and keeps it in history as the record of the switch.
func (k *Keeper) writeTakeover(t health.Takeover) error {
raw, err := health.EncodeTakeover(t)
if err != nil {
return err
}
_, rev, found, err := k.Store.Get(TakeoverState, health.TakeoverKey)
if err != nil {
return err
}
if found {
_, err = k.Store.Update(TakeoverState, health.TakeoverKey, json.RawMessage(raw), rev)
} else {
_, err = k.Store.Create(TakeoverState, health.TakeoverKey, json.RawMessage(raw))
}
if errors.Is(err, stdio.ErrChanged) {
return fmt.Errorf("the take-over key moved under this write: read it again and ask again")
}
if err != nil {
return err
}
k.Takeover(t)
k.Logf("the readers of conditions follow the %s from now (%s)", t.Follow, t.By)
return nil
}
@@ -0,0 +1,340 @@
package main
// **A module being delivered raises no condition until its delivery ends** (novox/hq ADR 0564 clause 13 and Limits 12;
// issue 1952 C7, C8). The keeper reads the assigner's wants and the nodes' haves, and judges a module's delivery on a
// node with mesh-sdk's delivery.MotionOf, as the view does. While it is delivered there, a finding about it there, or
// about what it serves, and a condition carried from the controller's bucket on those subjects, is kept in the state
// `held` and raises nothing. When the delivery ends, healthy at the want or failed, what still stands is raised once,
// saying it happened during the delivery; what went is dropped with a log line. The hold lasts at most the delivery's
// bound, a key of the state `bounds` whose time to live is the bound: its expiry raises what is held, saying the delivery
// did not end. A finding about another module or about the machine itself is never held. A condition already open is
// not held: it is no news.
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// The modules whose delivery holds what they serve beyond their own findings (Limits 12).
const (
controllerModule = "mesh-controller"
runtimeModule = "node-tools"
)
// controllerServes are the sources of what the controller serves while it is delivered: its listening, its lease, its
// writes and its facts (ADR 0505 §2 (a)), matched to mesh-controller whatever its key names.
var controllerServes = map[string]bool{"S4": true, "S7": true, "S12": true, "S13": true, "S14": true, "S25": true}
// about is a node and a module a finding or a condition is about, for the hold.
type about struct{ Node, Module string }
// aboutOf are the modules on nodes a condition's subject is about (Limits 12): a module on a machine, a provider on its
// node, a seat of the same name as its module on a machine, the controller's own work wherever mesh-controller is
// delivered, and every module's tools on a machine whose tool runner is delivered. Never the machine itself.
func (k *Keeper) aboutOf(kind, source string, s health.Subject) []about {
var out []about
parts := strings.Split(s.ID, ".")
switch s.Scope {
case health.ScopeModule:
if len(parts) >= 2 {
out = append(out, about{Node: parts[len(parts)-1], Module: strings.Join(parts[:len(parts)-1], ".")})
}
case health.ScopeProvider:
if len(parts) >= 2 {
out = append(out, about{Node: parts[1], Module: parts[0]})
}
case health.ScopeSeat:
// The module the placement says holds the seat on that node, or the seat's own name where none is read.
if len(parts) >= 2 {
node, seat := parts[len(parts)-1], strings.Join(parts[:len(parts)-1], ".")
module := k.holders[node+"."+seat]
if module == "" {
module = seat
}
out = append(out, about{Node: node, Module: module})
}
case health.ScopeMachine:
// The machine itself (presence, network, resolver, disk) is never held; its tools are, while its tool runner is
// delivered.
if source == "S11" || kind == "tools-silent" {
out = append(out, about{Node: s.Machine, Module: runtimeModule})
if s.Machine == "" && len(parts) > 0 {
out[len(out)-1].Node = parts[0]
}
}
return out
}
if controllerServes[source] || (s.Scope == health.ScopeCore && strings.HasPrefix(s.ID, "controller")) {
for hk := range k.wants {
if node, module := splitNodeModule(hk); module == controllerModule {
out = append(out, about{Node: node, Module: module})
}
}
}
if s.Machine != "" && s.Scope != health.ScopeMachine {
out = append(out, about{Node: s.Machine, Module: runtimeModule})
}
return out
}
func splitNodeModule(hk string) (string, string) {
node, module, _ := strings.Cut(hk, ".")
return node, module
}
// motion is the delivery of module on node, from its want and have.
func (k *Keeper) motion(node, module string) (delivery.Motion, delivery.Want) {
w, wanted := k.wants[node+"."+module]
h, has := k.haves[node+"."+module]
return delivery.MotionOf(w, wanted, h, has, "", false), w
}
// holdFor says whether key is held now, and as what. Called with mu held.
func (k *Keeper) holdFor(key string, f *health.Finding, carried *health.Condition, open bool) (held, bool) {
if open || (f == nil && carried == nil) {
return held{}, false
}
var kind, source string
var subject health.Subject
if f != nil {
kind, source, subject = f.Kind, f.Source, f.Subject
} else {
kind, source, subject = carried.Kind, carried.Source, carried.Subject
}
if _, raisedOut := k.outOfHold[key]; raisedOut {
return held{}, false
}
for _, a := range k.aboutOf(kind, source, subject) {
m, w := k.motion(a.Node, a.Module)
if !m.Delivering || m.Ended() || k.passed[a.Node+"."+a.Module] == w.FP {
continue
}
since := k.Now().UTC()
if was, ok := k.held[key]; ok {
since = was.Since
}
return held{Key: key, Node: a.Node, Module: a.Module, Since: since, Version: w.FP}, true
}
return held{}, false
}
// keepHeld keeps a held key in the state `held`, and starts its delivery's bound once.
func (k *Keeper) keepHeld(h held) error {
k.mu.Lock()
_, was := k.held[h.Key]
k.held[h.Key] = h
bound := k.bound(h.Module)
k.mu.Unlock()
if was {
return nil
}
if _, err := k.Store.Create(HeldState, h.Key, h); err != nil && !errors.Is(err, stdio.ErrChanged) {
return fmt.Errorf("keeping %s held: %w", h.Key, err)
}
b := boundKey(h.Node, h.Module)
err := k.Store.CreateTTL(BoundsState, b, map[string]any{"node": h.Node, "module": h.Module, "version": h.Version,
"bound-seconds": int(bound / time.Second)}, bound)
if err != nil && !errors.Is(err, stdio.ErrChanged) {
return fmt.Errorf("starting the bound of %s's delivery on %s: %w", h.Module, h.Node, err)
}
k.Logf("held %s while %s is delivered on %s (version %s), at most %s", h.Key, h.Module, h.Node, h.Version, bound)
return nil
}
// bound is a module's delivery bound: its manifest names none yet, so the default (clause 13 (d)). Each key of the state
// `bounds` is written with it as its own time to live, and its expiry is the bound passing.
func (k *Keeper) bound(string) time.Duration {
if k.Bound > 0 {
return k.Bound
}
return delivery.DefaultDeliveryBound
}
func boundKey(node, module string) string { return node + "." + module }
// dropHeld lets a held key go: raised, or gone with why said.
func (k *Keeper) dropHeld(key, why string) {
k.mu.Lock()
h, ok := k.held[key]
delete(k.held, key)
k.mu.Unlock()
if !ok {
return
}
if err := k.Store.Delete(HeldState, key); err != nil {
k.Logf("the held %s could not be removed from held: %v", key, err)
}
if why != "" {
k.Logf("dropped %s, held while %s was delivered on %s: %s", key, h.Module, h.Node, why)
}
}
// Want takes one change of the assigner's wants; Have one of the nodes' haves. Each may end a delivery, which raises or
// drops what it held.
func (k *Keeper) Want(key string, value []byte, deleted bool) error {
node, module, err := delivery.ParseWantKey(key)
if err != nil {
return nil // not a want of a module on a node (a held fingerprint's key)
}
k.mu.Lock()
k.init()
if deleted {
delete(k.wants, node+"."+module)
} else if w, err := delivery.DecodeWant(value); err == nil {
k.wants[node+"."+module] = w
} else {
k.Logf("the want %s is not readable: %v", key, err)
}
k.mu.Unlock()
return k.judge(node, module, "")
}
// Have takes one change of a node's have.
func (k *Keeper) Have(key string, value []byte, deleted bool) error {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode {
return nil
}
k.mu.Lock()
k.init()
if deleted {
delete(k.haves, node+"."+module)
} else if h, err := delivery.DecodeHave(value); err == nil {
k.haves[node+"."+module] = h
} else {
k.Logf("the have %s is not readable: %v", key, err)
}
k.mu.Unlock()
return k.judge(node, module, "")
}
// Placement takes one change of the placement: which module holds which seat on a node.
func (k *Keeper) Placement(key string, value []byte, deleted bool) {
node, err := delivery.ParsePlacementKey(key)
if err != nil {
return
}
k.mu.Lock()
defer k.mu.Unlock()
k.init()
for hk := range k.holders {
if strings.HasPrefix(hk, node+".") {
delete(k.holders, hk)
}
}
if deleted {
return
}
p, err := delivery.DecodePlacement(value)
if err != nil {
k.Logf("the placement of %s is not readable, so its seats' holders are not known: %v", node, err)
return
}
for _, h := range p.Holds {
k.holders[node+"."+h.Claim] = h.Module
}
}
// BoundPassed is a delivery's bound key expiring: what it holds is raised, saying the delivery did not end.
func (k *Keeper) BoundPassed(key string) error {
node, module := splitNodeModule(key)
k.mu.Lock()
k.init()
_, w := k.motion(node, module)
holding := false
for _, h := range k.held {
holding = holding || (h.Node == node && h.Module == module)
}
if !holding {
k.mu.Unlock()
return nil
}
k.passed[key] = w.FP
k.mu.Unlock()
return k.judge(node, module, fmt.Sprintf("Its delivery of version %s on %s did not end within %s.", short(w.FP), node,
k.bound(module)))
}
// judge raises or drops what node and module hold when their delivery ended, or with line past its bound.
func (k *Keeper) judge(node, module, line string) error {
k.mu.Lock()
m, w := k.motion(node, module)
var keys []string
for key, h := range k.held {
if h.Node == node && h.Module == module {
keys = append(keys, key)
}
}
if line == "" && !m.Ended() && m.Delivering {
k.mu.Unlock()
return nil
}
if line == "" {
line = fmt.Sprintf("It happened during the delivery of version %s of %s on %s.", short(w.FP), module, node)
if m.Failed {
line = fmt.Sprintf("It happened as the delivery of version %s of %s on %s failed.", short(w.FP), module, node)
}
delete(k.passed, node+"."+module)
}
for _, key := range keys {
k.outOfHold[key] = line
}
k.mu.Unlock()
sort.Strings(keys)
var problems []string
for _, key := range keys {
k.dropHeld(key, "")
if err := k.Settle(key); err != nil {
problems = append(problems, err.Error())
}
k.mu.Lock()
delete(k.outOfHold, key)
k.mu.Unlock()
}
if len(problems) > 0 {
return errors.New(strings.Join(problems, "; "))
}
return nil
}
func short(fp string) string {
if _, rest, ok := strings.Cut(fp, ":"); ok {
fp = rest
}
if len(fp) > 12 {
return fp[:12]
}
return fp
}
// Held is every key held now, for the read tool and compare.
func (k *Keeper) Held() []held {
k.mu.Lock()
defer k.mu.Unlock()
out := make([]held, 0, len(k.held))
for _, h := range k.held {
out = append(out, h)
}
sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
return out
}
// restoreHeld takes the state `held` as read at the start.
func (k *Keeper) restoreHeld(key string, value []byte) {
var h held
if json.Unmarshal(value, &h) != nil {
return
}
k.mu.Lock()
k.init()
k.held[key] = h
k.mu.Unlock()
}
@@ -0,0 +1,129 @@
package main
import (
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// The fingerprints and composition of mesh-controller on novox, copied from its have (mesh-delivery-view.node_haves,
// 2026-10-29): the delivery replayed moves it from the running fingerprint to the next, through the node-engine's phases
// (delivery.go Phase*). No tool reads a have's or a want's history yet, so the sequence is the node-engine's documented
// order, not a capture; the values are captured.
const (
ctrlRunning = "src2:fcaa7d7ccb35ea2beed5b9e4b87f4c24c99452244d49f4786d4022e5c543f39c"
ctrlComposed = "sha256:e9a4de75c9ca70a32cef3137cb0e6692b0f9558b699013caaee399d88dd1bac0"
ctrlNext = "src2:0d1e2f3a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9001a2b3c4d5e6"
ctrlNextComp = "sha256:1f2e3d4c5b6a79881726354453627180918a7b6c5d4e3f2a1b0c9d8e7f605142"
)
// ADR 0564 clause 13 and Limits 12 (issue 1952 C7, C8): a delivery of mesh-controller on novox replayed want by have.
// Its own findings and the carried controller-deaf raise nothing during it and none after when they went; one still
// standing at the end is raised once with its during-the-delivery line; one past the bound with its did-not-end line; a
// failed delivery raises what stands at once; another module's and the machine's own are raised at once; compare lists a
// held key as held.
func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) {
want := func(fp, composed string) []byte {
raw, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp, Composed: composed})
return raw
}
have := func(fp, composed string, phase delivery.Phase, h delivery.Health) []byte {
raw, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp, Composed: composed, Phase: phase, Health: h})
return raw
}
deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4",
Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self"})
for _, c := range []struct {
name string
// end is the have the delivery ends with, or nil for a delivery past its bound.
end []byte
standing bool
said string
line string
}{
{name: "gone by its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK)},
{name: "still standing at its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK),
standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "during the delivery of version 0d1e2f3a4b5c"},
{name: "past its bound", standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "did not end within"},
{name: "failed", end: have(ctrlNext, ctrlNextComp, delivery.PhaseFailed, delivery.HealthFailed), standing: true,
said: "raised module.mesh-controller.novox.unhealthy", line: "as the delivery of version 0d1e2f3a4b5c"},
} {
r := newRig()
_ = r.k.Want("want.novox.mesh-controller", want(ctrlRunning, ctrlComposed), false)
_ = r.k.Have("novox.mesh-controller", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false)
// The delivery starts: the assigner wants the next fingerprint; the node applies it.
_ = r.k.Want("want.novox.mesh-controller", want(ctrlNext, ctrlNextComp), false)
_ = r.k.Have("novox.mesh-controller", have(ctrlNext, ctrlNextComp, delivery.PhaseApplying, delivery.HealthUnknown), false)
own, ownRaw := finding(t, watchman, health.ScopeModule, "mesh-controller.novox", "unhealthy", "health", health.Warning, r.now)
other, otherRaw := finding(t, watchman, health.ScopeModule, "postgres.novox", "unhealthy", "health", health.Warning, r.now)
machine, machineRaw := finding(t, watchman, health.ScopeMachine, "novox", "silent", "S1", health.Urgent, r.now)
for _, f := range []struct {
key string
raw []byte
}{{own, ownRaw}, {other, otherRaw}, {machine, machineRaw}} {
if err := r.k.Finding(watchman, f.key, f.raw, false); err != nil {
t.Fatal(err)
}
}
if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil {
t.Fatal(err)
}
// The delivery's bound is its own key's time to live, the module's bound (the default where it names none).
if r.st.ttls["bounds/novox.mesh-controller"] != delivery.DefaultDeliveryBound {
t.Errorf("%s: the bound key lives %s", c.name, r.st.ttls["bounds/novox.mesh-controller"])
}
if got := r.changes(); got != "raised "+other+", raised "+machine {
t.Errorf("%s: during the delivery said %q: another module's and the machine's at once, nothing of its own", c.name, got)
}
// C8: compare lists the held keys as held, never as a difference.
cmp, _ := r.k.Compare()
held := 0
for _, d := range cmp.Apart {
if strings.HasPrefix(d.Reason, "held while mesh-controller is delivered on novox") {
held++
}
}
if held != 2 {
t.Errorf("%s: compare lists %d held, %+v", c.name, held, cmp)
}
// The carried condition about the controller goes within the delivery; its own finding goes unless it stands.
_ = r.k.Controller("core.controller.deaf", nil, true)
if !c.standing {
_ = r.k.Finding(watchman, own, nil, true)
}
if c.end != nil {
_ = r.k.Have("novox.mesh-controller", c.end, false)
} else {
r.now = r.now.Add(delivery.DefaultDeliveryBound)
_ = r.k.BoundPassed("novox.mesh-controller")
}
if got := r.changes(); got != c.said {
t.Errorf("%s: at its end said %q, want %q", c.name, got, c.said)
}
if co, open := r.open(own); c.standing != open || (open && !strings.Contains(co.Explanation, c.line)) {
t.Errorf("%s: its own condition is %v: %q", c.name, open, co.Explanation)
}
if _, open := r.open("core.controller.deaf"); open || len(r.k.Held()) != 0 {
t.Errorf("%s: the carried deaf condition was raised, or something is still held: %+v", c.name, r.k.Held())
}
}
// A seat's finding is about the module the placement says holds the seat on that node (the lead, 2026-10-29): held
// while the assigner is delivered on ace, though the seat is named mesh-placement.
r := newRig()
placed, err := delivery.EncodePlacement(delivery.Placement{Node: "ace", Modules: []delivery.PlacedModule{{Module: "mesh-delivery-assigner"}},
Holds: []delivery.Holding{{Claim: "mesh-placement", Scope: "mesh", Module: "mesh-delivery-assigner"}}})
if err != nil {
t.Fatal(err)
}
r.k.Placement("placement.ace", placed, false)
_ = r.k.Want("want.ace.mesh-delivery-assigner", want(ctrlNext, ctrlNextComp), false)
_ = r.k.Have("ace.mesh-delivery-assigner", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false)
seat, seatRaw := finding(t, selfCheck, health.ScopeSeat, "mesh-placement.ace", "holder-silent", "D3", health.Warning, r.now)
_ = r.k.Finding(selfCheck, seat, seatRaw, false)
if got := r.changes(); got != "" || len(r.k.Held()) != 1 || r.k.Held()[0].Module != "mesh-delivery-assigner" {
t.Errorf("a seat's finding while its holder is delivered: said %q, held %+v", got, r.k.Held())
}
}
@@ -0,0 +1,73 @@
package main
// **Where each condition kind lives after the take-over** (novox/hq ADR 0564 clauses 4, 6, 7 and 9, the Context's probe
// table; issue 1952 C3, C5): data, so a kind's home is a row and never code. `compare` reads it to tell a difference from
// a kind that ends with the controller or that the controller never raises, and the keeper reads it to carry, until the
// lease write, the controller's conditions of the kinds that end with it.
//
// The rows below are what ADR 0564 decides, by the controller's source of each kind as the live controller says it
// (mesh-controller.conditions, read 2026-10-29). **About twenty-five kinds the controller raises have no home in the
// record** (L4.md, "Missing decision"): they have no row here on purpose, and `compare` names each as "no home decided"
// until the lead's decision adds its row.
// Home is where one condition kind lives.
type Home struct {
// Source is the controller's row or probe that raises it today, for whoever reads the table.
Source string
// Controller says the controller raises this kind today: a kind it never raises is the keeper's alone and is no
// difference in compare.
Controller bool
// Observer is the module whose findings raise it after the take-over; empty for a kind that ends.
Observer string
// Carried says it ends with the controller's lease and is carried from the controller's bucket until then
// (clause 9: S4, S7, S12, S13, S14, S25, D5, D9, H-controller).
Carried bool
// Replaced says the kind itself ends and another observes its fault (D1, D10, the stuck probe): listed apart in
// compare and not carried.
Replaced bool
}
// The kinds' homes, by kind.
var Homes = map[string]Home{
// The watchman's watchdogs (clause 4).
"silent": {Source: "S1", Controller: true, Observer: watchman},
"max-deliveries": {Source: "S9", Controller: true, Observer: watchman},
"slow-consumer": {Source: "S9", Controller: true, Observer: watchman},
"tools-silent": {Source: "S11", Controller: true, Observer: watchman},
"apply-failing": {Source: "S21", Controller: true, Observer: watchman},
"module-unhealthy": {Source: "health", Controller: true, Observer: watchman},
"stuck": {Source: "DM", Controller: true, Observer: watchman},
// The self-check's probes (clause 6, the Context's table).
"holder-silent": {Source: "D3", Controller: true, Observer: selfCheck},
"archives-missing": {Source: "D4", Controller: true, Observer: selfCheck},
"consumer-wrong": {Source: "D6", Controller: true, Observer: selfCheck},
"data-unmeasured": {Source: "D13", Controller: true, Observer: selfCheck},
"agent-can-become-root": {Source: "DA", Controller: true, Observer: selfCheck},
"root-not-free": {Source: "D-root", Controller: true, Observer: selfCheck},
"probe-failed": {Source: "doctor", Controller: true, Observer: selfCheck},
// The healers' own (clause 7, Limits 2).
"healer-wanted": {Source: "S15", Controller: true, Observer: healers},
"healers-braked": {Source: "healers", Controller: true, Observer: healers},
// The kinds that end with the controller and are carried until its lease write (clauses 4, 9).
"controller-deaf": {Source: "S4", Controller: true, Carried: true},
"lease-lost": {Source: "S12", Controller: true, Carried: true},
// The kinds that end, their faults observed otherwise (the Context's table).
"core-behind": {Source: "D10", Controller: true, Replaced: true},
}
// The observers, as the table names them.
const (
watchman = "mesh-delivery-watchman"
selfCheck = "mesh-self-check"
healers = "mesh-healers"
)
// homeOf is the home of a condition's kind, and whether the table has one.
func (k *Keeper) homeOf(kind string) (Home, bool) {
homes := k.Homes
if homes == nil {
homes = Homes
}
h, ok := homes[kind]
return h, ok
}
@@ -0,0 +1,665 @@
package main
// The condition keeper (novox/hq ADR 0564 clauses 2, 3, 7, 8, 9 and 13; issue 1952 C2, C3, C7): the one writer of the
// mesh's open conditions. It watches every observer's `findings`, the healers' `attempts`, the controller's lease and,
// until the lease write, the controller's `conditions`, and the assigner's wants and the nodes' haves for the hold. From
// all of them it derives, per key, the condition that should be open, and writes the difference by compare-and-set,
// saying each transition as the controller says it (go/health) and keeping it in `history`. Nothing clears a condition
// but its finding going (ADR 0231 §14).
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// The module, its seat, and the states it keeps.
const (
Module = health.KeeperModule
Seat = "mesh-conditions"
ConditionsState = health.ConditionsState
TakeoverState = health.TakeoverState
HistoryState = "history"
HeldState = "held"
BoundsState = "bounds"
// AskedState keeps the keeper's asks of the operator (go/asker).
AskedState = "asked"
)
// HistoryKeptFor is how long a transition is kept (clause 3).
const HistoryKeptFor = 90 * 24 * time.Hour
// Observers are the modules whose `findings` the keeper reads (clause 2; ADR 0565 §8 for the assigner's).
var Observers = []string{health.WatchmanModule, health.SelfCheckModule, health.HealersModule, delivery.AssignerModule}
// The other states it reads, as `<module>.<name>`.
const (
AttemptsRead = health.HealersModule + "." + health.AttemptsState
ControllerConditions = "mesh-controller.conditions"
ControllerLease = "mesh-controller.lease"
ControllerLeaseKey = "holder"
WantsRead = delivery.AssignerModule + ".wants"
// PlacementRead is the mesh-placement seat's placement, read for which module holds a seat on a node (hold.go; the
// lead, 2026-10-29, for ADR 0564 clause 2's inputs).
PlacementRead = delivery.PlacementRead
HavesRead = "node-engine.have"
ConditionWrites = 8
)
// Store is the bus as the keeper uses it: its own states, written by compare-and-set, and a key with its own time to
// live for a hold's bound.
type Store interface {
Get(state, key string) (value []byte, revision uint64, found bool, err error)
Create(state, key string, value any) (uint64, error)
Update(state, key string, value any, revision uint64) (uint64, error)
Delete(state, key string) error
Keys(state string) ([]string, error)
CreateTTL(state, key string, value any, ttl time.Duration) error
}
// held is a finding or a carried condition kept while its module is delivered (clause 13 (b)).
type held struct {
Key string `json:"key"`
Node string `json:"node"`
Module string `json:"module"`
Since time.Time `json:"since"`
Version string `json:"version"`
Wanted json.RawMessage `json:"-"`
}
// clearedMemo is what a cleared condition leaves for ReopenWithin: the same fault again is a reopening.
type clearedMemo struct {
at time.Time
count int
began time.Time
gaps []health.Gap
silenced *health.Silence
tried []health.Tried
}
// Keeper is the condition keeper's state and its writes.
type Keeper struct {
Store Store
// Say publishes one event on the seat; History keeps one transition.
Say func(e health.Event) error
Now func() time.Time
Logf func(format string, a ...any)
// Homes says, by kind, where each controller kind lives after the take-over (homes.go); Homes when nil.
Homes map[string]Home
// Bound is a delivery's hold bound; delivery.DefaultDeliveryBound when zero (hold.go).
Bound time.Duration
mu sync.Mutex
findings map[string]map[string]health.Finding
blind map[string]string
attempts map[string]health.Attempt
ctrl map[string]health.Condition
leaseOut bool
takeover health.Takeover
wants map[string]delivery.Want
haves map[string]delivery.Have
// holders are, by `<node>.<seat>`, the module the placement says holds the seat there.
holders map[string]string
stuck map[string]string
held map[string]held
// passed are the deliveries whose bound passed, by node and module, at the fingerprint wanted then; outOfHold the
// line a key raised out of a hold says (hold.go).
passed map[string]string
outOfHold map[string]string
cleared map[string]clearedMemo
seq uint64
}
func (k *Keeper) init() {
if k.findings == nil {
k.findings, k.blind, k.attempts, k.ctrl = map[string]map[string]health.Finding{}, map[string]string{},
map[string]health.Attempt{}, map[string]health.Condition{}
k.wants, k.haves, k.stuck, k.held, k.cleared = map[string]delivery.Want{}, map[string]delivery.Have{},
map[string]string{}, map[string]held{}, map[string]clearedMemo{}
k.takeover = health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController}
k.passed, k.outOfHold, k.holders = map[string]string{}, map[string]string{}, map[string]string{}
}
if k.Now == nil {
k.Now = time.Now
}
if k.Logf == nil {
k.Logf = func(string, ...any) {}
}
}
// Finding takes one change of an observer's finding: a key put raises or changes its condition, a key deleted clears
// it (clause 2). A value that is not a finding is the observer's fault, said and left as it was.
func (k *Keeper) Finding(observer, key string, value []byte, deleted bool) error {
k.mu.Lock()
k.init()
if k.findings[observer] == nil {
k.findings[observer] = map[string]health.Finding{}
}
if deleted {
delete(k.findings[observer], key)
} else {
f, err := health.DecodeFinding(value)
if err != nil {
k.mu.Unlock()
k.Logf("%s's finding %s is not one the keeper can raise, so it is left as it stood: %v", observer, key, err)
return nil
}
if f.Key != key {
k.mu.Unlock()
k.Logf("%s's finding under %s names the key %s, so it is left as it stood", observer, key, f.Key)
return nil
}
k.findings[observer][key] = f
}
k.mu.Unlock()
return k.Settle(key)
}
// Blind says an observer's findings cannot be read: its keys stay as they were, and nothing is cleared for not seeing
// (ADR 0227 §4). Empty why says it is read again.
func (k *Keeper) Blind(observer, why string) {
k.mu.Lock()
defer k.mu.Unlock()
k.init()
if why == "" {
delete(k.blind, observer)
} else {
k.blind[observer] = why
}
}
// Attempt takes one change of the healers' `attempts`: the condition it names gets its tried, resolver and escalation
// (clause 7).
func (k *Keeper) Attempt(key string, value []byte, deleted bool) error {
k.mu.Lock()
k.init()
var condition string
if old, ok := k.attempts[key]; ok {
condition = old.Condition
}
if deleted {
delete(k.attempts, key)
} else {
var a health.Attempt
if err := json.Unmarshal(value, &a); err != nil || a.Condition == "" {
k.mu.Unlock()
k.Logf("the healers' attempt %s is not readable, so it is left out: %v", key, err)
return nil
}
k.attempts[key], condition = a, a.Condition
}
k.mu.Unlock()
if condition == "" {
return nil
}
return k.Settle(condition)
}
// Controller takes one change of the controller's `conditions` bucket (clauses 8, 9): read until the lease write.
func (k *Keeper) Controller(key string, value []byte, deleted bool) error {
k.mu.Lock()
k.init()
if deleted {
delete(k.ctrl, key)
} else {
var c health.Condition
if err := json.Unmarshal(value, &c); err != nil {
k.mu.Unlock()
k.Logf("the controller's condition %s is not readable, so nothing is carried from it: %v", key, err)
return nil
}
k.ctrl[key] = c
}
k.mu.Unlock()
return k.Settle(key)
}
// Lease takes the controller's lease key as read (clause 8, Limits 8): once it is handed to a holder that is no
// controller, nothing more is carried from the controller's bucket, and the kinds that ended with it clear.
func (k *Keeper) Lease(away bool) error {
k.mu.Lock()
k.init()
was := k.leaseOut
k.leaseOut = away
keys := make([]string, 0, len(k.ctrl))
for key := range k.ctrl {
keys = append(keys, key)
}
k.mu.Unlock()
if was == away {
return nil
}
return k.settleAll(keys)
}
// LeaseAway says whether the controller's lease is handed away.
func (k *Keeper) LeaseAway() bool {
k.mu.Lock()
defer k.mu.Unlock()
return k.leaseOut
}
// Takeover takes the keeper's own take-over key as read.
func (k *Keeper) Takeover(t health.Takeover) {
k.mu.Lock()
defer k.mu.Unlock()
k.init()
k.takeover = t
}
// Settle derives the condition key should be and writes the difference (clause 2), retried while another write moves
// it under this one.
func (k *Keeper) Settle(key string) error {
var err error
for i := 0; i < ConditionWrites; i++ {
if err = k.settleOnce(key); !errors.Is(err, stdio.ErrChanged) {
return err
}
}
return fmt.Errorf("the condition %s kept moving under this write, %d tries: %w", key, ConditionWrites, err)
}
func (k *Keeper) settleAll(keys []string) error {
sort.Strings(keys)
var problems []string
for _, key := range keys {
if err := k.Settle(key); err != nil {
problems = append(problems, err.Error())
}
}
if len(problems) > 0 {
return errors.New(strings.Join(problems, "; "))
}
return nil
}
// wanted is what the inputs say key should be: a finding of an observer, or a condition carried from the controller's
// bucket of a kind that ends with the controller, until the lease write (clause 9). Nil when nothing says it.
func (k *Keeper) wanted(key string) (*health.Finding, *health.Condition) {
observers := make([]string, 0, len(k.findings))
for o := range k.findings {
observers = append(observers, o)
}
sort.Strings(observers)
for _, o := range observers {
if f, ok := k.findings[o][key]; ok {
return &f, nil
}
}
if c, ok := k.ctrl[key]; ok && !k.leaseOut && k.endsWithController(c) {
return nil, &c
}
return nil, nil
}
func (k *Keeper) endsWithController(c health.Condition) bool {
h, ok := k.homeOf(c.Kind)
return ok && h.Carried
}
// settleOnce is one try of Settle.
func (k *Keeper) settleOnce(key string) error {
now := k.Now().UTC()
raw, rev, open, err := k.Store.Get(ConditionsState, key)
if err != nil {
return fmt.Errorf("reading the condition %s: %w", key, err)
}
var was health.Condition
if open {
if err := json.Unmarshal(raw, &was); err != nil {
return fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
}
}
k.mu.Lock()
k.init()
f, carried := k.wanted(key)
hold, holding := k.holdFor(key, f, carried, open)
k.mu.Unlock()
if f == nil && carried == nil {
k.dropHeld(key, "its finding went while its module was delivered")
if !open {
return nil
}
return k.clear(key, was, rev, now, "its finding went")
}
if holding {
return k.keepHeld(hold)
}
k.dropHeld(key, "")
next := k.derive(key, was, open, f, carried, now)
events := transitions(was, next, open)
if len(events) == 0 && open && sameBody(was, next) {
return nil
}
if open {
_, err = k.Store.Update(ConditionsState, key, next, rev)
} else {
_, err = k.Store.Create(ConditionsState, key, next)
}
if err != nil {
return err
}
k.mu.Lock()
delete(k.cleared, key)
k.mu.Unlock()
for _, e := range events {
e.Condition, e.At, e.Event = next, now, health.EventFor(e.Change)
e.Show = k.show(next, carried != nil)
k.tell(e)
}
return nil
}
func (k *Keeper) show(c health.Condition, carried bool) string {
if carried {
return health.ShowController(c.Key)
}
return health.ShowKeeper(c.Key)
}
// derive is the condition key should be now, from what was open and what the inputs say.
func (k *Keeper) derive(key string, was health.Condition, open bool, f *health.Finding, carried *health.Condition, now time.Time) health.Condition {
var c health.Condition
switch {
case carried != nil:
// Raised and cleared as the controller's bucket says, with its show and actions (Limits 9).
c = *carried
c.Epoch = 0
c.LooksUnread = ""
case open:
c = was
k.fromFinding(&c, *f, now)
default:
c = health.Condition{Key: key, Raised: now, Count: 1, Resolver: health.ResolverSelf}
k.fromFinding(&c, *f, now)
k.mu.Lock()
if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= health.ReopenWithin {
c.Count = before.count + 1
c.First = before.began
c.Gaps = append(append([]health.Gap(nil), before.gaps...), health.Gap{Cleared: before.at, Reopened: now})
if over := len(c.Gaps) - health.KeptGaps; over > 0 {
c.First, c.Gaps = c.Gaps[over-1].Reopened, c.Gaps[over:]
}
if before.silenced != nil && now.Before(before.silenced.Until) {
c.Silenced = before.silenced
}
c.Tried = before.tried
}
k.mu.Unlock()
}
if carried == nil {
k.overlay(&c, now)
}
if !open {
k.mu.Lock()
if line := k.outOfHold[key]; line != "" {
c.Explanation = strings.TrimSpace(c.Explanation + " " + line)
}
k.mu.Unlock()
}
if c.Silenced != nil && !now.Before(c.Silenced.Until) {
c.Silenced = nil
}
return c
}
// fromFinding is what the condition says of its finding: its words, subject, severity and source, and its evidence
// newest first (clause 2). A healer's escalation is not undone by the finding saying a warning again.
func (k *Keeper) fromFinding(c *health.Condition, f health.Finding, now time.Time) {
escalated := c.Escalated()
c.Kind, c.Subject, c.Summary, c.Source = f.Kind, f.Subject, f.Summary, f.Source
c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = f.Headline, f.Explanation, f.Resolved, f.Needs, f.Actions
if !(escalated && f.Severity != health.Urgent) {
c.Severity = f.Severity
}
if !escalated {
c.Resolver = f.Resolver
if c.Resolver == "" {
c.Resolver = health.ResolverSelf
}
}
if len(f.Evidence) > 0 {
c.Evidence = append([]health.Evidence(nil), f.Evidence...)
c.LastObserved = f.Evidence[0].At
} else if c.Evidence == nil {
c.Evidence = []health.Evidence{{At: f.At, Said: f.Summary}}
c.LastObserved = f.At
}
c.Observations = len(c.Evidence)
if c.Actions == nil {
c.Actions = []health.Action{}
}
_ = now
}
// overlay writes the healers' attempts into the condition (clause 7) and, until the lease write, what the controller's
// bucket holds for the same key (clause 8): its tried, its resolver and escalation, and its silence where the keeper has
// none of its own.
func (k *Keeper) overlay(c *health.Condition, now time.Time) {
k.mu.Lock()
var mine []health.Attempt
for _, a := range k.attempts {
if a.Condition == c.Key {
mine = append(mine, a)
}
}
ctrl, carry := k.ctrl[c.Key]
carry = carry && !k.leaseOut
k.mu.Unlock()
sort.Slice(mine, func(i, j int) bool { return mine[i].Begun.Before(mine[j].Begun) })
if carry && len(ctrl.Tried) > 0 {
c.Tried = append([]health.Tried(nil), ctrl.Tried...)
if ctrl.Escalated() {
c.Resolver, c.Severity = health.ResolverOperator, health.Urgent
} else if ctrl.Resolver != "" && ctrl.Resolver != health.ResolverSelf {
c.Resolver = ctrl.Resolver
}
}
for _, a := range mine {
t := a.Tried()
if !containsTried(c.Tried, t) {
c.Tried = append(c.Tried, t)
}
switch a.Outcome {
case health.OutcomeEscalated:
c.Resolver, c.Severity = health.ResolverOperator, health.Urgent
default:
if c.Resolver != health.ResolverOperator {
c.Resolver = health.ResolverHealer(a.Healer)
}
}
}
if len(c.Tried) > health.KeptTried {
c.Tried = c.Tried[len(c.Tried)-health.KeptTried:]
}
if carry && c.Silenced == nil && ctrl.Silenced != nil && now.Before(ctrl.Silenced.Until) {
c.Silenced = ctrl.Silenced
}
}
func containsTried(list []health.Tried, t health.Tried) bool {
for _, x := range list {
if x.At.Equal(t.At) && x.By == t.By && x.What == t.What {
return true
}
}
return false
}
// transitions are the events between what was open and what is now, in the controller's order: raised or reopened;
// else severity, resolver, silence and words changes. An observation that changes none of these says nothing.
func transitions(was, now health.Condition, open bool) []health.Event {
if !open {
change := health.ChangeRaised
if now.Count > 1 {
change = health.ChangeReopened
}
return []health.Event{{Change: change}}
}
var out []health.Event
if was.Severity != now.Severity {
out = append(out, health.Event{Change: health.ChangeSeverity, Was: string(was.Severity)})
}
if was.Resolver != now.Resolver {
out = append(out, health.Event{Change: health.ChangeResolver, Was: was.Resolver})
}
switch {
case was.Silenced == nil && now.Silenced != nil:
out = append(out, health.Event{Change: health.ChangeSilenced, Why: now.Silenced.Why})
case was.Silenced != nil && now.Silenced == nil:
out = append(out, health.Event{Change: health.ChangeUnsilenced})
}
if was.Needs != now.Needs || !sameActions(was.Actions, now.Actions) {
out = append(out, health.Event{Change: health.ChangeWords, Was: was.Needs})
}
return out
}
func sameActions(a, b []health.Action) bool {
x, _ := json.Marshal(a)
y, _ := json.Marshal(b)
return string(x) == string(y) || (len(a) == 0 && len(b) == 0)
}
func sameBody(a, b health.Condition) bool {
x, _ := json.Marshal(a)
y, _ := json.Marshal(b)
return string(x) == string(y)
}
// clear removes an open condition whose finding went, and says it (clause 2).
func (k *Keeper) clear(key string, was health.Condition, rev uint64, now time.Time, why string) error {
// A delete at the revision read: a write that moved it meanwhile is read again.
if _, err := k.Store.Update(ConditionsState, key, json.RawMessage("null"), rev); err != nil {
return err
}
if err := k.Store.Delete(ConditionsState, key); err != nil {
return err
}
k.mu.Lock()
k.cleared[key] = clearedMemo{at: now, count: was.Count, began: was.Began(), gaps: was.Gaps, silenced: was.Silenced, tried: was.Tried}
k.mu.Unlock()
cleared := now
k.tell(health.Event{Condition: was, Event: health.EventCleared, At: now, Change: health.ChangeCleared, Why: why,
Cleared: &cleared, Show: k.show(was, false)})
return nil
}
// tell keeps the transition in history and says it on the seat. A transition not said is said in the log: the condition
// and its history still hold it.
func (k *Keeper) tell(e health.Event) {
k.mu.Lock()
k.seq++
hk := fmt.Sprintf("%020d-%06d", e.At.UnixNano(), k.seq)
k.mu.Unlock()
if _, err := k.Store.Create(HistoryState, hk, e); err != nil {
k.Logf("the %s of %s could not be kept in history: %v", e.Change, e.Key, err)
}
if k.Say == nil {
return
}
if err := k.Say(e); err != nil {
k.Logf("the %s of %s could NOT be said on the seat: %v", e.Change, e.Key, err)
}
}
// Open is every open condition, ordered as `conditions` says them.
func (k *Keeper) Open() ([]health.Condition, error) {
keys, err := k.Store.Keys(ConditionsState)
if err != nil {
return nil, err
}
var out []health.Condition
for _, key := range keys {
raw, _, found, err := k.Store.Get(ConditionsState, key)
if err != nil {
return nil, err
}
if !found || string(raw) == "null" {
continue
}
var c health.Condition
if err := json.Unmarshal(raw, &c); err != nil {
return nil, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
}
out = append(out, c)
}
health.Order(out)
return out, nil
}
// Resync settles every key any input names or any open condition holds (clause 12): what differs is raised or cleared.
func (k *Keeper) Resync() (int, error) {
open, err := k.Open()
if err != nil {
return 0, err
}
seen := map[string]bool{}
for _, c := range open {
seen[c.Key] = true
}
k.mu.Lock()
k.init()
for o, fs := range k.findings {
if k.blind[o] != "" {
continue
}
for key := range fs {
seen[key] = true
}
}
for key := range k.ctrl {
seen[key] = true
}
k.mu.Unlock()
keys := make([]string, 0, len(seen))
for key := range seen {
keys = append(keys, key)
}
return len(keys), k.settleAll(keys)
}
// Silence stops a condition's messages for a while, a hand act with why (clause 3, Limits 1): kept as its transition in
// history, naming who and why.
func (k *Keeper) Silence(key string, d time.Duration, by, why string) (health.Condition, error) {
if strings.TrimSpace(why) == "" {
return health.Condition{}, errors.New("a silence says why")
}
if d <= 0 || d > health.MaxSilence {
return health.Condition{}, fmt.Errorf("a silence lasts more than nothing and at most %s, not %s", health.MaxSilence, d)
}
for i := 0; i < ConditionWrites; i++ {
raw, rev, found, err := k.Store.Get(ConditionsState, key)
if err != nil {
return health.Condition{}, err
}
if !found || string(raw) == "null" {
return health.Condition{}, fmt.Errorf("no condition %s is open", key)
}
var c health.Condition
if err := json.Unmarshal(raw, &c); err != nil {
return health.Condition{}, err
}
now := k.Now().UTC()
c.Silenced = &health.Silence{Until: now.Add(d), By: by, Why: why, Since: now}
if _, err := k.Store.Update(ConditionsState, key, c, rev); errors.Is(err, stdio.ErrChanged) {
continue
} else if err != nil {
return health.Condition{}, err
}
k.tell(health.Event{Condition: c, Event: health.EventChanged, At: now, Change: health.ChangeSilenced,
Why: why + " (" + by + ")", Show: health.ShowKeeper(key)})
return c, nil
}
return health.Condition{}, fmt.Errorf("the condition %s kept moving under the silence", key)
}
@@ -0,0 +1,314 @@
package main
import (
"encoding/json"
"fmt"
"sort"
"strings"
"sync"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// fakeStore is the bus's states in memory, with revisions, as the runtime answers them.
type fakeStore struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
next uint64
// ttls are the times to live the keys were created with.
ttls map[string]time.Duration
}
func newFakeStore() *fakeStore {
return &fakeStore{vals: map[string][]byte{}, revs: map[string]uint64{}}
}
func (f *fakeStore) Get(state, key string) ([]byte, uint64, bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
v, ok := f.vals[state+"/"+key]
return v, f.revs[state+"/"+key], ok, nil
}
func (f *fakeStore) put(state, key string, value any) uint64 {
raw, _ := json.Marshal(value)
f.next++
f.vals[state+"/"+key], f.revs[state+"/"+key] = raw, f.next
return f.next
}
func (f *fakeStore) Create(state, key string, value any) (uint64, error) {
f.mu.Lock()
defer f.mu.Unlock()
if _, ok := f.vals[state+"/"+key]; ok {
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
}
return f.put(state, key, value), nil
}
func (f *fakeStore) Update(state, key string, value any, rev uint64) (uint64, error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.revs[state+"/"+key] != rev {
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
}
return f.put(state, key, value), nil
}
func (f *fakeStore) Delete(state, key string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.vals, state+"/"+key)
delete(f.revs, state+"/"+key)
return nil
}
func (f *fakeStore) Keys(state string) ([]string, error) {
f.mu.Lock()
defer f.mu.Unlock()
var out []string
for k := range f.vals {
if s, key, _ := strings.Cut(k, "/"); s == state {
out = append(out, key)
}
}
sort.Strings(out)
return out, nil
}
func (f *fakeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
f.mu.Lock()
if f.ttls == nil {
f.ttls = map[string]time.Duration{}
}
f.ttls[state+"/"+key] = ttl
f.mu.Unlock()
_, err := f.Create(state, key, value)
return err
}
// rig is a keeper over a fake store, its clock and what it said.
type rig struct {
k *Keeper
st *fakeStore
now time.Time
said []health.Event
}
func newRig() *rig {
r := &rig{st: newFakeStore(), now: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)}
r.k = &Keeper{Store: r.st, Now: func() time.Time { return r.now }, Say: func(e health.Event) error {
r.said = append(r.said, e)
return nil
}}
return r
}
func (r *rig) changes() string {
var out []string
for _, e := range r.said {
out = append(out, e.Change+" "+e.Key)
}
r.said = nil
return strings.Join(out, ", ")
}
func (r *rig) open(key string) (health.Condition, bool) {
raw, _, ok, _ := r.st.Get(ConditionsState, key)
var c health.Condition
if ok {
_ = json.Unmarshal(raw, &c)
}
return c, ok
}
// finding is an observer's finding as mesh-sdk's layout writes it (health.EncodeFinding).
func finding(t *testing.T, observer, scope, id, kind, source string, sev health.Severity, at time.Time) (string, []byte) {
t.Helper()
f := health.Finding{Key: health.Key(scope, id, kind), Kind: kind, Subject: health.Subject{Scope: scope, ID: id},
Severity: sev, Summary: kind + " of " + id, Headline: kind, Explanation: "It is " + kind + ".", Source: source,
Observer: observer, At: at, Evidence: []health.Evidence{{At: at, Said: kind}}}
raw, err := health.EncodeFinding(f)
if err != nil {
t.Fatal(err)
}
return f.Key, raw
}
// ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer
// clears nothing; the healers' attempts and the controller's escalation reach the condition; a reopening within ten
// minutes is the same fault; a silence is a transition in history.
func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) {
r := newRig()
key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now)
steps := []struct {
name string
do func() error
said string
check func(c health.Condition, open bool) string
}{
{name: "a finding written raises", do: func() error { return r.k.Finding(watchman, key, raw, false) },
said: "raised machine.g14.silent", check: func(c health.Condition, open bool) string {
if !open || c.Severity != health.Warning || c.Resolver != health.ResolverSelf || c.Epoch != 0 || c.Count != 1 {
return fmt.Sprintf("%+v", c)
}
return ""
}},
{name: "the same finding again says nothing", do: func() error { return r.k.Finding(watchman, key, raw, false) }},
{name: "an observer gone blind clears nothing", do: func() error { r.k.Blind(watchman, "refused"); _, err := r.k.Resync(); return err },
check: func(c health.Condition, open bool) string {
if !open {
return "cleared by blindness"
}
return ""
}},
{name: "a healer's attempt reaches tried and the resolver", do: func() error {
a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "asserted", Outcome: health.OutcomeActed, Begun: r.now})
return r.k.Attempt("h3-1", a, false)
}, said: "resolver machine.g14.silent", check: func(c health.Condition, _ bool) string {
if c.Resolver != "healer:H3" || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" {
return fmt.Sprintf("%+v", c)
}
return ""
}},
{name: "an escalation makes it urgent, the operator's", do: func() error {
a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "handed over", Outcome: health.OutcomeEscalated,
Begun: r.now.Add(time.Minute)})
return r.k.Attempt("h3-2", a, false)
}, said: "severity machine.g14.silent, resolver machine.g14.silent", check: func(c health.Condition, _ bool) string {
if c.Severity != health.Urgent || c.Resolver != health.ResolverOperator || len(c.Tried) != 2 {
return fmt.Sprintf("%+v", c)
}
return ""
}},
{name: "a silence is a transition with who and why", do: func() error {
_, err := r.k.Silence(key, time.Hour, "jochen", "known, a cable")
return err
}, said: "silenced machine.g14.silent"},
{name: "the finding deleted clears", do: func() error { return r.k.Finding(watchman, key, nil, true) },
said: "cleared machine.g14.silent", check: func(_ health.Condition, open bool) string {
if open {
return "still open"
}
return ""
}},
{name: "raised again within ten minutes is a reopening", do: func() error {
r.now = r.now.Add(5 * time.Minute)
return r.k.Finding(watchman, key, raw, false)
}, said: "reopened machine.g14.silent", check: func(c health.Condition, _ bool) string {
if c.Count != 2 || len(c.Gaps) != 1 {
return fmt.Sprintf("%+v", c)
}
return ""
}},
// Until the lease write the controller's escalation for the same key reaches the condition (clause 8).
{name: "the controller's escalation is carried", do: func() error {
ctrl, _ := json.Marshal(health.Condition{Key: key, Kind: "silent", Severity: health.Urgent, Resolver: health.ResolverOperator,
Tried: []health.Tried{{At: r.now, What: "x", Outcome: "escalated", By: "healer H1"}}})
_ = r.k.Attempt("h3-1", nil, true)
_ = r.k.Attempt("h3-2", nil, true)
return r.k.Controller(key, ctrl, false)
}, check: func(c health.Condition, _ bool) string {
if c.Resolver != health.ResolverOperator || c.Severity != health.Urgent {
return fmt.Sprintf("%+v", c)
}
return ""
}},
}
for _, s := range steps {
if err := s.do(); err != nil {
t.Fatalf("%s: %v", s.name, err)
}
if got := r.changes(); s.said != "" && got != s.said || s.said == "" && got != "" && s.name != "the controller's escalation is carried" {
t.Errorf("%s: said %q, want %q", s.name, got, s.said)
}
if s.check != nil {
c, open := r.open(key)
if why := s.check(c, open); why != "" {
t.Errorf("%s: %s", s.name, why)
}
}
}
// Every transition is kept in history, and each event carries the keeper's show.
keys, _ := r.st.Keys(HistoryState)
if len(keys) < 7 {
t.Errorf("history keeps %d transitions", len(keys))
}
raw0, _, _, _ := r.st.Get(HistoryState, keys[0])
if !strings.Contains(string(raw0), `"show":"mesh-conditions.conditions key=machine.g14.silent"`) {
t.Errorf("the first transition is %s", raw0)
}
// A carried kind (it ends with the controller) is raised and cleared as the controller's bucket says, until the
// lease is handed away, with the controller's show (clause 9, Limits 9).
deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4",
Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self", Epoch: 9})
if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil {
t.Fatal(err)
}
if c, open := r.open("core.controller.deaf"); !open || c.Epoch != 0 || r.said[0].Show != "mesh-controller.conditions key=core.controller.deaf" {
t.Errorf("a carried kind is %+v, said %+v", c, r.said)
}
r.said = nil
if err := r.k.Lease(true); err != nil {
t.Fatal(err)
}
if _, open := r.open("core.controller.deaf"); open || r.changes() != "cleared core.controller.deaf" {
t.Error("a carried kind outlived the lease write")
}
}
// ADR 0564 clause 9 (issue 1952 C5, C8; the lead 2026-10-29): compare counts only what both sides can raise. A key held
// while its module is delivered, a kind that ends with the controller and a kind the controller never raises are listed
// apart; a kind with no home decided is a difference naming the gap; take-over is refused while compare differs, and
// hand-back after the lease write.
func TestCompareCountsOnlyWhatBothSidesRaise(t *testing.T) {
r := newRig()
put := func(key, kind string) []byte {
raw, _ := json.Marshal(health.Condition{Key: key, Kind: kind, Severity: health.Warning, Source: "x"})
return raw
}
key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now)
_ = r.k.Finding(watchman, key, raw, false)
_ = r.k.Controller(key, put(key, "silent"), false)
only, onlyRaw := finding(t, delivery.AssignerModule, health.ScopeSeat, "mesh-placement.ace", "states-unbound", "say", health.Warning, r.now)
_ = r.k.Finding(delivery.AssignerModule, only, onlyRaw, false)
_ = r.k.Controller("core.controller.deaf", put("core.controller.deaf", "controller-deaf"), false)
_ = r.k.Controller("delivery.x.stalled", put("delivery.x.stalled", "stalled"), false)
_ = r.k.Controller("seat.vault.ace.holder-silent", put("seat.vault.ace.holder-silent", "holder-silent"), false)
cmp, err := r.k.Compare()
if err != nil {
t.Fatal(err)
}
reasons := map[string]string{}
for _, d := range append(append([]Difference(nil), cmp.Differences...), cmp.Apart...) {
reasons[d.Key] = d.Reason
}
for key, says := range map[string]string{
only: "keeper only",
"core.controller.deaf": "carried",
"delivery.x.stalled": "no home is decided for the kind stalled",
"seat.vault.ace.holder-silent": "mesh-self-check has not written its finding",
} {
if !strings.Contains(reasons[key], says) {
t.Errorf("%s: %q, want it to say %q", key, reasons[key], says)
}
}
if cmp.Same != 1 || len(cmp.Differences) != 2 {
t.Errorf("compare: %+v", cmp)
}
if _, err := r.k.TakeOver(nil, "jochen", "now"); err == nil || !strings.Contains(err.Error(), "2 difference") {
t.Errorf("take-over while compare differs: %v", err)
}
if _, err := r.k.HandBack("jochen", "back"); err != nil {
t.Errorf("hand-back before the lease write: %v", err)
}
_ = r.k.Lease(true)
if _, err := r.k.HandBack("jochen", "back"); err == nil {
t.Error("hand-back after the lease write was taken")
}
}
@@ -0,0 +1,164 @@
// mesh-conditions: the condition keeper, holder of the seat mesh-conditions (novox/hq ADR 0564 clauses 1-3, 9, 13). A
// Go bundle served by a runtime of its own, on its own bus account (`runs-as`), so that its account alone writes its
// states: `conditions` (one key per open condition), `history` (every transition, ninety days), `takeover` (the key the
// readers follow), `held` and `bounds` (what a delivery holds, and for how long). It watches every observer's
// `findings`, the healers' `attempts`, the assigner's wants, the nodes' haves and the controller's lease and, until the
// lease write, its `conditions`; it says every transition on its seat with the controller's body. stdout is the MCP
// channel; what this module says, it says on stderr.
package main
import (
"fmt"
"os"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/health"
)
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
// runtimeStore is the bus, through the runtime that launched this bundle.
type runtimeStore struct{}
func (runtimeStore) Get(state, key string) ([]byte, uint64, bool, error) {
e, err := stdio.State(state).Get(key)
if err != nil || e == nil {
return nil, 0, false, err
}
return e.Value, e.Revision, true, nil
}
func (runtimeStore) Create(state, key string, value any) (uint64, error) {
return stdio.State(state).Create(key, value)
}
func (runtimeStore) Update(state, key string, value any, revision uint64) (uint64, error) {
return stdio.State(state).Update(key, value, revision)
}
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
// CreateTTL creates the key with its own time to live (`bounds` is key-ttl); its expiry is handed over as OpExpire.
func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
_, err := stdio.State(state).CreateTTL(key, value, ttl)
return err
}
// sayOnSeat says a transition on the seat, as the controller says it on its own (ADR 0564 clause 3).
func sayOnSeat(e health.Event) error {
return stdio.SeatPublish("mesh.seat."+Seat+".event."+e.Event, e, "")
}
// NewKeeper is the keeper over the runtime.
func NewKeeper() *Keeper {
return &Keeper{Store: runtimeStore{}, Say: sayOnSeat, Now: time.Now, Logf: logf}
}
// watch keeps a watch of one state, said blind while it cannot be read: a state not on the bus yet (an observer not
// delivered) is read again when it is.
func watch(k *Keeper, state, observer string, handle func(stdio.StateChange) error) {
go func() {
for wait := 5 * time.Second; ; wait = min(wait*2, 5*time.Minute) {
err := stdio.State(state).Watch("", handle)
if err == nil {
if observer != "" {
k.Blind(observer, "")
}
return
}
if observer != "" {
k.Blind(observer, err.Error())
}
logf("%s cannot be watched (%v); watched again in %s", state, err, wait)
time.Sleep(wait)
}
}()
}
// Keep starts every watch the keeper reads by (clauses 2, 8, 9, 13).
func Keep(k *Keeper) {
for _, s := range []string{HeldState} {
keys, _ := stdio.State(s).Keys()
for _, key := range keys {
if e, err := stdio.State(s).Get(key); err == nil && e != nil {
k.restoreHeld(key, e.Value)
}
}
}
if e, err := stdio.State(TakeoverState).Get(health.TakeoverKey); err == nil {
var raw []byte
if e != nil {
raw = e.Value
}
if t, err := health.DecodeTakeover(raw, e != nil); err == nil {
k.Takeover(t)
}
}
said := func(err error) error {
if err != nil {
logf("%v", err)
}
return nil
}
watch(k, ControllerLease, "", func(c stdio.StateChange) error {
if c.Key != ControllerLeaseKey {
return nil
}
away, why := buscompose.LeaseHandedAway(c.Value, c.Op == "put", nil)
logf("the controller's lease: %s", why)
return said(k.Lease(away))
})
watch(k, WantsRead, "", func(c stdio.StateChange) error { return said(k.Want(c.Key, c.Value, c.Op != "put")) })
watch(k, HavesRead, "", func(c stdio.StateChange) error { return said(k.Have(c.Key, c.Value, c.Op != "put")) })
watch(k, AttemptsRead, health.HealersModule+" attempts", func(c stdio.StateChange) error {
return said(k.Attempt(c.Key, c.Value, c.Op != "put"))
})
watch(k, ControllerConditions, "", func(c stdio.StateChange) error {
if k.LeaseAway() {
return nil
}
return said(k.Controller(c.Key, c.Value, c.Op != "put"))
})
for _, o := range Observers {
watch(k, o+"."+health.FindingsState, o, func(c stdio.StateChange) error {
return said(k.Finding(o, c.Key, c.Value, c.Op != "put"))
})
}
watch(k, BoundsState, "", func(c stdio.StateChange) error {
if c.Op == stdio.OpExpire {
return said(k.BoundPassed(c.Key))
}
return nil
})
watch(k, PlacementRead, "", func(c stdio.StateChange) error {
k.Placement(c.Key, c.Value, c.Op != stdio.OpPut)
return nil
})
}
func main() {
k := NewKeeper()
client := asker.New(AskedState)
client.Module = Module
go func() {
<-stdio.Served()
Keep(k)
if err := client.Listen(k.Warranted, func(h asker.Handled, w asks.Warrant, err error) {
if err != nil {
logf("the answer to %s (%s): %v", w.Ask, h, err)
}
}); err != nil {
logf("the operator's answers cannot be heard: %v", err)
}
}()
if err := stdio.Serve("", Tools(k, client)); err != nil {
logf("%v", err)
os.Exit(1)
}
}
@@ -0,0 +1,69 @@
package main
import (
"os"
"path/filepath"
"slices"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
)
// ADR 0564 clauses 1, 2, 3 and 12 (issue 1952 C1): the manifest says what the code does. It claims the seat
// mesh-conditions once for the mesh with the verbs mesh-sdk defines for it; reads every observer's findings, the healers'
// attempts, the controller's conditions and lease, the wants and the haves, and never the healers' would; keeps its
// states, history ninety days and each delivery's bound as long as the bound; and every state has a tool that reads it,
// every act a tool that starts it.
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatalf("the manifest is refused: %v", err)
}
seat, _ := catalogue.SeatNamed(Seat)
if m.Module != Module || m.RunsAs != Module || len(m.Claims) != 1 || m.Claims[0].Name != Seat ||
!slices.Equal(Verbs, catalogue.VerbNames(seat.Serves)) {
t.Errorf("module %s runs as %s and claims %+v; the seat serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(seat.Serves))
}
reads := []string{ControllerConditions, ControllerLease, WantsRead, HavesRead, AttemptsRead, PlacementRead}
for _, o := range Observers {
reads = append(reads, o+"."+health.FindingsState)
}
slices.Sort(reads)
got := slices.Clone(m.Reads)
slices.Sort(got)
if !slices.Equal(got, reads) || slices.Contains(m.Reads, health.HealersModule+"."+health.WouldState) {
t.Errorf("reads %v, want %v and never the healers' would", got, reads)
}
ttl, keyTTL := map[string]int{}, map[string]bool{}
for _, s := range m.State {
ttl[s.Name], keyTTL[s.Name] = s.TTLSeconds, s.KeyTTL
}
// Each bound is its own key's time to live (the runtime's per-key TTL), never the bucket's age.
if ttl[HistoryState] != int(HistoryKeptFor.Seconds()) || ttl[BoundsState] != 0 || !keyTTL[BoundsState] {
t.Errorf("history is kept %d s, a bound's bucket ages %d s, its keys have their own time to live: %v",
ttl[HistoryState], ttl[BoundsState], keyTTL[BoundsState])
}
// Clause 12: each state with the tool that reads it, each act with the tool that starts it.
readBy := map[string]string{ConditionsState: Seat + ".conditions", HistoryState: HistoryTool, TakeoverState: TakeoverTool,
HeldState: HeldTool, BoundsState: HeldTool, AskedState: Seat + ".take-over", "a resync": Seat + ".resync",
"what it reads": InputsTool}
tools := map[string]bool{}
for _, tool := range Tools(&Keeper{}, nil) {
tools[tool.Name] = true
}
for name := range ttl {
if readBy[name] == "" {
t.Errorf("the state %s has no tool that reads it", name)
}
}
for what, tool := range readBy {
if !tools[tool] {
t.Errorf("%s is read or started by %s, which is not served", what, tool)
}
}
}
@@ -0,0 +1,2 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
modules/mesh-conditions/module.json
@@ -0,0 +1,252 @@
package main
// The seat's verbs and the keeper's own tools (novox/hq ADR 0564 clauses 3, 9 and 12; issue 1952 C4): `conditions`
// answers as the controller's verb answers, so a caller changes only the seat it asks; and a read of every state the
// keeper keeps and of what it reads, so a person sees all it holds and a recovery is a call.
import (
"encoding/json"
"fmt"
"slices"
"strconv"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Verbs are the seat's verbs, as the manifest claims them.
var Verbs = []string{"conditions", "silence", "compare", "take-over", "hand-back", "resync"}
// The keeper's own read tools.
const (
HistoryTool = "conditions_history"
HeldTool = "conditions_held"
TakeoverTool = "conditions_takeover"
InputsTool = "conditions_inputs"
)
// briefEvidence is how many observations a condition carries in a list: its newest (the controller's, issue 314).
const briefEvidence = 1
// inBrief is a condition in a list: its newest evidence, with how many it keeps and how to read it whole.
type inBrief struct {
health.Condition
EvidenceKept int `json:"evidence-kept,omitempty"`
More string `json:"more,omitempty"`
}
func obj(props map[string]any, required ...string) map[string]any {
out := map[string]any{"type": "object", "properties": props}
if len(required) > 0 {
out["required"] = required
}
return out
}
func str(description string) map[string]any {
return map[string]any{"type": "string", "description": description}
}
// Tools are what the runtime serves.
func Tools(k *Keeper, a Asks) []stdio.Tool {
return []stdio.Tool{
{Name: Seat + ".conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, then " +
"oldest, raised from the observers' findings and cleared when the finding goes, never by hand. Given a key, that " +
"one whole; with history, every transition lately.",
Input: obj(map[string]any{"key": str("a condition's key"), "scope": str("only this scope"),
"severity": str("urgent or warning"), "machine": str("only those about this machine"),
"history": str("\"true\": every transition lately"), "days": str("with history: days back (7, at most 90)")}),
Run: func(args map[string]any) (any, error) { return k.ConditionsVerb(args) }},
{Name: Seat + ".silence", Description: "Stop one condition's messages for a while, a hand act with why, kept as its " +
"transition in history; it stays open.",
Input: obj(map[string]any{"key": str("the condition's key"), "for": str("30m, 4h, 2d; at most 7d"),
"why": str("why"), "cause": str("the cause in a word")}, "key", "for", "why"),
Run: func(args map[string]any) (any, error) {
d, err := duration(strArg(args, "for"))
if err != nil {
return nil, err
}
return k.Silence(strArg(args, "key"), d, stdio.CallerOf(args), strArg(args, "why"))
}},
{Name: Seat + ".compare", Description: "The keeper's open conditions against the controller's, each difference with " +
"its reason; what ends with the controller, what only the keeper raises and what is held listed apart.",
Run: func(map[string]any) (any, error) { return k.Compare() }},
{Name: Seat + ".take-over", Description: "Ask the operator's Approve to switch the readers of conditions to the " +
"keeper; refused while compare differs.",
Input: obj(map[string]any{"why": str("why now")}, "why"),
Run: func(args map[string]any) (any, error) {
return k.TakeOver(a, stdio.CallerOf(args), strArg(args, "why"))
}},
{Name: Seat + ".hand-back", Description: "Switch the readers of conditions back to the controller, until its lease " +
"is handed away.",
Input: obj(map[string]any{"why": str("why")}, "why"),
Run: func(args map[string]any) (any, error) { return k.HandBack(stdio.CallerOf(args), strArg(args, "why")) }},
{Name: Seat + ".resync", Description: "Read every finding again and raise and clear what differs.",
Run: func(map[string]any) (any, error) {
n, err := k.Resync()
return map[string]any{"settled": n}, err
}},
{Name: HistoryTool, Description: "The keeper's history: every transition it said, newest first, at most limit.",
Input: obj(map[string]any{"limit": str("how many (default 50)"), "key": str("only this condition")}),
Run: func(args map[string]any) (any, error) {
return k.History(strArg(args, "key"), intArg(args, "limit", 50))
}},
{Name: HeldTool, Description: "What the keeper holds while its module is delivered (ADR 0564 clause 13): each key " +
"with its module, node, version and since when, and the deliveries' bounds.",
Run: func(map[string]any) (any, error) {
bounds, err := k.Store.Keys(BoundsState)
return map[string]any{"held": k.Held(), "bounds": bounds}, err
}},
{Name: TakeoverTool, Description: "The take-over key: which source the router, the grants part and mesh-watcher hear.",
Run: func(map[string]any) (any, error) {
raw, _, found, err := k.Store.Get(TakeoverState, health.TakeoverKey)
if err != nil {
return nil, err
}
return health.DecodeTakeover(raw, found)
}},
{Name: InputsTool, Description: "What the keeper reads, as it stands: each observer's findings counted, an observer it " +
"cannot read and why, the healers' attempts, the controller's conditions and whether its lease is handed away.",
Run: func(map[string]any) (any, error) { return k.Inputs(), nil }},
}
}
// ConditionsVerb is `conditions`: the list, one whole, or the history.
func (k *Keeper) ConditionsVerb(args map[string]any) (any, error) {
key := strArg(args, "key")
if strArg(args, "history") == "true" {
days := intArg(args, "days", 7)
if days < 1 || days > 90 {
return nil, fmt.Errorf("days is from 1 to 90, not %d", days)
}
return k.HistorySince(key, k.Now().Add(-time.Duration(days)*24*time.Hour), days)
}
if key != "" {
raw, _, found, err := k.Store.Get(ConditionsState, key)
if err != nil {
return nil, err
}
if !found || string(raw) == "null" {
return nil, fmt.Errorf("no condition %s is open", key)
}
var c health.Condition
return c, json.Unmarshal(raw, &c)
}
scope, severity, machine := strArg(args, "scope"), strArg(args, "severity"), strArg(args, "machine")
if severity != "" && severity != string(health.Urgent) && severity != string(health.Warning) {
return nil, fmt.Errorf("a severity is urgent or warning, not %q", severity)
}
open, err := k.Open()
if err != nil {
return nil, err
}
list := []inBrief{}
counted := map[string]int{}
for _, c := range open {
if (scope != "" && c.Subject.Scope != scope) || (severity != "" && string(c.Severity) != severity) ||
(machine != "" && !concerns(c, machine)) {
continue
}
b := inBrief{Condition: c}
if len(c.Evidence) > briefEvidence {
b.EvidenceKept, b.Evidence, b.More = len(c.Evidence), c.Evidence[:briefEvidence:briefEvidence], health.ShowKeeper(c.Key)
}
list = append(list, b)
counted[c.Source+" "+c.Kind]++
}
return map[string]any{"conditions": list, "open": len(open), "counted": counted,
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; each with " +
"its newest evidence — `conditions key=<key>` gives one whole"}, nil
}
// concerns says whether a condition is about a machine: it names it, or its key does.
func concerns(c health.Condition, machine string) bool {
return c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
slices.Contains(strings.Split(c.Key, "."), machine)
}
// History is the newest transitions, at most limit, of one key or all.
func (k *Keeper) History(key string, limit int) ([]health.Event, error) {
all, err := k.HistorySince(key, time.Time{}, 0)
if err != nil {
return nil, err
}
events := all["history"].([]health.Event)
slices.Reverse(events)
if len(events) > limit {
events = events[:limit]
}
return events, nil
}
// HistorySince is every transition since a moment, oldest first, as the controller's verb answers it.
func (k *Keeper) HistorySince(key string, since time.Time, days int) (map[string]any, error) {
keys, err := k.Store.Keys(HistoryState)
if err != nil {
return nil, err
}
events := []health.Event{}
for _, hk := range keys {
raw, _, found, err := k.Store.Get(HistoryState, hk)
if err != nil || !found {
continue
}
var e health.Event
if json.Unmarshal(raw, &e) != nil || e.At.Before(since) || (key != "" && e.Key != key) {
continue
}
events = append(events, e)
}
return map[string]any{"history": events, "days": days}, nil
}
// Inputs is what the keeper reads, as it stands.
func (k *Keeper) Inputs() map[string]any {
k.mu.Lock()
defer k.mu.Unlock()
k.init()
findings := map[string]int{}
for o, fs := range k.findings {
findings[o] = len(fs)
}
return map[string]any{"findings": findings, "blind": k.blind, "attempts": len(k.attempts),
"controller-conditions": len(k.ctrl), "lease-handed-away": k.leaseOut, "follow": k.takeover.Follow,
"wants": len(k.wants), "haves": len(k.haves)}
}
func strArg(a map[string]any, name string) string {
switch v := a[name].(type) {
case string:
return strings.TrimSpace(v)
case bool:
return strconv.FormatBool(v)
case float64:
return strconv.FormatFloat(v, 'f', -1, 64)
}
return ""
}
func intArg(a map[string]any, name string, def int) int {
if n, err := strconv.Atoi(strArg(a, name)); err == nil {
return n
}
return def
}
// duration reads 30m, 4h or 2d.
func duration(s string) (time.Duration, error) {
if n, ok := strings.CutSuffix(s, "d"); ok {
days, err := strconv.Atoi(n)
if err != nil {
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
}
return time.Duration(days) * 24 * time.Hour, nil
}
d, err := time.ParseDuration(s)
if err != nil {
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
}
return d, nil
}
@@ -0,0 +1,335 @@
//go:build integration
package main
// The condition keeper whole against a real bus (novox/hq ADR 0564 clauses 2 and 3, issue 1952 C6): the built binary,
// played over its real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a
// real nats-server's key-value buckets, watches handed over as the runtime hands them, mesh/seat.publish onto its
// subjects). An observer's finding written to its bucket raises a condition the keeper keeps in its bucket and says as
// condition-raised with the controller's body; the seat's verb lists it; the finding deleted clears it, said as
// condition-cleared.
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
type wire struct {
ID json.RawMessage `json:"id,omitempty"`
Method string `json:"method,omitempty"`
Params json.RawMessage `json:"params,omitempty"`
Result json.RawMessage `json:"result,omitempty"`
Error *struct {
Message string `json:"message"`
} `json:"error,omitempty"`
}
// player is the node's tool runtime, as far as the keeper needs it, over a real bus.
type player struct {
ctx context.Context
nc *nats.Conn
js jetstream.JetStream
mu sync.Mutex
in io.Writer
next int
pending map[string]chan wire
}
func (p *player) send(m map[string]any) {
raw, _ := json.Marshal(m)
p.mu.Lock()
defer p.mu.Unlock()
_, _ = p.in.Write(append(raw, '\n'))
}
func (p *player) answer(id json.RawMessage, result any, err error) {
if err != nil {
p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
return
}
p.send(map[string]any{"id": id, "result": result})
}
func (p *player) serve(out io.Reader) {
scanner := bufio.NewScanner(out)
scanner.Buffer(make([]byte, 1<<20), 1<<24)
for scanner.Scan() {
var m wire
if json.Unmarshal(scanner.Bytes(), &m) != nil {
continue
}
if m.Method == "" {
var key string
_ = json.Unmarshal(m.ID, &key)
p.mu.Lock()
ch := p.pending[key]
delete(p.pending, key)
p.mu.Unlock()
if ch != nil {
ch <- m
}
continue
}
go p.handle(m)
}
}
// kv is a state's bucket: the keeper's own by its local name, another's by `<module>.<name>`.
func (p *player) kv(state string) (jetstream.KeyValue, error) {
if owner, name, read := strings.Cut(state, "."); read {
return p.js.KeyValue(p.ctx, owner+"_"+name)
}
return p.js.KeyValue(p.ctx, Module+"_"+state)
}
func (p *player) handle(m wire) {
var a struct {
State string `json:"state"`
Key string `json:"key"`
Value json.RawMessage `json:"value"`
Revision uint64 `json:"revision"`
Subject string `json:"subject"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &a)
kv, kvErr := p.kv(a.State)
switch m.Method {
case "mesh/state.get":
if kvErr != nil {
p.answer(m.ID, nil, kvErr)
return
}
e, err := kv.Get(p.ctx, a.Key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
p.answer(m.ID, nil, nil)
return
}
if err != nil {
p.answer(m.ID, nil, err)
return
}
p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
case "mesh/state.create", "mesh/state.update", "mesh/state.put":
if kvErr != nil {
p.answer(m.ID, nil, kvErr)
return
}
var rev uint64
var err error
switch m.Method {
case "mesh/state.create":
rev, err = kv.Create(p.ctx, a.Key, a.Value)
case "mesh/state.update":
rev, err = kv.Update(p.ctx, a.Key, a.Value, a.Revision)
default:
rev, err = kv.Put(p.ctx, a.Key, a.Value)
}
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32010, "message": "changed: " + err.Error()}})
return
}
p.answer(m.ID, map[string]any{"revision": rev}, err)
case "mesh/state.keys":
if kvErr != nil {
p.answer(m.ID, nil, kvErr)
return
}
keys, err := kv.Keys(p.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
keys, err = []string{}, nil
}
p.answer(m.ID, keys, err)
case "mesh/state.delete":
err := kvErr
if err == nil {
err = kv.Delete(p.ctx, a.Key)
}
p.answer(m.ID, map[string]any{}, err)
case "mesh/state.watch":
if kvErr != nil {
p.answer(m.ID, nil, kvErr)
return
}
w, err := kv.WatchAll(p.ctx)
if err != nil {
p.answer(m.ID, nil, err)
return
}
var watchID string
_ = json.Unmarshal(m.ID, &watchID)
current := true
for e := range w.Updates() {
if e == nil {
current = false
p.answer(m.ID, map[string]any{}, nil)
continue
}
op := "put"
if e.Operation() != jetstream.KeyValuePut {
op = "delete"
}
p.mu.Lock()
p.next++
id := fmt.Sprintf("w-%d", p.next)
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watchID,
"change": map[string]any{"state": a.State, "key": e.Key(), "op": op, "value": json.RawMessage(nullIfEmpty(e.Value())),
"revision": e.Revision(), "current": current}}})
}
case "mesh/seat.publish":
p.answer(m.ID, map[string]any{"sequence": 1}, p.nc.Publish(a.Subject, a.Body))
case "mesh/subscribe":
p.answer(m.ID, map[string]any{}, nil)
default:
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
}
}
func nullIfEmpty(b []byte) []byte {
if len(b) == 0 {
return []byte("null")
}
return b
}
func (p *player) call(name string, args map[string]any) (json.RawMessage, error) {
p.mu.Lock()
p.next++
id := fmt.Sprintf("c-%d", p.next)
ch := make(chan wire, 1)
p.pending[id] = ch
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args,
"_meta": map[string]any{"mesh/caller": "person.jochen"}}})
select {
case m := <-ch:
if m.Error != nil {
return nil, errors.New(m.Error.Message)
}
var r struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
IsError bool `json:"isError"`
}
if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 {
return nil, fmt.Errorf("%s answered %s", name, m.Result)
}
if r.IsError {
return nil, errors.New(r.Content[0].Text)
}
return json.RawMessage(r.Content[0].Text), nil
case <-time.After(60 * time.Second):
return nil, fmt.Errorf("%s did not answer in 60s", name)
}
}
// 1952/C6: an observer's finding raises a condition on the bus, said with the controller's body, and its deletion clears
// it.
func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
bin := filepath.Join(t.TempDir(), Module)
build := exec.Command("go", "build", "-o", bin, ".")
build.Stderr = os.Stderr
if err := build.Run(); err != nil {
t.Fatalf("the keeper does not build: %v", err)
}
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
buckets := []string{"mesh-controller_conditions", "mesh-controller_lease", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_placement",
"mesh-healers_attempts"}
for _, o := range Observers {
buckets = append(buckets, o+"_findings")
}
for _, s := range []string{ConditionsState, HistoryState, TakeoverState, HeldState, BoundsState, AskedState} {
buckets = append(buckets, Module+"_"+s)
}
for _, b := range buckets {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
t.Fatal(err)
}
}
events, err := nc.SubscribeSync("mesh.seat." + Seat + ".event.>")
if err != nil {
t.Fatal(err)
}
cmd := exec.CommandContext(ctx, bin)
cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module)
stdin, _ := cmd.StdinPipe()
stdout, _ := cmd.StdoutPipe()
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
go p.serve(stdout)
// The first call serves; the watches start after it.
if _, err := p.call(InputsTool, map[string]any{}); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
at := time.Now().UTC()
f := health.Finding{Key: "machine.g14.silent", Kind: "silent", Subject: health.Subject{Scope: health.ScopeMachine, ID: "g14",
Machine: "g14"}, Severity: health.Warning, Summary: "g14 is silent", Headline: "g14 is offline",
Explanation: "g14 said nothing for three minutes.", Resolved: "g14 is back", Source: "S1",
Observer: health.WatchmanModule, At: at, Evidence: []health.Evidence{{At: at, Said: "no presence for 3m"}}}
raw, err := health.EncodeFinding(f)
if err != nil {
t.Fatal(err)
}
findings, _ := js.KeyValue(ctx, health.WatchmanModule+"_findings")
if _, err := findings.Put(ctx, f.Key, raw); err != nil {
t.Fatal(err)
}
msg, err := events.NextMsg(10 * time.Second)
if err != nil {
t.Fatalf("no condition-raised was said: %v", err)
}
var e health.Event
if err := json.Unmarshal(msg.Data, &e); err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-raised" ||
e.Key != f.Key || e.Change != health.ChangeRaised || e.Show != health.ShowKeeper(f.Key) || e.Headline != f.Headline {
t.Fatalf("said %s on %s", msg.Data, msg.Subject)
}
listed, err := p.call(Seat+".conditions", map[string]any{})
if err != nil || !strings.Contains(string(listed), `"key":"machine.g14.silent"`) {
t.Fatalf("conditions answered %s, %v", listed, err)
}
if err := findings.Delete(ctx, f.Key); err != nil {
t.Fatal(err)
}
msg, err = events.NextMsg(10 * time.Second)
if err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-cleared" {
t.Fatalf("no condition-cleared was said: %v", err)
}
conditions, _ := js.KeyValue(ctx, Module+"_"+ConditionsState)
if _, err := conditions.Get(ctx, f.Key); !errors.Is(err, jetstream.ErrKeyNotFound) {
t.Errorf("the cleared condition is still kept: %v", err)
}
}
+22
View File
@@ -0,0 +1,22 @@
module mesh-conditions
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
+27
View File
@@ -0,0 +1,27 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+84
View File
@@ -0,0 +1,84 @@
{
"module": "mesh-conditions",
"version": "1",
"runs-as": "mesh-conditions",
"claims": [
{
"name": "mesh-conditions",
"scope": "mesh",
"serves": [
"conditions",
"silence",
"compare",
"take-over",
"hand-back",
"resync"
]
}
],
"uses": [
"operator-channel"
],
"reads": [
"mesh-delivery-watchman.findings",
"mesh-self-check.findings",
"mesh-healers.findings",
"mesh-delivery-assigner.findings",
"mesh-healers.attempts",
"mesh-controller.conditions",
"mesh-controller.lease",
"mesh-delivery-assigner.wants",
"node-engine.have",
"mesh-delivery-assigner.placement"
],
"state": [
"conditions",
{
"name": "history",
"ttl-seconds": 7776000
},
"takeover",
"held",
{
"name": "bounds",
"key-ttl": true
},
"asked"
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
"secrets-owner": "mesh-conditions",
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-conditions",
"shell": "/usr/bin/nologin",
"home": "/var/lib/mesh-conditions"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "mesh-conditions",
"place": ".",
"unknown": "program"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-conditions",
"binary": "mesh-conditions",
"loads": [
"mesh-conditions"
]
}
]
}
}
+18
View File
@@ -61,3 +61,21 @@ them, newest first; with `proposal`, one read whole, with whether the router's a
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
## The switch from the controller's records (novox/hq ADR 0565 §3, §4, §13)
- `mesh-config_declared`: each placed module's declarations at the trunk, with the manifest and commit they come
from, written only when they differ.
- `mesh-config_takeover`: the key `switch` names the source that holds: none (the interim since mesh-catalog #558:
declarations from the modules' own manifests, the list of predefined keys and pull requests' config from the
controller), `indexer` after the operator's take-over (the list and pull requests' config from the indexer's
`predefined` and `config-candidates`; the controller's `pull-declares` refused), `controller` after a hand-back
(declarations, list and pull requests' config from the controller's records). Each take-over ask is kept beside it.
- `config_compare` lists every difference between the own sources and the controller's records: each module's
declarations, the list, each open pull request's config, and every placed module's value on every node resolved
both ways. `config_take_over` asks the operator's Approve, refused while any difference stands and switched on the
Approve only if none stands then. `config_hand_back` returns to the controller's records until its lease is handed
away. `config_declared` and `config_resync` read and write again what it keeps.
- From the controller's lease write a trusted value is set only through propose and the operator's Approve.
- Its sets, clears and warrants are recorded in the hand-act log of the seat `mesh-hand-acts`, falling back to the
controller's while that seat has no holder and the controller holds its lease.
@@ -275,7 +275,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) {
return nil, fmt.Errorf("%s is refused for %s at %s: %s", show(value), call.Key, level,
strings.Join(problems, "; "))
}
if trusted && call.Caller != ControllerCaller && call.approval == nil {
if trusted && call.approval == nil && !c.atTheTerminal(call) {
return nil, fmt.Errorf("%s is trusted (it decides what runs or who can reach what), so it is set only at "+
"the controller's terminal or on the operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: "+
"propose it instead, with config's action propose", call.Key, call.Caller)
@@ -293,7 +293,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) {
if !had {
return nil, fmt.Errorf("nothing is set for %s at %s, so there is nothing to clear", call.Key, level)
}
if trusted && call.Caller != ControllerCaller && call.approval == nil {
if trusted && call.approval == nil && !c.atTheTerminal(call) {
return nil, fmt.Errorf("%s is trusted, so it is cleared only at the controller's terminal or on the "+
"operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: propose the clear instead, with "+
"config's action propose and clear", call.Key, call.Caller)
@@ -326,6 +326,13 @@ func (c *Config) change(call Call) (*SetAnswer, error) {
return answer, nil
}
// atTheTerminal says a call is the controller's relay of its terminal, which holds only until the controller's lease is
// handed away: from the lease write a trusted value is set only through propose and the operator's Approve, whoever
// calls (novox/hq ADR 0565 §13).
func (c *Config) atTheTerminal(call Call) bool {
return call.Caller == ControllerCaller && !c.leaseAway()
}
// declared is the rule a value at a level is held to, and whether it is trusted; refused, naming the declared
// keys, for a key the module or the list of predefined keys does not declare (ADR 0348 §10).
func (c *Config) declared(s Sources, level Level, key string) (rule, bool, error) {
+22 -7
View File
@@ -19,6 +19,8 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/handacts"
)
// Seat is the role this module holds, once for the mesh, and Verb its one verb.
@@ -73,9 +75,21 @@ func (runtimeStore) Read(log string) ([]LogEntry, error) {
func (runtimeStore) Emit(event string, body any) error { return stdio.Emit(event, body) }
// HandAct records a change in the controller's hand-act log (to-be 45 §7) through its seat's verb.
// recorder records into the hand-act log of the seat mesh-hand-acts, falling back to the controller's while the seat
// has no holder that answers and the controller still holds its lease (novox/hq ADR 0565 §11).
var recorder = handacts.Recorder{Ask: stdio.Ask, LeaseAway: func() bool {
e, err := stdio.State(LeaseRead).Get(buscompose.ControllerLeaseKey)
var raw []byte
if e != nil {
raw = e.Value
}
away, _ := buscompose.LeaseHandedAway(raw, e != nil, err)
return away
}}
// HandAct records a change in the hand-act log (to-be 45 §7).
func (runtimeStore) HandAct(what, why, cause string) error {
_, err := stdio.Ask("seat:mesh-controller.hand-act", map[string]any{"what": what, "why": why, "cause": cause})
_, err := recorder.Record(handacts.Request{What: what, Why: why, Cause: cause})
return err
}
@@ -90,7 +104,7 @@ func (runtimeAsks) Publish(subject string, body any, id string) error {
func (runtimeAsks) Record(id string) (*stdio.StateEntry, error) { return stdio.SeatRecord("", id) }
func (runtimeAsks) Kinds() ([]stdio.KindHeld, error) { return stdio.SeatKinds() }
func (runtimeAsks) Warranted(ask string) error {
_, err := stdio.Ask("seat:mesh-controller.warranted", map[string]any{"asker": Module, "ask": ask})
_, err := recorder.RecordWarrant(Module, ask)
return err
}
@@ -118,7 +132,8 @@ func resolving(c *Config) {
}
// A build's manifest is written before a want names the build, so a want, a trunk module or a placement moving is
// every change of what a module declares where (issue 1953); the builds themselves are read, not watched.
for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead} {
for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead, IndexerPredefinedRead,
CandidatesRead, MirrorDeclaredRead, TakeoverState} {
go func(read string) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
time.Sleep(wait)
@@ -221,9 +236,9 @@ func hearing(c *Config) {
}
}
// Tools is the seat's verb as the runtime serves it.
// Tools is the seat's verb and the module's own tools of the switch, as the runtime serves them.
func Tools(c *Config) []stdio.Tool {
return []stdio.Tool{{
return append(switchTools(c), stdio.Tool{
Name: Seat + "." + Verb,
Description: "Set, clear, list or read the history of config values (novox/hq ADR 0348). A value is set at " +
"one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " +
@@ -268,7 +283,7 @@ func Tools(c *Config) []stdio.Tool {
call.Caller = stdio.CallerOf(args)
return c.Do(call)
},
}}
})
}
// callOf reads a call's arguments strictly: an argument it does not know is refused, not ignored.
@@ -0,0 +1,73 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
// Every state, log and read the manifest declares has a tool that reads it, and every act the module takes by itself
// a tool that starts it (novox/hq ADR 0565 §15): the verb config's actions, and the switch's tools.
func TestEveryStateAndActHasItsTool(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m struct {
State []string `json:"state"`
Logs []string `json:"logs"`
Reads []string `json:"reads"`
Tools []string `json:"tools"`
Claims []struct {
Name string `json:"name"`
Serves []string `json:"serves"`
} `json:"claims"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
via := map[string]string{
ValuesState: "config list", ResolvedState: "config list", PredefinedState: "config list",
ProposalsState: "config proposals", PullsState: "config list", KeptState: "config list", HistoryLog: "config history",
DeclaredState: DeclaredTool, TakeoverState: DeclaredTool,
PlacementRead: CompareTool, WantsRead: CompareTool, BuildsRead: CompareTool, ModulesRead: DeclaredTool,
ManifestsRead: DeclaredTool, PredefinedRead: CompareTool, MirrorDeclaredRead: CompareTool,
IndexerPredefinedRead: CompareTool, CandidatesRead: CompareTool, LeaseRead: HandBackTool,
"a resolution": ResyncTool, "the indexer's candidates taken": ResyncTool, "a take-over decided": TakeOverTool,
}
served := map[string]bool{}
for _, tool := range Tools(&Config{}) {
served[tool.Name] = true
}
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || strings.Join(m.Claims[0].Serves, ",") != strings.Join(SeatVerbs, ",") {
t.Errorf("claims %+v, want %s serving %v", m.Claims, Seat, SeatVerbs)
}
for _, verb := range SeatVerbs {
if !served[Seat+"."+verb] {
t.Errorf("the seat's verb %s is not served", verb)
}
}
for _, name := range m.Tools {
if !served[name] {
t.Errorf("the manifest lists the tool %s, which is not served", name)
}
}
for _, list := range [][]string{m.State, m.Logs, m.Reads} {
for _, s := range list {
if _, ok := via[s]; !ok {
t.Errorf("%s has no tool that reads it", s)
}
}
}
for what, tool := range via {
name, _, _ := strings.Cut(tool, " ")
if name == "config" {
name = Seat + "." + Verb
}
if !served[name] {
t.Errorf("%s is read or started by %s, which is not served", what, tool)
}
}
}
@@ -758,6 +758,7 @@ func (c *Config) Decide() ([]string, error) {
for _, r := range ready {
said = append(said, c.decide(r.p, r.w))
}
said = append(said, c.decideTakeovers()...)
c.recordWarrants(now)
return said, nil
}
+29 -24
View File
@@ -10,6 +10,8 @@ import (
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A config value held for the open pull request that introduces its key (novox/hq ADR 0502, issue 1826).
@@ -61,31 +63,22 @@ type PullDecl struct {
At time.Time `json:"at"`
}
// KeptValue is an approved held value, kept apart from the values mesh-config resolves.
type KeptValue struct {
Pull string `json:"pull"`
Head string `json:"head"`
Digest string `json:"declaration"`
Node string `json:"node"`
Module string `json:"module"`
Key string `json:"key"`
Value json.RawMessage `json:"value"`
By string `json:"by"`
Via string `json:"via"`
Why string `json:"why"`
// Approved is who approved it, as the proposal kept it.
Approved string `json:"approved"`
Proposal string `json:"proposal"`
At time.Time `json:"at"`
}
// KeptValue is an approved held value, kept apart from the values mesh-config resolves: mesh-sdk's layout, which the
// checker and the assigner's composition check read (novox/hq ADR 0565 §5).
type KeptValue = delivery.KeptValue
// keptKey is a held value's key in `kept` (delivery.HeldValueKey), the pull request's key form kept for one that is
// not one.
func keptKey(pull, node, module, key string) string {
if k, err := delivery.HeldValueKey(pull, node, module, key); err == nil {
return k
}
return pullKey(pull) + "." + node + "." + module + "." + key
}
func (k KeptValue) stateKey() string { return keptKey(k.Pull, k.Node, k.Module, k.Key) }
func keptStateKey(k KeptValue) string { return keptKey(k.Pull, k.Node, k.Module, k.Key) }
func (k KeptValue) level() Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} }
func keptLevel(k KeptValue) Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} }
// declDigest is the digest of what a declaration holds a value to: a held value stays only while it is the same.
func declDigest(d Declaration) string {
@@ -151,6 +144,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) {
return nil, fmt.Errorf("pull-declares is the controller's: it reads a pull request's manifests and tells "+
"mesh-config what the head declares; %s is not the controller", call.Caller)
}
if sw, err := c.source(); err != nil {
return nil, err
} else if sw.Source == SourceIndexer {
return nil, errors.New("pull-declares is refused: since the take-over mesh-config reads a pull request's config " +
"declarations from the indexer's config-candidates (novox/hq ADR 0565 §4)")
}
if _, _, _, err := parsePull(call.Pull); err != nil {
return nil, err
}
@@ -185,6 +184,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) {
pull := strings.TrimSpace(call.Pull)
c.mu.Lock()
defer c.mu.Unlock()
return c.declarePull(pull, head, declares)
}
// declarePull keeps what a pull request's head declares and judges again the values held for it; from the
// controller's pull-declares, or after the take-over from the indexer's config candidates. The caller holds the lock.
func (c *Config) declarePull(pull, head string, declares map[string]map[string]Declaration) (*PullDeclaresAnswer, error) {
if err := c.Store.Put(PullsState, pullKey(pull), PullDecl{Pull: pull, Head: head, Declares: declares,
At: c.now()}); err != nil {
return nil, fmt.Errorf("the declarations were not kept: %v", err)
@@ -235,7 +240,7 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) {
// heldChange records a hold, a drop or a merge's set of a held value in the history, the hand-act log and
// config-changed; what could not be done is answered, not dropped.
func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string {
level := kv.level()
level := keptLevel(kv)
ch := Change{Action: action, Level: &level, Key: kv.Key, Value: kv.Value, By: kv.By, Via: kv.Via, Why: why,
Approved: approved, Pull: kv.Pull, Head: kv.Head, At: c.now(), Moved: []Moved{}}
unsaid := c.record(level.ValueKey(kv.Key), ch)
@@ -252,8 +257,8 @@ func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string
// dropKept removes a held value, saying why.
func (c *Config) dropKept(kv KeptValue, why string) []string {
if err := c.Store.Delete(KeptState, kv.stateKey()); err != nil {
return []string{"the held value " + kv.stateKey() + " was not dropped: " + err.Error()}
if err := c.Store.Delete(KeptState, keptStateKey(kv)); err != nil {
return []string{"the held value " + keptStateKey(kv) + " was not dropped: " + err.Error()}
}
c.logf("[mesh-config] dropped %s of %s on %s held for %s: %s", kv.Key, kv.Module, kv.Node, kv.Pull, why)
return c.heldChange(kv, ActionDrop, "dropped, nothing was set: "+why, "")
@@ -408,7 +413,7 @@ func (c *Config) keep(p Proposal) (string, error) {
kv := KeptValue{Pull: p.Pull, Head: pd.Head, Digest: p.DeclDigest, Node: p.Level.Node, Module: p.Level.Module,
Key: p.Key, Value: p.Value, By: "the operator", Via: p.From, Why: p.Why, Approved: p.Approved, Proposal: p.ID,
At: c.now()}
if err := c.Store.Put(KeptState, kv.stateKey(), kv); err != nil {
if err := c.Store.Put(KeptState, keptStateKey(kv), kv); err != nil {
return "", fmt.Errorf("the value was not kept: %v", err)
}
unsaid := c.heldChange(kv, ActionHold, "held for "+p.Pull+" until it merges: "+p.Why, p.Approved)
@@ -471,7 +476,7 @@ func (c *Config) settlePull(pull string, merged bool, head string) []string {
said = append(said, c.dropKept(kv, kv.Pull+" merged, but "+why)...)
continue
}
vk := kv.level().ValueKey(kv.Key)
vk := keptLevel(kv).ValueKey(kv.Key)
v := Value{Value: compact(kv.Value), By: kv.By, Via: kv.Via, Why: kv.Why + " (held for " + kv.Pull +
", merged)", Approved: kv.Approved, At: c.now()}
if err := c.Store.Put(ValuesState, vk, v); err != nil {
@@ -0,0 +1,2 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
modules/mesh-config/module.json
+44 -14
View File
@@ -12,10 +12,19 @@ import (
// Reading the sources, writing the resolved values, and publishing the list of predefined keys.
// sources reads everything a resolution reads, from the bus as it is now: which modules are placed where from the
// seat mesh-placement, and each placed module's declarations from its own manifests as built, never from the
// controller (issue 1953, ADR 0565 §3).
// sources reads everything a resolution reads, from the bus as it is now, from the source the switch names
// (takeover.go): which modules are placed where from the seat mesh-placement, and each placed module's declarations
// from its own manifests as built (issue 1953, ADR 0565 §3), or after a hand-back from the controller's mirror.
func (c *Config) sources() (Sources, error) {
sw, err := c.source()
if err != nil {
return Sources{}, err
}
return c.sourcesFrom(sw.Source)
}
// sourcesFrom reads the sources with declarations from one source: the own (interim and indexer) or the mirror.
func (c *Config) sourcesFrom(source string) (Sources, error) {
s := Sources{Declared: map[string]ModuleDeclarations{}, OnNode: map[string]ModuleDeclarations{},
Predefined: map[string]Published{}, Values: map[string]Value{}}
var problems []string
@@ -25,6 +34,9 @@ func (c *Config) sources() (Sources, error) {
if err := c.readDeclarations(&s); err != nil {
return s, fmt.Errorf("the modules' declarations cannot be read: %w", err)
}
if source == SourceController {
c.mirrorDeclarations(&s)
}
for _, u := range s.Unresolved {
problems = append(problems, fmt.Sprintf("%s on %s: %s", u.Module, u.Node, u.Why))
}
@@ -350,6 +362,10 @@ type Resolution struct {
Moved []Moved `json:"moved"`
Published string `json:"published,omitempty"`
Withheld string `json:"withheld,omitempty"`
// Declared are the modules whose `declared` key was written, and Candidates what the indexer's config candidates
// did after the take-over.
Declared []string `json:"declared,omitempty"`
Candidates []string `json:"candidates,omitempty"`
}
// Reconcile publishes the newest list of predefined keys and brings mesh-config_resolved to what the sources
@@ -374,26 +390,39 @@ func (c *Config) Reconcile() (Resolution, error) {
c.record("resolved", Change{Action: "resolve", Why: "an assignment, a declaration or the list of predefined " +
"keys changed", At: c.now(), Moved: moved})
}
if written, derr := c.writeDeclared(s); derr != nil {
c.logf("[mesh-config] declared is not all written: %v", derr)
} else if len(written) > 0 {
r.Declared = written
}
if sw, serr := c.source(); serr == nil && sw.Source == SourceIndexer {
r.Candidates = c.takeCandidates()
}
return r, err
}
// publish makes mesh-config_predefined hold the list the controller kept, when it is newer than the one published
// (ADR 0348 §7: the newest trunk build wins). **A list dropping a key that still holds a value is withheld**: the
// publish makes mesh-config_predefined hold the list its source keeps — the indexer's after the take-over, the
// controller's mirror otherwise — when it is newer than the one published (ADR 0348 §7: the newest trunk build wins;
// the indexer never keeps an older merge's, so its list is newer when its commit differs). **A list dropping a key that still holds a value is withheld**: the
// list published stays, and why is said in every listing and in the log, because a value for a key no longer on
// the list could neither be resolved nor cleared by name (§7: removing a key still holding a value is refused).
// It answers what it published, empty when nothing changed.
func (c *Config) publish() (string, error) {
e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey)
sw, err := c.source()
if err != nil {
return "", fmt.Errorf("the controller's list cannot be read: %w", err)
return "", err
}
if e == nil {
read, whose := c.mirrorList, "the controller's list"
if sw.Source == SourceIndexer {
read, whose = c.indexerList, "the indexer's list"
}
list, found, err := read()
if err != nil {
return "", fmt.Errorf("%s cannot be read: %w", whose, err)
}
if !found {
return "", nil // no list kept yet
}
var list PredefinedList
if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil {
return "", fmt.Errorf("the controller's list is not one: %s", e.Value)
}
current := map[string]Published{}
if err := c.each(PredefinedState, func(key string, raw []byte) error {
var p Published
@@ -405,11 +434,12 @@ func (c *Config) publish() (string, error) {
}); err != nil {
return "", err
}
generation := 0
generation, sameCommit := 0, len(current) > 0
for _, p := range current {
generation = max(generation, p.Generation)
sameCommit = sameCommit && p.Commit == list.Commit
}
if len(current) > 0 && generation >= list.Generation {
if len(current) > 0 && (sameCommit || (list.Generation > 0 && generation >= list.Generation)) {
c.withheld = ""
return "", nil
}
@@ -0,0 +1,79 @@
package main
// The tools of the switch (novox/hq ADR 0565 §4, §15): compare, take-over and hand-back, served as the seat's verbs
// and as the module's own tools beside them, the read of `declared` and resync.
import (
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The switch's tools.
const (
CompareTool = "config_compare"
TakeOverTool = "config_take_over"
HandBackTool = "config_hand_back"
DeclaredTool = "config_declared"
ResyncTool = "config_resync"
)
// SeatVerbs are the seat's verbs, as the manifest claims them.
var SeatVerbs = []string{Verb, CompareVerb, TakeOverVerb, HandBackVerb}
func switchTools(c *Config) []stdio.Tool {
own := ownSwitchTools(c)
byName := map[string]stdio.Tool{}
for _, t := range own {
byName[t.Name] = t
}
for verb, tool := range map[string]string{CompareVerb: CompareTool, TakeOverVerb: TakeOverTool, HandBackVerb: HandBackTool} {
t := byName[tool]
t.Name = Seat + "." + verb
own = append(own, t)
}
return own
}
func ownSwitchTools(c *Config) []stdio.Tool {
why := map[string]any{"type": "object", "properties": map[string]any{
"why": map[string]any{"type": "string", "description": "why, in a sentence"}}, "required": []string{"why"}}
whyOf := func(a map[string]any) string { s, _ := a["why"].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: CompareTool,
Description: "Every difference between config's own sources and the controller's records (novox/hq ADR 0565 §4): " +
"each placed module's declarations (declared against the mirror declared-config), the list of predefined " +
"keys (the indexer's against the mirror's), each open pull request's config (the indexer's candidate " +
"against the controller's pull-declares), and every placed module's value on every node resolved both " +
"ways; each with its reason. The take-over is refused while any stands.",
Run: func(map[string]any) (any, error) { return c.Compare() }},
{Name: TakeOverTool,
Description: "Ask the operator's Approve, under mesh-config's own name, to switch config to its own sources: " +
"the indexer's list of predefined keys and pull request config, refusing the controller's pull-declares. " +
"Refused while compare shows a difference; on Approve switched if nothing differs then.",
Input: why,
Run: func(a map[string]any) (any, error) {
return c.TakeOver(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)})
}},
{Name: HandBackTool,
Description: "Return config to the controller's records (its mirrors declared-config and predefined, and " +
"pull-declares) until the controller hands its lease away; gone after it. Recorded as a hand act.",
Input: why,
Run: func(a map[string]any) (any, error) {
return c.HandBack(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)})
}},
{Name: DeclaredTool,
Description: "Each placed module's declarations as config keeps them (its state declared): the manifest and " +
"commit they come from and how many keys, or one module whole; and which source holds now.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "one module, whole"}}},
Run: func(a map[string]any) (any, error) {
m, _ := a["module"].(string)
return c.Declared(strings.TrimSpace(m))
}},
{Name: ResyncTool,
Description: "Read every source again and write what differs: the published list of predefined keys, the " +
"resolved values, declared, and after the take-over the indexer's config candidates.",
Run: func(map[string]any) (any, error) { return c.Resync() }},
}
}
@@ -0,0 +1,771 @@
package main
// Config's records out of the controller (novox/hq ADR 0565 §3, §4, §13, §15, and the progressive insight of
// 2026-10-29): mesh-config keeps each placed module's declarations in its own state `declared`; `compare` lists every
// difference between its own sources and the controller's mirrors — each module's declarations, the predefined list,
// each open pull request's config declarations, and every placed module's resolved value on every node both ways;
// `take-over` asks the operator's Approve under mesh-config's own name and, refused while anything differs, writes the
// key `takeover.switch` naming the indexer; `hand-back` returns to the mirrors until the lease write.
//
// **Which source holds** is the key `switch` alone, read by mesh-config alone:
// - none written (the interim of mesh-catalog #558): declarations from each placed module's own manifests, the list
// of predefined keys from the controller's mirror, a pull request's declarations from the controller's
// pull-declares;
// - `indexer` (taken over): declarations as before, the list from the indexer's `predefined`, a pull request's from
// the indexer's `config-candidates`; pull-declares refused;
// - `controller` (handed back): declarations from the mirror `declared-config`, the list from the mirror,
// pull-declares taken.
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The states and reads of the switch.
const (
// DeclaredState is each placed module's declarations at the trunk, one key per module (ADR 0565 §3).
DeclaredState = "declared"
// TakeoverState holds the key SwitchKey and each take-over ask, `ask.<id>`.
TakeoverState = "takeover"
SwitchKey = "switch"
// MirrorDeclaredRead is the controller's mirror of every registered module's declarations, one key per module:
// what `compare` holds the own source to, and the source after a hand-back.
MirrorDeclaredRead = "mesh-controller.declared-config"
// IndexerPredefinedRead and CandidatesRead are the indexer's states (ADR 0565 §2).
IndexerPredefinedRead = delivery.IndexerModule + "." + delivery.IndexerPredefinedState
CandidatesRead = delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState
// LeaseRead is the controller's lease: after its write hand-back is gone and the terminal sets nothing (§13).
LeaseRead = buscompose.ControllerLease
)
// The sources the switch names.
const (
SourceInterim = "interim"
SourceIndexer = "indexer"
SourceController = "controller"
)
// TakeOverVerb and HandBackVerb are the seat's verbs.
const (
TakeOverVerb = "take-over"
HandBackVerb = "hand-back"
CompareVerb = "compare"
)
// Switch is the key `takeover.switch`: the source that holds, who chose it, why and on which ask.
type Switch struct {
Source string `json:"source"`
At time.Time `json:"at"`
By string `json:"by"`
Why string `json:"why"`
Ask string `json:"ask,omitempty"`
}
// source is the source that holds now.
func (c *Config) source() (Switch, error) {
e, err := c.Store.Get(TakeoverState, SwitchKey)
if err != nil {
return Switch{}, fmt.Errorf("the take-over's switch cannot be read: %w", err)
}
if e == nil {
return Switch{Source: SourceInterim}, nil
}
var s Switch
if err := json.Unmarshal(e.Value, &s); err != nil || (s.Source != SourceIndexer && s.Source != SourceController) {
return Switch{}, fmt.Errorf("the take-over's switch is not one: %s", e.Value)
}
return s, nil
}
// leaseAway says the controller's lease is handed away (ADR 0561 §3's reading).
func (c *Config) leaseAway() bool {
e, err := c.Store.Get(LeaseRead, buscompose.ControllerLeaseKey)
var raw []byte
if e != nil {
raw = e.Value
}
away, _ := buscompose.LeaseHandedAway(raw, e != nil, err)
return away
}
// ---- declared --------------------------------------------------------------------------------------------
// DeclaredKept is one key of `declared`: a module's declarations as the manifest the indexer's `modules.<module>`
// names holds them, with that manifest's digest and commit (ADR 0565 §3).
type DeclaredKept struct {
Module string `json:"module"`
Manifest string `json:"manifest"`
Commit string `json:"commit,omitempty"`
Config map[string]Declaration `json:"settings"`
}
// trunkKept is a module's declarations at the trunk with the manifest they come from.
func (c *Config) trunkKept(module string) (DeclaredKept, error) {
key, err := delivery.ModulesKey(module)
if err != nil {
return DeclaredKept{}, err
}
e, err := c.Store.Get(ModulesRead, key)
if err != nil || e == nil {
return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s cannot be read (%v)", module, err)
}
var m struct {
Manifest string `json:"manifest"`
Commit string `json:"commit"`
}
if err := json.Unmarshal(e.Value, &m); err != nil || m.Manifest == "" {
return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s names no manifest", module)
}
d, err := c.manifestDeclarations(module, m.Manifest)
if err != nil {
return DeclaredKept{}, err
}
return DeclaredKept{Module: module, Manifest: m.Manifest, Commit: m.Commit, Config: d.Config}, nil
}
// writeDeclared makes `declared` hold each placed module's trunk declarations, writing a key only when it differs
// (ADR 0289 §1) and removing one of a module placed nowhere. It answers the modules written.
func (c *Config) writeDeclared(s Sources) ([]string, error) {
held := map[string][]byte{}
if err := c.each(DeclaredState, func(key string, raw []byte) error {
held[key] = raw
return nil
}); err != nil {
return nil, err
}
var written []string
var problems []error
placed := map[string]bool{}
for _, module := range modulesOf(s.Assignments) {
placed[module] = true
k, err := c.trunkKept(module)
if err != nil {
continue // unresolved, said by the resolution; its key stays as it was
}
raw, _ := json.Marshal(k)
if bytes.Equal(raw, held[module]) {
continue
}
if err := c.Store.Put(DeclaredState, module, k); err != nil {
problems = append(problems, fmt.Errorf("%s: %w", module, err))
continue
}
written = append(written, module)
}
for module := range held {
if !placed[module] {
if err := c.Store.Delete(DeclaredState, module); err != nil {
problems = append(problems, fmt.Errorf("%s: %w", module, err))
}
}
}
return written, errors.Join(problems...)
}
// mirrorDeclarations makes the sources' declarations the controller's mirror's, for every placed module: the source
// after a hand-back, and the other side of compare.
func (c *Config) mirrorDeclarations(s *Sources) {
s.Declared, s.OnNode, s.Unresolved = map[string]ModuleDeclarations{}, map[string]ModuleDeclarations{}, nil
for _, module := range modulesOf(s.Assignments) {
e, err := c.Store.Get(MirrorDeclaredRead, module)
var d ModuleDeclarations
if err == nil && e != nil {
d, err = ParseModuleDeclarations(e.Value)
} else if err == nil {
err = errors.New("the mirror holds no key for it")
}
if err != nil {
for _, a := range s.Assignments {
if a.Module == module {
s.Unresolved = append(s.Unresolved, Unresolved{Node: a.Node, Module: module,
Why: "the controller's mirror declared-config: " + err.Error()})
}
}
continue
}
s.Declared[module] = d
}
}
// ---- the predefined list and the pull requests from the indexer ------------------------------------------
// indexerList is the indexer's predefined list as the mirror's list holds one: the keys, and the commit.
func (c *Config) indexerList() (PredefinedList, bool, error) {
e, err := c.Store.Get(IndexerPredefinedRead, delivery.PredefinedListKey)
if err != nil || e == nil {
return PredefinedList{}, false, err
}
var kept delivery.PredefinedKept
if err := json.Unmarshal(e.Value, &kept); err != nil {
return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not one: %w", err)
}
list := PredefinedList{Keys: map[string]PredefinedKey{}, Commit: kept.Commit}
if err := json.Unmarshal(kept.List, &list.Keys); err != nil {
return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not predefined.json: %w", err)
}
return list, true, nil
}
// mirrorList is the controller's mirror's list.
func (c *Config) mirrorList() (PredefinedList, bool, error) {
e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey)
if err != nil || e == nil {
return PredefinedList{}, false, err
}
var list PredefinedList
if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil {
return PredefinedList{}, false, fmt.Errorf("the controller's list is not one: %s", e.Value)
}
return list, true, nil
}
// candidates is every open pull request's config declarations as the indexer keeps them, by pull key.
func (c *Config) candidates() (map[string]PullDecl, error) {
out := map[string]PullDecl{}
err := c.each(CandidatesRead, func(key string, raw []byte) error {
var cand delivery.ConfigCandidate
if err := json.Unmarshal(raw, &cand); err != nil || cand.Pull == "" {
return fmt.Errorf("the indexer's config candidate %s is not one", key)
}
p := PullDecl{Pull: cand.Pull, Head: cand.Head, Declares: map[string]map[string]Declaration{}, At: cand.At}
for module, keys := range cand.Declares {
raw, _ := json.Marshal(keys)
var d map[string]Declaration
if err := json.Unmarshal(raw, &d); err != nil {
return fmt.Errorf("the indexer's config candidate %s declares %s unreadably", key, module)
}
for k, decl := range d {
if !decl.HasDefault() {
decl.Default = nil
}
d[k] = decl
}
p.Declares[module] = d
}
out[pullKey(cand.Pull)] = p
return nil
})
return out, err
}
// takeCandidates is the indexer's config candidates taken in place of pull-declares, after the take-over: each one
// whose head or declarations differ from what `pulls` holds is declared again, its held values judged again (ADR 0502
// §6). The caller holds the lock.
func (c *Config) takeCandidates() []string {
cands, err := c.candidates()
if err != nil {
return []string{"the indexer's config candidates cannot be read: " + err.Error()}
}
var s Sources
if err := c.readPulls(&s); err != nil {
return []string{err.Error()}
}
var said []string
for _, k := range sortedKeys(cands) {
p := cands[k]
if have, ok := s.Pulls[k]; ok && have.Head == p.Head && declaresDigest(have.Declares) == declaresDigest(p.Declares) {
continue
}
a, err := c.declarePull(p.Pull, p.Head, p.Declares)
if err != nil {
said = append(said, fmt.Sprintf("%s from the indexer: %v", p.Pull, err))
continue
}
said = append(said, a.Said+" (from the indexer)")
}
return said
}
// declaresDigest is what a pull request's declarations hold a value to, in one encoding.
func declaresDigest(d map[string]map[string]Declaration) string {
flat := map[string]string{}
for m, keys := range d {
for k, decl := range keys {
flat[m+"."+k] = declDigest(decl)
}
}
raw, _ := json.Marshal(flat)
return string(raw)
}
// ---- compare ----------------------------------------------------------------------------------------------
// Difference is one thing the own sources and the controller's mirrors say differently, and why.
type Difference struct {
What string `json:"what"`
Module string `json:"module,omitempty"`
Node string `json:"node,omitempty"`
Key string `json:"key,omitempty"`
Pull string `json:"pull,omitempty"`
Own json.RawMessage `json:"own,omitempty"`
Mirror json.RawMessage `json:"mirror,omitempty"`
Why string `json:"why"`
}
// The kinds of difference.
const (
DiffDeclared = "declared"
DiffPredefined = "predefined"
DiffPull = "pull"
DiffValue = "value"
)
// Comparison is what compare answers.
type Comparison struct {
Source string `json:"source"`
Differences []Difference `json:"differences"`
Count int `json:"count"`
Said string `json:"said"`
}
// Compare lists every difference (ADR 0565 §4 and its insight). The caller holds no lock.
func (c *Config) Compare() (*Comparison, error) {
c.mu.Lock()
defer c.mu.Unlock()
return c.compare()
}
// compare is Compare; the caller holds the lock.
func (c *Config) compare() (*Comparison, error) {
sw, err := c.source()
if err != nil {
return nil, err
}
own, err := c.sourcesFrom(SourceInterim)
if err != nil {
return nil, fmt.Errorf("mesh-config's own sources cannot be read: %w", err)
}
mirror := own
mirror.Declared, mirror.OnNode, mirror.Unresolved = nil, nil, nil
c.mirrorDeclarations(&mirror)
var diffs []Difference
// Each placed module's declarations: `declared` against the mirror's key for it.
for _, module := range modulesOf(own.Assignments) {
var ownRaw []byte
if e, err := c.Store.Get(DeclaredState, module); err == nil && e != nil {
var k DeclaredKept
if json.Unmarshal(e.Value, &k) == nil {
ownRaw, _ = json.Marshal(normal(k.Config))
}
}
var mirrorRaw []byte
if d, ok := mirror.Declared[module]; ok {
mirrorRaw, _ = json.Marshal(normal(d.Config))
}
switch {
case ownRaw == nil && mirrorRaw == nil:
diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Why: "neither side holds its declarations"})
case ownRaw == nil:
diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Mirror: mirrorRaw,
Why: "the mirror holds its declarations and declared does not"})
case mirrorRaw == nil:
diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw,
Why: "declared holds its declarations and the mirror does not"})
case !bytes.Equal(ownRaw, mirrorRaw):
diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw, Mirror: mirrorRaw,
Why: "the trunk's manifest declares it otherwise than the mirror"})
}
}
// The predefined list: the indexer's against the mirror's.
il, iok, ierr := c.indexerList()
ml, mok, merr := c.mirrorList()
switch {
case ierr != nil || merr != nil:
diffs = append(diffs, Difference{What: DiffPredefined, Why: "a list cannot be read: " + errors.Join(ierr, merr).Error()})
case !iok || !mok:
if iok != mok {
diffs = append(diffs, Difference{What: DiffPredefined, Why: fmt.Sprintf("the indexer holds a list: %v; the "+
"mirror holds one: %v", iok, mok)})
}
default:
for _, k := range unionKeys(il.Keys, ml.Keys) {
a, aok := il.Keys[k]
b, bok := ml.Keys[k]
ar, _ := json.Marshal(a)
br, _ := json.Marshal(b)
switch {
case !aok:
diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Mirror: br, Why: "only the mirror's list holds it"})
case !bok:
diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Why: "only the indexer's list holds it"})
case !bytes.Equal(ar, br):
diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Mirror: br,
Why: "the two lists declare it otherwise"})
}
}
}
// Each open pull request: the indexer's candidate against what the controller told by pull-declares.
cands, cerr := c.candidates()
if cerr != nil {
diffs = append(diffs, Difference{What: DiffPull, Why: "the indexer's config candidates cannot be read: " + cerr.Error()})
}
for _, k := range unionKeys(cands, own.Pulls) {
a, aok := cands[k]
b, bok := own.Pulls[k]
switch {
case !aok:
diffs = append(diffs, Difference{What: DiffPull, Pull: b.Pull, Why: "the controller told its declarations and the indexer holds no candidate"})
case !bok:
diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the indexer holds a candidate and the controller told nothing"})
case a.Head != b.Head:
diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: fmt.Sprintf("the indexer's candidate is at %s, "+
"the controller told %s", short(a.Head), short(b.Head))})
case declaresDigest(a.Declares) != declaresDigest(b.Declares):
diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the two declare its config otherwise"})
}
}
// Every placed module's value on every node, resolved both ways.
ownValues, mirrorValues := Resolve(own), Resolve(mirror)
for _, k := range unionKeys(ownValues, mirrorValues) {
a, aok := ownValues[k]
b, bok := mirrorValues[k]
ar, _ := json.Marshal(a)
br, _ := json.Marshal(b)
node, module, key := a.Node, a.Module, a.Key
if !aok {
node, module, key = b.Node, b.Module, b.Key
}
d := Difference{What: DiffValue, Node: node, Module: module, Key: key}
switch {
case !aok:
d.Mirror, d.Why = br, "only the mirror's declarations resolve it on this node"
case !bok:
d.Own, d.Why = ar, "only mesh-config's own declarations (with this node's build) resolve it"
case !bytes.Equal(ar, br):
d.Own, d.Mirror, d.Why = ar, br, "it resolves to another value or source"
default:
continue
}
diffs = append(diffs, d)
}
for _, u := range mirror.Unresolved {
diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from the mirror: " + u.Why})
}
for _, u := range own.Unresolved {
diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from its own source: " + u.Why})
}
sort.SliceStable(diffs, func(i, j int) bool {
a, b := diffs[i], diffs[j]
return a.What+a.Module+a.Node+a.Key+a.Pull < b.What+b.Module+b.Node+b.Key+b.Pull
})
if diffs == nil {
diffs = []Difference{}
}
cmp := &Comparison{Source: sw.Source, Differences: diffs, Count: len(diffs)}
if len(diffs) == 0 {
cmp.Said = "nothing differs: the take-over may be asked"
} else {
cmp.Said = fmt.Sprintf("%d difference(s): the take-over is refused while any stands", len(diffs))
}
return cmp, nil
}
// normal is declarations in one encoding: a null default as none.
func normal(d map[string]Declaration) map[string]Declaration {
out := map[string]Declaration{}
for k, decl := range d {
if !decl.HasDefault() {
decl.Default = nil
} else {
decl.Default = compact(decl.Default)
}
out[k] = decl
}
return out
}
func unionKeys[A, B any](a map[string]A, b map[string]B) []string {
seen := map[string]bool{}
for k := range a {
seen[k] = true
}
for k := range b {
seen[k] = true
}
return sortedKeys(seen)
}
// ---- take-over and hand-back ---------------------------------------------------------------------------------
// TakeoverAsk is one take-over asked of the operator, `takeover.ask.<id>`: what the ask is composed from again, and
// where it stands.
type TakeoverAsk struct {
ID string `json:"id"`
From string `json:"from"`
Why string `json:"why"`
At time.Time `json:"at"`
State string `json:"state"`
Outcome string `json:"outcome,omitempty"`
Decided time.Time `json:"decided,omitzero"`
Answered string `json:"answered,omitempty"`
Recorded bool `json:"recorded,omitempty"`
}
func askKey(id string) string { return "ask." + id }
// act is what an answer to the take-over binds.
func (t TakeoverAsk) act(option string) asks.Act {
return asks.Act{"verb": Seat + "." + TakeOverVerb, "option": option, "ask": t.ID, "from": t.From,
"at": t.At.UTC().Format(time.RFC3339Nano)}
}
// ask is the take-over's ask, composed from it alone.
func (t TakeoverAsk) ask() asks.Ask {
why, _ := sayable("Why: " + t.Why)
a := asks.Ask{ID: t.ID, Headline: "Switch config to its own sources",
Explanation: strings.Join([]string{
"mesh-config would read the list of predefined keys and each pull request's config from the delivery's " +
"indexer, no longer from the controller. Nothing any node resolves changes: compare showed no difference.",
point(why)}, "\n"),
Who: asks.Operator, Expires: t.At.Add(ProposalLasts),
OnExpiry: "nothing changes; config keeps reading the controller's records", About: "config.takeover",
Details: "Read what was compared with mesh-config's compare verb, through the mesh MCP server. hand-back returns " +
"to the controller's records until the controller's lease is handed away."}
for _, o := range []struct{ id, label, does string }{
{OptionApprove, "Approve", "config switches to its own sources at once, if nothing differs then"},
{OptionDecline, "Decline", "nothing changes"},
} {
binds, _ := asks.ActDigest(t.act(o.id))
a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds})
}
return a
}
// TakeoverAnswer is what take-over and hand-back answer.
type TakeoverAnswer struct {
Said string `json:"said"`
Ask string `json:"ask,omitempty"`
Source string `json:"source"`
}
// TakeOver asks the operator's Approve to switch to the own sources, refused while compare shows a difference.
func (c *Config) TakeOver(call Call) (*TakeoverAnswer, error) {
if strings.TrimSpace(call.Why) == "" {
return nil, errors.New("say why the take-over is asked now: it is shown to the operator")
}
if call.Caller == "" || call.Caller == Unproven {
return nil, errors.New("the call names no proven caller")
}
if c.Asks == nil {
return nil, errors.New("the operator cannot be asked from here, so nothing is switched")
}
c.mu.Lock()
defer c.mu.Unlock()
sw, err := c.source()
if err != nil {
return nil, err
}
if sw.Source == SourceIndexer {
return &TakeoverAnswer{Source: sw.Source, Said: "config reads its own sources already, since " +
sw.At.UTC().Format(time.RFC3339) + " (" + sw.By + ")"}, nil
}
cmp, err := c.compare()
if err != nil {
return nil, err
}
if cmp.Count > 0 {
first := cmp.Differences[0]
return nil, fmt.Errorf("the take-over is refused: compare shows %d difference(s), the first %s %s%s%s%s: %s. "+
"Nothing was asked", cmp.Count, first.What, first.Module, first.Node, first.Key, first.Pull, first.Why)
}
now := c.now()
var open []string
_ = c.each(TakeoverState, func(key string, raw []byte) error {
var t TakeoverAsk
if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting &&
now.Before(t.At.Add(ProposalLasts)) {
open = append(open, t.ID)
}
return nil
})
if len(open) > 0 {
return nil, fmt.Errorf("the take-over is asked already, as %s, and waits for the operator's answer", open[0])
}
t := TakeoverAsk{ID: "takeover-" + newProposalID(), From: call.Caller, Why: strings.TrimSpace(call.Why), At: now.UTC(),
State: ProposalWaiting}
ask := t.ask()
if err := ask.Check(now); err != nil {
return nil, err
}
if err := c.Store.Put(TakeoverState, askKey(t.ID), t); err != nil {
return nil, fmt.Errorf("the take-over could not be kept, so the operator was not asked: %w", err)
}
if err := c.Asks.Publish(asks.AskSubject(Module), ask, ask.ID); err != nil {
t.State, t.Outcome = ProposalUnasked, "the ask could not be published: "+err.Error()
_ = c.Store.Put(TakeoverState, askKey(t.ID), t)
return nil, fmt.Errorf("the operator could not be asked: %w", err)
}
return &TakeoverAnswer{Ask: t.ID, Source: sw.Source, Said: fmt.Sprintf("the operator is asked %q under "+
"mesh-config's own name; nothing changes until they approve, and on Approve config switches if nothing differs "+
"then", ask.Headline)}, nil
}
// decideTakeovers acts on each take-over ask the router has closed. The caller holds the lock.
func (c *Config) decideTakeovers() []string {
var said []string
var waiting []TakeoverAsk
_ = c.each(TakeoverState, func(key string, raw []byte) error {
var t TakeoverAsk
if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting {
waiting = append(waiting, t)
}
return nil
})
now := c.now()
for _, t := range waiting {
keep := func(state, outcome string) string {
t.State, t.Outcome, t.Decided = state, outcome, now
_ = c.Store.Put(TakeoverState, askKey(t.ID), t)
return "take-over " + t.ID + ": " + outcome
}
e, err := c.Asks.Record(t.ID)
if err != nil || e == nil {
if err == nil && now.After(t.At.Add(ProposalLasts+time.Hour)) {
said = append(said, keep(ProposalEnded, "the router keeps no record of its ask; nothing was switched"))
}
continue
}
var rec routerRecord
if json.Unmarshal(e.Value, &rec) != nil || rec.State == "open" || rec.Warrant == nil {
continue
}
w := *rec.Warrant
if w.Asker != Module || w.Ask != t.ID {
said = append(said, keep(ProposalRefused, "the router's record is for another ask; nothing was switched"))
continue
}
if w.Outcome != asks.OutcomeChosen {
said = append(said, keep(ProposalEnded, "the ask "+string(w.Outcome)+"; nothing was switched"))
continue
}
option, err := w.For(Module, t.ask())
if err == nil {
err = option.Performs(t.act(option.ID))
}
if err != nil {
said = append(said, keep(ProposalRefused, "the answer is refused: "+err.Error()))
continue
}
t.Answered = w.Says()
if option.ID != OptionApprove {
said = append(said, keep(ProposalDeclined, "declined by the operator; nothing was switched"))
continue
}
cmp, err := c.compare()
switch {
case err != nil:
said = append(said, keep(ProposalRefused, "approved, and compare could not be read, so nothing was switched: "+err.Error()))
continue
case cmp.Count > 0:
said = append(said, keep(ProposalRefused, fmt.Sprintf("approved, and refused: compare shows %d difference(s) "+
"now; nothing was switched, ask again once it is empty", cmp.Count)))
continue
}
sw := Switch{Source: SourceIndexer, At: now.UTC(), By: approvedTakeover(w), Why: t.Why, Ask: t.ID}
if err := c.Store.Put(TakeoverState, SwitchKey, sw); err != nil {
said = append(said, fmt.Sprintf("take-over %s: approved, and the switch could not be written (%v); read again "+
"on the next look", t.ID, err))
continue
}
if err := c.Asks.Warranted(t.ID); err == nil {
t.Recorded = true
}
said = append(said, keep(ProposalApproved, "approved by the operator: config reads its own sources from now"))
}
return said
}
func approvedTakeover(w asks.Warrant) string {
if w.By == nil {
return "the operator"
}
return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity)
}
// HandBack returns to the controller's records until the controller's lease is handed away.
func (c *Config) HandBack(call Call) (*TakeoverAnswer, error) {
if strings.TrimSpace(call.Why) == "" {
return nil, errors.New("say why config hands back: it is kept with the switch")
}
if call.Caller == "" || call.Caller == Unproven {
return nil, errors.New("the call names no proven caller")
}
c.mu.Lock()
defer c.mu.Unlock()
if c.leaseAway() {
return nil, errors.New("hand-back is gone: the controller handed its lease away, and its records end with it " +
"(novox/hq ADR 0565 §4)")
}
sw, err := c.source()
if err != nil {
return nil, err
}
if sw.Source == SourceController {
return &TakeoverAnswer{Source: sw.Source, Said: "config reads the controller's records already"}, nil
}
now := c.now()
back := Switch{Source: SourceController, At: now.UTC(), By: call.Caller, Why: strings.TrimSpace(call.Why)}
if err := c.Store.Put(TakeoverState, SwitchKey, back); err != nil {
return nil, fmt.Errorf("the switch could not be written: %w", err)
}
answer := &TakeoverAnswer{Source: back.Source, Said: "config reads the controller's records again: its mirrors " +
"declared-config and predefined, and pull-declares"}
if err := c.Store.HandAct("handed config back to the controller's records", back.Why, "config-hand-back"); err != nil {
answer.Said += "; the hand-act log does not hold it: " + err.Error()
}
return answer, nil
}
// Declared is the read of `declared`: every module's key, or one whole.
func (c *Config) Declared(module string) (any, error) {
if module != "" {
e, err := c.Store.Get(DeclaredState, module)
if err != nil {
return nil, err
}
if e == nil {
return nil, fmt.Errorf("declared holds no %s: it is placed nowhere, or its manifest cannot be read", module)
}
return json.RawMessage(e.Value), nil
}
rows := map[string]any{}
err := c.each(DeclaredState, func(key string, raw []byte) error {
var k DeclaredKept
if json.Unmarshal(raw, &k) == nil {
rows[key] = map[string]any{"manifest": k.Manifest, "commit": k.Commit, "keys": len(k.Config)}
}
return nil
})
sw, _ := c.source()
return map[string]any{"declared": rows, "count": len(rows), "source": sw}, err
}
// Resync reads every source again and writes what differs: the published list, the resolved values, `declared`, and
// after the take-over the indexer's config candidates (ADR 0565 §15).
func (c *Config) Resync() (any, error) {
r, err := c.Reconcile()
if err != nil {
return r, err
}
c.mu.Lock()
defer c.mu.Unlock()
out := map[string]any{"resolution": r}
if sw, err := c.source(); err == nil && sw.Source == SourceIndexer {
out["candidates"] = c.takeCandidates()
}
return out, nil
}
@@ -0,0 +1,238 @@
package main
import (
"encoding/json"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The trunk's manifests aMesh declares, as the controller's mirror declared-config holds the same declarations.
const (
claudeDeclared = `{"module":"claude-code","settings":{
"managed":{"kind":"","default":{},"why":"","type":"object","description":"managed settings","trusted":true,"predefined":"agent-managed-settings"}}}`
oldDeclared = `{"module":"old","settings":{"workers":{"kind":"preference","default":4,"why":"two per core"}}}`
)
// anAgreeingMesh is aProposingMesh whose controller mirrors say what mesh-config's own sources say: the mirror's
// declarations of every placed module, and the indexer's predefined list equal to the mirror's.
func anAgreeingMesh(t *testing.T) (*Config, *memStore, *standIn) {
t.Helper()
c, m, r := aProposingMesh(t)
m.mirror(t, MirrorDeclaredRead, "dunst", dunstDeclared)
m.mirror(t, MirrorDeclaredRead, "claude-code", claudeDeclared)
m.mirror(t, MirrorDeclaredRead, "old", oldDeclared)
indexerList(t, m, "aaaaaaaaaaaa", `{"agent-managed-settings":{"type":"object","description":"managed settings","trusted":true},
"font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`)
if _, err := c.Reconcile(); err != nil {
t.Fatal(err)
}
return c, m, r
}
// indexerList is the indexer's `predefined.list` at a commit.
func indexerList(t *testing.T, m *memStore, commit, list string) {
t.Helper()
raw, _ := json.Marshal(delivery.PredefinedKept{V: 1, Commit: commit, List: json.RawMessage(list)})
m.mirror(t, IndexerPredefinedRead, delivery.PredefinedListKey, string(raw))
}
// candidate is the indexer's config candidate of a pull request.
func candidate(t *testing.T, m *memStore, pull, head, declares string) {
t.Helper()
key, err := delivery.ConfigCandidateKey(pull)
if err != nil {
t.Fatal(err)
}
m.mirror(t, CandidatesRead, key, `{"v":1,"pull":"`+pull+`","head":"`+head+`","declares":`+declares+`}`)
}
func kinds(cmp *Comparison) []string {
var out []string
for _, d := range cmp.Differences {
k := d.What + ":" + d.Module + d.Pull + d.Key
if d.Node != "" {
k += "@" + d.Node
}
if !slices.Contains(out, k) {
out = append(out, k)
}
}
return out
}
// compare lists every difference with its reason — a module's declarations, the predefined list, a pull request's
// config, and each value a node resolves both ways — and the take-over is refused while one stands (ADR 0565 §4 and
// its insight of 2026-10-29; issue 1953 L5-C4).
func TestCompareListsEveryDifferenceAndTheTakeOverWaitsForNone(t *testing.T) {
cases := []struct {
name string
change func(t *testing.T, m *memStore)
want []string
}{
{"all agree", func(*testing.T, *memStore) {}, nil},
{"the mirror declares a default otherwise", func(t *testing.T, m *memStore) {
m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1))
}, []string{"declared:dunst", "value:dunstwidth@g14", "value:dunstwidth@novox"}},
{"the mirror lacks a placed module", func(t *testing.T, m *memStore) { _ = m.Delete(MirrorDeclaredRead, "old") },
[]string{"declared:old", "value:old@g14", "value:oldworkers@g14"}},
{"a node's build adds a key the mirror lacks", func(t *testing.T, m *memStore) {
m.want(t, "g14", "dunst", "src2:"+strings.Repeat("ab", 32), strings.Replace(dunstDeclared, `"width"`, `"height":{"type":"number","default":5},"width"`, 1))
}, []string{"value:dunstheight@g14", "value:dunstheight@novox"}}, // a key only a build declares resolves on every node (#558)
{"the lists differ", func(t *testing.T, m *memStore) {
indexerList(t, m, "bbbbbbbbbbbb", `{"font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`)
}, []string{"predefined:agent-managed-settings"}},
{"a pull request only the indexer holds", func(t *testing.T, m *memStore) {
candidate(t, m, "novox/mesh-catalog#9", "h1", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`)
}, []string{"pull:novox/mesh-catalog#9"}},
}
for _, tc := range cases {
c, m, r := anAgreeingMesh(t)
tc.change(t, m)
cmp, err := c.Compare()
if err != nil {
t.Fatalf("%s: %v", tc.name, err)
}
if got := kinds(cmp); !slices.Equal(got, tc.want) {
t.Errorf("%s: compare lists %v, want %v (%+v)", tc.name, got, tc.want, cmp.Differences)
}
for _, d := range cmp.Differences {
if d.Why == "" {
t.Errorf("%s: a difference without its reason: %+v", tc.name, d)
}
}
a, err := c.TakeOver(Call{Why: "compare is empty", Caller: "person.jochen"})
if len(tc.want) > 0 {
if err == nil || !strings.Contains(err.Error(), "refused") || len(r.subjects) != 0 {
t.Errorf("%s: the take-over was asked while compare differs: %v", tc.name, err)
}
continue
}
if err != nil || a.Ask == "" || len(r.subjects) != 1 {
t.Errorf("%s: the take-over was not asked: %v", tc.name, err)
}
}
}
// The operator's Approve switches config to the indexer: pull-declares is refused, the list and the pull requests'
// config come from the indexer; hand-back returns to the mirrors until the lease write and is gone after it; from the
// lease write a trusted value is set only on an Approve; declared is written once per manifest (ADR 0565 §3, §4, §13).
func TestTheTakeOverAndTheHandBack(t *testing.T) {
c, m, r := anAgreeingMesh(t)
steps := []struct {
name string
do func() (string, error)
want string // what the answer or the error says
}{
{"declared is written when it differs, and once", func() (string, error) {
_ = m.Delete(DeclaredState, "dunst")
first, err := c.Reconcile()
again, _ := c.Reconcile()
return strings.Join(first.Declared, ",") + "|" + strings.Join(again.Declared, ","), err
}, "dunst|"},
{"pull-declares taken before the take-over", func() (string, error) {
a, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#7", Head: "h0",
Declares: map[string]any{}})
return answered(a), err
}, "keeps what"},
{"the take-over is asked", func() (string, error) {
_ = m.Delete(PullsState, "novox.mesh-catalog.7") // compare holds the told pull request against no candidate
a, err := c.TakeOver(Call{Why: "the release is delivered", Caller: "person.jochen"})
return answered(a), err
}, "is asked"},
{"the operator approves: switched to the indexer", func() (string, error) {
r.choose(t, r.lastAsk(t), OptionApprove, nil)
out, err := c.Decide()
return strings.Join(out, "; "), err
}, "reads its own sources from now"},
{"pull-declares refused after it, naming the indexer", func() (string, error) {
_, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#8", Head: "h1",
Declares: map[string]any{}})
return "", err
}, "config-candidates"},
{"the list comes from the indexer", func() (string, error) {
indexerList(t, m, "cccccccccccc", `{"font-size":{"type":"number","description":"text size","minimum":6,"maximum":72},
"agent-managed-settings":{"type":"object","description":"managed settings","trusted":true},
"theme":{"type":"string","description":"the desktop's theme"}}`)
res, err := c.Reconcile()
return res.Published, err
}, "theme"},
{"a pull request's config comes from the indexer", func() (string, error) {
candidate(t, m, "novox/mesh-catalog#9", "h9", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`)
res, err := c.Reconcile()
return strings.Join(res.Candidates, "; "), err
}, "from the indexer"},
{"hand-back returns to the mirrors", func() (string, error) {
a, err := c.HandBack(Call{Why: "a difference was found after the switch", Caller: "person.jochen"})
return answered(a), err
}, "controller's records again"},
{"after it, declarations resolve from the mirror", func() (string, error) {
m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1))
_, err := c.Reconcile()
return string(m.resolved(t, "g14", "dunst", "width").Value), err
}, "400"},
{"hand-back is gone after the lease write", func() (string, error) {
m.mirror(t, LeaseRead, "holder", `{"handed-to":"the operator's hand-over"}`)
_, err := c.HandBack(Call{Why: "again", Caller: "person.jochen"})
return "", err
}, "gone"},
{"the terminal sets no trusted value after the lease write", func() (string, error) {
_, err := c.Change(Call{Action: "set", Key: "managed", Module: "claude-code", Node: "g14", Value: map[string]any{},
Why: "x", Caller: ControllerCaller, By: "person.jochen"})
return "", err
}, "propose"},
}
for _, s := range steps {
got, err := s.do()
text := got
if err != nil {
text = err.Error()
}
if !strings.Contains(text, s.want) {
t.Errorf("%s: said %q, want %q", s.name, text, s.want)
}
}
if !slices.Contains(m.handActs, "handed config back to the controller's records | a difference was found after the switch | config-hand-back") {
t.Errorf("the hand-back is not recorded: %v", m.handActs)
}
if e, _ := m.Get(DeclaredState, "dunst"); e == nil || !strings.Contains(string(e.Value), `"manifest":"`) || !strings.Contains(string(e.Value), `"commit":"c1"`) {
t.Errorf("declared holds no dunst with its manifest: %v", e)
}
// An Approve that finds compare differing at the answer switches nothing.
c2, m2, r2 := anAgreeingMesh(t)
if _, err := c2.TakeOver(Call{Why: "now", Caller: "person.jochen"}); err != nil {
t.Fatal(err)
}
m2.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1))
r2.choose(t, r2.lastAsk(t), OptionApprove, nil)
out, _ := c2.Decide()
if sw, _ := c2.source(); sw.Source != SourceInterim || !strings.Contains(strings.Join(out, ""), "difference") {
t.Errorf("an Approve while compare differs switched: %s, %v", sw.Source, out)
}
}
// lastAsk is the newest ask the router took.
func (r *standIn) lastAsk(t *testing.T) string {
t.Helper()
r.mu.Lock()
defer r.mu.Unlock()
var newest string
var at time.Time
for id, a := range r.asks {
if newest == "" || a.Expires.After(at) {
newest, at = id, a.Expires
}
}
if newest == "" {
t.Fatal("the router took no ask")
}
return newest
}
func answered(a any) string {
raw, _ := json.Marshal(a)
return string(raw)
}
+10 -1
View File
@@ -2,4 +2,13 @@ module mesh-config
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
)
+26 -2
View File
@@ -1,2 +1,26 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+21 -3
View File
@@ -7,7 +7,10 @@
"name": "mesh-config",
"scope": "mesh",
"serves": [
"config"
"config",
"compare",
"take-over",
"hand-back"
]
}
],
@@ -23,6 +26,8 @@
"gitea.forge.pull-request.closed"
],
"invokes": [
"seat:mesh-hand-acts.hand-act",
"seat:mesh-hand-acts.warranted",
"seat:mesh-controller.hand-act",
"seat:mesh-controller.warranted",
"gitea.gitea_get_pull_request"
@@ -33,7 +38,18 @@
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.modules",
"mesh-delivery-indexer.manifests",
"mesh-controller.predefined"
"mesh-controller.predefined",
"mesh-controller.declared-config",
"mesh-delivery-indexer.predefined",
"mesh-delivery-indexer.config-candidates",
"mesh-controller.lease"
],
"tools": [
"config_compare",
"config_take_over",
"config_hand_back",
"config_declared",
"config_resync"
],
"state": [
"values",
@@ -41,7 +57,9 @@
"predefined",
"proposals",
"pulls",
"kept"
"kept",
"declared",
"takeover"
],
"logs": [
"history"
+2
View File
@@ -0,0 +1,2 @@
/mesh-hand-acts
/cmd/mesh-hand-acts/mesh-hand-acts
+26
View File
@@ -0,0 +1,26 @@
# mesh-hand-acts
Holds the seat `mesh-hand-acts` (novox/hq ADR 0565 §10, §11): the hand-act log, moved out of the controller. Go, one
binary (`cmd/mesh-hand-acts`), served by a runtime of its own on its own bus account (`runs-as`), so that its account
alone writes the log.
## What it keeps
- the log `acts` (the stream `LOG_mesh-hand-acts_acts`, a module log of ADR 0297, as ADR 0565 §10 has it a stream):
every act appended in its order under its key `act-<ns>-<n>` (`warrant-<asker>-<ask>` for a warrant), in the layout
of the controller's log. The log keeps each act 400 days (`max-age-days`) and refuses a new act when
full rather than drop one; `hand-acts` says how full it is against its cap. Each act is said as `act-recorded`.
- `mesh-hand-acts_warrants`: one key `<asker>.<ask>` per warrant, made once by create before its act is appended, so of
several instances one records it; a warrant whose act was lost is written again from the router's record.
## Its verbs
- `mesh-hand-acts.hand-act`: what, why, cause, the condition addressed, and the act's verb (a repair when not given;
`hand-act drill` for a drill, whose cause is always `drill`). Who did it is the caller the runtime proves.
- `mesh-hand-acts.warranted`: asker and ask. Read from the router's own record of that ask (`messenger.asks`), never
from the caller, and recorded once per ask; an ask still open, ended without a choice or another asker's is refused.
- `mesh-hand-acts.hand-acts`: both logs, this one and the controller's `mesh-controller_hand-acts` until its lease
ends, newest first, with the causes repaired by hand twice within a fortnight; `warrants=true` lists the warrants.
No act is written to both logs: every recorder calls this seat from its delivery, and the controller keeps its own
acts in its own bucket until it loses its lease.
@@ -0,0 +1,375 @@
package main
// The hand-act log (novox/hq ADR 0565 §10, §11): every act done by hand, with who, why and its cause, so a repair a
// person keeps making is found (S15) and a person's decision is read back. mesh-hand-acts holds the seat
// mesh-hand-acts, once for the mesh, and is the one writer of its log `acts` — a stream, as ADR 0565 §10 has it:
// the mesh's module log (ADR 0297), append-only, every act kept in its order under its key, refusing a new act when
// full rather than dropping an old one, so every act is held 400 days and longer — and of its bucket `warrants` (one
// key `<asker>.<ask>` made once by create). Each act is said as `act-recorded`.
//
// **Who did it is the runtime's word, never the caller's**: `by` is the bus user the runtime proved. A warrant is the
// router's word, never the caller's: `warranted` reads the router's own record of the asker's ask and records who
// chose, through which channel, with which proofs and which answer, once per ask. The controller's bucket
// `mesh-controller_hand-acts` stays its own until it loses its lease and is read, never written: `hand-acts` answers
// over both logs, and no act is written to both (ADR 0565 §11).
import (
"encoding/json"
"errors"
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/handacts"
)
// The seat, the log, the states and the event, as mesh-sdk's handacts names them.
const (
Module = handacts.Module
Seat = handacts.Seat
ActsLog = handacts.ActsLog
WarrantsState = handacts.WarrantsState
ControllerActs = handacts.ControllerActs
// RouterRecords is the router's record of every ask, `<asker>.<ask>` (ADR 0259): the warrant is read there.
RouterRecords = "messenger.asks"
RecordedEvent = handacts.EventRecorded
// RecordedSubject is where the seat says it: a seat's events are on mesh.seat.<seat>.event.<name>, which its
// consumers hear (catalogue.SeatSays).
RecordedSubject = "mesh.seat." + Seat + ".event." + RecordedEvent
VerbRecord = handacts.VerbRecord
VerbDrill = handacts.VerbDrill
VerbWarrant = handacts.VerbWarrant
CauseDrill = handacts.CauseDrill
CauseOperatorAnswer = handacts.CauseOperatorAnswer
)
// RepeatedWithin is how far back a cause counts as repeated (S15).
const RepeatedWithin = 14 * 24 * time.Hour
// Act is one entry of the log: mesh-sdk's handacts.Act, the controller's HandAct unchanged.
type Act = handacts.Act
// Store is the bus's buckets and this module's log as it reaches them, through its runtime. Create writes a key only
// when it has none, and says ErrExists otherwise.
type Store interface {
Get(state, key string) ([]byte, bool, error)
Create(state, key string, value any) error
Keys(state string) ([]string, error)
// Append adds an entry to the end of a log; Last is a key's newest entry; Entries every entry in order; Fill how
// many entries and bytes the log holds against its cap.
Append(log, key string, value any) error
Last(log, key string) ([]byte, bool, error)
Entries(log string) ([][]byte, error)
Fill(log string) (Fill, error)
}
// Fill is how full a log is: its entries, its bytes and its cap in bytes.
type Fill struct {
Entries uint64 `json:"entries"`
Bytes uint64 `json:"bytes"`
MaxBytes uint64 `json:"max-bytes"`
}
// ErrExists is a create of a key another writer made first.
var ErrExists = errors.New("the key was made by another writer first")
// Log is mesh-hand-acts: the log and what records into it.
type Log struct {
Store Store
// Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish).
Publish func(subject string, body any, id string) error
Now func() time.Time
Logf func(format string, args ...any)
}
var seq atomic.Uint64
// actKey is an act's key: its time and a sequence, so the keys list in order, as the controller's do.
func actKey(at time.Time) string {
return "act-" + strconv.FormatInt(at.UnixNano(), 10) + "-" + strconv.FormatUint(seq.Add(1), 10)
}
// Recorded is what a recorder is answered: the act as kept.
type Recorded struct {
Act Act `json:"act"`
Said string `json:"said"`
}
// HandAct records one act a caller says it did, by hand or on purpose: what, why, its cause and the condition it
// addresses. by is the caller the runtime proved. verb is the act's verb as the log names it: a repair by default, a
// drill with VerbDrill, or a recorder's own (`conditions silence`, `config set`), which S15 reads as it reads the
// controller's.
func (l *Log) HandAct(by, verb, what, why, cause, condition string) (Recorded, error) {
what, why, cause, verb = strings.TrimSpace(what), strings.TrimSpace(why), strings.TrimSpace(cause), strings.TrimSpace(verb)
switch {
case !proven(by):
return Recorded{}, fmt.Errorf("who acts is the caller the runtime proves, and it proved none (%q): nothing was recorded", by)
case what == "":
return Recorded{}, errors.New("a hand act says what was done: nothing was recorded")
case why == "":
return Recorded{}, errors.New("a hand act says why: nothing was recorded")
}
if verb == "" {
verb = VerbRecord
}
if verb == VerbDrill {
cause = CauseDrill
}
switch {
case cause == "":
return Recorded{}, errors.New("a hand act says its cause, in a word or a condition's kind: the word a second act " +
"for the same reason uses, which is how a repair done twice is found. Nothing was recorded")
case strings.EqualFold(cause, CauseDrill) && verb != VerbDrill:
return Recorded{}, errors.New("a drill is not recorded as a repair: record it with verb " + VerbDrill +
", which no healer is wanted for. Nothing was recorded")
}
at := l.Now().UTC()
act := Act{ID: actKey(at), At: at, By: by, Verb: verb, Args: []string{what}, Why: why, Cause: cause,
Condition: strings.TrimSpace(condition)}
if err := l.record(act); err != nil {
return Recorded{}, err
}
return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s did %q, because %q (cause: %s)", act.ID, by, what,
why, cause)}, nil
}
// record writes one act and says it. An act not said is still kept, and the log says so.
func (l *Log) record(act Act) error {
if err := l.Store.Append(ActsLog, act.ID, act); err != nil {
return fmt.Errorf("the act could not be recorded: %w", err)
}
if l.Publish != nil {
if err := l.Publish(RecordedSubject, act, act.ID); err != nil {
l.Logf("[%s] %s is recorded and could NOT be said as %s: %v", Module, act.ID, RecordedEvent, err)
}
}
return nil
}
// routerRecord is what warranted reads of the router's record of an ask.
type routerRecord struct {
State string `json:"state"`
Warrant *asks.Warrant `json:"warrant"`
Ask map[string]any `json:"ask"`
}
var askerModule = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
// Warrant is one ask's warrant as recorded: the key `<asker>.<ask>` of `warrants`, naming the act it recorded.
type Warrant struct {
Act string `json:"act"`
At time.Time `json:"at"`
By string `json:"recorded-at-the-word-of"`
}
// Warranted records an asker's act on the operator's warrant for its ask, read from the router's record, once per ask
// however often it is asked (ADR 0274, ADR 0259). The caller gives which ask and nothing else.
func (l *Log) Warranted(caller, asker, ask string) (Recorded, error) {
switch {
case !proven(caller):
return Recorded{}, fmt.Errorf("who asks is the caller the runtime proves, and it proved none (%q): nothing was recorded", caller)
case !askerModule.MatchString(asker):
return Recorded{}, fmt.Errorf("%q is not a module's name", asker)
case !asks.UsableID(ask):
return Recorded{}, fmt.Errorf("%q is not an ask's id", ask)
}
key := asker + "." + ask
claimed, found, err := l.Store.Get(WarrantsState, key)
if err != nil {
return Recorded{}, err
}
if found {
if r, ok, err := l.already(key, claimed); err != nil || ok {
return r, err
}
}
raw, found, err := l.Store.Get(RouterRecords, key)
if err != nil {
return Recorded{}, fmt.Errorf("the router's record of %s's ask %s cannot be read: %w", asker, ask, err)
}
var rec routerRecord
if found {
if err := json.Unmarshal(raw, &rec); err != nil {
return Recorded{}, fmt.Errorf("the router's record of %s's ask %s is not readable: %w", asker, ask, err)
}
}
w := rec.Warrant
switch {
case rec.State == "open" || rec.State == "held":
return Recorded{}, fmt.Errorf("%s's ask %s is still %s: nobody has answered it. Nothing was recorded", asker, ask, rec.State)
case !found || rec.State == "" || w == nil:
return Recorded{}, fmt.Errorf("the router holds no closed record of %s's ask %s. Nothing was recorded", asker, ask)
case w.Asker != asker || w.Ask != ask:
return Recorded{}, fmt.Errorf("the router's record is for %s's ask %s. Nothing was recorded", w.Asker, w.Ask)
case w.Outcome != asks.OutcomeChosen || w.By == nil:
return Recorded{}, fmt.Errorf("%s's ask %s ended %s: no person chose, so there is no warrant to record. Nothing was "+
"recorded", asker, ask, w.Outcome)
case w.AskDigest == "":
return Recorded{}, fmt.Errorf("the router's warrant for %s's ask %s names no ask digest. Nothing was recorded", asker, ask)
}
at := l.Now().UTC()
act := Act{ID: "warrant-" + asker + "-" + ask, At: w.At.UTC(), Verb: VerbWarrant,
Args: []string{fmt.Sprintf("the operator chose %s on %s's ask %s", w.Label, asker, ask)},
Why: fmt.Sprintf("%s (ask %s of %s)", w.Says(), ask, asker),
By: fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity),
Cause: CauseOperatorAnswer, Via: via(*w), Ask: ask, Proofs: w.Proofs,
RequestedBy: asker + ", recorded at the word of " + caller,
Outcome: "chosen; what " + asker + " did with it is in its own record"}
if claimed == nil {
// The claim first: of any number of instances asking at once, the one whose create lands records the act.
if err := l.Store.Create(WarrantsState, key, Warrant{Act: act.ID, At: at, By: caller}); errors.Is(err, ErrExists) {
return Recorded{Said: "another instance records " + act.ID + " now"}, nil
} else if err != nil {
return Recorded{}, fmt.Errorf("the warrant could not be recorded: %w", err)
}
}
if err := l.record(act); err != nil {
return Recorded{}, err
}
return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s, through %s", act.ID, act.Why, act.Via)}, nil
}
// already answers a warrant recorded before, when its act is in the log; false when the act was lost between the
// claim and the log, so it is read from the router again and written.
func (l *Log) already(key string, raw []byte) (Recorded, bool, error) {
var w Warrant
if err := json.Unmarshal(raw, &w); err != nil {
return Recorded{}, false, fmt.Errorf("the warrant %s is not readable: %w", key, err)
}
actRaw, found, err := l.Store.Last(ActsLog, w.Act)
if err != nil || !found {
return Recorded{}, false, err
}
var act Act
_ = json.Unmarshal(actRaw, &act)
return Recorded{Act: act, Said: "already recorded as " + act.ID + ": " + act.Why}, true, nil
}
func via(w asks.Warrant) string {
if w.By == nil {
return w.Channel
}
v := w.Channel + " (" + w.By.Kind + ")"
if w.By.Verified != "" {
v += ", " + w.By.Verified
}
return v
}
// proven is a bus user the runtime proved: never empty, never the SDK's unproven word.
func proven(by string) bool {
by = strings.TrimSpace(by)
return by != "" && !strings.EqualFold(by, "unproven") && !strings.HasPrefix(by, "(")
}
// Listing is what hand-acts answers: every act of both logs within the days asked, newest first, narrowed by cause
// and who, and the causes repaired by hand twice or more within a fortnight.
type Listing struct {
Acts []Act `json:"acts"`
Repeated map[string]int `json:"repeated"`
Days int `json:"days"`
// Unread says which log could not be read, so what is missing is said and never read as nothing.
Unread map[string]string `json:"unread,omitempty"`
Kept map[string]int `json:"kept"`
// Fill is how full this seat's log is against its cap: full, it refuses the next act.
Fill *Fill `json:"fill,omitempty"`
}
// HandActs reads both logs.
func (l *Log) HandActs(days int, cause, who string) Listing {
now := l.Now()
since := now.Add(-time.Duration(days) * 24 * time.Hour)
out := Listing{Acts: []Act{}, Repeated: map[string]int{}, Days: days, Kept: map[string]int{}}
var all []Act
for _, log := range []string{ActsLog, ControllerActs} {
acts, kept, err := l.read(log, now.Add(-max(time.Duration(days)*24*time.Hour, RepeatedWithin)))
if err != nil {
if out.Unread == nil {
out.Unread = map[string]string{}
}
out.Unread[log] = err.Error()
continue
}
out.Kept[log] = kept
all = append(all, acts...)
}
for _, a := range all {
if now.Sub(a.At) <= RepeatedWithin && a.Repair() {
out.Repeated[a.Cause]++
}
if a.At.Before(since) || (cause != "" && a.Cause != cause) || (who != "" && !strings.Contains(a.By, who)) {
continue
}
out.Acts = append(out.Acts, a)
}
for c, n := range out.Repeated {
if n < 2 {
delete(out.Repeated, c)
}
}
sort.SliceStable(out.Acts, func(i, j int) bool { return out.Acts[i].At.After(out.Acts[j].At) })
if f, err := l.Store.Fill(ActsLog); err == nil {
out.Fill = &f
}
return out
}
// read is every act of one log since a moment, and how many it keeps: this seat's log entry by entry, the
// controller's bucket by its keys, each of which names its time, so only those within are read whole.
func (l *Log) read(log string, since time.Time) ([]Act, int, error) {
var out []Act
keep := func(raw []byte, from string) {
var a Act
if json.Unmarshal(raw, &a) != nil || a.At.Before(since) {
return
}
a.Log = from
out = append(out, a)
}
if log == ActsLog {
entries, err := l.Store.Entries(ActsLog)
if err != nil {
return nil, 0, err
}
for _, raw := range entries {
keep(raw, handacts.LogSeat)
}
return out, len(entries), nil
}
keys, err := l.Store.Keys(log)
if err != nil {
return nil, 0, err
}
for _, k := range keys {
if at, ok := keyTime(k); ok && at.Before(since) {
continue
}
raw, found, err := l.Store.Get(log, k)
if err != nil {
return nil, len(keys), err
}
if found {
keep(raw, handacts.LogController)
}
}
return out, len(keys), nil
}
// keyTime is the time an act's key names (`act-<ns>-<n>`), when it names one.
func keyTime(key string) (time.Time, bool) {
parts := strings.Split(key, "-")
if len(parts) != 3 || parts[0] != "act" {
return time.Time{}, false
}
ns, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
return time.Time{}, false
}
return time.Unix(0, ns), true
}
@@ -0,0 +1,286 @@
package main
import (
"encoding/json"
"errors"
"sort"
"strings"
"sync"
"testing"
"time"
)
type entry struct {
key string
raw []byte
}
type memStore struct {
mu sync.Mutex
kv map[string]map[string][]byte
logs map[string][]entry
keysErr map[string]error
}
func newStore() *memStore {
return &memStore{kv: map[string]map[string][]byte{}, logs: map[string][]entry{}, keysErr: map[string]error{}}
}
func (s *memStore) Append(log, key string, value any) error {
raw, err := json.Marshal(value)
if err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
if err := s.keysErr[log]; err != nil {
return err
}
s.logs[log] = append(s.logs[log], entry{key, raw})
return nil
}
func (s *memStore) Last(log, key string) ([]byte, bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
for i := len(s.logs[log]) - 1; i >= 0; i-- {
if s.logs[log][i].key == key {
return s.logs[log][i].raw, true, nil
}
}
return nil, false, nil
}
func (s *memStore) Entries(log string) ([][]byte, error) {
s.mu.Lock()
defer s.mu.Unlock()
if err := s.keysErr[log]; err != nil {
return nil, err
}
var out [][]byte
for _, e := range s.logs[log] {
out = append(out, e.raw)
}
return out, nil
}
func (s *memStore) Fill(log string) (Fill, error) {
s.mu.Lock()
defer s.mu.Unlock()
return Fill{Entries: uint64(len(s.logs[log])), MaxBytes: 1 << 30}, nil
}
// logKeys are a log's keys in order.
func (s *memStore) logKeys(log string) []string {
s.mu.Lock()
defer s.mu.Unlock()
var out []string
for _, e := range s.logs[log] {
out = append(out, e.key)
}
return out
}
func (s *memStore) Get(state, key string) ([]byte, bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
v, ok := s.kv[state][key]
return v, ok, nil
}
func (s *memStore) put(state, key string, value any, create bool) error {
raw, err := json.Marshal(value)
if err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
if s.kv[state] == nil {
s.kv[state] = map[string][]byte{}
}
if _, ok := s.kv[state][key]; ok && create {
return ErrExists
}
s.kv[state][key] = raw
return nil
}
func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) }
func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) }
func (s *memStore) Keys(state string) ([]string, error) {
s.mu.Lock()
defer s.mu.Unlock()
if err := s.keysErr[state]; err != nil {
return nil, err
}
var out []string
for k := range s.kv[state] {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
func newLog() (*Log, *memStore, *[]Act, *time.Time) {
store := newStore()
var said []Act
now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
l := &Log{Store: store, Logf: func(string, ...any) {}, Now: func() time.Time { return now },
Publish: func(subject string, body any, id string) error {
if a, ok := body.(Act); ok && subject == "mesh.seat.mesh-hand-acts.event.act-recorded" && id == a.ID {
said = append(said, a)
}
return nil
}}
return l, store, &said, &now
}
// hand-act records what the proven caller says it did, once, and says it; it refuses what cannot be read back, a
// caller the runtime did not prove, and a drill recorded as a repair (ADR 0565 §10; issue 1953 L5-C1).
func TestHandActRecordsWhatTheProvenCallerDid(t *testing.T) {
cases := []struct {
name, by, verb, what, why, cause string
wantVerb, wantCause, err string
}{
{"a repair", "ace.mesh-config", "", "set dns", "it was wrong", "config", VerbRecord, "config", ""},
{"a recorder's own verb", "novox.mesh-conditions", "conditions silence", "silenced x", "known", "operator-answer",
"conditions silence", "operator-answer", ""},
{"a drill takes its cause", "novox.mesh-healers", VerbDrill, "stopped records", "does it raise", "", VerbDrill, CauseDrill, ""},
{"no proven caller", "unproven", "", "x", "y", "z", "", "", "proves"},
{"by said empty", "", "", "x", "y", "z", "", "", "proves"},
{"no why", "person.jochen", "", "x", "", "z", "", "", "says why"},
{"no what", "person.jochen", "", "", "y", "z", "", "", "says what"},
{"no cause", "person.jochen", "", "x", "y", "", "", "", "cause"},
{"a drill as a repair", "person.jochen", "", "x", "y", "drill", "", "", "not recorded as a repair"},
}
for _, tc := range cases {
l, store, said, _ := newLog()
r, err := l.HandAct(tc.by, tc.verb, tc.what, tc.why, tc.cause, "bus.x.missing")
keys := store.logKeys(ActsLog)
if tc.err != "" {
if err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 || len(*said) != 0 {
t.Errorf("%s: %v, kept %v", tc.name, err, keys)
}
continue
}
if err != nil || len(keys) != 1 || keys[0] != r.Act.ID || len(*said) != 1 {
t.Fatalf("%s: %v, kept %v, said %d", tc.name, err, keys, len(*said))
}
var kept Act
raw, _, _ := store.Last(ActsLog, keys[0])
_ = json.Unmarshal(raw, &kept)
if kept.By != tc.by || kept.Verb != tc.wantVerb || kept.Cause != tc.wantCause || kept.Condition != "bus.x.missing" ||
kept.Args[0] != tc.what || kept.Log != "" {
t.Errorf("%s: kept %+v", tc.name, kept)
}
}
}
// warranted records the router's word once per ask, never the caller's; it refuses an ask still open, one ended
// without a choice, another asker's, and one the router holds no record of (ADR 0565 §10, ADR 0259).
func TestWarrantedRecordsTheRoutersWordOnce(t *testing.T) {
chosen := `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"chosen","label":"Approve",
"by":{"who":"operator","kind":"telegram","identity":"42","verified":"P2"},"channel":"telegram","proofs":["P1","P2"],
"at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}`
cases := []struct {
name, record, asker, ask, err string
// claimed is a warrant claimed by an instance that stopped before its act reached the log.
claimed bool
}{
{"chosen", chosen, "claude-code", "a1", "", false},
{"claimed, its act lost: written now", chosen, "claude-code", "a1", "", true},
{"still open", `{"state":"open"}`, "claude-code", "a1", "still open", false},
{"no record", "", "claude-code", "a1", "no closed record", false},
{"expired", `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"expired","at":"2026-10-29T21:00:00Z"}}`,
"claude-code", "a1", "no person chose", false},
{"another asker's", strings.Replace(chosen, `"asker":"claude-code"`, `"asker":"mesh-config"`, 1), "claude-code", "a1",
"record is for mesh-config", false},
{"no digest", strings.Replace(chosen, `,"ask-digest":"sha256:x"`, "", 1), "claude-code", "a1", "no ask digest", false},
{"not a module", chosen, "Claude Code", "a1", "not a module", false},
}
for _, tc := range cases {
l, store, said, _ := newLog()
if tc.record != "" {
_ = store.Put(RouterRecords, "claude-code.a1", json.RawMessage(tc.record))
}
if tc.claimed {
_ = store.Create(WarrantsState, "claude-code.a1", Warrant{Act: "warrant-claude-code-a1"})
}
r, err := l.Warranted("ace.claude-code", tc.asker, tc.ask)
if tc.err != "" {
if keys := store.logKeys(ActsLog); err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 {
t.Errorf("%s: %v, kept %v", tc.name, err, keys)
}
continue
}
if err != nil || r.Act.Verb != VerbWarrant || r.Act.Cause != CauseOperatorAnswer ||
!strings.Contains(r.Act.By, "operator, as telegram identity 42") || r.Act.Via != "telegram (telegram), P2" ||
!strings.Contains(r.Act.RequestedBy, "ace.claude-code") || len(r.Act.Proofs) != 2 {
t.Fatalf("%s: %+v, %v", tc.name, r.Act, err)
}
// Asked again, by another instance: nothing more is recorded.
again, err := l.Warranted("g14.claude-code", "claude-code", "a1")
keys := store.logKeys(ActsLog)
if err != nil || !strings.HasPrefix(again.Said, "already recorded") || len(keys) != 1 || len(*said) != 1 {
t.Errorf("%s: a second warranted: %q, %v, kept %v, said %d", tc.name, again.Said, err, keys, len(*said))
}
}
}
// hand-acts answers over both logs, newest first, narrowed by days, cause and who, and counts each cause repaired by
// hand twice in a fortnight across both, never a person's decision; a log it cannot read is said, never read as empty.
func TestHandActsReadsBothLogs(t *testing.T) {
l, store, _, now := newLog()
put := func(log, verb, cause, by string, ago time.Duration) {
at := now.Add(-ago)
a := Act{At: at, Verb: verb, Cause: cause, By: by, Why: "x"}
if log == ActsLog {
_ = store.Append(log, actKey(at), a)
return
}
_ = store.Put(log, actKey(at), a)
}
put(ActsLog, VerbRecord, "relogin", "person.jochen", time.Hour)
put(ControllerActs, VerbRecord, "relogin", "shanks.claude-code", 3*24*time.Hour)
put(ControllerActs, VerbDrill, CauseDrill, "person.jochen", time.Hour)
put(ActsLog, VerbDrill, CauseDrill, "person.jochen", 2*time.Hour)
put(ActsLog, VerbWarrant, CauseOperatorAnswer, "the operator", time.Hour)
put(ActsLog, VerbWarrant, CauseOperatorAnswer, "the operator", 2*time.Hour)
put(ControllerActs, VerbRecord, "config", "ace.mesh-config", 30*24*time.Hour)
cases := []struct {
name string
days int
cause, who string
want int
}{
{"a fortnight", 14, "", "", 6},
{"a day", 1, "", "", 5},
{"forty days", 40, "", "", 7},
{"by cause", 14, "relogin", "", 2},
{"by who", 14, "", "claude-code", 1},
}
for _, tc := range cases {
got := l.HandActs(tc.days, tc.cause, tc.who)
if len(got.Acts) != tc.want || len(got.Repeated) != 1 || got.Repeated["relogin"] != 2 || got.Unread != nil {
t.Errorf("%s: %d acts, repeated %v, unread %v", tc.name, len(got.Acts), got.Repeated, got.Unread)
}
for i := 1; i < len(got.Acts); i++ {
if got.Acts[i].At.After(got.Acts[i-1].At) {
t.Errorf("%s: not newest first", tc.name)
}
}
}
if got := l.HandActs(14, "", ""); got.Acts[0].Log != Seat || got.Kept[ControllerActs] != 3 || got.Fill == nil ||
got.Fill.Entries != 4 {
t.Errorf("the log of an act or the kept counts are not said: %+v", got.Kept)
}
store.keysErr[ControllerActs] = errors.New("the bus is away")
if got := l.HandActs(14, "", ""); got.Unread[ControllerActs] == "" || len(got.Acts) != 4 {
t.Errorf("an unreadable log: %+v", got.Unread)
}
w, err := l.Warrants()
if err != nil || w.(map[string]any)["count"] != 0 {
t.Errorf("warrants answered %v, %v", w, err)
}
}
@@ -0,0 +1,84 @@
// mesh-hand-acts: the holder of the seat mesh-hand-acts (novox/hq ADR 0565 §10). A Go bundle served by a runtime of
// its own, on its own bus account (`runs-as`), so that its account alone writes the hand-act log (`acts`, its module
// log: a stream that keeps every act in order) and `warrants`. It reads the router's record of each ask for a warrant and the controller's log for
// `hand-acts`. stdout is the MCP channel; what this module says, it says on stderr.
package main
import (
"errors"
"fmt"
"os"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
// runtimeStore is the bus, through the runtime that launched this bundle.
type runtimeStore struct{}
func (runtimeStore) Get(state, key string) ([]byte, bool, error) {
e, err := stdio.State(state).Get(key)
if err != nil || e == nil {
return nil, false, err
}
return e.Value, true, nil
}
func (runtimeStore) Create(state, key string, value any) error {
_, err := stdio.State(state).Create(key, value)
if errors.Is(err, stdio.ErrChanged) {
return ErrExists
}
return err
}
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
func (runtimeStore) Append(log, key string, value any) error {
_, err := stdio.Log(log).Append(key, value)
return err
}
func (runtimeStore) Last(log, key string) ([]byte, bool, error) {
e, err := stdio.Log(log).Last(key)
if err != nil || e == nil {
return nil, false, err
}
return e.Value, true, nil
}
func (runtimeStore) Entries(log string) ([][]byte, error) {
var out [][]byte
for from := uint64(0); ; {
page, err := stdio.Log(log).Read(from, "", 1000)
if err != nil {
return nil, err
}
for _, e := range page.Entries {
out = append(out, e.Value)
}
if page.Next == 0 {
return out, nil
}
from = page.Next
}
}
func (runtimeStore) Fill(log string) (Fill, error) {
i, err := stdio.Log(log).Info()
if err != nil {
return Fill{}, err
}
return Fill{Entries: i.Entries, Bytes: i.Bytes, MaxBytes: i.MaxBytes}, nil
}
func main() {
l := &Log{Store: runtimeStore{}, Now: time.Now, Logf: logf,
Publish: stdio.SeatPublish}
if err := stdio.Serve("", Tools(l)); err != nil {
logf("%v", err)
os.Exit(1)
}
}
@@ -0,0 +1,76 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"slices"
"testing"
)
// The manifest says what the code does: it claims the seat mesh-hand-acts once for the mesh with its verbs (ADR 0565
// §1, §10), runs on its own account, says act-recorded, reads the controller's log and the router's records, keeps its
// acts as its log (a stream, §10) and warrants as a bucket that never ages, so a warrant is recorded once. Every state
// and log it keeps and reads has a tool that reads it (ADR 0565 §15).
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m struct {
Module string `json:"module"`
Runs string `json:"runs-as"`
Claims []struct {
Name string `json:"name"`
Scope string `json:"scope"`
Serves []string `json:"serves"`
} `json:"claims"`
Emits []string `json:"emits"`
Reads []string `json:"reads"`
Logs []struct {
Name string `json:"name"`
MaxAgeDays int `json:"max-age-days"`
} `json:"logs"`
State []json.RawMessage `json:"state"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
if m.Module != Module || m.Runs != Module {
t.Errorf("module %q runs as %q", m.Module, m.Runs)
}
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) {
t.Errorf("claims %+v", m.Claims)
}
// act-recorded is the seat's event, said on the seat's subject: a module lists no event of a seat it claims.
if len(m.Emits) != 0 || !slices.Equal(m.Reads, []string{ControllerActs, RouterRecords}) {
t.Errorf("emits %v, reads %v", m.Emits, m.Reads)
}
readBy := map[string]string{ActsLog: HandActsTool, WarrantsState: HandActsTool, ControllerActs: HandActsTool,
RouterRecords: WarrantedTool}
tools := map[string]bool{}
for _, tool := range Tools(&Log{}) {
tools[tool.Name] = true
}
// The acts are a log kept 400 days (ADR 0565 §10).
if len(m.Logs) != 1 || m.Logs[0].Name != ActsLog || m.Logs[0].MaxAgeDays != 400 {
t.Errorf("logs %+v, want the acts as a log kept 400 days", m.Logs)
}
for _, raw := range m.State {
var name string
if json.Unmarshal(raw, &name) != nil || name != WarrantsState {
t.Errorf("the state %s is not the warrants, or ages", raw)
}
if !tools[readBy[name]] {
t.Errorf("the state %s has no tool that reads it", name)
}
}
if !tools[readBy[ActsLog]] {
t.Errorf("the log %s has no tool that reads it", ActsLog)
}
for _, r := range m.Reads {
if !tools[readBy[r]] {
t.Errorf("the read %s has no tool that answers it", r)
}
}
}
@@ -0,0 +1,2 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
modules/mesh-hand-acts/module.json
@@ -0,0 +1,110 @@
package main
// The seat's verbs (novox/hq ADR 0565 §10, §15): hand-act and warranted record, hand-acts reads both logs and, with
// warrants, every warrant recorded. A recovery is a call: a warrant whose act was lost is written again by asking
// warranted again, from the router's record.
import (
"encoding/json"
"fmt"
"sort"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// Verbs are the seat's verbs, as the manifest claims them.
var Verbs = []string{"hand-act", "hand-acts", "warranted"}
// Tool names.
const (
HandActTool = Seat + ".hand-act"
HandActsTool = Seat + ".hand-acts"
WarrantedTool = Seat + ".warranted"
)
func obj(props map[string]any, required ...string) map[string]any {
m := map[string]any{"type": "object", "properties": props}
if len(required) > 0 {
m["required"] = required
}
return m
}
func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} }
// Tools are what the runtime serves.
func Tools(l *Log) []stdio.Tool {
return []stdio.Tool{
{Name: HandActTool,
Description: "Record one act done by hand (novox/hq ADR 0565 §10): what was done, why, its cause in a word or a " +
"condition's kind (the word a second act for the same reason uses, which is how a repair done twice is " +
"found) and the condition it addresses. Who did it is the caller the runtime proves. verb is the act's " +
"verb as the log names it: a repair when not given, `hand-act drill` for a drill, or a recorder's own " +
"(`conditions silence`, `config set`). Said as act-recorded.",
Input: obj(map[string]any{"what": str("what was done"), "why": str("why, in a sentence"),
"cause": str("the cause, in a word or a condition's kind"), "condition": str("the condition's key, if any"),
"verb": str("the act's verb in the log; a repair when not given")}, "what", "why"),
Run: func(a map[string]any) (any, error) {
return l.HandAct(stdio.CallerOf(a), arg(a, "verb"), arg(a, "what"), arg(a, "why"), arg(a, "cause"), arg(a, "condition"))
}},
{Name: HandActsTool,
Description: "What was done by hand lately, over both logs (this seat's and the controller's until its lease " +
"ends), newest first: each act with who, why, its cause and which log holds it, and the causes repaired by " +
"hand twice or more within a fortnight (a healer is wanted). Narrowed by cause and who; warrants=true lists " +
"every warrant recorded instead.",
Input: obj(map[string]any{"days": map[string]any{"type": "number", "description": "how many days back (default 14)"},
"cause": str("only this cause"), "who": str("only acts whose who contains this"),
"warrants": str("\"true\": every warrant recorded, `<asker>.<ask>` with its act")}),
Run: func(a map[string]any) (any, error) {
if strings.EqualFold(arg(a, "warrants"), "true") || a["warrants"] == true {
return l.Warrants()
}
return l.HandActs(days(a), arg(a, "cause"), arg(a, "who")), nil
}},
{Name: WarrantedTool,
Description: "Record an asker's act on the operator's warrant (ADR 0274, ADR 0259): read from the router's own " +
"record of that ask — who chose, through which channel, with which proofs and which answer — never from the " +
"caller, once per ask; refused for an ask still open, ended without a choice, or another asker's.",
Input: obj(map[string]any{"asker": str("the module that asked"), "ask": str("its ask's id")}, "asker", "ask"),
Run: func(a map[string]any) (any, error) {
return l.Warranted(stdio.CallerOf(a), arg(a, "asker"), arg(a, "ask"))
}},
}
}
// Warrants is every warrant recorded, by key.
func (l *Log) Warrants() (any, error) {
keys, err := l.Store.Keys(WarrantsState)
if err != nil {
return nil, fmt.Errorf("the warrants cannot be read: %w", err)
}
sort.Strings(keys)
rows := map[string]json.RawMessage{}
for _, k := range keys {
if raw, found, err := l.Store.Get(WarrantsState, k); err == nil && found {
rows[k] = raw
}
}
return map[string]any{"warrants": rows, "count": len(rows)}, nil
}
func arg(a map[string]any, k string) string {
s, _ := a[k].(string)
return strings.TrimSpace(s)
}
func days(a map[string]any) int {
switch v := a["days"].(type) {
case float64:
if v > 0 {
return int(v)
}
case string:
var n int
if _, err := fmt.Sscan(v, &n); err == nil && n > 0 {
return n
}
}
return 14
}
@@ -0,0 +1,405 @@
//go:build integration
package main
// mesh-hand-acts whole against a real bus (novox/hq ADR 0565 §10; issue 1953 L5-C2): the built binary, played over its
// real stdin and stdout as the node's tool runtime plays it (tools/call with the caller in _meta; mesh/state.* answered
// from a real nats-server's key-value buckets, mesh/log.* from the log's real stream, mesh/publish onto its subjects). A recorder's hand act is kept in the
// log on the bus under the proven caller and heard as act-recorded; a warrant the router recorded is recorded once.
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
type wire struct {
ID json.RawMessage `json:"id,omitempty"`
Method string `json:"method,omitempty"`
Params json.RawMessage `json:"params,omitempty"`
Result json.RawMessage `json:"result,omitempty"`
Error *struct {
Message string `json:"message"`
} `json:"error,omitempty"`
}
// player is the node's tool runtime, as far as the self-check needs it, over a real bus.
type player struct {
ctx context.Context
nc *nats.Conn
js jetstream.JetStream
mu sync.Mutex
in io.Writer
next int
pending map[string]chan wire
}
func (p *player) send(m map[string]any) {
raw, _ := json.Marshal(m)
p.mu.Lock()
defer p.mu.Unlock()
_, _ = p.in.Write(append(raw, '\n'))
}
func (p *player) answer(id json.RawMessage, result any, err error) {
if err != nil {
p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
return
}
p.send(map[string]any{"id": id, "result": result})
}
func (p *player) serve(out io.Reader) {
scanner := bufio.NewScanner(out)
scanner.Buffer(make([]byte, 1<<20), 1<<24)
for scanner.Scan() {
var m wire
if json.Unmarshal(scanner.Bytes(), &m) != nil {
continue
}
if m.Method == "" {
var key string
_ = json.Unmarshal(m.ID, &key)
p.mu.Lock()
ch := p.pending[key]
delete(p.pending, key)
p.mu.Unlock()
if ch != nil {
ch <- m
}
continue
}
go p.handle(m)
}
}
// kv is the bucket a state names: this module's own by its local name, another's as `<owner>.<name>`.
func (p *player) kv(state string) (jetstream.KeyValue, error) {
if owner, name, ok := strings.Cut(state, "."); ok {
return p.js.KeyValue(p.ctx, owner+"_"+name)
}
return p.js.KeyValue(p.ctx, Module+"_"+state)
}
func (p *player) handle(m wire) {
var a struct {
State string `json:"state"`
Key string `json:"key"`
Value json.RawMessage `json:"value"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &a)
switch m.Method {
case "mesh/state.get":
kv, err := p.kv(a.State)
if err != nil {
p.answer(m.ID, nil, err)
return
}
e, err := kv.Get(p.ctx, a.Key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
p.answer(m.ID, nil, nil)
return
}
if err != nil {
p.answer(m.ID, nil, err)
return
}
p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
case "mesh/state.put":
kv, err := p.kv(a.State)
if err == nil {
var rev uint64
if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil {
p.answer(m.ID, map[string]any{"revision": rev}, nil)
return
}
}
p.answer(m.ID, nil, err)
case "mesh/state.create":
kv, err := p.kv(a.State)
if err == nil {
var rev uint64
if rev, err = kv.Create(p.ctx, a.Key, a.Value); err == nil {
p.answer(m.ID, map[string]any{"revision": rev}, nil)
return
}
if errors.Is(err, jetstream.ErrKeyExists) {
p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32001, "message": "the key was written by another writer first"}})
return
}
}
p.answer(m.ID, nil, err)
case "mesh/state.watch":
kv, err := p.kv(a.State)
if err != nil {
p.answer(m.ID, nil, err)
return
}
key := a.Key
if key == "" {
key = ">"
}
w, err := kv.Watch(p.ctx, key)
if err != nil {
p.answer(m.ID, nil, err)
return
}
var watch string
_ = json.Unmarshal(m.ID, &watch)
current := true
for e := range w.Updates() {
if e == nil {
current = false
p.answer(m.ID, map[string]any{}, nil)
continue
}
op := "put"
if e.Operation() != jetstream.KeyValuePut {
op = "delete"
}
change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current}
if op == "put" {
change["value"] = json.RawMessage(e.Value())
}
p.mu.Lock()
p.next++
id := fmt.Sprintf("w-%d", p.next)
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}})
}
case "mesh/subscribe":
p.answer(m.ID, map[string]any{}, nil)
case "mesh/log.append", "mesh/log.read", "mesh/log.last", "mesh/log.info":
p.log(m)
case "mesh/state.keys":
kv, err := p.kv(a.State)
if err != nil {
p.answer(m.ID, nil, err)
return
}
keys, err := kv.Keys(p.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
keys, err = []string{}, nil
}
p.answer(m.ID, keys, err)
case "mesh/state.delete":
kv, err := p.kv(a.State)
if err == nil {
err = kv.Delete(p.ctx, a.Key)
}
p.answer(m.ID, map[string]any{}, err)
case "mesh/seat.publish":
var e struct {
Subject string `json:"subject"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &e)
p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body))
case "mesh/publish":
var e struct {
Key string `json:"key"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &e)
p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body))
default:
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
}
}
// log answers the runtime's log verbs from the log's real stream, LOG_<module>_<name> on mesh.log.<module>.<name>.<key>.
func (p *player) log(m wire) {
var a struct {
Log string `json:"log"`
Key string `json:"key"`
Value json.RawMessage `json:"value"`
From uint64 `json:"from"`
}
_ = json.Unmarshal(m.Params, &a)
stream := "LOG_" + Module + "_" + a.Log
subject := "mesh.log." + Module + "." + a.Log
s, err := p.js.Stream(p.ctx, stream)
if err != nil {
p.answer(m.ID, nil, err)
return
}
entry := func(r *jetstream.RawStreamMsg) map[string]any {
return map[string]any{"sequence": r.Sequence, "key": strings.TrimPrefix(r.Subject, subject+"."),
"value": json.RawMessage(r.Data), "at": r.Time}
}
switch m.Method {
case "mesh/log.append":
ack, err := p.js.Publish(p.ctx, subject+"."+a.Key, a.Value)
if err != nil {
p.answer(m.ID, nil, err)
return
}
p.answer(m.ID, map[string]any{"sequence": ack.Sequence}, nil)
case "mesh/log.last":
r, err := s.GetLastMsgForSubject(p.ctx, subject+"."+a.Key)
if errors.Is(err, jetstream.ErrMsgNotFound) {
p.answer(m.ID, nil, nil)
return
}
if err != nil {
p.answer(m.ID, nil, err)
return
}
p.answer(m.ID, entry(r), nil)
case "mesh/log.read":
info, err := s.Info(p.ctx)
if err != nil {
p.answer(m.ID, nil, err)
return
}
entries := []map[string]any{}
for seq := max(a.From, 1); seq <= info.State.LastSeq; seq++ {
if r, err := s.GetMsg(p.ctx, seq); err == nil {
entries = append(entries, entry(r))
}
}
p.answer(m.ID, map[string]any{"entries": entries, "next": 0, "last": info.State.LastSeq}, nil)
case "mesh/log.info":
info, err := s.Info(p.ctx)
if err != nil {
p.answer(m.ID, nil, err)
return
}
p.answer(m.ID, map[string]any{"entries": info.State.Msgs, "bytes": info.State.Bytes,
"max_bytes": info.Config.MaxBytes, "first": info.State.FirstSeq, "last": info.State.LastSeq}, nil)
}
}
func (p *player) call(name, caller string, args map[string]any) (json.RawMessage, error) {
p.mu.Lock()
p.next++
id := fmt.Sprintf("c-%d", p.next)
ch := make(chan wire, 1)
p.pending[id] = ch
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args,
"_meta": map[string]any{"mesh/caller": caller}}})
select {
case m := <-ch:
if m.Error != nil {
return nil, errors.New(m.Error.Message)
}
var r struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
IsError bool `json:"isError"`
}
if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 {
return nil, fmt.Errorf("%s answered %s", name, m.Result)
}
if r.IsError {
return nil, errors.New(r.Content[0].Text)
}
return json.RawMessage(r.Content[0].Text), nil
case <-time.After(60 * time.Second):
return nil, fmt.Errorf("%s did not answer in 60s", name)
}
}
// 1953/L5-C2: an act recorded through the seat is kept on the bus and heard as act-recorded.
func TestTheHandActLogWholeRecordsOnTheBus(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
dir := t.TempDir()
bin := filepath.Join(dir, Module)
build := exec.Command("go", "build", "-o", bin, ".")
build.Stderr = os.Stderr
if err := build.Run(); err != nil {
t.Fatalf("mesh-hand-acts does not build: %v", err)
}
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
for _, b := range []string{Module + "_" + WarrantsState, "mesh-controller_hand-acts", "messenger_asks"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
t.Fatal(err)
}
}
// The log as the mesh makes it (ADR 0297): a stream kept by limits, refusing when full, never deleting.
logStream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "LOG_" + Module + "_" + ActsLog,
Subjects: []string{"mesh.log." + Module + "." + ActsLog + ".>"}, MaxBytes: 64 << 20, Discard: jetstream.DiscardNew,
DenyDelete: true, DenyPurge: true, AllowDirect: true})
if err != nil {
t.Fatal(err)
}
recorded, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + RecordedEvent)
if err != nil {
t.Fatal(err)
}
cmd := exec.CommandContext(ctx, bin)
cmd.Env = append(os.Environ(), "MESH_NODE=novox", "MESH_MODULE="+Module)
stdin, _ := cmd.StdinPipe()
stdout, _ := cmd.StdoutPipe()
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
go p.serve(stdout)
raw, err := p.call(HandActTool, "ace.mesh-config", map[string]any{"what": "set dns on g14", "why": "it was wrong",
"cause": "config", "by": "someone else"})
if err != nil {
t.Fatal(err)
}
var r Recorded
if err := json.Unmarshal(raw, &r); err != nil || r.Act.By != "ace.mesh-config" {
t.Fatalf("hand-act answered %s", raw)
}
e, err := logStream.GetLastMsgForSubject(ctx, "mesh.log."+Module+"."+ActsLog+"."+r.Act.ID)
if err != nil || !strings.Contains(string(e.Data), `"by":"ace.mesh-config"`) {
t.Fatalf("the act is not kept in the log on the bus: %v", err)
}
msg, err := recorded.NextMsg(10 * time.Second)
if err != nil || !strings.Contains(string(msg.Data), r.Act.ID) {
t.Fatalf("act-recorded was not said: %v", err)
}
// A warrant the router recorded, asked twice: one act.
asks, _ := js.KeyValue(ctx, "messenger_asks")
if _, err := asks.Put(ctx, "claude-code.a1", []byte(`{"state":"closed","warrant":{"ask":"a1","asker":"claude-code",
"outcome":"chosen","label":"Approve","by":{"who":"operator","kind":"telegram","identity":"42"},"channel":"telegram",
"at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}`)); err != nil {
t.Fatal(err)
}
for _, caller := range []string{"ace.claude-code", "g14.claude-code"} {
if _, err := p.call(WarrantedTool, caller, map[string]any{"asker": "claude-code", "ask": "a1"}); err != nil {
t.Fatal(err)
}
}
if info, err := logStream.Info(ctx); err != nil || info.State.Msgs != 2 {
t.Errorf("the log keeps %v, want the act and one warrant (%v)", info, err)
}
listed, err := p.call(HandActsTool, "person.jochen", map[string]any{})
if err != nil || !strings.Contains(string(listed), "warrant-claude-code-a1") || !strings.Contains(string(listed), r.Act.ID) {
t.Errorf("hand-acts answered %s, %v", listed, err)
}
}
+22
View File
@@ -0,0 +1,22 @@
module mesh-hand-acts
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
+27
View File
@@ -0,0 +1,27 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+65
View File
@@ -0,0 +1,65 @@
{
"module": "mesh-hand-acts",
"version": "1",
"runs-as": "mesh-hand-acts",
"claims": [
{
"name": "mesh-hand-acts",
"scope": "mesh",
"serves": [
"hand-act",
"hand-acts",
"warranted"
]
}
],
"reads": [
"mesh-controller.hand-acts",
"messenger.asks"
],
"logs": [
{
"name": "acts",
"max-age-days": 400
}
],
"state": [
"warrants"
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
"secrets-owner": "mesh-hand-acts",
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-hand-acts",
"shell": "/usr/bin/nologin",
"home": "/var/lib/mesh-hand-acts"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "mesh-hand-acts",
"place": ".",
"unknown": "program"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-hand-acts",
"binary": "mesh-hand-acts",
"loads": [
"mesh-hand-acts"
]
}
]
}
}
+2
View File
@@ -0,0 +1,2 @@
/mesh-healers
/cmd/mesh-healers/mesh-healers
+46
View File
@@ -0,0 +1,46 @@
# mesh-healers
Holds the seat `mesh-healers` (novox/hq ADR 0564 §7, §8): the mesh's healers, moved out of the controller. Go, one
binary (`cmd/mesh-healers`), served by a runtime of its own on its own bus account (`runs-as`).
## What it does
It hears the condition keeper's `condition-raised` and `condition-changed` (seat `mesh-conditions`) on a durable
consumer. A condition whose kind a healer answers is acted on within the healer's budget, its settle and the mesh-wide
brake (twelve acts an hour). A repair is a call of the seat or tool that owns the act: H3 asks the grants part's
`grants_assert` for the module whose consumer is gone. H4 (the controller's own consumer) and H5 (the provider's own
repair) are registered and run elsewhere.
Each act is begun by a create of its key in `attempts` before it is made and finished after, so a holder that dies
mid-act has spent it and a second instance cannot begin it again. The condition keeper writes the condition's
`tried`, resolver and escalation from `attempts`; a healer writes no condition.
**Only once the controller's lease names no controller.** Until the write that hands the lease away, each act it would
take is written to `would`, never to `attempts`: it counts for no budget, settle or brake and reaches no condition.
Each key of `attempts` and `would` lives a week from its last write, its own time to live (the buckets are declared
`key-ttl`): nothing else deletes `would`, and once the lease is handed away nothing is written there. The healers' waits
are keys of `wakes` whose time to live is the wait: `settle.<condition key>` ends with a settle and judges the condition
again, `fortnight` ends when S15's oldest counted act leaves the fortnight and counts again. The watch of `wakes` hears
each expiry; no timer runs in the process. A wait still running is on the bus across a restart; one that expired while
no instance watched is judged at the start, once the lease is first read.
S15: it hears the seat mesh-hand-acts' `act-recorded` and watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a
fortnight; a drill and a person's decision by design never count.
## What it keeps
- `mesh-healers_attempts`: every act and escalation, each key kept a week.
- `mesh-healers_would`: the newest act it would take per healer and condition while the controller holds the lease,
each key kept a week.
- `mesh-healers_findings`: the brake, its own blindness, and S15.
- `mesh-healers_wakes`: the waits, each key living as long as it waits.
## Its verbs and tools
- `mesh-healers.healers`: the registry, the acts of the last days, the would-acts, the brake, whether it acts and why;
`recount=true` counts the hand-act log again.
- `mesh-healers.heal`: one act now on an open condition, by key and why.
- `mesh-healers.drill`: records a drill in the hand-act log of the seat `mesh-hand-acts` as `hand-act drill` (the
controller's `drill` while that seat has no holder and the controller holds its lease). S15 never counts a drill.
- `healers_findings`, `healers_would`, `healers_wakes`: the reads of its buckets.
@@ -0,0 +1,663 @@
package main
// The healers (novox/hq ADR 0564 §7, §8; ADR 0231): a known failure heals itself, under a brake, and every repair is
// said. The registry is compiled in; mesh-healers hears `condition-raised` and `condition-changed` of the seat
// mesh-conditions on a durable consumer, instead of looking every half minute. A repair is a call of the seat or tool
// that owns the act (H3: the grants part's grants_assert). Each act is begun by a create of its key in the bucket
// `attempts` before it is made and finished after, so a holder that dies mid-act has spent it and a second instance
// cannot begin the same act; budgets, settles and the mesh-wide brake count from there. Success is never the
// healer's to say: the condition clears when its observer no longer sees it, and the condition keeper writes `tried`,
// the resolver and the escalation into the condition from `attempts`.
//
// **Only once the controller's lease names no controller** (ADR 0564 §8): until then each act it would take is
// written to the bucket `would`, never to `attempts`, so a would-act counts for no budget, settle or brake, reaches
// no `tried` and escalates nothing. `would` keeps each key a week (its bucket's age): nothing else deletes it, and once
// the lease is handed away nothing is written there again.
import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Module and Seat are this module and the seat it holds once for the mesh.
const (
Module = "mesh-healers"
Seat = "mesh-healers"
)
// The states it keeps and reads, and the event it says.
const (
AttemptsState = health.AttemptsState
WouldState = health.WouldState
FindingsState = health.FindingsState
LeaseRead = buscompose.ControllerLease
LeaseKey = buscompose.ControllerLeaseKey
ConditionsRead = "mesh-conditions.conditions"
HandActsRead = "mesh-controller.hand-acts"
ActedEvent = health.EventHealerActed
// ActedSubject is where the seat says it, which its consumers hear (catalogue.SeatSays).
ActedSubject = "mesh.seat." + Seat + ".event." + ActedEvent
// RecordedEvent is the hand-act log's seat event, each act recorded there (ADR 0565 §11).
RecordedEvent = handacts.Seat + "." + handacts.EventRecorded
// The events it hears: the keeper's, on the seat mesh-conditions.
RaisedEvent = "mesh-conditions.condition-raised"
ChangedEvent = "mesh-conditions.condition-changed"
// AssertTool is the grants part's tool that asserts a module's buckets, logs and streams again.
AssertTool = "mesh-grants.grants_assert"
)
// The mesh-wide brake (to-be 45 §7): how many acts all healers together may take in an hour.
const (
BrakeLimit = 12
BrakeWindow = time.Hour
)
// The outcomes an attempt has.
const (
OutcomeActing = health.OutcomeActing
OutcomeActed = health.OutcomeActed
OutcomeFailed = health.OutcomeFailed
OutcomeEscalated = health.OutcomeEscalated
)
// Store is the bus's buckets as this module reaches them, through its runtime. CreateTTL writes a key only when it has
// none, and says ErrExists otherwise; it and PutTTL give the key its own time to live, at whose end the bus removes it
// and a watch of the bucket hears it expire (a bucket declared key-ttl).
type Store interface {
Get(state, key string) ([]byte, bool, error)
Put(state, key string, value any) error
CreateTTL(state, key string, value any, ttl time.Duration) error
PutTTL(state, key string, value any, ttl time.Duration) error
Delete(state, key string) error
Keys(state string) ([]string, error)
}
// KeptFor is how long an attempt and a would-act are kept: each key a week from its last write (ADR 0564 Limits).
const KeptFor = 7 * 24 * time.Hour
// WakesState holds the healers' waits as keys whose time to live is the wait: `settle.<condition key>` ends with a
// settle, `fortnight` when S15's oldest counted act leaves the fortnight. Its expiry, heard on the bucket's watch, is
// the wake: no timer runs in the process, and a restart loses nothing a key on the bus holds.
const WakesState = "wakes"
// The wakes' keys.
const (
settlePrefix = "settle."
fortnightKey = "fortnight"
)
// ErrExists is a create of a key another writer made first.
var ErrExists = errors.New("the key was made by another writer first")
// Condition is what a healer reads of a condition the keeper said: the controller's condition body, its looks aside.
type Condition struct {
Key string `json:"key"`
Kind string `json:"kind"`
Subject Subject `json:"subject"`
Severity string `json:"severity"`
Resolver string `json:"resolver"`
Tried []json.RawMessage `json:"tried,omitempty"`
}
// Escalated says a healer's budget was spent: the operator resolves it now, and no healer touches it again until
// observation clears it.
func (c Condition) Escalated() bool { return c.Resolver == "operator" && len(c.Tried) > 0 }
// Attempt, Would and Acted are mesh-sdk's health layouts: an act begun in `attempts` (its key
// `<healer>.<condition key>.<begun ns>` is the healers' own), an act written to `would` while the lease names a
// controller (one key per healer and condition, `<healer>.<condition key>`, its newest), and the body of healer-acted,
// epoch left out (ADR 0564 Limits 10).
type (
Attempt = health.Attempt
Would = health.Would
Acted = health.HealerActed
)
// Healer is one row of the registry.
type Healer struct {
ID string
Kinds []string
Condition string
Repair string
Then string
From string
Budget int
Window time.Duration
Settle time.Duration
// ActsIn is where the repair runs: here, or a module that repairs its own (H5), or the controller (H4).
ActsIn string
// applies says whether this condition is one it may act on, and what its budget counts against; why when not.
applies func(c Condition) (budget string, ok bool, why string)
// repair is the act: what it did, what came of it, or why it could not.
repair func(h *Healers, c Condition) (act, said string, err error)
}
// actsHere marks a healer whose repair mesh-healers makes.
const actsHere = "mesh-healers"
// Registry is to-be 45 §7's table, compiled in: every healer the mesh has, the ones mesh-healers runs with their act.
func Registry() []Healer {
return []Healer{
{ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"},
Condition: "a durable consumer of a module the bus does not hold (D6), or a seat's holder that does not answer (D3)",
Repair: "assert the module's buckets, logs and streams again through the grants part (grants_assert)",
Then: "resolver operator, urgent", From: "a controller restarted to make a missing object again (208, 248)",
Budget: 1, Window: time.Hour, Settle: 6 * time.Minute, ActsIn: actsHere,
applies: appliesToAModulesObject, repair: assertAgain},
{ID: "H4", Kinds: []string{"consumer-behind"},
Condition: "a durable consumer far behind its stream's head (D6) that the stream table marks resettable",
Repair: "re-make the consumer to deliver from now; the one marked resettable is the controller's own",
Then: "resolver operator, urgent", From: "a consumer re-made from now by hand (248)",
Budget: 1, Window: 24 * time.Hour, Settle: 6 * time.Minute,
ActsIn: "the controller, until its lease: the one consumer marked resettable is its own (ADR 0564 §7)"},
{ID: "H5", Kinds: []string{"provider-failing"},
Condition: "the identity provider's administrator refusing the mesh's secret (provider-failing)",
Repair: "the provider repairs it itself through the server's own bootstrap command (ADR 0224 §5)",
Then: "announced failing by the provider, braked from ten minutes doubling to six hours",
From: "the identity provider's admin reset through its bootstrap command (179)",
Budget: 1, Window: 10 * time.Minute, Settle: 10 * time.Minute,
ActsIn: "the provider module (keycloak), ADR 0224 §5"},
}
}
// Healers is mesh-healers: the registry, what its acts reach, and what it holds.
type Healers struct {
Store Store
Ask func(key string, body any) (json.RawMessage, error)
// Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish).
Publish func(subject string, body any, id string) error
Now func() time.Time
Logf func(format string, args ...any)
Registry []Healer
mu sync.Mutex
// lease is the controller's lease as last read: acting is on only while it is handed away.
acting bool
leaseWhy string
// declined is why each condition was last passed over, said once.
declined map[string]string
// acts are the hand-act log as watched, for S15.
acts map[string]HandAct
}
// LeaseHeard records what the lease says: acting is on only while it names a holder that is no controller (ADR 0564
// §8, Limits 8, as ADR 0561 §3's reader reads it).
func (h *Healers) LeaseHeard(raw []byte, found bool, readErr error) {
on, why := buscompose.LeaseHandedAway(raw, found, readErr)
h.mu.Lock()
flipped := on != h.acting || h.leaseWhy == ""
h.acting, h.leaseWhy = on, leaseWords(on, why)
h.mu.Unlock()
if flipped {
h.Logf("[%s] the healers %s: %s", Module, map[bool]string{true: "ACT", false: "act in shadow"}[on], h.leaseWhy)
}
}
// leaseWords says the lease's state as the healers read it; the SDK's words speak of minting.
func leaseWords(on bool, why string) string {
if on {
return "the controller's lease is handed away: the healers act"
}
if strings.Contains(why, "cannot be read") || strings.Contains(why, "not readable") {
return "the controller's lease cannot be read, so it is taken to name a controller: every act is written as would"
}
return "the controller's lease names a controller (or none is written, so one may take it): every act is written as would"
}
// Acting says whether the healers act, and why.
func (h *Healers) Acting() (bool, string) {
h.mu.Lock()
defer h.mu.Unlock()
if h.leaseWhy == "" {
return false, "the controller's lease has not been read yet: every act is written as would"
}
return h.acting, h.leaseWhy
}
// healerFor is the healer that acts here on a kind.
func (h *Healers) healerFor(kind string) (Healer, bool) {
for _, r := range h.Registry {
if r.repair != nil && slices.Contains(r.Kinds, kind) {
return r, true
}
}
return Healer{}, false
}
// Heard is a condition the keeper raised or changed: judged, and acted on, written as would, or passed over.
func (h *Healers) Heard(body []byte) error {
var c Condition
if err := json.Unmarshal(body, &c); err != nil {
h.Logf("[%s] a condition event is not readable, and is passed over: %v", Module, err)
return nil
}
_, err := h.Judge(c, "")
return err
}
// Judge is one condition judged by the healer registered for its kind. by names a person's `heal`, empty for an
// event. It answers what was done, in words.
func (h *Healers) Judge(c Condition, by string) (string, error) {
h.mu.Lock()
defer h.mu.Unlock()
row, ok := h.healerFor(c.Kind)
if !ok {
return "no healer here answers the kind " + c.Kind, nil
}
if c.Escalated() {
return "it is handed to the operator already: no healer touches it until observation clears it", nil
}
budget, applies, why := row.applies(c)
if !applies {
h.decline(c.Key, row.ID, why)
return fmt.Sprintf("healer %s leaves it alone: %s", row.ID, why), nil
}
delete(h.declined, c.Key)
now := h.Now()
if !h.acting {
w := Would{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: row.Repair, At: now.UTC(), Why: h.whyNot()}
if err := h.Store.PutTTL(WouldState, row.ID+"."+c.Key, w, KeptFor); err != nil {
return "", fmt.Errorf("the would-act of %s on %s is not written: %w", row.ID, c.Key, err)
}
return fmt.Sprintf("healer %s would act on %s: %s", row.ID, c.Key, w.Why), nil
}
all, err := h.attempts()
if err != nil {
h.blind(err)
return "", fmt.Errorf("the healers cannot read what they did, so they count no budget and act on nothing: %w", err)
}
h.see("blind", nil)
acts := actsWithin(all, now.Add(-BrakeWindow))
if braked, said := h.brakeHolds(acts, now); braked {
h.see("brake", []Observation{brakeFinding(acts, said)})
return "every healer has stopped: " + said, nil
}
h.see("brake", nil)
spent := spentOn(all, row, budget, now)
if n := len(spent); n > 0 && now.Sub(spent[n-1].Begun) < row.Settle {
h.wake(budget, spent[n-1].Begun.Add(row.Settle), c.Key)
return fmt.Sprintf("healer %s waits: its last act on it is still the observation's to judge until %s", row.ID,
spent[n-1].Begun.Add(row.Settle).UTC().Format(time.RFC3339)), nil
}
if len(spent) >= row.Budget {
return h.escalate(row, c, budget, spent)
}
if len(acts) >= BrakeLimit {
return "the brake holds from the next act on", nil
}
return h.act(row, c, budget, len(spent), by)
}
func (h *Healers) whyNot() string {
if h.leaseWhy == "" {
return "the controller's lease has not been read yet"
}
return h.leaseWhy
}
// act is one healer's act: begun by a create in attempts, made, finished, and said.
func (h *Healers) act(row Healer, c Condition, budget string, spent int, by string) (string, error) {
begun := h.Now().UTC()
key := attemptKey(row.ID, c.Key, begun)
if by == "" {
by = "healer " + row.ID
}
window := fmt.Sprintf("act %d of %d within %s", spent+1, row.Budget, row.Window)
a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: row.Repair, Outcome: OutcomeActing,
Begun: begun}
if by != "healer "+row.ID {
a.Said = "asked by " + by
}
if err := h.Store.CreateTTL(AttemptsState, key, a, KeptFor); err != nil {
if errors.Is(err, ErrExists) {
return "another instance began this act first", nil
}
return "", fmt.Errorf("healer %s did not act on %s: its attempt could not be begun: %w", row.ID, c.Key, err)
}
act, said, err := row.repair(h, c)
a.Outcome = OutcomeActed
if err != nil {
a.Outcome, said = OutcomeFailed, err.Error()
}
if act != "" {
a.Act = act
}
finished := h.Now().UTC()
if a.Said != "" {
said = a.Said + "; " + said
}
a.Said, a.Finished = said+" ("+window+")", &finished
if err := h.Store.PutTTL(AttemptsState, key, a, KeptFor); err != nil {
h.Logf("[%s] healer %s %s %s and could not record what came of it: %v", Module, row.ID, a.Outcome, c.Key, err)
}
h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: a.Act, Outcome: a.Outcome, Said: said, Budget: window})
h.wake(budget, begun.Add(row.Settle), c.Key)
return fmt.Sprintf("healer %s %s %s: %s — %s (%s)", row.ID, a.Outcome, c.Key, a.Act, said, window), nil
}
// escalate is a spent budget: the condition is the operator's now. Written once as an attempt the keeper turns into
// the condition's resolver and urgency (ADR 0564 §7); an escalated condition is passed over until it clears.
func (h *Healers) escalate(row Healer, c Condition, budget string, spent []Attempt) (string, error) {
var tried []string
for _, s := range spent {
tried = append(tried, fmt.Sprintf("%s at %s (%s)", s.Outcome, s.Begun.UTC().Format("15:04 MST"), firstLine(s.Said)))
}
said := fmt.Sprintf("its budget of %d act(s) within %s is spent and %s is still open: %s", row.Budget, row.Window,
c.Key, strings.Join(tried, "; "))
now := h.Now().UTC()
window := fmt.Sprintf("%d of %d within %s", len(spent), row.Budget, row.Window)
a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: "handed the condition to the operator",
Outcome: OutcomeEscalated, Said: said, Begun: now, Finished: &now}
if err := h.Store.CreateTTL(AttemptsState, attemptKey(row.ID, c.Key, now), a, KeptFor); err != nil && !errors.Is(err, ErrExists) {
return "", fmt.Errorf("healer %s could not record handing %s to the operator, and does not: %w", row.ID, c.Key, err)
}
h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: "handed to the operator", Outcome: OutcomeEscalated,
Said: said, Budget: window})
return fmt.Sprintf("healer %s handed %s to the operator: %s", row.ID, c.Key, said), nil
}
// tell says a heal as healer-acted. Not said is said in the log: the attempt holds it.
func (h *Healers) tell(e Acted) {
e.Event, e.At, e.By = ActedEvent, h.Now().UTC(), "healer "+e.Healer
e.Show = "mesh-conditions.conditions key=" + e.Condition
if h.Publish == nil {
return
}
if err := h.Publish(ActedSubject, e, e.Healer+"."+e.Condition+"."+e.Outcome+"."+strconv.FormatInt(e.At.UnixNano(), 10)); err != nil {
h.Logf("[%s] healer %s %s %s, and that could NOT be said on the bus: %v", Module, e.Healer, e.Outcome, e.Condition, err)
}
}
// Rearm is the healers' start, once the lease is first read: a settle whose key expired while no instance watched has
// no expiry left to hear, so each newest act whose settle ended within its window is judged at once. A settle still
// running keeps its key on the bus and needs nothing.
func (h *Healers) Rearm() {
all, err := h.attempts()
if err != nil {
h.Logf("[%s] what the healers did cannot be read, so no ended settle is judged: %v", Module, err)
return
}
now := h.Now()
newest := map[string]Attempt{} // the newest act per healer and budget; all is oldest first
for _, a := range all {
if a.Outcome != OutcomeEscalated {
newest[a.Healer+" "+a.BudgetKey] = a
}
}
var due []string
for _, a := range newest {
for _, r := range h.Registry {
if r.ID == a.Healer && r.repair != nil && !now.Before(a.Begun.Add(r.Settle)) && now.Sub(a.Begun) <= r.Window {
due = append(due, a.Condition)
}
}
}
sort.Strings(due)
for _, key := range due {
h.again(key)
}
}
// wake writes the wait for a settle's end as a key whose time to live is the wait; its expiry judges the condition
// again (WakeHeard). Written again with the same end, it is the same wait.
func (h *Healers) wake(_ string, at time.Time, key string) {
ttl := at.Sub(h.Now()).Round(time.Second) + time.Second
if err := h.Store.PutTTL(WakesState, settlePrefix+key, map[string]any{"condition": key, "at": at.UTC()}, ttl); err != nil {
h.Logf("[%s] the settle of %s is not written as a wait, so its end is judged at the next event: %v", Module, key, err)
}
}
// WakeHeard is a change of the wakes bucket: an expiry is a wait ended — a settle's, whose condition is judged again,
// or S15's fortnight, counted again.
func (h *Healers) WakeHeard(key, op string) {
if op != OpExpire {
return
}
switch {
case key == fortnightKey:
h.Wanted()
case strings.HasPrefix(key, settlePrefix):
h.again(strings.TrimPrefix(key, settlePrefix))
}
}
// OpExpire is the watch's word for a key whose own time to live ended.
const OpExpire = stdio.OpExpire
// again judges a condition again from the keeper's bucket, if it is still open.
func (h *Healers) again(key string) {
raw, found, err := h.Store.Get(ConditionsRead, key)
if err != nil || !found {
return
}
if _, err := h.Judge(conditionOf(raw), ""); err != nil {
h.Logf("[%s] %s judged again at its settle's end: %v", Module, key, err)
}
}
func conditionOf(raw []byte) Condition {
var c Condition
_ = json.Unmarshal(raw, &c)
return c
}
// decline remembers why a healer passed a condition over, said once.
func (h *Healers) decline(key, healer, why string) {
if h.declined == nil {
h.declined = map[string]string{}
}
if h.declined[key] != why {
h.Logf("[%s] healer %s leaves %s alone: %s", Module, healer, key, why)
}
h.declined[key] = why
}
func attemptKey(healer, condition string, begun time.Time) string {
return healer + "." + condition + "." + strconv.FormatInt(begun.UnixNano(), 10)
}
// attempts are every attempt kept, oldest first.
func (h *Healers) attempts() ([]Attempt, error) {
keys, err := h.Store.Keys(AttemptsState)
if err != nil {
return nil, err
}
var out []Attempt
for _, k := range keys {
raw, found, err := h.Store.Get(AttemptsState, k)
if err != nil {
return nil, err
}
if !found {
continue
}
var a Attempt
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the attempt %s is not readable: %w", k, err)
}
out = append(out, a)
}
sort.SliceStable(out, func(i, j int) bool { return out[i].Begun.Before(out[j].Begun) })
return out, nil
}
// actsWithin are the attempts since a moment that were acts: an escalation is a saying, and the brake does not count it.
func actsWithin(all []Attempt, since time.Time) []Attempt {
var out []Attempt
for _, a := range all {
if !a.Begun.Before(since) && a.Outcome != OutcomeEscalated {
out = append(out, a)
}
}
return out
}
// spentOn is one healer's acts against one budget key within its window, oldest first.
func spentOn(all []Attempt, row Healer, budget string, now time.Time) []Attempt {
var out []Attempt
for _, a := range actsWithin(all, now.Add(-row.Window)) {
if a.Healer == row.ID && a.BudgetKey == budget {
out = append(out, a)
}
}
return out
}
// brakeHolds says whether the mesh-wide brake holds: the limit reached within the hour, or the brake already written
// and an act within the hour still — it lets go an hour after the last act, not the first.
func (h *Healers) brakeHolds(acts []Attempt, now time.Time) (bool, string) {
said := fmt.Sprintf("%d act(s) by the healers in the last %s, the limit %d", len(acts), BrakeWindow, BrakeLimit)
if len(acts) >= BrakeLimit {
return true, said
}
_, held, _ := h.Store.Get(FindingsState, brakeKey)
if held && len(acts) > 0 {
last := acts[len(acts)-1].Begun
return true, fmt.Sprintf("%s; held until an hour after the last, at %s", said,
last.Add(BrakeWindow).UTC().Format("15:04 MST"))
}
return false, said
}
var brakeKey = health.Key(ScopeMesh, "healers", "braked")
// brakeFinding is the brake, as the urgent finding that says it.
func brakeFinding(acts []Attempt, said string) Observation {
counts := map[string]int{}
for _, a := range acts {
counts[a.Healer+" on "+a.Condition]++
}
var most []string
for what, n := range counts {
most = append(most, fmt.Sprintf("%s ×%d", what, n))
}
sort.Strings(most)
return Observation{Kind: "healers-braked", Token: "braked", Severity: Urgent, Subject: Subject{Scope: ScopeMesh, ID: "healers"},
Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " +
"healer looping, not repairing — " + said,
Said: strings.Join(most, ", "),
Headline: "All automatic repairs stopped",
Needs: "read the details and decide whether the repairs may run again.",
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which looks " +
"like a loop, so all of them stopped.",
Resolved: "Automatic repairs run again"}
}
// blind writes the healers' own blindness: they cannot read what they did.
func (h *Healers) blind(err error) {
h.see("blind", []Observation{{Kind: "probe-failed", Token: "failed", Severity: Warning,
Subject: Subject{Scope: ScopeProbe, ID: "healers"},
Summary: "the healers cannot read what they did, so they count no budget and act on nothing",
Said: firstLine(err.Error()), Headline: "A check of the mesh could not run",
Explanation: "The automatic repairs cannot read what they did, so they act on nothing until they can.",
Resolved: "The automatic repairs read what they did again"}})
}
// see writes the healers' own findings of one source, each only when it changes, and deletes those of the source it
// no longer sees (ADR 0564 §2).
func (h *Healers) see(source string, found []Observation) {
now := h.Now()
want := map[string]health.Finding{}
for _, o := range found {
o.Source = source
want[o.Key()] = o.Finding("", now)
}
keys, err := h.Store.Keys(FindingsState)
if err != nil {
h.Logf("[%s] its own findings cannot be read, so none is written or cleared: %v", Module, err)
return
}
for _, k := range keys {
raw, found, err := h.Store.Get(FindingsState, k)
if err != nil || !found {
continue
}
was, err := health.DecodeFinding(raw)
if err != nil || was.Source != source {
continue
}
if f, still := want[k]; still {
if says(was) == says(f) {
delete(want, k)
}
continue
}
if err := h.Store.Delete(FindingsState, k); err != nil {
h.Logf("[%s] the finding %s no longer seen is not deleted: %v", Module, k, err)
}
}
for k, f := range want {
raw, err := health.EncodeFinding(f)
if err == nil {
err = h.Store.Put(FindingsState, k, json.RawMessage(raw))
}
if err != nil {
h.Logf("[%s] the finding %s is not written: %v", Module, k, err)
}
}
}
// --- H3 --------------------------------------------------------------------------------------------------------
// moduleOf is the module whose objects a condition names: a consumer of a module's own (`<stream>.<module>`). The
// controller's own consumers are not a module's: the controller asserts them itself until its lease.
func moduleOf(c Condition) (string, string) {
stream, consumer, ok := strings.Cut(c.Subject.ID, ".")
if c.Kind != "consumer-lost" || !ok || stream == "" || consumer == "" {
return "", "it names no consumer of a module whose objects the grants part asserts"
}
if consumer == buscompose.ControllerName || strings.HasPrefix(consumer, buscompose.ControllerName+"-") {
return "", "the consumer is the controller's own, which the controller asserts itself while it holds its lease"
}
return consumer, ""
}
// appliesToAModulesObject is H3's: a module's consumer the bus does not hold, its budget per condition.
func appliesToAModulesObject(c Condition) (string, bool, string) {
if _, why := moduleOf(c); why != "" {
return "", false, why
}
return c.Key, true, ""
}
// assertAgain is H3: the grants part asserts the module's buckets, logs and streams again.
func assertAgain(h *Healers, c Condition) (string, string, error) {
module, why := moduleOf(c)
if module == "" {
return "", "", errors.New(why)
}
raw, err := h.Ask(AssertTool, map[string]any{"module": module})
if err != nil {
return "", "", fmt.Errorf("the grants part did not assert %s: %w", module, err)
}
var a struct {
Says string `json:"says"`
}
_ = json.Unmarshal(raw, &a)
return "asserted " + module + "'s buckets, logs and streams again through the grants part",
orSaid(a.Says, "the grants part answered") + "; the probe that raised it says on its next run whether it is there", nil
}
func orSaid(s, otherwise string) string {
if strings.TrimSpace(s) == "" {
return otherwise
}
return s
}
func firstLine(s string) string {
line, _, _ := strings.Cut(s, "\n")
return line
}
@@ -0,0 +1,496 @@
package main
import (
"encoding/json"
"errors"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/health"
)
type memStore struct {
mu sync.Mutex
kv map[string]map[string][]byte
keysErr map[string]error
ttl map[string]time.Duration
}
func newStore() *memStore {
return &memStore{kv: map[string]map[string][]byte{}, keysErr: map[string]error{}}
}
func (s *memStore) Get(state, key string) ([]byte, bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
v, ok := s.kv[state][key]
return v, ok, nil
}
func (s *memStore) put(state, key string, value any, create bool) error {
raw, err := json.Marshal(value)
if err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
if s.kv[state] == nil {
s.kv[state] = map[string][]byte{}
}
if _, ok := s.kv[state][key]; ok && create {
return ErrExists
}
s.kv[state][key] = raw
return nil
}
func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) }
func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) }
// CreateTTL and PutTTL keep the key's time to live beside it, for a test to read.
func (s *memStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
if err := s.put(state, key, value, true); err != nil {
return err
}
s.setTTL(state, key, ttl)
return nil
}
func (s *memStore) PutTTL(state, key string, value any, ttl time.Duration) error {
if err := s.put(state, key, value, false); err != nil {
return err
}
s.setTTL(state, key, ttl)
return nil
}
func (s *memStore) setTTL(state, key string, ttl time.Duration) {
s.mu.Lock()
defer s.mu.Unlock()
if s.ttl == nil {
s.ttl = map[string]time.Duration{}
}
s.ttl[state+"/"+key] = ttl
}
// ttlOf is a key's time to live as last written; zero for none.
func (s *memStore) ttlOf(state, key string) time.Duration {
s.mu.Lock()
defer s.mu.Unlock()
return s.ttl[state+"/"+key]
}
func (s *memStore) Delete(state, key string) error {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.kv[state], key)
return nil
}
func (s *memStore) Keys(state string) ([]string, error) {
s.mu.Lock()
defer s.mu.Unlock()
if err := s.keysErr[state]; err != nil {
return nil, err
}
var out []string
for k := range s.kv[state] {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
func (s *memStore) keys(state string) []string {
k, _ := s.Keys(state)
return k
}
func (s *memStore) attempt(t *testing.T, key string) Attempt {
t.Helper()
raw, _, _ := s.Get(AttemptsState, key)
var a Attempt
if err := json.Unmarshal(raw, &a); err != nil {
t.Fatalf("%s: %s", key, raw)
}
return a
}
// world is mesh-healers over memory, with its clock, its asks, what it said and the wakes it set.
type world struct {
h *Healers
store *memStore
now time.Time
asked []string
said []Acted
}
func newWorld() *world {
w := &world{store: newStore(), now: time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)}
w.fresh()
return w
}
// fresh is the healers started again over the same buckets: nothing of the last one's memory, its wakes forgotten.
func (w *world) fresh() {
w.h = &Healers{Store: w.store, Registry: Registry(), Logf: func(string, ...any) {},
Now: func() time.Time { return w.now },
Ask: func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
w.asked = append(w.asked, key+" "+string(raw))
return json.RawMessage(`{"says":"2 the bus lacks: created at this ask"}`), nil
},
Publish: func(subject string, body any, _ string) error {
if a, ok := body.(Acted); ok && subject == "mesh.seat.mesh-healers.event.healer-acted" {
w.said = append(w.said, a)
}
return nil
},
}
}
func (w *world) handAway() {
w.h.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil)
}
// lost is a module's consumer the bus does not hold, as D6 raises it.
func lost(module string) Condition {
return Condition{Key: "bus.EVENTS." + module + ".missing", Kind: "consumer-lost", Severity: "warning", Resolver: "self",
Subject: Subject{Scope: "bus", ID: "EVENTS." + module}}
}
func body(c Condition) []byte {
raw, _ := json.Marshal(map[string]any{"key": c.Key, "kind": c.Kind, "subject": c.Subject, "severity": c.Severity,
"resolver": c.Resolver, "tried": c.Tried, "event": "condition-raised", "show": "mesh-conditions.conditions key=" + c.Key})
return raw
}
// No act is made while the lease names a controller: each is written to would, never to attempts, so it counts for no
// budget or brake and nothing is asked or said; one is made from the write that hands the lease away (ADR 0564 §8).
func TestTheHealersActOnlyOnceTheLeaseNamesNoController(t *testing.T) {
cases := []struct {
name string
lease func(h *Healers)
acts bool
}{
{"never read", func(*Healers) {}, false},
{"absent", func(h *Healers) { h.LeaseHeard(nil, false, nil) }, false},
{"unreadable", func(h *Healers) { h.LeaseHeard(nil, false, errors.New("timeout")) }, false},
{"a controller holds it", func(h *Healers) {
h.LeaseHeard([]byte(`{"instance":"novox/1/2026-10-27","epoch":57}`), true, nil)
}, false},
{"handed away", func(h *Healers) { h.LeaseHeard([]byte(`{"handed-to":"the operator"}`), true, nil) }, true},
}
for _, tc := range cases {
w := newWorld()
tc.lease(w.h)
if err := w.h.Heard(body(lost("records"))); err != nil {
t.Fatalf("%s: %v", tc.name, err)
}
attempts, would := w.store.keys(AttemptsState), w.store.keys(WouldState)
if tc.acts {
if len(attempts) != 1 || len(would) != 0 || len(w.asked) != 1 || !strings.Contains(w.asked[0], `"module":"records"`) ||
len(w.said) != 1 || w.said[0].Outcome != OutcomeActed {
t.Errorf("%s: attempts %v, would %v, asked %v, said %+v", tc.name, attempts, would, w.asked, w.said)
continue
}
a := w.store.attempt(t, attempts[0])
if a.Outcome != OutcomeActed || a.Finished == nil || a.Healer != "H3" || a.Condition != "bus.EVENTS.records.missing" {
t.Errorf("%s: the attempt is %+v", tc.name, a)
}
continue
}
if len(attempts) != 0 || len(w.asked) != 0 || len(w.said) != 0 || !slices.Equal(would, []string{"H3.bus.EVENTS.records.missing"}) {
t.Errorf("%s: attempts %v, would %v, asked %v, said %v", tc.name, attempts, would, w.asked, w.said)
}
if _, braked, _ := w.store.Get(FindingsState, brakeKey); braked {
t.Errorf("%s: a would-act moved the brake", tc.name)
}
}
}
// An act waits out its settle, then its spent budget hands the condition to the operator once; an escalated condition
// is passed over; a second instance cannot begin an act already begun; the brake stops every healer past twelve acts
// in an hour and lets go an hour after the last (ADR 0231 §21, ADR 0564 §7).
func TestBudgetSettleEscalationAndBrake(t *testing.T) {
w := newWorld()
w.handAway()
c := lost("records")
_ = w.store.Put(ConditionsRead, c.Key, c)
settle := settlePrefix + c.Key
steps := []struct {
name string
after time.Duration
cond Condition
attempts int
outcome string // the newest said outcome, empty for none new
// how is what happens: an event, a restart over the same buckets, or the settle key's expiry heard.
how string
// wait is the settle key's time to live after the step, zero for no check.
wait time.Duration
}{
{"acts: its settle is a key that expires", 0, c, 1, OutcomeActed, "event", 6*time.Minute + time.Second},
{"within its settle: waits for the same end", 2 * time.Minute, c, 1, "", "event", 4*time.Minute + time.Second},
{"a restart within its settle: the key on the bus still waits", time.Minute, c, 1, "", "restart", 4*time.Minute + time.Second},
{"its key expires: judged again, budget spent, escalates", 4 * time.Minute, c, 2, OutcomeEscalated, "expire", 0},
{"escalated: passed over", time.Minute, Condition{Key: c.Key, Kind: c.Kind, Subject: c.Subject, Resolver: "operator",
Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, "", "event", 0},
}
for _, s := range steps {
w.now = w.now.Add(s.after)
before := len(w.said)
switch s.how {
case "restart":
w.fresh()
w.handAway()
w.h.Rearm()
case "expire":
w.h.WakeHeard(settle, OpExpire)
default:
if _, err := w.h.Judge(s.cond, ""); err != nil {
t.Fatalf("%s: %v", s.name, err)
}
}
if n := len(w.store.keys(AttemptsState)); n != s.attempts {
t.Errorf("%s: %d attempts, want %d", s.name, n, s.attempts)
}
if s.wait != 0 && w.store.ttlOf(WakesState, settle) != s.wait {
t.Errorf("%s: the settle waits %s, want %s", s.name, w.store.ttlOf(WakesState, settle), s.wait)
}
switch {
case s.outcome == "" && len(w.said) != before:
t.Errorf("%s: said %+v", s.name, w.said[before:])
case s.outcome != "" && (len(w.said) != before+1 || w.said[before].Outcome != s.outcome):
t.Errorf("%s: said %+v, want %s", s.name, w.said[before:], s.outcome)
}
}
// Each attempt is kept a week from its last write.
for _, k := range w.store.keys(AttemptsState) {
if ttl := w.store.ttlOf(AttemptsState, k); ttl != KeptFor {
t.Errorf("the attempt %s is kept %s, want a week", k, ttl)
}
}
// A settle whose key expired while no instance watched is judged at the start.
r := newWorld()
r.handAway()
_ = r.store.Put(ConditionsRead, c.Key, c)
if _, err := r.h.Judge(c, ""); err != nil {
t.Fatal(err)
}
r.now = r.now.Add(7 * time.Minute)
r.fresh()
r.handAway()
r.h.Rearm()
if len(r.said) != 2 || r.said[1].Outcome != OutcomeEscalated {
t.Errorf("a settle that ended unheard is not judged at the start: %+v", r.said)
}
// A second instance cannot begin the act an attempt's key already holds.
x := newWorld()
x.handAway()
_ = x.store.CreateTTL(AttemptsState, attemptKey("H3", lost("mail").Key, x.now.UTC()), Attempt{}, KeptFor)
if said, _ := x.h.Judge(lost("mail"), ""); !strings.Contains(said, "another instance") || len(x.asked) != 0 {
t.Errorf("a second instance acted: %q, asked %v", said, x.asked)
}
// The brake: twelve acts in the hour stop every healer, said as an urgent finding.
b := newWorld()
b.handAway()
for i := 0; i < BrakeLimit; i++ {
b.now = b.now.Add(time.Minute)
_ = b.store.CreateTTL(AttemptsState, attemptKey("H3", "x", b.now), Attempt{Healer: "H3", Condition: "x", BudgetKey: "x",
Outcome: OutcomeActed, Begun: b.now}, KeptFor)
}
if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "stopped") || len(b.asked) != 0 {
t.Errorf("past the brake: %q, asked %v", said, b.asked)
}
raw, braked, _ := b.store.Get(FindingsState, brakeKey)
f, _ := health.DecodeFinding(raw)
if !braked || f.Severity != Urgent || f.Kind != "healers-braked" || f.Observer != Module {
t.Errorf("the brake is written as %s", raw)
}
b.now = b.now.Add(55 * time.Minute) // fewer than twelve within the hour, but the brake is said: held
if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "held until") {
t.Errorf("the brake let go before an hour after the last act: %q", said)
}
b.now = b.now.Add(6 * time.Minute)
if _, err := b.h.Judge(lost("mail"), ""); err != nil || len(b.asked) != 1 {
t.Errorf("an hour after the last act the healers do not act again: %v, asked %v", err, b.asked)
}
if _, braked, _ := b.store.Get(FindingsState, brakeKey); braked {
t.Error("the brake's finding outlived the brake")
}
// Blind to what they did, they act on nothing and say so.
z := newWorld()
z.handAway()
z.store.keysErr[AttemptsState] = errors.New("the bus is away")
if _, err := z.h.Judge(lost("mail"), ""); err == nil || len(z.asked) != 0 {
t.Errorf("blind, the healers acted: %v", err)
}
if keys := z.store.keys(FindingsState); !slices.Equal(keys, []string{"probe.healers.failed"}) {
t.Errorf("blind, the findings are %v", keys)
}
}
// H3 answers a module's consumer the bus does not hold, never the controller's own or a condition naming no consumer;
// the registry holds every healer the mesh has, H3 run here, H4 and H5 run elsewhere (ADR 0231 §2, ADR 0564 §7).
func TestTheRegistryAndWhatH3Answers(t *testing.T) {
cases := []struct {
c Condition
holds bool
}{
{lost("records"), true},
{lost("controller"), false},
{lost("controller-merges"), false},
{Condition{Key: "seat.x.silent", Kind: "holder-silent", Subject: Subject{Scope: "seat", ID: "issue-tracker"}}, false},
{Condition{Key: "bus.EVENTS.records.behind", Kind: "consumer-behind", Subject: Subject{Scope: "bus", ID: "EVENTS.records"}}, false},
}
for _, tc := range cases {
w := newWorld()
w.handAway()
said, _ := w.h.Judge(tc.c, "")
if acted := len(w.asked) == 1; acted != tc.holds {
t.Errorf("%s: acted %v, want %v (%s)", tc.c.Key, acted, tc.holds, said)
}
}
runs := map[string]bool{}
for _, r := range Registry() {
if len(r.Kinds) == 0 || r.Budget < 1 || r.Window <= 0 || r.Settle <= 0 || r.Repair == "" || r.ActsIn == "" {
t.Errorf("%s is not a whole row: %+v", r.ID, r)
}
runs[r.ID] = r.repair != nil
}
if !runs["H3"] || runs["H4"] || runs["H5"] || len(runs) != 3 {
t.Errorf("the healers run here are %v, want H3 alone of H3, H4, H5", runs)
}
}
// S15: a cause repaired by hand twice within a fortnight is a finding; a decision by design never counts; it clears
// when fewer than two remain, and the healers wake when the oldest leaves the fortnight (ADR 0564 Limits 2).
func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) {
w := newWorld()
act := func(key, verb, cause string, ago time.Duration) {
raw, _ := json.Marshal(HandAct{ID: key, At: w.now.Add(-ago), By: "person.jochen", Verb: verb, Why: "x", Cause: cause})
w.h.HandActHeard(key, "put", raw)
}
act("a1", "broker consumer-reset", "consumer-behind", 10*24*time.Hour)
if keys := w.store.keys(FindingsState); len(keys) != 0 {
t.Fatalf("one act is a finding: %v", keys)
}
// The second of the cause is the seat's own act, heard as act-recorded: one in each log counts two (ADR 0565 §11).
seatAct, _ := json.Marshal(HandAct{ID: "a2", At: w.now.Add(-24 * time.Hour), By: "person.jochen",
Verb: "broker consumer-reset", Why: "x", Cause: "consumer-behind"})
w.h.ActRecorded(seatAct)
// A drill in each log counts none.
act("a3", "hand-act drill", "drill", time.Hour)
drill, _ := json.Marshal(HandAct{ID: "a4", At: w.now.Add(-2 * time.Hour), Verb: "hand-act drill", Why: "x", Cause: "drill"})
w.h.ActRecorded(drill)
act("a5", "conditions silence", "operator-answer", time.Hour)
act("a6", "conditions silence", "operator-answer", time.Hour)
act("a7", "hand-act record", "relogin", 20*24*time.Hour)
act("a8", "hand-act record", "relogin", time.Hour)
want := []string{"mesh.hand-acts.consumer-behind.healer-wanted"}
if keys := w.store.keys(FindingsState); !slices.Equal(keys, want) {
t.Fatalf("findings %v, want %v", keys, want)
}
raw, _, _ := w.store.Get(FindingsState, want[0])
if !strings.Contains(string(raw), "healer H4 answers this cause") {
t.Errorf("the finding does not say its healer is not enough: %s", raw)
}
if ttl := w.store.ttlOf(WakesState, fortnightKey); ttl != 4*24*time.Hour+time.Second {
t.Errorf("the fortnight waits %s, want the oldest act's leaving it in 4 days", ttl)
}
w.now = w.now.Add(4*24*time.Hour + time.Second)
w.h.Wanted()
if keys := w.store.keys(FindingsState); len(keys) != 0 {
t.Errorf("past the fortnight the finding stays: %v", keys)
}
w.h.HandActHeard("a5", "delete", nil)
if len(w.h.acts) != 7 {
t.Errorf("a deleted entry is kept: %d", len(w.h.acts))
}
}
// The verbs: heal acts on an open condition by key and says why it was asked; drill waits for the log's home; the reads
// answer every bucket.
func TestTheVerbsAndReads(t *testing.T) {
w := newWorld()
w.handAway()
c := lost("records")
_ = w.store.Put(ConditionsRead, c.Key, c)
if _, err := w.h.Heal(c.Key, "", "person.jochen"); err == nil {
t.Error("heal took no why")
}
if _, err := w.h.Heal("bus.none", "x", "person.jochen"); err == nil {
t.Error("heal acted on no open condition")
}
if _, err := w.h.Heal(c.Key, "it is gone again", "person.jochen"); err != nil {
t.Fatal(err)
}
keys := w.store.keys(AttemptsState)
if len(keys) != 1 || !strings.Contains(w.store.attempt(t, keys[0]).Said, "asked by person.jochen (it is gone again)") {
t.Errorf("heal's attempt is %v", keys)
}
answer, err := w.h.Answer(7, true)
if err != nil {
t.Fatal(err)
}
a := answer.(map[string]any)
if a["acting"] != true || len(a["acts"].([]Attempt)) != 1 || len(a["healers"].([]map[string]any)) != 3 {
t.Errorf("healers answered %v", a)
}
// drill records into mesh-hand-acts as a drill; while that seat has no holder, into the controller's drill until
// the lease is handed away (ADR 0565 §11).
drills := []struct {
name string
seatGone bool
leaseAway bool
args map[string]any
log, err string
}{
{"into the seat", false, true, map[string]any{"what": "stopped records", "why": "does it raise"}, handacts.LogSeat, ""},
{"no holder: the controller's", true, false, map[string]any{"what": "stopped records", "why": "x"}, handacts.LogController, ""},
{"no holder, lease away", true, true, map[string]any{"what": "stopped records", "why": "x"}, "", "handed its lease away"},
{"says nothing", false, true, map[string]any{}, "", "says what"},
}
for _, d := range drills {
x := newWorld()
if d.leaseAway {
x.handAway()
} else {
x.h.LeaseHeard([]byte(`{"instance":"novox/1","epoch":5}`), true, nil)
}
var asked []string
x.h.Ask = func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
asked = append(asked, key+" "+string(raw))
if d.seatGone && strings.HasPrefix(key, "seat:mesh-hand-acts.") {
return nil, errors.New("nats: no responders available for request")
}
return json.RawMessage(`{}`), nil
}
var got any
var err error
for _, tool := range Tools(x.h) {
if tool.Name == DrillTool {
got, err = tool.Run(d.args)
}
}
switch {
case d.err != "":
if err == nil || !strings.Contains(err.Error(), d.err) {
t.Errorf("drill %s: %v", d.name, err)
}
case err != nil || got.(map[string]any)["log"] != d.log:
t.Errorf("drill %s: %v, %v", d.name, got, err)
case d.log == handacts.LogSeat && !strings.Contains(asked[0], `"verb":"hand-act drill"`):
t.Errorf("drill %s asked %v", d.name, asked)
}
}
if _, err := w.h.read(WouldState, ""); err != nil {
t.Error(err)
}
}
@@ -0,0 +1,118 @@
// mesh-healers: the holder of the seat mesh-healers (novox/hq ADR 0564 §7, §8). A Go bundle served by a runtime of its
// own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `attempts` (every act begun,
// made and finished, and every escalation, kept a week), `would` (what it would have done while the controller holds
// its lease, kept a week) and `findings` (the brake, its blindness, and S15). It hears the condition keeper's
// condition-raised and condition-changed on a durable consumer, watches the controller's lease, and watches the
// controller's hand-act log for S15. stdout is the MCP channel; what this module says, it says on stderr.
package main
import (
"encoding/json"
"errors"
"fmt"
"os"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
// runtimeStore is the bus, through the runtime that launched this bundle.
type runtimeStore struct{}
func (runtimeStore) Get(state, key string) ([]byte, bool, error) {
e, err := stdio.State(state).Get(key)
if err != nil || e == nil {
return nil, false, err
}
return e.Value, true, nil
}
func (runtimeStore) Put(state, key string, value any) error {
_, err := stdio.State(state).Put(key, value)
return err
}
func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
_, err := stdio.State(state).CreateTTL(key, value, ttl)
if errors.Is(err, stdio.ErrChanged) {
return ErrExists
}
return err
}
func (runtimeStore) PutTTL(state, key string, value any, ttl time.Duration) error {
_, err := stdio.State(state).PutTTL(key, value, ttl)
return err
}
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs.
func watched(state, key string, handle func(stdio.StateChange) error) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
time.Sleep(wait)
if err := stdio.State(state).Watch(key, handle); err != nil {
logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait)
continue
}
return
}
}
func main() {
h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf,
Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) },
Publish: stdio.SeatPublish}
// The lease first: until it is read the healers write would-acts only. Once it is first read, a settle that ended
// while no instance watched is judged (Rearm); the waits still running are keys on the bus, heard on their expiry.
var started sync.Once
go watched(LeaseRead, LeaseKey, func(c stdio.StateChange) error {
h.LeaseHeard(c.Value, c.Op == "put", nil)
started.Do(func() { go h.Rearm() })
return nil
})
go watched(WakesState, "", func(c stdio.StateChange) error {
h.WakeHeard(c.Key, c.Op)
return nil
})
go watched(HandActsRead, "", func(c stdio.StateChange) error {
h.HandActHeard(c.Key, c.Op, c.Value)
return nil
})
// S15 over both logs: the seat's act-recorded on a durable consumer, beside the controller's bucket's watch.
go func() {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
time.Sleep(wait)
err := stdio.Subscribe(RecordedEvent, func(e stdio.Envelope) error {
h.ActRecorded(e.Body)
return nil
})
if err == nil {
return
}
logf("[%s] does not hear %s yet (%v); asking again in %s", Module, RecordedEvent, err, wait)
}
}()
for _, event := range []string{RaisedEvent, ChangedEvent} {
go func(event string) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
time.Sleep(wait)
err := stdio.Subscribe(event, func(e stdio.Envelope) error { return h.Heard(e.Body) })
if err == nil {
logf("[%s] hears %s", Module, event)
return
}
logf("[%s] does not hear %s yet (%v); asking again in %s", Module, event, err, wait)
}
}(event)
}
if err := stdio.Serve("", Tools(h)); err != nil {
logf("%v", err)
os.Exit(1)
}
}
@@ -0,0 +1,92 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"slices"
"strings"
"testing"
)
type manifest struct {
Module string `json:"module"`
RunsAs string `json:"runs-as"`
Claims []struct {
Name string `json:"name"`
Scope string `json:"scope"`
Serves []string `json:"serves"`
} `json:"claims"`
Emits []string `json:"emits"`
Consumes []string `json:"consumes"`
Invokes []string `json:"invokes"`
Reads []string `json:"reads"`
State []json.RawMessage `json:"state"`
}
// The manifest says what the code does: it claims the seat mesh-healers once for the mesh with its verbs (ADR 0564 §1,
// §7), runs on its own account, says healer-acted, hears the keeper's raised and changed, asks only the grants part's
// assert, reads the lease, the keeper's conditions and the hand-act log, and keeps attempts and would a week each.
// Every state it keeps has a tool that reads it, and every act it takes by itself a tool that starts it (ADR 0564 §12).
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m manifest
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
if m.Module != Module || m.RunsAs != Module {
t.Errorf("module %q runs as %q", m.Module, m.RunsAs)
}
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) {
t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs)
}
for _, c := range []struct {
name string
got, want []string
}{
{"emits (healer-acted is the seat's, said on its subject)", m.Emits, nil},
{"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent, RecordedEvent}},
{"invokes", m.Invokes, []string{AssertTool, "seat:mesh-hand-acts.hand-act", "seat:mesh-controller.drill"}},
{"reads", m.Reads, []string{LeaseRead, ConditionsRead, HandActsRead}},
} {
if !slices.Equal(c.got, c.want) {
t.Errorf("%s %v, want %v", c.name, c.got, c.want)
}
}
states := map[string]bool{}
for _, s := range m.State {
var d struct {
Name string `json:"name"`
KeyTTL bool `json:"key-ttl"`
}
if json.Unmarshal(s, &d.Name) != nil {
_ = json.Unmarshal(s, &d)
}
states[d.Name] = d.KeyTTL
}
// Each key of attempts and would lives a week from its last write (KeptFor, would has no other deleter), and each
// wait in wakes as long as it waits: their buckets take a key's own time to live.
if !states[AttemptsState] || !states[WouldState] || !states[WakesState] || states[FindingsState] {
t.Errorf("the states take a key's own time to live as %v, want attempts, would and wakes alone", states)
}
readBy := map[string]string{AttemptsState: HealersTool, WouldState: WouldTool, FindingsState: FindingsTool,
WakesState: WakesTool, "a settle's end": HealTool,
"an act on an event": HealTool, "S15's count": HealersTool + " recount=true", "the lease as read": HealersTool}
tools := map[string]bool{}
for _, tool := range Tools(&Healers{}) {
tools[tool.Name] = true
}
for state := range states {
if _, ok := readBy[state]; !ok {
t.Errorf("the state %s has no tool that reads it", state)
}
}
for what, tool := range readBy {
if name, _, _ := strings.Cut(tool, " "); !tools[name] {
t.Errorf("%s is read or started by %s, which is not served", what, tool)
}
}
}
@@ -0,0 +1,77 @@
package main
// What the healers observe of themselves, and the finding it is written as (novox/hq ADR 0564 §2): the brake, their
// blindness and S15, each written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the
// controller's key for the same condition.
import (
"encoding/json"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// The severities and scopes, as mesh-sdk's health names them.
const (
Urgent = health.Urgent
Warning = health.Warning
ScopeProbe = health.ScopeProbe
ScopeMesh = health.ScopeMesh
)
// Subject is what a finding is about.
type Subject = health.Subject
// Observation is one thing a probe sees wrong now.
type Observation struct {
Kind string
Token string
Subject Subject
Severity health.Severity
Summary string
Said string
Headline string
Explanation string
Resolved string
Needs string
// Source is what saw it: the brake, the healers' blindness, or S15.
Source string
}
// Key is where it is kept: `<scope>.<id>.<token>`, the controller's key for the same condition.
func (o Observation) Key() string {
token := o.Token
if token == "" {
token = o.Kind
}
return health.Key(o.Subject.Scope, o.Subject.ID, token)
}
// Finding is the observation as written by run at, its evidence the one line it said.
func (o Observation) Finding(run string, at time.Time) health.Finding {
f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary,
Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source,
Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}}
if o.Token != "" && o.Token != o.Kind {
f.Token = o.Token
}
said := o.Said
if said == "" {
said = o.Summary
}
f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said})
return f
}
// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write
// (ADR 0289 §1).
func says(f health.Finding) string {
f.At, f.Run, f.V = time.Time{}, "", 0
ev := make([]health.Evidence, len(f.Evidence))
for i, e := range f.Evidence {
ev[i] = health.Evidence{Said: e.Said}
}
f.Evidence = ev
raw, _ := json.Marshal(f)
return string(raw)
}
@@ -0,0 +1,2 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
modules/mesh-healers/module.json
@@ -0,0 +1,230 @@
package main
// The seat's verbs and the module's own reads (novox/hq ADR 0564 §7, §12): `healers` is the registry, what the healers
// did and would do, the brake and whether they act; `heal` takes one act now on a condition, within budget and brake;
// `drill` waits for the hand-act log's home; and a read of each bucket it keeps, so a recovery is a call.
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/handacts"
)
// Verbs are the seat's verbs, as the manifest claims them.
var Verbs = []string{"healers", "heal", "drill"}
// Tool names.
const (
HealersTool = Seat + ".healers"
HealTool = Seat + ".heal"
DrillTool = Seat + ".drill"
FindingsTool = "healers_findings"
WouldTool = "healers_would"
WakesTool = "healers_wakes"
)
func obj(props map[string]any) map[string]any {
return map[string]any{"type": "object", "properties": props}
}
// Tools are what the runtime serves.
func Tools(h *Healers) []stdio.Tool {
return []stdio.Tool{
{Name: HealersTool,
Description: "The mesh's healers (novox/hq ADR 0564 §7, §8): the registry, every act and escalation of the last " +
"days (the bucket attempts), whether the mesh-wide brake holds, whether they act or only write what they " +
"would do while the controller holds its lease, and the would-acts. recount=true counts the hand-act log " +
"again for S15 (a cause repaired by hand twice in a fortnight).",
Input: obj(map[string]any{
"days": map[string]any{"type": "number", "description": "how many days of acts back (default 7)"},
"recount": map[string]any{"type": "boolean", "description": "count the hand-act log again now"}}),
Run: func(a map[string]any) (any, error) { return h.Answer(intArg(a, "days", 7), boolArg(a, "recount")) }},
{Name: HealTool,
Description: "One healer's act now on an open condition, by its key, within its budget and the brake, as an " +
"event would have it — or, while the controller holds its lease, the act it would take written as would. " +
"Says what was done.",
Input: obj(map[string]any{
"key": map[string]any{"type": "string", "description": "the condition's key"},
"why": map[string]any{"type": "string", "description": "why, in a sentence"}}),
Run: func(a map[string]any) (any, error) {
return h.Heal(strArg(a, "key"), strArg(a, "why"), stdio.CallerOf(a))
}},
{Name: DrillTool,
Description: "Record a drill: something broken on purpose to see the mesh raise and clear it, with what it " +
"tests, in the hand-act log of the seat mesh-hand-acts (the controller's drill while that seat has no " +
"holder and the controller holds its lease). A drill is never a repair: S15 never counts it.",
Input: obj(map[string]any{"what": map[string]any{"type": "string", "description": "what was done on purpose"},
"why": map[string]any{"type": "string", "description": "what the drill tests"},
"condition": map[string]any{"type": "string", "description": "the condition it is meant to raise, if any"}}),
Run: func(a map[string]any) (any, error) {
return h.Drill(strArg(a, "what"), strArg(a, "why"), strArg(a, "condition"))
}},
{Name: FindingsTool,
Description: "What the healers find of themselves (their bucket findings): the brake, their blindness, and " +
"each cause repaired by hand twice in a fortnight (S15); or one whole by key.",
Input: obj(map[string]any{"key": map[string]any{"type": "string"}}),
Run: func(a map[string]any) (any, error) { return h.read(FindingsState, strArg(a, "key")) }},
{Name: WakesTool,
Description: "What the healers wait for (their bucket wakes): each settle still running (`settle.<condition key>`) " +
"and S15's fortnight end (`fortnight`), each a key whose time to live is the wait and whose expiry wakes them; " +
"or one whole by key. heal acts on a condition now, and healers with recount=true counts S15 now.",
Input: obj(map[string]any{"key": map[string]any{"type": "string"}}),
Run: func(a map[string]any) (any, error) { return h.read(WakesState, strArg(a, "key")) }},
{Name: WouldTool,
Description: "What the healers would have done while the controller holds its lease (their bucket would, each " +
"key kept a week): one per healer and condition, its newest; or one whole by key.",
Input: obj(map[string]any{"key": map[string]any{"type": "string"}}),
Run: func(a map[string]any) (any, error) { return h.read(WouldState, strArg(a, "key")) }},
}
}
// Drill records a drill (ADR 0565 §11, which ends ADR 0564 Limits 1's refusal): through the seat mesh-hand-acts as
// the act `hand-act drill`, and while that seat has no holder that answers and the controller still holds its lease,
// through the controller's own `drill`, which records it as a drill too. It answers which log took it.
func (h *Healers) Drill(what, why, condition string) (any, error) {
if what == "" || why == "" {
return nil, errors.New("a drill says what was done on purpose and what it tests: nothing was recorded")
}
if h.Ask == nil {
return nil, errors.New("the hand-act log cannot be reached from here: nothing was recorded")
}
body := map[string]any{"what": what, "why": why, "verb": handacts.VerbDrill}
controller := map[string]any{"what": what, "why": why}
if condition != "" {
body["condition"], controller["condition"] = condition, condition
}
answer, err := h.Ask("seat:"+handacts.Seat+".hand-act", body)
if err == nil {
return map[string]any{"log": handacts.LogSeat, "answer": answer}, nil
}
if !handacts.NoResponder(err) {
return nil, fmt.Errorf("%s.hand-act refused the drill: %w", handacts.Seat, err)
}
if acting, _ := h.Acting(); acting {
return nil, fmt.Errorf("%s has no holder that answers, and the controller handed its lease away: nothing was "+
"recorded (%w)", handacts.Seat, err)
}
answer, cerr := h.Ask("seat:mesh-controller.drill", controller)
if cerr != nil {
return nil, fmt.Errorf("%s has no holder that answers (%v), and the controller's drill refused too: %w", handacts.Seat,
err, cerr)
}
return map[string]any{"log": handacts.LogController, "answer": answer}, nil
}
// Heal is the verb heal: one condition judged now by its healer.
func (h *Healers) Heal(key, why, caller string) (any, error) {
if key == "" || why == "" {
return nil, errors.New("heal names the condition's key and says why")
}
raw, found, err := h.Store.Get(ConditionsRead, key)
if err != nil {
return nil, fmt.Errorf("the open conditions cannot be read: %w", err)
}
if !found {
return nil, fmt.Errorf("no condition is open under %s", key)
}
said, err := h.Judge(conditionOf(raw), caller+" ("+why+")")
if err != nil {
return nil, err
}
return map[string]any{"said": said}, nil
}
// Answer is the verb healers.
func (h *Healers) Answer(days int, recount bool) (any, error) {
if recount {
h.Wanted()
}
now := h.Now()
all, err := h.attempts()
if err != nil {
return nil, fmt.Errorf("what the healers did cannot be read: %w", err)
}
var recent []Attempt
for _, a := range all {
if now.Sub(a.Begun) <= time.Duration(days)*24*time.Hour {
recent = append(recent, a)
}
}
sort.SliceStable(recent, func(i, j int) bool { return recent[i].Begun.After(recent[j].Begun) })
braked, said := h.brakeHolds(actsWithin(all, now.Add(-BrakeWindow)), now)
var rows []map[string]any
for _, r := range h.Registry {
rows = append(rows, map[string]any{"id": r.ID, "kinds": r.Kinds, "condition": r.Condition, "repair": r.Repair,
"budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle),
"then": r.Then, "acts-in": r.ActsIn, "from": r.From})
}
acting, why := h.Acting()
would, err := h.read(WouldState, "")
if err != nil {
would = map[string]any{"unread": err.Error()}
}
h.mu.Lock()
declined := map[string]string{}
for k, v := range h.declined {
declined[k] = v
}
h.mu.Unlock()
if recent == nil {
recent = []Attempt{}
}
return map[string]any{"healers": rows, "acts": recent, "days": days, "acting": acting, "why": why, "would": would,
"brake": map[string]any{"holds": braked, "said": said, "limit": BrakeLimit, "window": BrakeWindow.String()},
"declined": declined,
"note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"}, nil
}
// read is every key of a bucket with its value, or one whole.
func (h *Healers) read(state, key string) (any, error) {
if key != "" {
raw, found, err := h.Store.Get(state, key)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("nothing is kept under %s in %s", key, state)
}
return json.RawMessage(raw), nil
}
keys, err := h.Store.Keys(state)
if err != nil {
return nil, err
}
rows := map[string]json.RawMessage{}
for _, k := range keys {
if raw, found, err := h.Store.Get(state, k); err == nil && found {
rows[k] = raw
}
}
return map[string]any{state: rows, "count": len(rows)}, nil
}
func strArg(a map[string]any, k string) string {
s, _ := a[k].(string)
return strings.TrimSpace(s)
}
// boolArg reads a flag as a boolean or as "true".
func boolArg(a map[string]any, k string) bool {
switch v := a[k].(type) {
case bool:
return v
case string:
return strings.EqualFold(strings.TrimSpace(v), "true")
}
return false
}
func intArg(a map[string]any, k string, otherwise int) int {
if n, ok := a[k].(float64); ok && n > 0 {
return int(n)
}
return otherwise
}
@@ -0,0 +1,166 @@
package main
// S15: a cause repaired by hand twice within a fortnight is a healer wanted (ADR 0231 §27, §28; ADR 0564 Limits 2).
// The hand-act log is the controller's bucket mesh-controller_hand-acts, a key-value bucket with no events: mesh-healers
// watches it, the one of the four holders granted its read, counts each cause again on every put, and writes the
// finding `healer-wanted` per cause. **A heal is never a hand act**, and an act that is a person's decision by design
// (a drill above all) never counts. The finding clears when fewer than two acts of its cause remain within the
// fortnight: the key `fortnight` of the healers' wakes expires when the oldest counted act leaves it. From the log's home (issue 1953) they
// watch that instead.
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/handacts"
)
// Within is how far back a repeated cause counts.
const Within = 14 * 24 * time.Hour
// HandAct is what S15 reads of one entry of the controller's hand-act log (mesh-controller internal/link HandAct).
type HandAct struct {
ID string `json:"id"`
At time.Time `json:"at"`
By string `json:"by"`
Verb string `json:"verb"`
Args []string `json:"arguments,omitempty"`
Why string `json:"why"`
Cause string `json:"cause"`
Kind string `json:"kind,omitempty"`
}
// decided are the verbs whose entries are a person's decision by design, with the causes they hold for (every cause
// when none): the controller's handActVerbs, read as S15 reads them. A verb not listed counts as a repair.
var decided = map[string][]string{
"warrant": nil,
"seat": {"seat-passed"},
"conditions silence": {"operator-answer"},
"hand-act record": {"drill"},
"secret issued": nil,
"hand-act drill": nil,
"retire approve": nil,
"retire reject": nil,
"cleanup delete": nil,
"dead-letters deliver": nil,
"dead-letters drop": nil,
"collect": nil,
"mirrors": nil,
"move-placement": nil,
"move-config": nil,
"bus upgrade": nil,
"upgrade release-backlog": nil,
"secret accept": nil,
"secret rotate": {"leaked-in-logs"},
"rotate": {"leaked-in-logs"},
}
// personsDecision is whether an act is a person's decision by design, by the verb that recorded it.
func personsDecision(a HandAct) bool {
causes, ok := decided[a.Verb]
if !ok {
return false
}
return len(causes) == 0 || slices.Contains(causes, a.Cause)
}
// ActRecorded is one act the seat mesh-hand-acts recorded, heard as act-recorded: counted beside the controller's
// log's, under a key of its own log, so an act is counted once whichever log holds it (ADR 0565 §11).
func (h *Healers) ActRecorded(body []byte) {
var a HandAct
if json.Unmarshal(body, &a) != nil || a.ID == "" {
return
}
h.HandActHeard(handacts.LogSeat+":"+a.ID, "put", body)
}
// HandActHeard is one change of the hand-act log: kept or forgotten, and every cause counted again.
func (h *Healers) HandActHeard(key, op string, value []byte) {
h.mu.Lock()
if h.acts == nil {
h.acts = map[string]HandAct{}
}
if op != "put" {
delete(h.acts, key)
} else {
var a HandAct
if json.Unmarshal(value, &a) == nil {
h.acts[key] = a
}
}
h.mu.Unlock()
h.Wanted()
}
// Wanted counts every cause repaired by hand within the fortnight and writes a finding for each counted twice or more,
// deleting the rest; its wait `fortnight` expires when the oldest counted act of a wanted cause leaves the fortnight.
func (h *Healers) Wanted() []Observation {
h.mu.Lock()
defer h.mu.Unlock()
now := h.Now()
byCause := map[string][]HandAct{}
for _, a := range h.acts {
if personsDecision(a) || now.Sub(a.At) > Within {
continue
}
byCause[a.Cause] = append(byCause[a.Cause], a)
}
var causes []string
for c, acts := range byCause {
if len(acts) >= 2 {
causes = append(causes, c)
}
}
sort.Strings(causes)
var found []Observation
var next time.Time
for _, cause := range causes {
acts := byCause[cause]
sort.Slice(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) })
var lines []string
for _, a := range acts {
lines = append(lines, fmt.Sprintf("%s %s by %s: %s", a.At.UTC().Format("2006-01-02 15:04"),
strings.TrimSpace(a.Verb+" "+strings.Join(a.Args, " ")), a.By, a.Why))
}
newest := acts[len(acts)-1]
wanted := "a healer is wanted for it"
for _, r := range h.Registry {
if slices.Contains(r.Kinds, cause) {
wanted = fmt.Sprintf("healer %s answers this cause and a person still repaired it: its reach or its budget "+
"is not enough", r.ID)
}
}
words := strings.NewReplacer(".", " ", "_", " ").Replace(cause)
found = append(found, Observation{Kind: "healer-wanted", Token: "healer-wanted", Severity: Warning,
Subject: Subject{Scope: ScopeMesh, ID: "hand-acts." + cause},
Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause, len(acts),
int(Within.Hours()/24), newest.By, wanted),
Said: strings.Join(lines, "; "),
Headline: capital(words) + " keeps being fixed by hand",
Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is wanted "+
"for it. Nothing is broken now.", words, len(acts), int(Within.Hours()/24)),
Resolved: "Resolved: no more hand repairs of " + words})
if leaves := acts[0].At.Add(Within); next.IsZero() || leaves.Before(next) {
next = leaves
}
}
h.see("S15", found)
if !next.IsZero() {
ttl := next.Sub(now).Round(time.Second) + time.Second
if err := h.Store.PutTTL(WakesState, fortnightKey, map[string]any{"at": next.UTC()}, ttl); err != nil {
h.Logf("[%s] the fortnight's end is not written as a wait, so it is counted at the next act: %v", Module, err)
}
}
return found
}
func capital(s string) string {
if s == "" {
return s
}
return strings.ToUpper(s[:1]) + s[1:]
}
@@ -0,0 +1,423 @@
//go:build integration
package main
// mesh-healers whole against a real bus (novox/hq ADR 0564 §7, §8; issue 1952 H3): the built binary, played over its
// real stdin and stdout as the node's tool runtime plays it (mesh/state.* and watches answered from a real nats-server's
// key-value buckets, the keeper's events handed over as the runtime hands them, the grants part's assert answered). An
// induced failure, a module's consumer gone, is only written as would while the controller holds its lease; from the
// write that hands the lease away a healer acts, its attempt on the bus and healer-acted said; past twelve acts in the
// hour the brake stops every healer, said as an urgent finding on the bus.
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
type wire struct {
ID json.RawMessage `json:"id,omitempty"`
Method string `json:"method,omitempty"`
Params json.RawMessage `json:"params,omitempty"`
Result json.RawMessage `json:"result,omitempty"`
Error *struct {
Message string `json:"message"`
} `json:"error,omitempty"`
}
// player is the node's tool runtime, as far as the self-check needs it, over a real bus.
type player struct {
ctx context.Context
nc *nats.Conn
js jetstream.JetStream
mu sync.Mutex
in io.Writer
next int
pending map[string]chan wire
asked []string
}
func (p *player) send(m map[string]any) {
raw, _ := json.Marshal(m)
p.mu.Lock()
defer p.mu.Unlock()
_, _ = p.in.Write(append(raw, '\n'))
}
func (p *player) answer(id json.RawMessage, result any, err error) {
if err != nil {
p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
return
}
p.send(map[string]any{"id": id, "result": result})
}
func (p *player) serve(out io.Reader) {
scanner := bufio.NewScanner(out)
scanner.Buffer(make([]byte, 1<<20), 1<<24)
for scanner.Scan() {
var m wire
if json.Unmarshal(scanner.Bytes(), &m) != nil {
continue
}
if m.Method == "" {
var key string
_ = json.Unmarshal(m.ID, &key)
p.mu.Lock()
ch := p.pending[key]
delete(p.pending, key)
p.mu.Unlock()
if ch != nil {
ch <- m
}
continue
}
go p.handle(m)
}
}
// kv is the bucket a state names: this module's own by its local name, another's as `<owner>.<name>`.
func (p *player) kv(state string) (jetstream.KeyValue, error) {
if owner, name, ok := strings.Cut(state, "."); ok {
return p.js.KeyValue(p.ctx, owner+"_"+name)
}
return p.js.KeyValue(p.ctx, Module+"_"+state)
}
// deliver hands the bundle one event as the runtime does from its consumer.
func (p *player) deliver(key string, body []byte) {
p.mu.Lock()
p.next++
id := fmt.Sprintf("e-%d", p.next)
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": map[string]any{
"key": key, "node": "ace", "body": json.RawMessage(body)}}})
}
func (p *player) handle(m wire) {
var a struct {
State string `json:"state"`
Key string `json:"key"`
Value json.RawMessage `json:"value"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &a)
switch m.Method {
case "mesh/state.get":
kv, err := p.kv(a.State)
if err != nil {
p.answer(m.ID, nil, err)
return
}
e, err := kv.Get(p.ctx, a.Key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
p.answer(m.ID, nil, nil)
return
}
if err != nil {
p.answer(m.ID, nil, err)
return
}
p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
case "mesh/state.put":
kv, err := p.kv(a.State)
if err == nil {
var rev uint64
if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil {
p.answer(m.ID, map[string]any{"revision": rev}, nil)
return
}
}
p.answer(m.ID, nil, err)
case "mesh/state.create":
kv, err := p.kv(a.State)
if err == nil {
var rev uint64
if rev, err = kv.Create(p.ctx, a.Key, a.Value); err == nil {
p.answer(m.ID, map[string]any{"revision": rev}, nil)
return
}
if errors.Is(err, jetstream.ErrKeyExists) {
p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32001, "message": "the key was written by another writer first"}})
return
}
}
p.answer(m.ID, nil, err)
case "mesh/state.watch":
kv, err := p.kv(a.State)
if err != nil {
p.answer(m.ID, nil, err)
return
}
key := a.Key
if key == "" {
key = ">"
}
w, err := kv.Watch(p.ctx, key)
if err != nil {
p.answer(m.ID, nil, err)
return
}
var watch string
_ = json.Unmarshal(m.ID, &watch)
current := true
for e := range w.Updates() {
if e == nil {
current = false
p.answer(m.ID, map[string]any{}, nil)
continue
}
op := "put"
if e.Operation() != jetstream.KeyValuePut {
op = "delete"
}
change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current}
if op == "put" {
change["value"] = json.RawMessage(e.Value())
}
p.mu.Lock()
p.next++
id := fmt.Sprintf("w-%d", p.next)
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}})
}
case "mesh/subscribe":
p.answer(m.ID, map[string]any{}, nil)
case "mesh/state.keys":
kv, err := p.kv(a.State)
if err != nil {
p.answer(m.ID, nil, err)
return
}
keys, err := kv.Keys(p.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
keys, err = []string{}, nil
}
p.answer(m.ID, keys, err)
case "mesh/state.delete":
kv, err := p.kv(a.State)
if err == nil {
err = kv.Delete(p.ctx, a.Key)
}
p.answer(m.ID, map[string]any{}, err)
case "mesh/seat.publish":
var e struct {
Subject string `json:"subject"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &e)
p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body))
case "mesh/publish":
var e struct {
Key string `json:"key"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &e)
p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body))
case "mesh/ask":
var ask struct {
Key string `json:"key"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &ask)
p.mu.Lock()
p.asked = append(p.asked, ask.Key+" "+string(ask.Body))
p.mu.Unlock()
if ask.Key != AssertTool {
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
return
}
p.answer(m.ID, map[string]any{"says": "1 the bus lacks: created at this ask"}, nil)
default:
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
}
}
func (p *player) call(name string, args map[string]any) (json.RawMessage, error) {
p.mu.Lock()
p.next++
id := fmt.Sprintf("c-%d", p.next)
ch := make(chan wire, 1)
p.pending[id] = ch
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args,
"_meta": map[string]any{"mesh/caller": "person.jochen"}}})
select {
case m := <-ch:
if m.Error != nil {
return nil, errors.New(m.Error.Message)
}
var r struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
IsError bool `json:"isError"`
}
if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 {
return nil, fmt.Errorf("%s answered %s", name, m.Result)
}
if r.IsError {
return nil, errors.New(r.Content[0].Text)
}
return json.RawMessage(r.Content[0].Text), nil
case <-time.After(60 * time.Second):
return nil, fmt.Errorf("%s did not answer in 60s", name)
}
}
func eventually(t *testing.T, what string, ok func() bool) {
t.Helper()
for end := time.Now().Add(30 * time.Second); time.Now().Before(end); time.Sleep(50 * time.Millisecond) {
if ok() {
return
}
}
t.Fatalf("never: %s", what)
}
func raised(key, module string) []byte {
raw, _ := json.Marshal(map[string]any{"key": key, "kind": "consumer-lost", "severity": "warning", "resolver": "self",
"subject": map[string]any{"scope": "bus", "id": "EVENTS." + module}, "event": "condition-raised",
"show": health.ShowKeeper(key)})
return raw
}
// 1952/H3: an induced failure; the healer acts only from the lease write, and brakes.
func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
dir := t.TempDir()
bin := filepath.Join(dir, Module)
build := exec.Command("go", "build", "-o", bin, ".")
build.Stderr = os.Stderr
if err := build.Run(); err != nil {
t.Fatalf("mesh-healers does not build: %v", err)
}
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
for _, b := range []string{Module + "_" + AttemptsState, Module + "_" + WouldState, Module + "_" + FindingsState, Module + "_" + WakesState,
"mesh-controller_lease", "mesh-conditions_conditions", "mesh-controller_hand-acts"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
t.Fatal(err)
}
}
bucket := func(name string) jetstream.KeyValue {
kv, err := js.KeyValue(ctx, name)
if err != nil {
t.Fatal(err)
}
return kv
}
keys := func(name string) []string {
k, _ := bucket(name).Keys(ctx)
return k
}
lease := bucket("mesh-controller_lease")
if _, err := lease.Put(ctx, LeaseKey, []byte(`{"instance":"novox/1/2026-10-27","epoch":57}`)); err != nil {
t.Fatal(err)
}
acted, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + ActedEvent)
if err != nil {
t.Fatal(err)
}
cmd := exec.CommandContext(ctx, bin)
cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module)
stdin, _ := cmd.StdinPipe()
stdout, _ := cmd.StdoutPipe()
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
go p.serve(stdout)
healers := func() map[string]any {
raw, err := p.call(HealersTool, map[string]any{})
if err != nil {
return nil
}
var a map[string]any
_ = json.Unmarshal(raw, &a)
return a
}
eventually(t, "the lease read as a controller's", func() bool {
a := healers()
return a != nil && strings.Contains(fmt.Sprint(a["why"]), "names a controller")
})
// The controller holds the lease: the induced failure is written as would, never acted on.
p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records"))
eventually(t, "the would-act written", func() bool { return len(keys(Module+"_"+WouldState)) == 1 })
if k := keys(Module + "_" + AttemptsState); len(k) != 0 {
t.Fatalf("acted under the controller's lease: %v", k)
}
// The write that hands the lease away: the same failure is acted on.
if _, err := lease.Put(ctx, LeaseKey, []byte(`{"handed-to":"the operator's hand-over"}`)); err != nil {
t.Fatal(err)
}
eventually(t, "the healers acting", func() bool { return healers()["acting"] == true })
p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records"))
eventually(t, "the act begun and finished on the bus", func() bool {
k := keys(Module + "_" + AttemptsState)
if len(k) != 1 {
return false
}
e, err := bucket(Module+"_"+AttemptsState).Get(ctx, k[0])
return err == nil && strings.Contains(string(e.Value()), `"outcome":"acted"`)
})
msg, err := acted.NextMsg(10 * time.Second)
if err != nil || !strings.Contains(string(msg.Data), `"healer":"H3"`) {
t.Fatalf("healer-acted was not said: %v", err)
}
p.mu.Lock()
asked := strings.Join(p.asked, "\n")
p.mu.Unlock()
if !strings.Contains(asked, AssertTool+` {"module":"records"}`) {
t.Errorf("the grants part was asked %s", asked)
}
// Eleven more acts within the hour: the twelfth was the limit, and the next condition meets the brake.
attempts := bucket(Module + "_" + AttemptsState)
for i := 0; i < BrakeLimit-1; i++ {
begun := time.Now().Add(-time.Duration(i+1) * time.Minute).UTC()
raw, _ := json.Marshal(Attempt{Healer: "H3", Condition: fmt.Sprintf("bus.EVENTS.m%d.missing", i), BudgetKey: "x",
Outcome: OutcomeActed, Begun: begun})
if _, err := attempts.Put(ctx, attemptKey("H3", fmt.Sprintf("bus.EVENTS.m%d.missing", i), begun), raw); err != nil {
t.Fatal(err)
}
}
p.deliver(RaisedEvent, raised("bus.EVENTS.mail.missing", "mail"))
eventually(t, "the brake written as a finding on the bus", func() bool {
e, err := bucket(Module+"_"+FindingsState).Get(ctx, brakeKey)
if err != nil {
return false
}
f, err := health.DecodeFinding(e.Value())
return err == nil && f.Severity == health.Urgent && f.Kind == "healers-braked"
})
if n := len(keys(Module + "_" + AttemptsState)); n != BrakeLimit {
t.Errorf("past the brake the attempts are %d, want %d", n, BrakeLimit)
}
}
+22
View File
@@ -0,0 +1,22 @@
module mesh-healers
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
+27
View File
@@ -0,0 +1,27 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+82
View File
@@ -0,0 +1,82 @@
{
"module": "mesh-healers",
"version": "1",
"runs-as": "mesh-healers",
"claims": [
{
"name": "mesh-healers",
"scope": "mesh",
"serves": [
"healers",
"heal",
"drill"
]
}
],
"consumes": [
"mesh-conditions.condition-raised",
"mesh-conditions.condition-changed",
"mesh-hand-acts.act-recorded"
],
"invokes": [
"mesh-grants.grants_assert",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-controller.drill"
],
"reads": [
"mesh-controller.lease",
"mesh-conditions.conditions",
"mesh-controller.hand-acts"
],
"state": [
{
"name": "attempts",
"key-ttl": true
},
{
"name": "would",
"key-ttl": true
},
"findings",
{
"name": "wakes",
"key-ttl": true
}
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
"secrets-owner": "mesh-healers",
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-healers",
"shell": "/usr/bin/nologin",
"home": "/var/lib/mesh-healers"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "mesh-healers",
"place": ".",
"unknown": "program"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-healers",
"binary": "mesh-healers",
"loads": [
"mesh-healers"
]
}
]
}
}
+2
View File
@@ -0,0 +1,2 @@
/mesh-provisioner
/cmd/mesh-provisioner/mesh-provisioner
@@ -0,0 +1,204 @@
package main
// The bus as the holder composes from it (frombus.Reader), read through this module's runtime with its own `reads`:
// placement, each node's record in its own have key, the indexer's modules and manifests, the builder's builds and
// mesh-config's resolved values. A plan names no value, so no own secret and no credential is read (compose.Planning).
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// busReader reads the bus for frombus through a Store.
type busReader struct {
s Store
fetch func(ctx context.Context, url string) ([]byte, error)
}
func (r busReader) each(read string, visit func(key string, value []byte)) error {
keys, err := r.s.Keys(read)
if err != nil {
return fmt.Errorf("%s: %w", read, err)
}
for _, k := range keys {
e, err := r.s.Get(read, k)
if err != nil {
return fmt.Errorf("%s %s: %w", read, k, err)
}
if e != nil {
visit(k, e.Value)
}
}
return nil
}
func (r busReader) Placements(context.Context) (map[string]delivery.Placement, error) {
out := map[string]delivery.Placement{}
err := r.each(PlacementRead, func(k string, v []byte) {
node, err := delivery.ParsePlacementKey(k)
if err != nil {
return
}
if p, err := delivery.DecodePlacement(v); err == nil && p.Node == node {
out[node] = p
}
})
return out, err
}
func (r busReader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
out := map[string]delivery.NodeRecord{}
err := r.each(HaveRead, func(k string, v []byte) {
node, _, isNode, err := delivery.ParseHaveKey(k)
if err != nil || !isNode {
return
}
if h, err := delivery.DecodeHaveNode(v); err == nil && h.Record != nil {
out[node] = *h.Record
}
})
return out, err
}
func (r busReader) Modules(context.Context) (map[string]delivery.Module, error) {
out := map[string]delivery.Module{}
err := r.each(ModulesRead, func(k string, v []byte) {
module, err := delivery.ParseModulesKey(k)
if err != nil {
return
}
if m, err := delivery.DecodeModule(v); err == nil {
out[module] = m
}
})
return out, err
}
func (r busReader) Manifest(_ context.Context, digest string) ([]byte, error) {
key, err := delivery.ManifestsKey(digest)
if err != nil {
return nil, err
}
e, err := r.s.Get(ManifestsRead, key)
if err != nil {
return nil, err
}
if e == nil {
return nil, fmt.Errorf("the indexer holds no manifest %s", digest)
}
m, err := delivery.DecodeManifest(e.Value)
if err != nil {
return nil, err
}
return []byte(m.JSON), nil
}
func (r busReader) Values(_ context.Context, node, module string) (map[string]any, error) {
var out map[string]any
prefix := node + "." + module + "."
err := r.each(ResolvedRead, func(k string, v []byte) {
name, ok := strings.CutPrefix(k, prefix)
if !ok || strings.Contains(name, ".") {
return
}
if value, laid := catalogue.LaidValue(v); laid {
if out == nil {
out = map[string]any{}
}
out[name] = value
}
})
return out, err
}
func (busReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { return nil, nil }
func (busReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
return nil, nil, nil
}
func (r busReader) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return nil, err
}
e, err := r.s.Get(BuildsRead, key)
if err != nil || e == nil {
return nil, err
}
b, err := delivery.DecodeBuilderBuild(e.Value)
if err != nil {
return nil, err
}
return &b, nil
}
// archiveWithin bounds one read of a templates archive.
const archiveWithin = 30 * time.Second
func (r busReader) Archive(ctx context.Context, url, digest string) ([]byte, error) {
if r.fetch == nil {
return nil, fmt.Errorf("this holder reads no archive (%s)", url)
}
fetching, cancel := context.WithTimeout(ctx, archiveWithin)
defer cancel()
body, err := r.fetch(fetching, url)
if err != nil {
return nil, err
}
sum := sha256.Sum256(body)
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
return nil, fmt.Errorf("%s was recorded as %s and what arrived is %s", url, digest, got)
}
return body, nil
}
func (r busReader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
all, err := r.Values(ctx, node, module)
var out map[string]any
for _, k := range keys {
if v, ok := all[k]; ok {
if out == nil {
out = map[string]any{}
}
out[k] = v
}
}
return out, err
}
// httpFetch reads an archive from the artifact store over HTTP.
func httpFetch(ctx context.Context, url string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%s answered %s", url, resp.Status)
}
return io.ReadAll(io.LimitReader(resp.Body, 256<<20))
}
// FromBus is the holder's composer over the bus: frombus's plans of a node, at the builds keptAt names.
func FromBus(s Store) Composer {
r := busReader{s: s, fetch: httpFetch}
return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) {
return frombus.Plans(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt})
}
}
@@ -0,0 +1,673 @@
package main
// The seat's holder (novox/hq ADR 0570 clauses 1 to 5, 7; issue 1954 H1-H3): mesh-provisioner composes every placed
// module's provisioning plan and every provision from bus state, on change events and never on a timer, and keeps them
// in its states `plans` (`<node>.<module>`) and `provisions` (`<node>.<consumer>.<provision>[.<local>]`), which it alone
// writes.
//
// - **Composed per node.** A change of a node's placement or of one of its wants composes that node; a change of the
// indexer's module composes the nodes the module is placed on. Each node is composed twice: at the builds its wants
// name (its plans, what its client applies) and at the newest build of each placed module, so a build that adds a
// requirement has its key before its want (clause 2). A provision is needed while either needs it (clause 5).
// - **A key's state comes from its provider's outcome alone** (clause 3, 7): the provider's own entry for that
// consumer, `<provider>_bound` under the same key: present is provided (the vault's generation carried into the
// reference, clause 6), refused is failed with why, waits is waits with what. The holder never sets provided by
// itself: a key of a provider that writes no outcome stays asked.
// - **Retiring is asked** (clause 5): a key no longer needed is `retiring`, one approve ask under mesh-provisioner's
// name says what goes from which provider on which node; the warrant goes into the key and the provider acts on it;
// a decline is kept and not asked again while nothing changes; a key needed again returns to its outcome's state.
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// The states it keeps, beside Registry, Needs, Applied and Collect.
const (
ProvisionsState = provisions.ProvisionsState
PlansState = provisions.PlansState
)
// The reads it composes from.
const (
PlacementRead = delivery.PlacementRead
WantsRead = delivery.AssignerModule + ".wants"
ModulesRead = delivery.IndexerModule + ".modules"
ManifestsRead = delivery.IndexerModule + ".manifests"
BuildsRead = delivery.BuilderModule + ".builds"
HaveRead = "node-engine.have"
ResolvedRead = "mesh-config.resolved"
)
// Providers are the providing modules that keep an entry per consumer, `<module>_bound`, read as `<module>.bound`: the
// outcome of each provision they make. A provider not named here keeps none.
var Providers = []string{"gitea", "keycloak", "postgres", "route-adapter", "mesh-vault"}
// boundRead is a provider's entries as this module reads them.
func boundRead(module string) string { return module + ".bound" }
// Composer is each module's provisioning plan on a node, composed from the bus: at the builds keptAt names
// (module → fingerprint), the newest build of every other placed module.
type Composer func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error)
// Asker is the operator's channel as this module asks on it: publish an ask under its own name, and read the router's
// record of it.
type Asker interface {
Publish(subject string, body any, id string) error
Record(id string) ([]byte, bool, error)
}
// Holder is the seat's holder.
type Holder struct {
Store Store
Compose Composer
Asks Asker
Now func() time.Time
Logf func(format string, args ...any)
mu sync.Mutex
}
// Composed is what one node's composition did.
type Composed struct {
Node string `json:"node"`
Plans []string `json:"plans-written,omitempty"`
Gone []string `json:"plans-removed,omitempty"`
Changed []string `json:"provisions-changed,omitempty"`
Asked []string `json:"retiring-asked,omitempty"`
Problems []string `json:"problems,omitempty"`
}
// ComposeNode composes one node: its plans and its provisions.
func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) {
h.mu.Lock()
defer h.mu.Unlock()
out := Composed{Node: node}
wanted, err := h.wantedOn(node)
if err != nil {
return out, err
}
atWants, err := h.Compose(ctx, node, wanted)
if err != nil {
return out, fmt.Errorf("the plans of %s cannot be composed: %w", node, err)
}
newest, err := h.Compose(ctx, node, nil)
if err != nil {
return out, fmt.Errorf("the plans of %s at the newest builds cannot be composed: %w", node, err)
}
if err := h.writePlans(node, atWants, &out); err != nil {
out.Problems = append(out.Problems, err.Error())
}
needed := map[string]provisions.Provision{}
for _, plans := range []map[string]catalogue.Plan{atWants, newest} {
for module, plan := range plans {
for _, p := range plan.Provisions {
pv := provisions.Provision{Provision: p.Provision, Local: p.Local,
Consumer: provisions.End{Module: module, Node: node},
Provider: provisions.End{Module: p.ProviderModule, Node: p.Provider}}
key, err := pv.Key()
if err != nil {
out.Problems = append(out.Problems, err.Error())
continue
}
if _, seen := needed[key]; !seen {
needed[key] = pv
}
}
}
}
h.judgeNode(node, needed, &out)
return out, nil
}
// wantedOn is the fingerprint each want on a node names, by module.
func (h *Holder) wantedOn(node string) (map[string]string, error) {
out := map[string]string{}
keys, err := h.Store.Keys(WantsRead)
if err != nil {
return nil, fmt.Errorf("the wants cannot be read: %w", err)
}
for _, k := range keys {
n, module, err := delivery.ParseWantKey(k)
if err != nil || n != node {
continue
}
e, err := h.Store.Get(WantsRead, k)
if err != nil || e == nil {
continue
}
if w, err := delivery.DecodeWant(e.Value); err == nil && !w.Removed && w.FP != "" {
out[module] = w.FP
}
}
return out, nil
}
// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there.
func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *Composed) error {
var problems []error
for _, module := range sortedKeys(plans) {
key, err := provisions.PlanKey(node, module)
if err != nil {
problems = append(problems, err)
continue
}
raw, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: plans[module]})
if err != nil {
problems = append(problems, err)
continue
}
if e, err := h.Store.Get(PlansState, key); err == nil && e != nil && bytes.Equal(e.Value, raw) {
continue
}
if err := h.Store.Put(PlansState, key, json.RawMessage(raw)); err != nil {
problems = append(problems, fmt.Errorf("the plan %s: %w", key, err))
continue
}
out.Plans = append(out.Plans, key)
}
keys, err := h.Store.Keys(PlansState)
if err != nil {
return errors.Join(append(problems, err)...)
}
for _, k := range keys {
n, module, ok := strings.Cut(k, ".")
if !ok || n != node {
continue
}
if _, still := plans[module]; still {
continue
}
if err := h.Store.Delete(PlansState, k); err != nil {
problems = append(problems, err)
continue
}
out.Gone = append(out.Gone, k)
}
return errors.Join(problems...)
}
// judgeNode brings the node's provision keys to what is needed and what each provider said.
func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, out *Composed) {
held := h.held(node)
for _, key := range unionKeys(needed, held) {
want, isNeeded := needed[key]
have, isHeld := held[key]
var next provisions.Provision
switch {
case isNeeded:
next = want
if isHeld {
next.Reference = have.Reference
}
h.outcome(key, &next)
case have.State == provisions.Retiring:
next = have
if next.Ask == "" && next.Warrant == nil {
if err := h.ask(&next, key); err != nil {
out.Problems = append(out.Problems, err.Error())
} else {
out.Asked = append(out.Asked, key)
}
}
if gone, why := h.retired(key, have); gone {
if err := h.Store.Delete(ProvisionsState, key); err != nil {
out.Problems = append(out.Problems, err.Error())
} else {
out.Changed = append(out.Changed, key+" (retired: "+why+")")
}
continue
}
default:
next = have
next.State, next.Why = provisions.Retiring, fmt.Sprintf("no build wanted or placed on %s requires %s of %s on %s "+
"any more", node, have.Provision, have.Provider.Module, have.Provider.Node)
next.Warrant = nil
if err := h.ask(&next, key); err != nil {
out.Problems = append(out.Problems, err.Error())
} else {
out.Asked = append(out.Asked, key)
}
}
if isHeld && same(have, next) {
continue
}
next.At = h.Now().UTC()
raw, err := provisions.Encode(next)
if err != nil {
out.Problems = append(out.Problems, err.Error())
continue
}
if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil {
out.Problems = append(out.Problems, fmt.Sprintf("%s: %v", key, err))
continue
}
out.Changed = append(out.Changed, key+" "+next.State)
}
}
// held is the node's provision keys as kept.
func (h *Holder) held(node string) map[string]provisions.Provision {
out := map[string]provisions.Provision{}
keys, err := h.Store.Keys(ProvisionsState)
if err != nil {
return out
}
for _, k := range keys {
if !strings.HasPrefix(k, node+".") {
continue
}
e, err := h.Store.Get(ProvisionsState, k)
if err != nil || e == nil {
continue
}
if p, err := provisions.Decode(e.Value); err == nil {
out[k] = p
}
}
return out
}
// same says two provisions say the same, their time aside.
func same(a, b provisions.Provision) bool {
a.At, b.At, a.V, b.V = time.Time{}, time.Time{}, 0, 0
ra, _ := json.Marshal(a)
rb, _ := json.Marshal(b)
return bytes.Equal(ra, rb)
}
// entry is a provider's outcome for one consumer, as its entry says it: the provider helper's Entry, read for what
// the holder needs and nothing more.
type entry struct {
// State and Why are the provider's outcome (provider.Entry, ADR 0570 clause 3): provided, failed or waits.
State string `json:"state,omitempty"`
Why string `json:"why,omitempty"`
Refused string `json:"refused,omitempty"`
Waits string `json:"waits,omitempty"`
VaultName string `json:"vault-name,omitempty"`
Generation int `json:"generation,omitempty"`
}
// outcome sets a needed provision's state from its provider's entry: provided, failed or waits; asked while it has
// none. The holder never sets provided by itself (clause 3): a provider whose entries it does not read keeps the key
// asked, said so.
func (h *Holder) outcome(key string, p *provisions.Provision) {
p.Ask, p.Warrant = "", nil
if !keepsEntries(p.Provider.Module) {
h.haveOutcome(p)
return
}
e, err := h.Store.Get(boundRead(p.Provider.Module), key)
if err != nil {
p.State, p.Why = provisions.Asked, "the provider's entries cannot be read: "+err.Error()
return
}
if e == nil {
p.State, p.Why = provisions.Asked, ""
return
}
var en entry
if err := json.Unmarshal(e.Value, &en); err != nil {
p.State, p.Why = provisions.Asked, "the provider's entry is not readable"
return
}
switch {
case en.State == provisions.Failed:
p.State, p.Why = provisions.Failed, orSaid(en.Why, en.Refused, "the provider said it failed")
case en.State == provisions.Waits:
p.State, p.Why = provisions.Waits, orSaid(en.Why, en.Waits, "the provider waits for a person")
case en.State == "" && en.Refused != "":
p.State, p.Why = provisions.Failed, en.Refused
case en.State == "" && en.Waits != "":
p.State, p.Why = provisions.Waits, en.Waits
case en.State != "" && en.State != provisions.Provided:
p.State, p.Why = provisions.Asked, "the provider's entry says "+en.State+", which no state of a provision is"
return
default:
p.State, p.Why = provisions.Provided, ""
if en.VaultName != "" || en.Generation > 0 {
p.Reference = &provisions.Reference{ID: en.VaultName, Generation: en.Generation}
}
}
}
// haveOutcome is the outcome of a provider that makes no credential per consumer: its own have on its node (the lead's
// reading of ADR 0570 clause 3, recorded as an insight). Running with health ok at the build its want names is
// provided; failed is failed with the have's words; anything else is asked, waiting for the provider to run.
func (h *Holder) haveOutcome(p *provisions.Provision) {
where := p.Provider.Module + " on " + p.Provider.Node
key := p.Provider.Node + "." + p.Provider.Module
e, err := h.Store.Get(HaveRead, key)
if err != nil {
p.State, p.Why = provisions.Asked, "the have of "+where+" cannot be read: "+err.Error()
return
}
if e == nil {
p.State, p.Why = provisions.Asked, "waits for "+where+" to run: it says nothing yet"
return
}
have, err := delivery.DecodeHave(e.Value)
if err != nil {
p.State, p.Why = provisions.Asked, "the have of "+where+" is not readable"
return
}
wantFP := ""
if we, err := h.Store.Get(WantsRead, "want."+key); err == nil && we != nil {
if w, err := delivery.DecodeWant(we.Value); err == nil && !w.Removed {
wantFP = w.FP
}
}
switch {
case have.Phase == delivery.PhaseFailed || have.Health == delivery.HealthFailed:
p.State, p.Why = provisions.Failed, orSaid(have.Detail, where+" failed")
case have.Phase == delivery.PhaseRunning && have.Health == delivery.HealthOK && (wantFP == "" || have.FP == wantFP):
p.State, p.Why = provisions.Provided, ""
default:
p.State, p.Why = provisions.Asked, fmt.Sprintf("waits for %s to run healthy at the build its want names (it is %s, "+
"health %s)", where, have.Phase, have.Health)
}
}
func keepsEntries(module string) bool {
for _, m := range Providers {
if m == module {
return true
}
}
return false
}
// retired says a retiring key the operator approved is retired: its provider's entry is gone.
func (h *Holder) retired(key string, p provisions.Provision) (bool, string) {
if p.Warrant == nil || p.Warrant.Option != OptionApprove {
return false, ""
}
if !keepsEntries(p.Provider.Module) {
return false, ""
}
e, err := h.Store.Get(boundRead(p.Provider.Module), key)
if err != nil || e != nil {
return false, ""
}
return true, "its provider removed its credential on the operator's warrant"
}
// ---- the retiring ask ----------------------------------------------------------------------------------------------
// The options a retiring ask offers.
const (
OptionApprove = "approve"
OptionDecline = "decline"
)
// askLasts is how long a retiring ask stays open.
const askLasts = asks.ApproveLasts - 10*time.Minute
// retiringAsk is the ask of one retiring provision, composed from the key alone.
func retiringAsk(key string, p provisions.Provision, at time.Time, id string) asks.Ask {
what := fmt.Sprintf("%s's access to %s on %s, for %s on %s", p.Consumer.Module, p.Provision, p.Provider.Node,
p.Consumer.Module, p.Consumer.Node)
a := asks.Ask{ID: id, Headline: "Remove " + p.Consumer.Module + "'s " + p.Provision + " access?",
Explanation: fmt.Sprintf("Nothing wanted or placed on %s needs %s any more. On Approve, %s removes %s: what it owns "+
"is handed to the resource's owner and its credential goes; the resource and its data stay.", p.Consumer.Node,
p.Provision, p.Provider.Module, what),
Who: asks.Operator, Expires: at.Add(askLasts), OnExpiry: "nothing is removed; it is asked again when it changes",
About: "provision." + key, Details: "The provision " + key + " of mesh-provisioner's state provisions."}
for _, o := range []struct{ id, label, does string }{
{OptionApprove, "Remove", "the provider removes the credential; the resource and its data stay"},
{OptionDecline, "Keep", "nothing is removed, and it is not asked again while nothing changes"},
} {
binds, _ := asks.ActDigest(retireAct(key, id, o.id))
a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds})
}
return a
}
func retireAct(key, id, option string) asks.Act {
return asks.Act{"verb": "mesh-provisioner.retire", "provision": key, "ask": id, "option": option}
}
// ask raises the retiring ask of a key, under mesh-provisioner's own name; one already raised stands.
func (h *Holder) ask(p *provisions.Provision, key string) error {
if p.Ask != "" {
return nil
}
if h.Asks == nil {
return errors.New("the operator cannot be asked from here, so " + key + " is retiring and nothing is removed")
}
now := h.Now().UTC()
id := "retire-" + strings.ReplaceAll(key, ".", "-") + "-" + fmt.Sprint(now.Unix())
a := retiringAsk(key, *p, now, id)
if err := a.Check(now); err != nil {
return err
}
if err := h.Asks.Publish(asks.AskSubject(Module), a, id); err != nil {
return fmt.Errorf("the retiring ask of %s could not be published: %w", key, err)
}
p.Ask = id
return nil
}
// Decide reads the router's record of each retiring key's ask and writes the warrant into the key (clause 5): the
// provider acts on an Approve; a Decline is kept, and not asked again while nothing changes.
func (h *Holder) Decide() []string {
h.mu.Lock()
defer h.mu.Unlock()
if h.Asks == nil {
return nil
}
keys, err := h.Store.Keys(ProvisionsState)
if err != nil {
return []string{"the provisions cannot be read: " + err.Error()}
}
var said []string
for _, k := range keys {
e, err := h.Store.Get(ProvisionsState, k)
if err != nil || e == nil {
continue
}
p, err := provisions.Decode(e.Value)
if err != nil || p.State != provisions.Retiring || p.Ask == "" || p.Warrant != nil {
continue
}
raw, found, err := h.Asks.Record(p.Ask)
if err != nil || !found {
continue
}
var rec struct {
State string `json:"state"`
Ask *asks.Ask `json:"ask"`
Warrant *asks.Warrant `json:"warrant"`
}
if json.Unmarshal(raw, &rec) != nil || rec.State == "open" || rec.Warrant == nil || rec.Ask == nil {
continue
}
w := *rec.Warrant
if w.Asker != Module || w.Ask != p.Ask {
continue
}
option := ""
if w.Outcome == asks.OutcomeChosen {
o, err := w.For(Module, *rec.Ask)
if err == nil {
err = o.Performs(retireAct(k, p.Ask, o.ID))
}
if err != nil {
said = append(said, k+": the answer is refused: "+err.Error())
continue
}
option = o.ID
} else {
// Ended with no choice: asked again at the next change.
p.Ask = ""
h.put(k, p)
said = append(said, k+": the ask "+string(w.Outcome)+"; asked again when it changes")
continue
}
by := "the operator"
if w.By != nil {
by = fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity)
}
p.Warrant = &provisions.Warrant{Ask: p.Ask, Option: option, By: by, At: w.At.UTC()}
h.put(k, p)
said = append(said, k+": "+option+" by "+by)
}
return said
}
func (h *Holder) put(key string, p provisions.Provision) {
p.At = h.Now().UTC()
if raw, err := provisions.Encode(p); err == nil {
if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil {
h.Logf("[%s] %s is not written: %v", Module, key, err)
}
}
}
// ---- what a change touches --------------------------------------------------------------------------------------
// NodesOfPlacement is the node a placement key names.
func NodesOfPlacement(key string) []string {
if n, err := delivery.ParsePlacementKey(key); err == nil {
return []string{n}
}
return nil
}
// NodesOfWant is the node a want key names.
func NodesOfWant(key string) []string {
if n, _, err := delivery.ParseWantKey(key); err == nil {
return []string{n}
}
return nil
}
// NodesOfModule is every node a module is placed on, and every node whose plans name it as a provider: what a change
// of the module's manifest touches.
func (h *Holder) NodesOfModule(module string) []string {
nodes := map[string]bool{}
if keys, err := h.Store.Keys(PlacementRead); err == nil {
for _, k := range keys {
n, err := delivery.ParsePlacementKey(k)
if err != nil {
continue
}
e, err := h.Store.Get(PlacementRead, k)
if err != nil || e == nil {
continue
}
if p, err := delivery.DecodePlacement(e.Value); err == nil {
for _, m := range p.Modules {
if m.Module == module {
nodes[n] = true
}
}
}
}
}
if keys, err := h.Store.Keys(ProvisionsState); err == nil {
for _, k := range keys {
if e, err := h.Store.Get(ProvisionsState, k); err == nil && e != nil {
if p, err := provisions.Decode(e.Value); err == nil && p.Provider.Module == module {
nodes[p.Consumer.Node] = true
}
}
}
}
return sortedKeys(nodes)
}
// NodeOfEntry is the consumer's node a provider's entry key names.
func NodeOfEntry(key string) []string {
if n, _, _, _, err := provisions.ParseKey(key); err == nil {
return []string{n}
}
return nil
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
func unionKeys[A, B any](a map[string]A, b map[string]B) []string {
seen := map[string]bool{}
for k := range a {
seen[k] = true
}
for k := range b {
seen[k] = true
}
return sortedKeys(seen)
}
// Rejudge sets one key's state again from its provider's outcome: what a change of a provider's entry touches. A
// retiring key whose provider removed its entry on the operator's warrant goes.
func (h *Holder) Rejudge(key string) (string, error) {
h.mu.Lock()
defer h.mu.Unlock()
e, err := h.Store.Get(ProvisionsState, key)
if err != nil || e == nil {
return "", err
}
have, err := provisions.Decode(e.Value)
if err != nil {
return "", err
}
if have.State == provisions.Retiring {
if gone, why := h.retired(key, have); gone {
return key + " retired: " + why, h.Store.Delete(ProvisionsState, key)
}
return "", nil
}
next := have
h.outcome(key, &next)
if same(have, next) {
return "", nil
}
h.put(key, next)
return key + " " + next.State, nil
}
// RetireNeeds deletes every key of the retired state `needs`, which the clients no longer write (clause 1): run once at
// the start.
func (h *Holder) RetireNeeds() (int, error) {
keys, err := h.Store.Keys(Needs)
if err != nil {
return 0, err
}
n := 0
for _, k := range keys {
if err := h.Store.Delete(Needs, k); err != nil {
return n, err
}
n++
}
return n, nil
}
// orSaid is the first of the words that says something.
func orSaid(words ...string) string {
for _, w := range words {
if strings.TrimSpace(w) != "" {
return w
}
}
return ""
}
@@ -0,0 +1,276 @@
package main
import (
"context"
"encoding/json"
"go/ast"
"go/parser"
"go/token"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// router is the operator's channel as the holder asks on it.
type router struct {
asked map[string]asks.Ask
records map[string][]byte
}
func (r *router) Publish(_ string, body any, id string) error {
r.asked[id] = body.(asks.Ask)
return nil
}
func (r *router) Record(id string) ([]byte, bool, error) {
raw, ok := r.records[id]
return raw, ok, nil
}
// answer is the operator's answer to an ask.
func (r *router) answer(t *testing.T, id, option string) {
t.Helper()
a := r.asked[id]
o, ok := a.Option(option)
if !ok {
t.Fatalf("the ask %s offers no %s", id, option)
}
w := asks.Warrant{Ask: id, Asker: Module, About: a.About, Outcome: asks.OutcomeChosen, Option: o.ID, Label: o.Label,
Level: o.Level, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42"}, Channel: "telegram",
At: time.Date(2026, 10, 29, 23, 0, 0, 0, time.UTC), AskDigest: a.Digest()}
r.records[id], _ = json.Marshal(map[string]any{"state": "closed", "ask": a, "warrant": w})
}
// mesh is the holder over memory, its composer answering each node from tables the test sets: the plans at the wanted
// builds and at the newest.
type mesh struct {
h *Holder
s *memStore
r *router
atWant map[string]map[string]catalogue.Plan
atNewest map[string]map[string]catalogue.Plan
composed []string
}
func newMesh() *mesh {
m := &mesh{s: newStore(), r: &router{asked: map[string]asks.Ask{}, records: map[string][]byte{}},
atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{}}
now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {},
Now: func() time.Time { now = now.Add(time.Second); return now },
Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) {
m.composed = append(m.composed, node)
if keptAt != nil {
return m.atWant[node], nil
}
return m.atNewest[node], nil
}}
return m
}
// needs makes a module on a node require provisions in its wanted build (and the same in its newest unless set).
func planOf(provs ...catalogue.PlanProvision) catalogue.Plan {
return catalogue.Plan{Provisions: provs}
}
var (
pgDB = catalogue.PlanProvision{Provision: "postgres-database", Provider: "ace", ProviderModule: "postgres"}
oidc = catalogue.PlanProvision{Provision: "oidc-client", Provider: "novox", ProviderModule: "keycloak"}
split = catalogue.PlanProvision{Provision: "split-dns", Provider: "g14", ProviderModule: "systemd-resolved"}
)
func (m *mesh) state(t *testing.T, key string) provisions.Provision {
t.Helper()
e, _ := m.s.Get(ProvisionsState, key)
if e == nil {
return provisions.Provision{}
}
p, err := provisions.Decode(e.Value)
if err != nil {
t.Fatal(err)
}
return p
}
func (m *mesh) compose(t *testing.T, node string) Composed {
t.Helper()
c, err := m.h.ComposeNode(context.Background(), node)
if err != nil {
t.Fatal(err)
}
return c
}
// A node composes its plans and provisions from what is wanted and placed; a key's state follows its provider's entry
// alone; the vault's generation reaches the reference; a
// quiet pass writes nothing (ADR 0570 clauses 1-3, 6, 7; issue 1954 H1, H2).
func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) {
m := newMesh()
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, split)}
m.atNewest["g14"] = m.atWant["g14"]
key := "g14.gitea.postgres-database"
steps := []struct {
name string
entry string // the provider's entry written before the step, "-" to delete
state string
why string
}{
{"composed: asked", "", provisions.Asked, ""},
{"the provider waits for a person", `{"waits":"the database's owner to accept"}`, provisions.Waits, "owner"},
{"the provider refused", `{"refused":"no room on ace"}`, provisions.Failed, "no room"},
{"the provider says failed in its state", `{"state":"failed","why":"the disk is full"}`, provisions.Failed, "disk"},
{"the provider made it", `{"state":"provided","vault-name":"pair/gitea-postgres","generation":3}`, provisions.Provided, ""},
}
for _, s := range steps {
if s.entry != "" {
m.s.write(t, "postgres.bound", key, s.entry)
}
m.compose(t, "g14")
p := m.state(t, key)
if p.State != s.state || !strings.Contains(p.Why, s.why) || p.Provider.Module != "postgres" || p.Provider.Node != "ace" {
t.Errorf("%s: %+v", s.name, p)
}
}
if p := m.state(t, key); p.Reference == nil || p.Reference.ID != "pair/gitea-postgres" || p.Reference.Generation != 3 {
t.Errorf("the vault's generation does not reach the reference: %+v", p.Reference)
}
// A provider making no credential per consumer reports through its own have (clause 3): never provided by the
// holder alone.
split := "g14.gitea.split-dns"
for _, row := range []struct{ have, want, state, why string }{
{"", "", provisions.Asked, "says nothing yet"},
{`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"applying","health":"unknown"}`, "", provisions.Asked, "applying"},
{`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Asked, "build its want"},
{`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"failed","health":"failed","detail":"the unit did not start"}`, "", provisions.Failed, "did not start"},
{`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Provided, ""},
} {
if row.have != "" {
m.s.write(t, HaveRead, "g14.systemd-resolved", row.have)
}
if row.want != "" {
m.s.write(t, WantsRead, "want.g14.systemd-resolved", row.want)
}
m.compose(t, "g14")
if p := m.state(t, split); p.State != row.state || !strings.Contains(p.Why, row.why) {
t.Errorf("the provider's have %s: %+v, want %s", row.have, p, row.state)
}
}
if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil {
t.Error("the plan is not written")
}
if c := m.compose(t, "g14"); len(c.Plans)+len(c.Changed)+len(c.Asked) != 0 {
t.Errorf("a quiet pass wrote %+v", c)
}
// An entry's change judges its key alone, no composition.
before := len(m.composed)
m.s.write(t, "postgres.bound", key, `{"refused":"the login is locked"}`)
if said, err := m.h.Rejudge(key); err != nil || !strings.Contains(said, provisions.Failed) || len(m.composed) != before {
t.Errorf("rejudged %q, %v, composed %d more", said, err, len(m.composed)-before)
}
}
// A provision no build wanted or placed needs is retiring with one ask naming provider, node and what goes; one composed
// ahead of its want raises none; a change removing one requirement and adding another retires only the removed one; the
// warrant goes into the key and the key goes once the provider removed its entry; a decline is kept and not asked again;
// a provision needed again returns from retiring (ADR 0570 clause 5; issue 1954 H3).
func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) {
m := newMesh()
pg, kc := "g14.gitea.postgres-database", "g14.gitea.oidc-client"
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)}
m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, oidc)} // the newest build adds oidc ahead of its want
m.s.write(t, "postgres.bound", pg, `{}`)
steps := []struct {
name string
change func()
asked []string
states map[string]string
removed bool
}{
{"a key composed ahead of its want is asked, not retiring", func() {}, nil,
map[string]string{pg: provisions.Provided, kc: provisions.Asked}, false},
{"one requirement removed and another added: only the removed retires", func() {
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
}, []string{pg}, map[string]string{pg: provisions.Retiring, kc: provisions.Asked}, false},
{"composed again: no second ask", func() {}, nil, map[string]string{pg: provisions.Retiring}, false},
{"needed again before the warrant: back to its outcome", func() {
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc, pgDB)}
}, nil, map[string]string{pg: provisions.Provided}, false},
{"gone again: asked again", func() { m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} },
[]string{pg}, map[string]string{pg: provisions.Retiring}, false},
}
for _, s := range steps {
s.change()
c := m.compose(t, "g14")
if !slices.Equal(c.Asked, s.asked) {
t.Errorf("%s: asked %v, want %v", s.name, c.Asked, s.asked)
}
for k, st := range s.states {
if p := m.state(t, k); p.State != st {
t.Errorf("%s: %s is %s, want %s (%s)", s.name, k, p.State, st, p.Why)
}
}
}
p := m.state(t, pg)
a := m.r.asked[p.Ask]
if !strings.Contains(a.Explanation, "postgres") || !strings.Contains(a.Explanation, "ace") ||
!strings.Contains(a.Explanation, "data stay") {
t.Errorf("the ask does not say what goes from which provider on which node: %+v", a)
}
// Nothing before the warrant: the provider's entry stands and the key stays.
if said := m.h.Decide(); len(said) != 0 || m.state(t, pg).Warrant != nil {
t.Errorf("decided before an answer: %v", said)
}
m.r.answer(t, p.Ask, OptionApprove)
m.h.Decide()
if w := m.state(t, pg).Warrant; w == nil || w.Option != OptionApprove || !strings.Contains(w.By, "42") {
t.Fatalf("the warrant is not in the key: %+v", w)
}
if said, _ := m.h.Rejudge(pg); said != "" || m.state(t, pg).State != provisions.Retiring {
t.Errorf("retired while the provider's entry stands: %q", said)
}
_ = m.s.Delete("postgres.bound", pg)
if said, _ := m.h.Rejudge(pg); !strings.Contains(said, "retired") || m.state(t, pg).State != "" {
t.Errorf("not retired once the provider removed its entry: %q", said)
}
// A decline is kept, and nothing is asked again while nothing changes.
d := newMesh()
d.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)}
d.compose(t, "g14")
d.atWant["g14"] = map[string]catalogue.Plan{}
d.compose(t, "g14")
d.r.answer(t, d.state(t, pg).Ask, OptionDecline)
d.h.Decide()
if c := d.compose(t, "g14"); len(c.Asked) != 0 || d.state(t, pg).Warrant.Option != OptionDecline {
t.Errorf("a decline was asked again: %+v", c)
}
}
// The holder holds no ticker: it composes on change events alone (ADR 0570 clause 2; issue 1954 H1).
func TestTheHolderHoldsNoTimer(t *testing.T) {
files, _ := filepath.Glob("*.go")
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
file, err := parser.ParseFile(token.NewFileSet(), f, nil, 0)
if err != nil {
t.Fatal(err)
}
ast.Inspect(file, func(n ast.Node) bool {
if sel, ok := n.(*ast.SelectorExpr); ok {
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "time" &&
slices.Contains([]string{"NewTicker", "Tick", "AfterFunc", "NewTimer"}, sel.Sel.Name) {
t.Errorf("%s uses time.%s: the holder composes on events alone", f, sel.Sel.Name)
}
}
return true
})
}
}
@@ -0,0 +1,104 @@
package main
// The holder's tools (novox/hq ADR 0570 Consequences, ADR 0564 §12's rule): a read of every key of `provisions` and
// `plans`, one key whole, each state's fill, composing one node now, and reading the operator's answers now.
import (
"context"
"encoding/json"
"fmt"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// The holder's tools.
const (
ProvisionsTool = "provisioner_provisions"
PlansTool = "provisioner_plans"
ComposeTool = "provisioner_compose"
DecideTool = "provisioner_decide"
)
func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} }
func holderTools(h *Holder) []stdio.Tool {
arg := func(a map[string]any, k string) string { s, _ := a[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: ProvisionsTool,
Description: "Every provision the seat keeps (novox/hq ADR 0570): its key <node>.<consumer>.<provision>[.<local>], " +
"provider, consumer and state (asked, provided, waits, failed, retiring, with why), narrowed by node and " +
"state, with how many keys and bytes the state holds; or one key whole.",
Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one key, whole"),
"node": str("a consumer's node"), "state": str("asked, provided, waits, failed or retiring")}},
Run: func(a map[string]any) (any, error) {
return h.read(ProvisionsState, arg(a, "key"), arg(a, "node"), arg(a, "state"))
}},
{Name: PlansTool,
Description: "Every module's provisioning plan the seat composed, <node>.<module>, narrowed by node, with how many " +
"keys and bytes the state holds; or one plan whole.",
Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one plan, <node>.<module>"),
"node": str("a node")}},
Run: func(a map[string]any) (any, error) { return h.read(PlansState, arg(a, "key"), arg(a, "node"), "") }},
{Name: ComposeTool,
Description: "Compose one node's plans and provisions now from the bus, as a change event would, and say what " +
"was written, removed, changed and asked.",
Input: map[string]any{"type": "object", "properties": map[string]any{"node": str("the node")},
"required": []string{"node"}},
Run: func(a map[string]any) (any, error) {
if arg(a, "node") == "" {
return nil, fmt.Errorf("compose names the node")
}
return h.ComposeNode(context.Background(), arg(a, "node"))
}},
{Name: DecideTool,
Description: "Read the operator's answers to every retiring provision's ask now, writing each warrant into its " +
"key, as the router's word on an ask would.",
Run: func(map[string]any) (any, error) { return map[string]any{"said": h.Decide()}, nil }},
}
}
// read is a state's keys narrowed, or one whole, with the state's fill.
func (h *Holder) read(state, key, node, want string) (any, error) {
if key != "" {
e, err := h.Store.Get(state, key)
if err != nil {
return nil, err
}
if e == nil {
return nil, fmt.Errorf("%s holds no %s", state, key)
}
return json.RawMessage(e.Value), nil
}
keys, err := h.Store.Keys(state)
if err != nil {
return nil, err
}
rows := []map[string]any{}
bytes := 0
for _, k := range keys {
e, err := h.Store.Get(state, k)
if err != nil || e == nil {
continue
}
bytes += len(e.Value)
if node != "" && !strings.HasPrefix(k, node+".") {
continue
}
row := map[string]any{"key": k}
if state == ProvisionsState {
p, err := provisions.Decode(e.Value)
if err != nil {
row["unreadable"] = err.Error()
} else {
if want != "" && p.State != want {
continue
}
row["state"], row["provider"], row["why"] = p.State, p.Provider.Module+" on "+p.Provider.Node, p.Why
}
}
rows = append(rows, row)
}
return map[string]any{state: rows, "count": len(rows), "fill": map[string]int{"keys": len(keys), "bytes": bytes}}, nil
}
@@ -1,23 +1,28 @@
// mesh-provisioner: the provisioner's server, holder of the mesh-provisioner seat once for the mesh (novox/hq ADR
// 0349 clause 3, design 57 P4). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR
// 0259 §8), keeping four buckets: mesh-provisioner_registry (which domain answers which kind of placeholder, its
// alone), and mesh-provisioner_needs, _applied and _collect, which each machine's provisioner client writes for its
// own machine (the controller grants the client those keys alone, issue 670). stdout is the MCP channel; what this
// module says, it says on stderr.
// 0349 clause 3, ADR 0570). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR 0259
// §8), keeping mesh-provisioner_provisions and _plans (every provision and every plan, composed from the bus on change
// events, its alone), _registry (which domain answers which kind of placeholder, its alone), and _applied and
// _collect, which each machine's provisioner client writes for its own machine; _needs is retired and emptied at its
// start. stdout is the MCP channel; what this module says, it says on stderr.
package main
import (
"context"
"fmt"
"os"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Seat is the role this module holds, once for the mesh, and Verb its one verb.
const (
Seat = "mesh-provisioner"
Verb = "provisioning"
Seat = "mesh-provisioner"
Module = "mesh-provisioner"
Verb = "provisioning"
)
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
@@ -42,30 +47,149 @@ func (runtimeStore) Delete(state, key string) error { return stdio.State(state).
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
// runtimeAsks is the operator's channel through the runtime: asks published under this module's own name, and the
// router's record of one read under that name.
type runtimeAsks struct{}
func (runtimeAsks) Publish(subject string, body any, id string) error {
return stdio.SeatPublish(subject, body, id)
}
func (runtimeAsks) Record(id string) ([]byte, bool, error) {
e, err := stdio.SeatRecord("", id)
if err != nil || e == nil {
return nil, false, err
}
return e.Value, true, nil
}
// work is the nodes a change touched, composed one at a time by one worker: no timer, a change's event is the wake.
type work struct {
mu sync.Mutex
pending map[string]bool
wake chan struct{}
}
func (w *work) add(nodes ...string) {
if len(nodes) == 0 {
return
}
w.mu.Lock()
for _, n := range nodes {
w.pending[n] = true
}
w.mu.Unlock()
select {
case w.wake <- struct{}{}:
default:
}
}
func (w *work) run(h *Holder) {
for range w.wake {
w.mu.Lock()
nodes := sortedKeys(w.pending)
w.pending = map[string]bool{}
w.mu.Unlock()
for _, n := range nodes {
c, err := h.ComposeNode(context.Background(), n)
switch {
case err != nil:
logf("[%s] %s is not composed: %v", Module, n, err)
case len(c.Plans)+len(c.Gone)+len(c.Changed)+len(c.Asked)+len(c.Problems) > 0:
logf("[%s] composed %s: %+v", Module, n, c)
}
}
}
}
// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs.
func watched(state string, handle func(stdio.StateChange) error) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
time.Sleep(wait)
if err := stdio.State(state).Watch("", handle); err != nil {
logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait)
continue
}
return
}
}
func main() {
go publishing(runtimeStore{})
if err := stdio.Serve("", Tools(runtimeStore{})); err != nil {
s := runtimeStore{}
h := &Holder{Store: s, Compose: FromBus(s), Asks: runtimeAsks{}, Now: time.Now, Logf: logf}
w := &work{pending: map[string]bool{}, wake: make(chan struct{}, 1)}
go w.run(h)
go starting(h)
go watched(PlacementRead, func(c stdio.StateChange) error { w.add(NodesOfPlacement(c.Key)...); return nil })
go watched(WantsRead, func(c stdio.StateChange) error { w.add(NodesOfWant(c.Key)...); return nil })
go watched(ModulesRead, func(c stdio.StateChange) error {
if module, err := delivery.ParseModulesKey(c.Key); err == nil {
w.add(h.NodesOfModule(module)...)
}
return nil
})
for _, provider := range Providers {
go watched(boundRead(provider), func(c stdio.StateChange) error {
if said, err := h.Rejudge(c.Key); err != nil {
logf("[%s] %s is not judged again: %v", Module, c.Key, err)
} else if said != "" {
logf("[%s] %s", Module, said)
}
return nil
})
}
go watched(Registry, func(c stdio.StateChange) error {
if c.Op != "put" {
if _, _, err := Publish(s); err != nil {
logf("[%s] the registry is not written back: %v", Module, err)
}
}
return nil
})
go func() {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
time.Sleep(wait)
err := stdio.Subscribe(asks.Seat+".decided."+Module, func(stdio.Envelope) error {
for _, said := range h.Decide() {
logf("[%s] %s", Module, said)
}
return nil
})
if err == nil {
return
}
logf("[%s] does not hear the operator's answers yet (%v); asking again in %s", Module, err, wait)
}
}()
if err := stdio.Serve("", append(Tools(s), holderTools(h)...)); err != nil {
logf("%v", err)
os.Exit(1)
}
}
// publishing writes the registry once the runtime answers, and again every hour, so a registry emptied by hand or
// a bucket made again is written back without a restart. The runtime answers only once Serve runs, so the first
// pass is tried again until it lands.
func publishing(s Store) {
for wait := 2 * time.Second; ; {
time.Sleep(wait)
written, deleted, err := Publish(s)
switch {
case err != nil:
logf("[mesh-provisioner] the registry is not written yet (%v); trying again in %s", err, wait)
wait = min(wait*2, time.Minute)
continue
case len(written) > 0 || len(deleted) > 0:
logf("[mesh-provisioner] the registry: wrote %v, deleted %v", written, deleted)
// starting writes the registry, deletes the retired needs and reads the operator's answers once the runtime answers,
// each tried again until it lands.
func starting(h *Holder) {
<-stdio.Served()
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
written, deleted, err := Publish(h.Store)
if err == nil {
if len(written)+len(deleted) > 0 {
logf("[%s] the registry: wrote %v, deleted %v", Module, written, deleted)
}
break
}
wait = time.Hour
logf("[%s] the registry is not written yet (%v); trying again in %s", Module, err, wait)
time.Sleep(wait)
}
if n, err := h.RetireNeeds(); err != nil {
logf("[%s] the retired needs are not all deleted: %v", Module, err)
} else if n > 0 {
logf("[%s] deleted %d key(s) of the retired needs", Module, n)
}
for _, said := range h.Decide() {
logf("[%s] %s", Module, said)
}
}
@@ -106,9 +106,9 @@ func TestTheRegistryAnswersEveryKindATemplateMayName(t *testing.T) {
for kind, bucket := range map[string]string{"config": "mesh-config_resolved", "collected": "mesh-provisioner_" + Collect,
"shell": "mesh-provisioner_" + Collect, "environment": "mesh-provisioner_" + Collect,
"contribution": "mesh-provisioner_" + Collect,
"dir": "mesh-controller_assignments", "access": "mesh-controller_assignments",
"port": "mesh-controller_assignments", "seat": "mesh-controller_assignments",
"machine": "mesh-controller_assignments", "backup": "mesh-controller_assignments"} {
"dir": "mesh-provisioner_plans", "access": "mesh-provisioner_plans",
"port": "mesh-provisioner_plans", "seat": "mesh-provisioner_plans",
"machine": "mesh-provisioner_plans", "backup": "mesh-provisioner_plans"} {
i := slices.IndexFunc(TheRegistry, func(a Answer) bool { return a.Kind == kind })
if TheRegistry[i].Bucket != bucket {
t.Errorf("%s is found in %s, want %s", kind, TheRegistry[i].Bucket, bucket)
@@ -288,6 +288,8 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) {
Serves []string `json:"serves"`
} `json:"claims"`
State []string `json:"state"`
Reads []string `json:"reads"`
Tools []string `json:"tools"`
OwnSecrets map[string]string `json:"own-secrets"`
}
if err := json.Unmarshal(raw, &m); err != nil {
@@ -300,7 +302,39 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) {
!slices.Equal(m.Claims[0].Serves, []string{Verb}) {
t.Errorf("the claim: %+v", m.Claims)
}
if want := []string{Registry, Needs, Applied, Collect}; !slices.Equal(m.State, want) {
// needs stays declared this release so its first start can empty it (ADR 0570 clause 1).
if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState}; !slices.Equal(m.State, want) {
t.Errorf("the state is %v, want %v", m.State, want)
}
// Every state and act has its tool (ADR 0570 Consequences): the verb reads the clients' states, the holder's tools
// its own and the composition it does on a change.
served := map[string]bool{}
for _, tool := range append(Tools(newStore()), holderTools(&Holder{})...) {
served[tool.Name] = true
}
for what, tool := range map[string]string{Registry: Seat + "." + Verb, Needs: Seat + "." + Verb,
Applied: Seat + "." + Verb, Collect: Seat + "." + Verb, ProvisionsState: ProvisionsTool, PlansState: PlansTool,
"a composition on a change": ComposeTool, "a decision on an answer": DecideTool} {
if !served[tool] {
t.Errorf("%s has no tool: %s is not served", what, tool)
}
}
for _, tool := range m.Tools {
if !served[tool] {
t.Errorf("the manifest lists %s, which is not served", tool)
}
}
for _, r := range []string{PlacementRead, WantsRead, ModulesRead, ManifestsRead, BuildsRead, HaveRead, ResolvedRead} {
if !slices.Contains(m.Reads, r) {
t.Errorf("the manifest does not read %s, which the composition reads", r)
}
}
for _, p := range Providers {
if !slices.Contains(m.Reads, boundRead(p)) {
t.Errorf("the manifest does not read %s's entries", p)
}
}
if slices.Contains(m.Reads, "mesh-controller.assignments") {
t.Error("the manifest reads the controller's assignments (ADR 0570 clause 2)")
}
}
@@ -0,0 +1,2 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
modules/mesh-provisioner/module.json
@@ -48,16 +48,15 @@ type Answer struct {
Decision string `json:"decision"`
}
// placement is the controller's: where a module runs, its directories, accesses, ports, seats and the machine's
// facts (ADR 0349 clause 6). On the bus, in the provisioner's plan of each module the controller handed to it, which
// its assignment carries (mesh-controller internal/inventory Provisioning, issue 671), or for a module only pulled in
// by another, its value of its own under the same key (issue 765): a client never reads the node's declaration,
// which the node-engine keeps for root alone.
// placement is the seat's: where a module runs, its directories, accesses, ports, seats and the machine's facts (ADR
// 0349 clause 6), in the provisioning plan this seat composes of each module on each node from the bus, its state
// `plans` (novox/hq ADR 0570 clauses 1, 2): a client never reads the node's declaration, which the node-engine keeps for
// root alone, nor the controller's assignments.
func placement(kind, what string) Answer {
return Answer{Kind: kind, Domain: "mesh-controller", From: FromBucket, Bucket: "mesh-controller_assignments",
return Answer{Kind: kind, Domain: Module, From: FromBucket, Bucket: Module + "_" + PlansState,
Key: "<node>.<module>",
Why: what + ": placement, the controller's, in the provisioner's plan its assignment carries (provisioning." +
"placement)", Decision: "novox/hq ADR 0349 clause 6"}
Why: what + ": placement, in the provisioning plan the seat mesh-provisioner composes of the module on the node " +
"(plans, plan.placement)", Decision: "novox/hq ADR 0570 clause 1"}
}
// oldCollection is a namespace read as a collection of its own name (novox/hq ADR 0349 clause 13, design 57 L1).
+1 -1
View File
@@ -2,4 +2,4 @@ module mesh-provisioner
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791
+14 -2
View File
@@ -1,2 +1,14 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+26 -1
View File
@@ -11,11 +11,36 @@
]
}
],
"uses": [
"operator-channel"
],
"reads": [
"mesh-delivery-assigner.placement",
"mesh-delivery-assigner.wants",
"mesh-delivery-indexer.modules",
"mesh-delivery-indexer.manifests",
"mesh-delivery-builder.builds",
"node-engine.have",
"mesh-config.resolved",
"gitea.bound",
"keycloak.bound",
"postgres.bound",
"route-adapter.bound",
"mesh-vault.bound"
],
"tools": [
"provisioner_provisions",
"provisioner_plans",
"provisioner_compose",
"provisioner_decide"
],
"state": [
"registry",
"needs",
"applied",
"collect"
"collect",
"provisions",
"plans"
],
"own-secrets": {
"broker": "${dir:state}/broker"
+2
View File
@@ -0,0 +1,2 @@
/mesh-self-check
/cmd/mesh-self-check/mesh-self-check
+33
View File
@@ -0,0 +1,33 @@
# mesh-self-check
Holds the seat `mesh-self-check` (novox/hq ADR 0564 §6): the mesh's self-check, moved out of the controller. Go, one
binary (`cmd/mesh-self-check`), served by a runtime of its own on its own bus account (`runs-as`), so that its account
alone writes its buckets.
## What it does
Every five minutes (the first run a minute after it starts) it runs its registry of probes, each within thirty
seconds. A probe is one live invariant of a design, with its assertion as the controller had it, reading its facts on
the bus (the bus module's tools for the bus's own state). A probe whose facts have no home on the bus yet answers
`deferred`, naming them: never a pass. A probe that cannot run is said in the verdict at once, and written as a finding
of its own (`probe.<id>.failed`) when the next run cannot run it either.
What a probe finds is written as a finding, one key `<scope>.<id>.<kind>` per fault, only when what it says changes, and
deleted when a run no longer sees it. The condition keeper (`mesh-conditions`) raises and clears conditions from them.
A finding one look can be wrong about (a consumer behind after a burst) is written when two runs in a row see it.
Every run ends with the event `checked`, the body of the controller's `doctor-heartbeat`, which the watchman (S10) and
mesh-watcher on another machine hear: a self-check that stops checking is said through a path that does not pass
through it.
## What it keeps
- `mesh-self-check_findings`: what it finds wrong now.
- `mesh-self-check_verdict`: the key `last`, the last run whole.
- `mesh-self-check_verdicts`: every run (each probe's findings at most twenty), kept a week by the bucket's age.
## Its verbs and tools
- `mesh-self-check.doctor`: the last verdict; `run=true` runs now; `probe=<id>` one probe whole.
- `mesh-self-check.probes`: the registry, with what each deferred probe lacks.
- `self_check_findings`, `self_check_verdicts`: the reads of its buckets.
@@ -0,0 +1,57 @@
// mesh-self-check: the holder of the seat mesh-self-check (novox/hq ADR 0564 §6). A Go bundle served by a runtime of
// its own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `findings` (what it finds
// wrong now, which the condition keeper raises and clears conditions from), `verdict` (the last run) and `verdicts`
// (every run, kept a week). It runs the probes every five minutes and says `checked` at the end of each run. stdout is
// the MCP channel; what this module says, it says on stderr.
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
// runtimeStore is the bus, through the runtime that launched this bundle.
type runtimeStore struct{}
func (runtimeStore) Get(state, key string) ([]byte, bool, error) {
e, err := stdio.State(state).Get(key)
if err != nil || e == nil {
return nil, false, err
}
return e.Value, true, nil
}
func (runtimeStore) Put(state, key string, value any) error {
_, err := stdio.State(state).Put(key, value)
return err
}
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
// NewChecker is the self-check over the runtime.
func NewChecker() *Checker {
return &Checker{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, Host: os.Getenv("MESH_NODE"),
Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) },
Publish: stdio.SeatPublish}
}
func main() {
c := NewChecker()
go func() {
<-stdio.Served()
c.Keep(context.Background())
}()
if err := stdio.Serve("", Tools(c)); err != nil {
logf("%v", err)
os.Exit(1)
}
}
@@ -0,0 +1,101 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"slices"
"testing"
)
type manifest struct {
Module string `json:"module"`
RunsAs string `json:"runs-as"`
Claims []struct {
Name string `json:"name"`
Scope string `json:"scope"`
Serves []string `json:"serves"`
} `json:"claims"`
Emits []string `json:"emits"`
Invokes []string `json:"invokes"`
Reads []string `json:"reads"`
State []json.RawMessage `json:"state"`
}
func readManifest(t *testing.T) manifest {
t.Helper()
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m manifest
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
return m
}
// The manifest says what the code does: it claims the seat mesh-self-check once for the mesh with the verbs it serves
// (ADR 0564 §1, §6), runs on its own account, says checked, asks only what its probes ask, keeps a week of verdicts,
// and reads no bucket of the controller's. Every state it keeps has a tool that reads it, and every act it takes by
// itself a tool that starts it by hand (ADR 0564 §12).
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
m := readManifest(t)
if m.Module != Module || m.RunsAs != Module {
t.Errorf("module %q runs as %q", m.Module, m.RunsAs)
}
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) {
t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs)
}
// checked is the seat's event, said on the seat's subject: a module lists no event of a seat it claims.
if len(m.Emits) != 0 {
t.Errorf("emits %v", m.Emits)
}
if !slices.Equal(m.Invokes, []string{StreamsTool, ClosedTool, SignalsVerb}) {
t.Errorf("invokes %v, want what the probes ask", m.Invokes)
}
if len(m.Reads) != 0 {
t.Errorf("reads %v: the self-check reads its facts through the bus module's tools", m.Reads)
}
states := map[string]int{}
for _, s := range m.State {
var d struct {
Name string `json:"name"`
TTL int `json:"ttl-seconds"`
}
if json.Unmarshal(s, &d.Name) != nil {
_ = json.Unmarshal(s, &d)
}
states[d.Name] = d.TTL
}
if states[VerdictsState] != 7*24*60*60 {
t.Errorf("verdicts are kept %d s, want a week", states[VerdictsState])
}
// ADR 0564 §12: each state with the tool that reads it, each act with the tool that starts it.
readBy := map[string]string{FindingsState: FindingsTool, VerdictState: DoctorTool, VerdictsState: VerdictsTool,
"the run": DoctorTool + " run=true"}
tools := map[string]bool{}
for _, tool := range Tools(&Checker{}) {
tools[tool.Name] = true
}
for state := range states {
if _, ok := readBy[state]; !ok {
t.Errorf("the state %s has no tool that reads it", state)
}
}
for what, tool := range readBy {
name, _, _ := cutSpace(tool)
if !tools[name] {
t.Errorf("%s is read or started by %s, which is not served", what, tool)
}
}
}
func cutSpace(s string) (string, string, bool) {
for i, r := range s {
if r == ' ' {
return s[:i], s[i+1:], true
}
}
return s, "", false
}
@@ -0,0 +1,142 @@
package main
// What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an
// Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong
// about it. It is written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the controller's key
// for the same condition, so the condition the keeper raises from it and the one the controller raised share one key.
import (
"encoding/json"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// The severities and scopes, as mesh-sdk's health names them.
const (
Urgent = health.Urgent
Warning = health.Warning
ScopeBus = health.ScopeBus
ScopeCore = health.ScopeCore
ScopeProbe = health.ScopeProbe
ScopeMesh = health.ScopeMesh
)
// Subject is what a finding is about.
type Subject = health.Subject
// Observation is one thing a probe sees wrong now.
type Observation struct {
Kind string
Token string
Subject Subject
Severity health.Severity
Summary string
Said string
Headline string
Explanation string
Resolved string
Needs string
// Source is the probe that saw it, or the self-check itself for a probe that could not run.
Source string
// Confirm says one look can be wrong about it: written only when two runs in a row see it (issue 277).
Confirm bool
}
// Key is where it is kept: `<scope>.<id>.<token>`, the controller's key for the same condition.
func (o Observation) Key() string {
token := o.Token
if token == "" {
token = o.Kind
}
return health.Key(o.Subject.Scope, o.Subject.ID, token)
}
// Finding is the observation as written by run at, its evidence the one line it said.
func (o Observation) Finding(run string, at time.Time) health.Finding {
f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary,
Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source,
Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}}
if o.Token != "" && o.Token != o.Kind {
f.Token = o.Token
}
said := o.Said
if said == "" {
said = o.Summary
}
f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said})
return f
}
// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write
// (ADR 0289 §1).
func says(f health.Finding) string {
f.At, f.Run, f.V = time.Time{}, "", 0
ev := make([]health.Evidence, len(f.Evidence))
for i, e := range f.Evidence {
ev[i] = health.Evidence{Said: e.Said}
}
f.Evidence = ev
raw, _ := json.Marshal(f)
return string(raw)
}
// worded gives a finding the operator's words registered for its kind, where it says none of its own.
func worded(f Observation) Observation {
w, ok := words[f.Kind]
if !ok {
return f
}
if f.Headline == "" {
f.Headline = w.Headline
}
if f.Explanation == "" {
f.Explanation = w.Explanation
}
if f.Resolved == "" {
f.Resolved = w.Resolved
}
if f.Needs == "" {
f.Needs = w.Needs
}
return f
}
type plain struct{ Headline, Explanation, Resolved, Needs string }
// words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the
// controller's name for itself replaced by the self-check's where the words spoke of it.
var words = map[string]plain{
"consumer-lost": {Headline: "A listener on the bus is gone",
Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.",
Resolved: "The listener is back"},
"consumer-wrong": {Headline: "A listener on the bus is set up wrong",
Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets it up again.",
Resolved: "The listener is set up right again"},
"consumer-behind": {Headline: "Messages pile up for a listener",
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A healer " +
"restarts it if it stays.",
Resolved: "The listener caught up"},
"stream-wrong": {Headline: "Part of the bus's storage is wrong",
Needs: "check the machine the bus runs on; the details say what is missing.",
Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
Resolved: "The bus's storage is right again"},
"bus-reconnects": {Headline: "A client keeps losing the bus",
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. While " +
"it reconnects, what it says and what it is asked waits.",
Resolved: "Resolved: the client stays connected"},
"core-unhealthy": {Headline: "The bus is not healthy",
Needs: "check the machine the bus runs on; the details say what fails.",
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds and tool " +
"calls may fail until it recovers.",
Resolved: "The bus is healthy again"},
"watchdogs-silent": {Headline: "The mesh's watchdogs stopped",
Needs: "restart the watchman if this stays, from the mesh MCP server.",
Explanation: "The watchdogs, which notice when something expected does not happen, do not answer, so missed " +
"signals are not noticed.",
Resolved: "The watchdogs run again"},
"probe-failed": {Headline: "A check of the mesh could not run",
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs again " +
"every few minutes.",
Resolved: "The check runs again"},
}
@@ -0,0 +1,448 @@
package main
// The registry (novox/hq ADR 0564 §6 and the probe table of its Context): the probes that move from the controller,
// each with its assertion unchanged, reading its facts where they live on the bus. A probe whose facts have no home
// on the bus yet answers deferred, naming them, and is never a pass (tracked by issue 1952). Not registered, as the
// table says: D1 (nodes compose their own; a refusal is the watchman's S21), D10 (a want ahead of its have is the
// watchman's pulse), D5, D9 and H-controller (they end with the controller), stuck (the watchman's own finding) and
// the bus step (its home is issue 1953's).
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// The tools a probe asks: the bus module's reads of its own server, and the watchdogs' seat.
const (
StreamsTool = "nats.nats_streams"
ClosedTool = "nats.nats_closed_connections"
SignalsVerb = "seat:mesh-watchdogs.signals"
notOnTheBus = "the bus module's tools (nats_streams) answer this only once asked; "
storeOnly = "held only in the controller's store today"
behindBound = 1000
reconnectMax = 12
)
// Registry is the self-check's registry, in to-be 45's order. **The registry is the design's live form**: a probe
// that moves here is a row added, and the registry test holds it to ADR 0564's table.
func Registry() []Probe {
return []Probe{
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
From: "issue 262", Kind: "resolver-wrong",
Deferred: "the machines' names and the resolver holders' addresses to ask: the overlay's addresses are " + storeOnly},
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every machine that " +
"is heard from", From: "issues 208, 218", Kind: "holder-silent",
Deferred: "the seats on record with their holders per machine, and which machines are heard from: the seat " +
"records are " + storeOnly + ", and the watchman's presence has no read granted to this module"},
{ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253", Kind: "archives-unheld",
Deferred: "the artifact store's kept archives with the manifests that hold them: the archive index is " + storeOnly},
{ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " +
"stream's head", From: "issues 248, 266", Kind: "consumer-wrong",
Raises: []string{"consumer-lost", "consumer-behind"}, Run: probeConsumers},
{ID: "D7", Asserts: "every stream the mesh defines exists with its definition", From: "issue 208",
Kind: "stream-wrong", Run: probeStreams},
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
From: "issue 238", Kind: "own-address-banned",
Deferred: "the machines' private addresses and endpoints: the overlay is " + storeOnly},
{ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding its " +
"cleanup", From: "ADR 0230", Kind: "cleanup-waiting",
Deferred: "the retirement records of providers' consumers: " + storeOnly},
{ID: "D12", Asserts: "every consumer of a provision that keeps its data is bound where it was last sent, or moves " +
"by a pin", From: "issue 273, ADR 0232", Kind: "binding-moved",
Deferred: "the bindings of provisions to their consumers: " + storeOnly},
{ID: "D13", Asserts: "every item of data a machine declares is measured, is there, holds what it held, and is " +
"backed up within its bound", From: "issue 273, ADR 0233", Kind: "data-shrank",
Deferred: "the data items' measurements and their history: " + storeOnly},
{ID: "D15", Asserts: "no user of the bus had its connection dropped more than twelve times in the last hour: the " +
"bus module's nats_closed_connections", From: "issue 327", Kind: "bus-reconnects", Run: probeReconnects},
{ID: "DA", Asserts: "every machine that names an agent account has it judged, on its node-engine's newest " +
"statement, unable to become root without a person", From: "ADR 0266, ADR 0259 §8",
Kind: "agent-can-become-root",
Deferred: "the node-engines' statements of the agent account's judgement: " + storeOnly},
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
"proving its sender runs", From: "ADR 0259 §8", Kind: "root-not-free",
Deferred: "the node-engines' statements of root and the tools that run as root: " + storeOnly},
{ID: "D-fill", Asserts: "every module's bucket and log holds less than three quarters of its cap; one said " +
"filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501",
Kind: "bucket-or-log-filling", Raises: []string{"bucket-or-log-unread"},
Deferred: "each bucket's and log's cap: the bus module's nats_buckets answers bytes and no cap, and the " +
"declared caps are " + storeOnly},
{ID: "D-heard", Asserts: "no grant of a restricted stream names a module that composes nothing for it",
From: "ADR 0376 §2", Kind: "grant-composes-nothing",
Deferred: "the grants of restricted streams with what each machine composed for them: " + storeOnly},
{ID: "DW", Asserts: "the watchdogs answer: the watchman holds the seat mesh-watchdogs and answers signals",
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Run: probeWatchdogs},
{ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses it " +
"to the bus's machine and back", From: "ADR 0236, to-be 45 §8", Kind: "core-unhealthy", Run: probeBusHealth},
}
}
// --- the bus's own state, as its module answers it ---------------------------------------------------------------
// streamsAnswer is what nats_streams answers.
type streamsAnswer struct {
Detail []streamRow `json:"detail"`
}
type streamRow struct {
Name string `json:"name"`
Subjects []string `json:"subjects"`
Retention string `json:"retention"`
MaxPerSubject int64 `json:"max_per_subject"`
Consumers []consumerRow `json:"consumer_detail"`
}
type consumerRow struct {
Name string `json:"name"`
Kind string `json:"kind"`
Filters []string `json:"filters"`
Pending int64 `json:"pending"`
Unacked int64 `json:"unacknowledged"`
MaxDeliver int64 `json:"max_deliver"`
}
// errNoStream is a stream the bus module says it does not hold.
var errNoStream = errors.New("no such stream")
// streams asks the bus module for every stream, or for one with its consumers.
func (c *Checker) streams(stream string) ([]streamRow, error) {
args := map[string]any{}
if stream != "" {
args["stream"] = stream
}
raw, err := c.Ask(StreamsTool, args)
if err != nil {
if stream != "" && strings.Contains(err.Error(), "no stream called") {
return nil, errNoStream
}
return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err)
}
var a streamsAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", StreamsTool, err)
}
return a.Detail, nil
}
// consumerWords is a consumer as the mesh says it, with its name.
func consumerWords(c buscompose.Consumer) string {
return fmt.Sprintf("the consumer %s on %s", c.Name, c.Stream)
}
// expectedConsumers are the durable consumers the mesh defines, by stream.
func expectedConsumers() map[string][]buscompose.Consumer {
out := map[string][]buscompose.Consumer{}
for _, c := range buscompose.MeshConsumers() {
out[c.Stream] = append(out[c.Stream], c)
}
return out
}
// probeConsumers is D6: every durable consumer the mesh defines exists as defined, and is near its stream's head.
func probeConsumers(ctx context.Context, c *Checker) ([]Observation, error) {
var out []Observation
byStream := expectedConsumers()
streams := make([]string, 0, len(byStream))
for s := range byStream {
streams = append(streams, s)
}
sort.Strings(streams)
for _, stream := range streams {
if ctx.Err() != nil {
return nil, ctx.Err()
}
rows, err := c.streams(stream)
held := map[string]consumerRow{}
switch {
case errors.Is(err, errNoStream):
case err != nil:
return nil, err
default:
for _, r := range rows {
for _, cr := range r.Consumers {
held[cr.Name] = cr
}
}
}
for _, want := range byStream[stream] {
who := want.Stream + "." + want.Name
have, ok := held[want.Name]
if !ok {
out = append(out, Observation{Kind: "consumer-lost", Token: "missing", Severity: Warning,
Subject: Subject{Scope: ScopeBus, ID: who},
Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(want)),
Said: "no consumer " + want.Name + " on " + want.Stream})
continue
}
if differs := consumerDiffers(want, have); differs != "" {
out = append(out, Observation{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who},
Summary: fmt.Sprintf("%s is not as the mesh defines it: %s", consumerWords(want), differs),
Said: differs})
}
if want.Stream != "NODES" && have.Pending > behindBound {
// One look at a consumer a burst put behind is not one that stays behind: held for the next run,
// so healer H4 resets nothing on a single sample (novox/hq issue 277).
out = append(out, Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true,
Subject: Subject{Scope: ScopeBus, ID: who},
Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(want), have.Pending),
Said: fmt.Sprintf("%d pending, %d handed out and not settled", have.Pending, have.Unacked)})
}
}
}
return sorted(out), nil
}
// consumerDiffers is what about a consumer on the bus is not as defined, of what the bus module says of it: its
// filters, its shape (push or pull) and how often it hands a message over. Empty when nothing is.
func consumerDiffers(want buscompose.Consumer, have consumerRow) string {
var differs []string
haveFilters := slices.Clone(have.Filters)
wantFilters := slices.Clone(want.Filters)
sort.Strings(haveFilters)
sort.Strings(wantFilters)
if !slices.Equal(haveFilters, wantFilters) {
differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters))
}
if want.MaxDeliver != 0 && have.MaxDeliver != int64(want.MaxDeliver) {
differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, want.MaxDeliver))
}
if wantPush, havePush := want.Push || want.Queue != "", have.Kind == "push"; wantPush != havePush {
differs = append(differs, "delivers by the other shape (push or pull) than defined")
}
return strings.Join(differs, "; ")
}
// probeStreams is D7: every stream the mesh defines exists as defined.
func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) {
rows, err := c.streams("")
if err != nil {
return nil, err
}
held := map[string]streamRow{}
for _, r := range rows {
held[r.Name] = r
}
var out []Observation
for _, s := range buscompose.MeshStreams() {
have, ok := held[s.Name]
if !ok {
out = append(out, Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name},
Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)),
Said: "no stream " + s.Name})
continue
}
if differs := streamDiffers(s, have); differs != "" {
out = append(out, Observation{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name},
Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs),
Said: differs})
}
}
return sorted(out), nil
}
// streamDiffers is what about a stream on the bus is not as defined, of what the bus module says of it: its
// subjects, its retention and how many it keeps per subject. Empty when nothing is.
func streamDiffers(want buscompose.Stream, have streamRow) string {
var differs []string
haveSubjects := slices.Clone(have.Subjects)
wantSubjects := slices.Clone(want.Subjects)
sort.Strings(haveSubjects)
sort.Strings(wantSubjects)
if !slices.Equal(haveSubjects, wantSubjects) {
differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects))
}
retention, perSubject := "limits", int64(want.MaxMsgsPerSubject)
switch want.Retention {
case buscompose.RetentionWorkQueue:
retention = "workqueue"
case buscompose.RetentionLastPerSubject:
perSubject = 1
}
if have.Retention != "" && have.Retention != retention {
differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention))
}
if perSubject != 0 && have.MaxPerSubject != perSubject {
differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxPerSubject, perSubject))
}
return strings.Join(differs, "; ")
}
// --- D15 -----------------------------------------------------------------------------------------------------
// closedAnswer is what nats_closed_connections answers.
type closedAnswer struct {
Hours float64 `json:"hours"`
Reaches bool `json:"reaches"`
Users []struct {
User string `json:"user"`
Closed int `json:"closed"`
Dropped int `json:"dropped"`
Names []struct {
Name string `json:"name"`
Dropped int `json:"dropped"`
Reasons map[string]int `json:"reasons"`
} `json:"names"`
} `json:"users"`
}
// probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour.
func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) {
raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1})
if err != nil {
return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err)
}
var read closedAnswer
if err := json.Unmarshal(raw, &read); err != nil {
return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", ClosedTool, err)
}
hours := read.Hours
if hours <= 0 {
hours = 1
}
var out []Observation
for _, u := range read.Users {
if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax {
continue // refused before it logged in is nobody's reconnect loop
}
reasons := map[string]int{}
var names []string
for _, n := range u.Names {
if n.Dropped == 0 {
continue
}
names = append(names, fmt.Sprintf("%q ×%d", n.Name, n.Dropped))
for r, k := range n.Reasons {
if r != "Client Closed" {
reasons[r] += k
}
}
}
var why []string
for r, k := range reasons {
why = append(why, fmt.Sprintf("%s ×%d", r, k))
}
sort.Strings(why)
partial := ""
if !read.Reaches {
partial = " (at least: the server's record of closed connections does not reach back the whole hour)"
}
who, machine := busUserWords(u.User)
out = append(out, Observation{Kind: "bus-reconnects", Severity: Warning,
Subject: Subject{Scope: ScopeBus, ID: u.User, Machine: machine},
Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+
"reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax),
Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped, u.Closed,
hours, strings.Join(names, ", "), strings.Join(why, ", ")),
Headline: capital(who) + " keeps losing the bus",
Explanation: capital(fmt.Sprintf("%s lost its connection to the bus %d times in the last hour and connected "+
"again each time. While it reconnects, what it says and what it is asked waits.", who, u.Dropped)),
Resolved: "Resolved: " + who + " stays connected"})
}
return sorted(out), nil
}
// busUserWords is a bus user as the operator says it, and the machine it is on.
func busUserWords(user string) (string, string) {
switch {
case user == "controller":
return "the controller", ""
case strings.HasPrefix(user, "node."):
m := strings.TrimPrefix(user, "node.")
return "the node-engine on " + m, m
}
if m, module, ok := strings.Cut(user, "."); ok {
if module == "node-tools" {
return "the tool runner on " + m, m
}
return module + " on " + m, m
}
return "a client of the bus", ""
}
// --- DW and H-bus --------------------------------------------------------------------------------------------
// probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the
// watchman's pulse on `checked`, is the other half.
func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) {
if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil {
return []Observation{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"},
Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " +
"late signal is being said",
Said: firstLine(err.Error())}}, nil
}
return nil, nil
}
// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a request crosses it
// to the bus module's machine and back. The question to the bus module is that request: an answer is its round trip,
// and no answer is no pass.
func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) {
rows, err := c.streams("")
if err != nil {
return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err)
}
held := map[string]bool{}
for _, r := range rows {
held[r.Name] = true
}
var problems []string
for _, s := range buscompose.MeshStreams() {
if !held[s.Name] {
problems = append(problems, "the stream "+s.Name+" is missing")
}
}
byStream := expectedConsumers()
streams := make([]string, 0, len(byStream))
for s := range byStream {
streams = append(streams, s)
}
sort.Strings(streams)
for _, stream := range streams {
if !held[stream] {
continue // said above
}
if ctx.Err() != nil {
return nil, ctx.Err()
}
detail, err := c.streams(stream)
if err != nil && !errors.Is(err, errNoStream) {
return nil, err
}
have := map[string]bool{}
for _, r := range detail {
for _, cr := range r.Consumers {
have[cr.Name] = true
}
}
for _, want := range byStream[stream] {
if !have[want.Name] {
problems = append(problems, consumerWords(want)+" is missing")
}
}
}
if len(problems) == 0 {
return nil, nil
}
return []Observation{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"},
Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil
}
// sorted is findings in a stated order, so two runs over one mesh say the same.
func sorted(out []Observation) []Observation {
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
return out
}
func capital(s string) string {
if s == "" {
return s
}
return strings.ToUpper(s[:1]) + s[1:]
}
@@ -0,0 +1,176 @@
package main
import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// bus is the bus module's answers as a probe asks them: every mesh stream and consumer as defined, unless changed.
type bus struct {
drop map[string]bool // stream or stream.consumer missing
change func(rows map[string]*streamRow)
closed string
signals error
asksFail error
}
func (b bus) rows() map[string]*streamRow {
rows := map[string]*streamRow{}
for _, s := range buscompose.MeshStreams() {
if b.drop[s.Name] {
continue
}
retention := "limits"
per := int64(s.MaxMsgsPerSubject)
switch s.Retention {
case buscompose.RetentionWorkQueue:
retention = "workqueue"
case buscompose.RetentionLastPerSubject:
per = 1
}
rows[s.Name] = &streamRow{Name: s.Name, Subjects: s.Subjects, Retention: retention, MaxPerSubject: per}
}
for _, c := range buscompose.MeshConsumers() {
r := rows[c.Stream]
if r == nil || b.drop[c.Stream+"."+c.Name] {
continue
}
kind := "pull"
if c.Push || c.Queue != "" {
kind = "push"
}
r.Consumers = append(r.Consumers, consumerRow{Name: c.Name, Kind: kind, Filters: c.Filters, MaxDeliver: int64(c.MaxDeliver)})
}
if b.change != nil {
b.change(rows)
}
return rows
}
func (b bus) ask(key string, body any) (json.RawMessage, error) {
if b.asksFail != nil {
return nil, b.asksFail
}
switch key {
case StreamsTool:
stream, _ := body.(map[string]any)["stream"].(string)
var detail []streamRow
for name, r := range b.rows() {
if stream == "" {
row := *r
row.Consumers = nil
detail = append(detail, row)
} else if name == stream {
detail = append(detail, *r)
}
}
if stream != "" && len(detail) == 0 {
return nil, errors.New("no stream called " + stream)
}
return json.Marshal(map[string]any{"detail": detail})
case ClosedTool:
if b.closed == "" {
return json.RawMessage(`{"hours":1,"reaches":true,"users":[]}`), nil
}
return json.RawMessage(b.closed), nil
case SignalsVerb:
if b.signals != nil {
return nil, b.signals
}
return json.RawMessage(`{"signals":[]}`), nil
}
return nil, errors.New("nats: no responders available for request")
}
func keysOf(found []Observation) []string {
var out []string
for _, f := range found {
out = append(out, f.Key())
}
return out
}
// Each probe that moved judges the bus module's answers with its assertion unchanged (ADR 0564 §6; issue 1952 SC2).
func TestTheBusProbes(t *testing.T) {
cases := []struct {
name string
probe func(context.Context, *Checker) ([]Observation, error)
bus bus
want []string
err string
}{
{"D6 all as defined", probeConsumers, bus{}, nil, ""},
{"D6 a consumer gone", probeConsumers, bus{drop: map[string]bool{"DEAD_LETTER_NOTICES.controller": true}},
[]string{"bus.DEAD_LETTER_NOTICES.controller.missing"}, ""},
{"D6 a stream gone takes its consumers", probeConsumers, bus{drop: map[string]bool{"CONTROL": true}},
[]string{"bus.CONTROL.controller.missing"}, ""},
{"D6 redefined and far behind", probeConsumers, bus{change: func(rows map[string]*streamRow) {
for i, c := range rows["EVENTS"].Consumers {
if c.Name == "controller" {
rows["EVENTS"].Consumers[i].MaxDeliver, rows["EVENTS"].Consumers[i].Pending = 9, 5000
}
}
}}, []string{"bus.EVENTS.controller.behind", "bus.EVENTS.controller.redefined"}, ""},
{"D6 the bus module silent", probeConsumers, bus{asksFail: errors.New("no responders")}, nil, "did not answer"},
{"D7 all as defined", probeStreams, bus{}, nil, ""},
{"D7 missing and redefined", probeStreams, bus{drop: map[string]bool{"NODES": true}, change: func(rows map[string]*streamRow) {
rows["EVENTS"].Retention = "workqueue"
}}, []string{"bus.EVENTS.stream-redefined", "bus.NODES.stream-missing"}, ""},
{"D15 quiet", probeReconnects, bus{}, nil, ""},
{"D15 a loop", probeReconnects, bus{closed: `{"hours":1,"reaches":true,"users":[
{"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]},
{"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`},
[]string{"bus.g14.node-tools.bus-reconnects"}, ""},
{"DW answers", probeWatchdogs, bus{}, nil, ""},
{"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.silent"}, ""},
{"H-bus healthy", probeBusHealth, bus{}, nil, ""},
{"H-bus a stream and a consumer missing", probeBusHealth,
bus{drop: map[string]bool{"NODES": true, "EVENTS.controller-merges": true}}, []string{"bus.mesh.unhealthy"}, ""},
{"H-bus no round trip", probeBusHealth, bus{asksFail: errors.New("timeout")}, nil, "no request crossed"},
}
for _, tc := range cases {
c := &Checker{Ask: tc.bus.ask}
found, err := tc.probe(context.Background(), c)
if tc.err != "" {
if err == nil || !strings.Contains(err.Error(), tc.err) {
t.Errorf("%s: error %v, want %q", tc.name, err, tc.err)
}
continue
}
if err != nil {
t.Errorf("%s: %v", tc.name, err)
continue
}
if got := keysOf(found); !slices.Equal(got, tc.want) {
t.Errorf("%s: found %v, want %v", tc.name, got, tc.want)
}
}
}
// The registry holds exactly the probes ADR 0564's table moves here, each with a kind, and a deferred one names what
// it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4).
func TestTheRegistryIsADR0564sTable(t *testing.T) {
moved := []string{"D2", "D3", "D4", "D6", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard",
"DW", "H-bus"}
var ids []string
for _, p := range Registry() {
ids = append(ids, p.ID)
if p.Kind == "" || p.Asserts == "" {
t.Errorf("%s has no kind or assertion", p.ID)
}
if (p.Run == nil) == (p.Deferred == "") {
t.Errorf("%s is neither run nor deferred naming what it lacks", p.ID)
}
}
slices.Sort(ids)
slices.Sort(moved)
if !slices.Equal(ids, moved) {
t.Errorf("the registry holds %v, ADR 0564 moves %v", ids, moved)
}
}
@@ -0,0 +1,2 @@
# What these tests read outside their package, as paths from the repository's root (issue 1936).
modules/mesh-self-check/module.json
@@ -0,0 +1,394 @@
package main
// The self-check (novox/hq ADR 0564 §6): the registry of probes, each a live invariant of a design, run every five
// minutes, each probe within thirty seconds. What a probe finds is written as a finding in this module's bucket
// `findings`, which the condition keeper raises and clears conditions from (ADR 0564 §2); every run is kept in the
// bucket `verdicts` for a week and the last as the key `last` of `verdict`, and is said as `checked`, which the
// watchman and mesh-watcher hear (S10, ADR 0227 §6). A probe that errors or does not finish is never a pass: it is
// said in the verdict at once and written as a finding of its own when the next run cannot run it either.
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// Module and Seat are this module and the seat it holds once for the mesh.
const (
Module = "mesh-self-check"
Seat = "mesh-self-check"
)
// The states it keeps, and the event that ends every run.
const (
FindingsState = health.FindingsState
VerdictState = "verdict"
VerdictsState = "verdicts"
LastKey = "last"
CheckedEvent = health.EventChecked
// CheckedSubject is where the seat says it: a seat's events are on mesh.seat.<seat>.event.<name>, which its
// consumers hear (catalogue.SeatSays).
CheckedSubject = "mesh.seat." + Seat + ".event." + CheckedEvent
)
// The clocks: the self-check's run is the one timer among the four holders (ADR 0564 §1).
var (
Every = 5 * time.Minute
FirstAfter = time.Minute
Within = 30 * time.Second
)
// The verdicts a probe can have, as mesh-sdk's health names them.
const (
VerdictPass = health.VerdictPassed
VerdictFail = health.VerdictFailed
VerdictFailedToRun = health.VerdictFailedToRun
VerdictDeferred = health.VerdictDeferred
)
// sourceSelf is what writes a probe's failure to run: the self-check itself.
const sourceSelf = "doctor"
// Store is the bus's buckets as this module reaches them, through its runtime.
type Store interface {
Get(state, key string) ([]byte, bool, error)
Put(state, key string, value any) error
Delete(state, key string) error
Keys(state string) ([]string, error)
}
// Probe is one live invariant: its id, what it asserts, the kind it raises, and the run; or why it waits.
type Probe struct {
ID string
Asserts string
From string
Kind string
Raises []string
// Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6):
// answered deferred, never a pass.
Deferred string
Run func(ctx context.Context, c *Checker) ([]Observation, error)
}
// ProbeVerdict, Counts and Run are a run as mesh-sdk's health holds it: the body of `checked`, the controller's
// doctor-heartbeat body unchanged (ADR 0564 §6), Controller naming the machine that ran it.
type (
ProbeVerdict = health.ProbeVerdict
Counts = health.CheckedCounts
Run = health.Checked
)
// Checker is the self-check: the registry and what its probes reach.
type Checker struct {
Store Store
Ask func(key string, body any) (json.RawMessage, error)
// Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish).
Publish func(subject string, body any, id string) error
Now func() time.Time
Logf func(format string, args ...any)
Host string
Registry []Probe
running sync.Mutex
mu sync.Mutex
n uint64
last *Run
// seen are the keys the run before saw, confirmed or not: a finding a single look can be wrong about is
// written only when two runs in a row see it, or while it is already written (novox/hq issue 277).
seen map[string]bool
// unsaid counts the runs whose `checked` could not be said since the start.
unsaid int
}
// LastRun is the last run, nil before the first.
func (c *Checker) LastRun() *Run {
c.mu.Lock()
defer c.mu.Unlock()
return c.last
}
type result struct {
found []Observation
err error
took time.Duration
}
// RunOnce runs every probe of the registry, writes what changed in `findings`, keeps the verdict and says `checked`.
// One run at a time: a person's run during a scheduled one waits for it.
func (c *Checker) RunOnce(ctx context.Context, why string) Run {
c.running.Lock()
defer c.running.Unlock()
c.mu.Lock()
c.n++
n := c.n
c.mu.Unlock()
started := c.Now()
run := Run{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
IntervalSeconds: int(Every / time.Second), Controller: c.Host, Why: why}
results := make([]result, len(c.Registry))
var wg sync.WaitGroup
for i, p := range c.Registry {
if p.Run == nil {
continue
}
wg.Add(1)
go func(i int, p Probe) {
defer wg.Done()
results[i] = c.probe(ctx, p)
}(i, p)
}
wg.Wait()
written, readErr := c.written()
seen := map[string]bool{}
want := map[string]Observation{} // every finding this run writes, by key
ran := map[string]bool{} // the sources whose findings this run judged whole
var blind []Observation
for i, p := range c.Registry {
v := ProbeVerdict{ID: p.ID}
r := results[i]
switch {
case p.Run == nil:
v.Verdict = VerdictDeferred
v.Error = "deferred: " + p.Deferred
run.Counts.Deferred++
case r.err != nil:
v.Verdict, v.Error = VerdictFailedToRun, r.err.Error()
run.Counts.FailedToRun++
blind = append(blind, Observation{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true,
Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf,
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
Said: firstLine(r.err.Error())})
default:
ran[p.ID] = true
for _, f := range r.found {
if f.Kind == "" {
f.Kind = p.Kind
}
f.Source = p.ID
f = worded(f)
key := f.Key()
seen[key] = true
if f.Confirm && !c.seenBefore(key) && !written.has(key) {
v.Unconfirmed = append(v.Unconfirmed, f.Summary)
continue
}
want[key] = f
v.Found = append(v.Found, f.Summary)
}
if len(v.Found) == 0 {
v.Verdict = VerdictPass
run.Counts.Passed++
} else {
v.Verdict = VerdictFail
run.Counts.Failed++
}
}
if p.Run != nil {
v.Took = r.took.Round(time.Millisecond).String()
}
run.Probes = append(run.Probes, v)
}
ran[sourceSelf] = true
for _, f := range blind {
f = worded(f)
key := f.Key()
seen[key] = true
if c.seenBefore(key) || written.has(key) {
want[key] = f
}
}
if readErr != nil {
// Blind to its own bucket: nothing is written or deleted (ADR 0227 §4), and the log says so.
c.Logf("[%s] its findings cannot be read, so none is written or cleared this run: %v", Module, readErr)
} else if err := c.write(written, want, ran, run.Run); err != nil {
c.Logf("[%s] what this run found is not all written: %v", Module, err)
}
c.mu.Lock()
c.seen = seen
c.mu.Unlock()
run.At = c.Now().UTC()
run.Took = run.At.Sub(started).Round(time.Millisecond).String()
c.mu.Lock()
run.Unsaid = c.unsaid
c.mu.Unlock()
c.keep(run)
return run
}
// probe runs one probe within its bound; a panic or an overrun is an error, never a pass.
func (c *Checker) probe(ctx context.Context, p Probe) result {
probing, cancel := context.WithTimeout(ctx, Within)
defer cancel()
began := c.Now()
done := make(chan result, 1)
go func() {
defer func() {
if r := recover(); r != nil {
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
}
}()
found, err := p.Run(probing, c)
done <- result{found: found, err: err}
}()
select {
case r := <-done:
r.took = c.Now().Sub(began)
return r
case <-probing.Done():
return result{err: fmt.Errorf("it did not finish within %s", Within), took: c.Now().Sub(began)}
}
}
func (c *Checker) seenBefore(key string) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.seen[key]
}
// writtenFindings are the findings in the bucket, by key.
type writtenFindings map[string]health.Finding
func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok }
// written reads every finding this module has written.
func (c *Checker) written() (writtenFindings, error) {
keys, err := c.Store.Keys(FindingsState)
if err != nil {
return writtenFindings{}, err
}
out := writtenFindings{}
for _, k := range keys {
raw, found, err := c.Store.Get(FindingsState, k)
if err != nil {
return out, err
}
if !found {
continue
}
f, err := health.DecodeFinding(raw)
if err != nil {
return out, fmt.Errorf("the finding %s is not readable: %w", k, err)
}
out[k] = f
}
return out, nil
}
// write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding
// of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4).
func (c *Checker) write(written writtenFindings, want map[string]Observation, ran map[string]bool, run string) error {
var errs []string
keys := make([]string, 0, len(want))
for k := range want {
keys = append(keys, k)
}
sort.Strings(keys)
now := c.Now()
for _, k := range keys {
f := want[k].Finding(run, now)
if was, ok := written[k]; ok && says(was) == says(f) {
continue
}
raw, err := health.EncodeFinding(f)
if err == nil {
err = c.Store.Put(FindingsState, k, json.RawMessage(raw))
}
if err != nil {
errs = append(errs, k+": "+err.Error())
}
}
for k, f := range written {
if _, still := want[k]; still || !ran[f.Source] {
continue
}
if err := c.Store.Delete(FindingsState, k); err != nil {
errs = append(errs, k+": "+err.Error())
}
}
if len(errs) > 0 {
return fmt.Errorf("%s", strings.Join(errs, "; "))
}
return nil
}
// keep keeps the run as the last verdict and in the week's verdicts, and says it as `checked`. Not said is said in
// the log: the watchman's S10 says the self-check silent when `checked` stops.
func (c *Checker) keep(run Run) {
c.mu.Lock()
c.last = &run
c.mu.Unlock()
if err := c.Store.Put(VerdictState, LastKey, run); err != nil {
c.Logf("[%s] the run %s is not kept as the last verdict: %v", Module, run.Run, err)
}
if err := c.Store.Put(VerdictsState, run.Run, Overview(run)); err != nil {
c.Logf("[%s] the run %s is not kept among the week's verdicts: %v", Module, run.Run, err)
}
if c.Publish == nil {
return
}
if err := c.Publish(CheckedSubject, run, run.Run); err != nil {
c.mu.Lock()
c.unsaid++
c.mu.Unlock()
c.Logf("[%s] the run %s could NOT be said as checked, so the watchers will say the self-check is silent: %v",
Module, run.Run, err)
}
}
// overviewFindings is how many findings of one probe an overview carries (novox/hq issue 314).
const overviewFindings = 20
// Overview is a run with each probe's findings at most overviewFindings, with their count and how to read them all.
func Overview(run Run) Run {
probes := make([]ProbeVerdict, len(run.Probes))
for i, p := range run.Probes {
total := len(p.Found) + len(p.Unconfirmed)
if total > overviewFindings {
p.FoundCount = len(p.Found)
if len(p.Found) > overviewFindings {
p.Found = p.Found[:overviewFindings:overviewFindings]
}
if room := overviewFindings - len(p.Found); len(p.Unconfirmed) > room {
p.Unconfirmed = p.Unconfirmed[:room:room]
}
p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `%s.doctor probe=%s` gives them all",
p.FoundCount, total-p.FoundCount, len(p.Found)+len(p.Unconfirmed), Seat, p.ID)
}
probes[i] = p
}
run.Probes = probes
return run
}
// Keep runs the registry on its schedule until ctx ends: the first run FirstAfter the start, then every Every.
func (c *Checker) Keep(ctx context.Context) {
select {
case <-ctx.Done():
return
case <-time.After(FirstAfter):
}
tick := time.NewTicker(Every)
defer tick.Stop()
for {
c.RunOnce(ctx, "the schedule")
select {
case <-ctx.Done():
return
case <-tick.C:
}
}
}
func firstLine(s string) string {
line, _, _ := strings.Cut(s, "\n")
return line
}
@@ -0,0 +1,255 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// memStore is the bus's buckets in memory, counting the writes.
type memStore struct {
mu sync.Mutex
kv map[string]map[string][]byte
puts map[string]int
keysErr error
}
func newStore() *memStore {
return &memStore{kv: map[string]map[string][]byte{}, puts: map[string]int{}}
}
func (s *memStore) Get(state, key string) ([]byte, bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
v, ok := s.kv[state][key]
return v, ok, nil
}
func (s *memStore) Put(state, key string, value any) error {
raw, err := json.Marshal(value)
if err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
if s.kv[state] == nil {
s.kv[state] = map[string][]byte{}
}
s.kv[state][key] = raw
s.puts[state+"/"+key]++
return nil
}
func (s *memStore) Delete(state, key string) error {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.kv[state], key)
return nil
}
func (s *memStore) Keys(state string) ([]string, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.keysErr != nil && state == FindingsState {
return nil, s.keysErr
}
var out []string
for k := range s.kv[state] {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
func (s *memStore) findingKeys() []string {
keys, _ := s.Keys(FindingsState)
return keys
}
// checker is a self-check over memory, with a registry the test drives.
func checker(store *memStore, registry ...Probe) (*Checker, *[]string) {
var said []string
now := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
c := &Checker{Store: store, Registry: registry, Host: "novox", Logf: func(string, ...any) {},
Now: func() time.Time { now = now.Add(time.Second); return now },
Publish: func(subject string, body any, id string) error {
raw, _ := json.Marshal(body)
said = append(said, subject+" "+id+" "+string(raw))
return nil
}}
return c, &said
}
// A run writes what a probe finds, writes nothing it already wrote, and deletes what it no longer sees; a probe that
// could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can
// be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1).
func TestARunWritesFindingsAsTheyChange(t *testing.T) {
var found []Observation
var fail error
steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { return found, fail }}
deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"}
missing := Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"},
Summary: "the stream EVENTS is not on the bus"}
behind := Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true,
Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"}
store := newStore()
c, said := checker(store, steady, deferred)
steps := []struct {
name string
found []Observation
fail error
want []string // finding keys written after the run
verdict string
missingPs int // puts of the missing finding so far
}{
{"a finding is written", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
{"the same again is no write", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
{"one look can be wrong: held", []Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
{"seen twice: written", []Observation{missing, behind}, nil,
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1},
{"blind once: clears nothing, held", nil, errors.New("timed out"),
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1},
{"blind twice: its own finding", nil, errors.New("timed out"),
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing", "probe.P1.failed"}, VerdictFailedToRun, 1},
{"no longer seen: cleared", nil, nil, nil, VerdictPass, 1},
}
for _, s := range steps {
found, fail = s.found, s.fail
run := c.RunOnce(context.Background(), "the test")
if got := store.findingKeys(); !slices.Equal(got, s.want) {
t.Errorf("%s: findings %v, want %v", s.name, got, s.want)
}
if run.Probes[0].Verdict != s.verdict {
t.Errorf("%s: P1 %s, want %s", s.name, run.Probes[0].Verdict, s.verdict)
}
if run.Probes[1].Verdict != VerdictDeferred || !strings.Contains(run.Probes[1].Error, "overlay") {
t.Errorf("%s: a deferred probe answered %+v", s.name, run.Probes[1])
}
if n := store.puts[FindingsState+"/bus.EVENTS.stream-missing"]; n != s.missingPs {
t.Errorf("%s: the unchanged finding was written %d times", s.name, n)
}
}
raw, _, _ := store.Get(FindingsState, "probe.P1.failed")
if raw != nil {
t.Errorf("the blind finding outlived the probe running again: %s", raw)
}
// Every run is kept, the last as `last`, and said as checked with the controller's heartbeat body.
if keys, _ := store.Keys(VerdictsState); len(keys) != len(steps) {
t.Errorf("verdicts kept %d runs, want %d", len(keys), len(steps))
}
// Said on the seat's subject, where its consumers hear it, de-duplicated by the run's id.
if len(*said) != len(steps) || !strings.HasPrefix((*said)[0], "mesh.seat.mesh-self-check.event.checked doctor-") {
t.Fatalf("said %v", *said)
}
_, rest, _ := strings.Cut((*said)[0], " ")
_, rest, _ = strings.Cut(rest, " ")
var body map[string]any
_ = json.Unmarshal([]byte(rest), &body)
for _, field := range []string{"run", "at", "started", "took", "interval-seconds", "counts", "probes", "controller", "why"} {
if _, ok := body[field]; !ok {
t.Errorf("checked's body lacks %s, which doctor-heartbeat carries: %v", field, body)
}
}
last, _, _ := store.Get(VerdictState, LastKey)
var r Run
_ = json.Unmarshal(last, &r)
if r.Counts.Passed != 1 || r.Counts.Deferred != 1 {
t.Errorf("the last verdict counts %+v", r.Counts)
}
// A finding carries the operator's words of its kind, the probe that found it and who observed it.
found, fail = []Observation{missing}, nil
c.RunOnce(context.Background(), "the test")
raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing")
f, err := health.DecodeFinding(raw)
if err != nil || f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" ||
f.Explanation == "" || f.Token != "stream-missing" || len(f.Evidence) != 1 || f.Run == "" {
t.Errorf("the written finding is %+v", f)
}
// Blind to its own bucket, a run writes and clears nothing.
store.keysErr = errors.New("the bus is away")
found = nil
c.RunOnce(context.Background(), "the test")
store.keysErr = nil
if got := store.findingKeys(); !slices.Equal(got, []string{"bus.EVENTS.stream-missing"}) {
t.Errorf("a run blind to its findings changed them: %v", got)
}
}
// A probe that panics or overruns its bound could not run: never a pass.
func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) {
within := Within
Within = 50 * time.Millisecond
t.Cleanup(func() { Within = within })
for name, run := range map[string]func(context.Context, *Checker) ([]Observation, error){
"panic": func(context.Context, *Checker) ([]Observation, error) { panic("boom") },
"overrun": func(ctx context.Context, _ *Checker) ([]Observation, error) { time.Sleep(time.Second); return nil, nil },
} {
c, _ := checker(newStore(), Probe{ID: "P", Kind: "k", Run: run})
c.Now = time.Now
r := c.RunOnce(context.Background(), "the test")
if r.Probes[0].Verdict != VerdictFailedToRun || r.Counts.Passed != 0 {
t.Errorf("%s: %+v", name, r.Probes[0])
}
}
}
// The verb answers the last run, one probe whole, a run now, and before any run says when one comes; the reads answer
// every state the module keeps (ADR 0564 §12).
func TestTheVerbsAndReads(t *testing.T) {
store := newStore()
many := make([]Observation, 25)
for i := range many {
many[i] = Observation{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"}
}
c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong",
Run: func(context.Context, *Checker) ([]Observation, error) { return many, nil }})
if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") {
t.Fatalf("before a run: %v", err)
}
if _, err := c.Doctor(true, "", "asked by person.jochen"); err != nil {
t.Fatal(err)
}
over, _ := c.Doctor(false, "", "")
if v := over["run"].(Run).Probes[0]; len(v.Found) != overviewFindings || v.More == "" {
t.Errorf("the overview carries %d findings, more %q", len(v.Found), v.More)
}
whole, _ := c.Doctor(false, "d7", "")
if v := whole["run"].(Run).Probes[0]; len(v.Found) != 25 {
t.Errorf("one probe whole carries %d findings", len(v.Found))
}
if _, err := c.Doctor(false, "D99", ""); err == nil {
t.Error("an unknown probe answered")
}
// A fresh checker over the same bucket answers the last verdict from it.
fresh, _ := checker(store)
if _, err := fresh.Doctor(false, "", ""); err != nil {
t.Errorf("the last verdict is not read from its bucket: %v", err)
}
fs, err := c.Findings("")
if err != nil || fs.(map[string]any)["count"] != 25 {
t.Errorf("findings answered %v, %v", fs, err)
}
if one, err := c.Findings("bus.S03.stream-wrong"); err != nil || !strings.Contains(string(one.(json.RawMessage)), "S03") {
t.Errorf("one finding answered %v, %v", one, err)
}
vs, err := c.Verdicts("", 0)
if err != nil || len(vs.(map[string]any)["verdicts"].([]map[string]any)) != 1 {
t.Errorf("verdicts answered %v, %v", vs, err)
}
if _, err := c.Verdicts("doctor-1-1", 0); err == nil {
t.Error("an unknown run answered")
}
if p := c.Probes()["probes"].([]map[string]any); len(p) != 1 || p[0]["id"] != "D7" {
t.Errorf("probes answered %v", p)
}
}
@@ -0,0 +1,216 @@
package main
// The seat's verbs and the module's own tools (novox/hq ADR 0564 §6, §12): `doctor` answers the last verdict at once,
// runs now, or gives one probe's verdict whole; `probes` is the registry; and a read of each state it keeps — the
// findings, the last verdict and the week's verdicts — so a person sees all it holds and a recovery is a call.
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// Verbs are the seat's verbs, as the manifest claims them.
var Verbs = []string{"doctor", "probes"}
// Tool names: the seat's verbs, and the module's own reads.
const (
DoctorTool = Seat + ".doctor"
ProbesTool = Seat + ".probes"
FindingsTool = "self_check_findings"
VerdictsTool = "self_check_verdicts"
)
func obj(props map[string]any) map[string]any {
return map[string]any{"type": "object", "properties": props}
}
// Tools are what the runtime serves.
func Tools(c *Checker) []stdio.Tool {
return []stdio.Tool{
{Name: DoctorTool,
Description: "The mesh's self-check (novox/hq ADR 0564 §6): the last run's verdict at once, each probe passed, " +
"failed with what it found, could not run (never a pass) or deferred with the facts it lacks; run=true runs " +
"every probe now and answers that run; probe=<id> gives one probe's verdict whole, every finding.",
Input: obj(map[string]any{
"run": map[string]any{"type": "boolean", "description": "run every probe now"},
"probe": map[string]any{"type": "string", "description": "one probe's verdict, whole, e.g. D6"}}),
Run: func(a map[string]any) (any, error) {
return c.Doctor(boolArg(a, "run"), strArg(a, "probe"), "asked by "+stdio.CallerOf(a))
}},
{Name: ProbesTool,
Description: "The self-check's registry: each probe's id, what it asserts, where it came from, the condition kinds " +
"it raises, and for one deferred the facts it lacks on the bus.",
Run: func(map[string]any) (any, error) { return c.Probes(), nil }},
{Name: FindingsTool,
Description: "What the self-check finds wrong now, as the condition keeper reads it (its bucket findings): every " +
"finding's key, kind, severity, the probe that found it and its summary; or one finding whole by key.",
Input: obj(map[string]any{"key": map[string]any{"type": "string", "description": "one finding's key"}}),
Run: func(a map[string]any) (any, error) { return c.Findings(strArg(a, "key")) }},
{Name: VerdictsTool,
Description: "The self-check's runs of the last week (its bucket verdicts, each kept a week) newest first, each " +
"with its counts; or one run whole by its id. The last run is the doctor verb's.",
Input: obj(map[string]any{
"run": map[string]any{"type": "string", "description": "one run's id, e.g. doctor-1793284800-12"},
"limit": map[string]any{"type": "number", "description": "at most this many, newest (default 50)"}}),
Run: func(a map[string]any) (any, error) { return c.Verdicts(strArg(a, "run"), intArg(a, "limit", 50)) }},
}
}
// Doctor is the verb: the last verdict, a run now, or one probe of either.
func (c *Checker) Doctor(run bool, probe, why string) (map[string]any, error) {
var r Run
switch {
case run:
r = c.RunOnce(context.Background(), why)
case c.LastRun() != nil:
r = *c.LastRun()
default:
raw, found, err := c.Store.Get(VerdictState, LastKey)
if err != nil {
return nil, fmt.Errorf("the last verdict cannot be read: %w", err)
}
if !found {
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after it starts, then "+
"every %s — run=true runs it now", FirstAfter, Every)
}
if err := json.Unmarshal(raw, &r); err != nil {
return nil, fmt.Errorf("the last verdict is not readable: %w", err)
}
}
if probe != "" {
one, ok := oneProbe(r, probe)
if !ok {
return nil, fmt.Errorf("the run %s has no probe %q — %s lists them", r.Run, probe, ProbesTool)
}
r = one
} else {
r = Overview(r)
}
return map[string]any{"run": r, "age": c.Now().Sub(r.At).Round(time.Second).String(),
"note": "a probe that could not run is never a pass; each finding is a condition the keeper raises until a run no " +
"longer sees it, and one a single look can be wrong about is written when two runs in a row see it"}, nil
}
func oneProbe(r Run, id string) (Run, bool) {
for _, p := range r.Probes {
if strings.EqualFold(p.ID, id) {
r.Probes = []ProbeVerdict{p}
return r, true
}
}
return Run{}, false
}
// Probes is the registry as the verb answers it.
func (c *Checker) Probes() map[string]any {
var out []map[string]any
for _, p := range c.Registry {
row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind}
if len(p.Raises) > 0 {
row["raises"] = p.Raises
}
if p.Run == nil {
row["deferred"] = p.Deferred
}
out = append(out, row)
}
return map[string]any{"probes": out, "every": Every.String(), "each within": Within.String()}
}
// Findings are the findings written now, or one whole.
func (c *Checker) Findings(key string) (any, error) {
if key != "" {
raw, found, err := c.Store.Get(FindingsState, key)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("no finding is written under %s now", key)
}
return json.RawMessage(raw), nil
}
written, err := c.written()
if err != nil {
return nil, fmt.Errorf("the findings cannot be read: %w", err)
}
rows := []map[string]any{}
for k, f := range written {
rows = append(rows, map[string]any{"key": k, "kind": f.Kind, "severity": f.Severity, "probe": f.Source,
"summary": f.Summary})
}
sort.Slice(rows, func(i, j int) bool { return rows[i]["key"].(string) < rows[j]["key"].(string) })
return map[string]any{"findings": rows, "count": len(rows)}, nil
}
// Verdicts are the week's runs newest first, or one whole.
func (c *Checker) Verdicts(run string, limit int) (any, error) {
if run != "" {
raw, found, err := c.Store.Get(VerdictsState, run)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("no run %s is kept: runs are kept a week", run)
}
return json.RawMessage(raw), nil
}
keys, err := c.Store.Keys(VerdictsState)
if err != nil {
return nil, fmt.Errorf("the week's verdicts cannot be read: %w", err)
}
sort.Slice(keys, func(i, j int) bool { return runOrder(keys[i]) > runOrder(keys[j]) })
total := len(keys)
if limit > 0 && len(keys) > limit {
keys = keys[:limit]
}
rows := []map[string]any{}
for _, k := range keys {
raw, found, err := c.Store.Get(VerdictsState, k)
if err != nil || !found {
continue
}
var r Run
if json.Unmarshal(raw, &r) == nil {
rows = append(rows, map[string]any{"run": r.Run, "at": r.At, "why": r.Why, "counts": r.Counts})
}
}
return map[string]any{"verdicts": rows, "kept": total, "kept-for": "a week"}, nil
}
// runOrder orders run ids `doctor-<unix>-<n>` by time, then number.
func runOrder(id string) string {
parts := strings.Split(id, "-")
if len(parts) != 3 {
return id
}
return fmt.Sprintf("%020s-%020s", parts[1], parts[2])
}
func strArg(a map[string]any, k string) string {
s, _ := a[k].(string)
return strings.TrimSpace(s)
}
// boolArg reads a flag as a boolean or as the seat's "true".
func boolArg(a map[string]any, k string) bool {
switch v := a[k].(type) {
case bool:
return v
case string:
return strings.EqualFold(strings.TrimSpace(v), "true")
}
return false
}
func intArg(a map[string]any, k string, otherwise int) int {
if n, ok := a[k].(float64); ok && n > 0 {
return int(n)
}
return otherwise
}
@@ -0,0 +1,298 @@
//go:build integration
package main
// The self-check whole against a real bus (novox/hq ADR 0564 §6, issue 1952 SC1): the built binary, played over its
// real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a real
// nats-server's key-value buckets, mesh/publish onto its subjects, and the bus module's nats_streams answered from
// that server's own streams). One run asked through the seat's verb finds the mesh's streams missing on a bare bus,
// writes them as findings into its bucket on the bus, keeps the verdict, and says checked.
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
type wire struct {
ID json.RawMessage `json:"id,omitempty"`
Method string `json:"method,omitempty"`
Params json.RawMessage `json:"params,omitempty"`
Result json.RawMessage `json:"result,omitempty"`
Error *struct {
Message string `json:"message"`
} `json:"error,omitempty"`
}
// player is the node's tool runtime, as far as the self-check needs it, over a real bus.
type player struct {
ctx context.Context
nc *nats.Conn
js jetstream.JetStream
mu sync.Mutex
in io.Writer
next int
pending map[string]chan wire
}
func (p *player) send(m map[string]any) {
raw, _ := json.Marshal(m)
p.mu.Lock()
defer p.mu.Unlock()
_, _ = p.in.Write(append(raw, '\n'))
}
func (p *player) answer(id json.RawMessage, result any, err error) {
if err != nil {
p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
return
}
p.send(map[string]any{"id": id, "result": result})
}
func (p *player) serve(out io.Reader) {
scanner := bufio.NewScanner(out)
scanner.Buffer(make([]byte, 1<<20), 1<<24)
for scanner.Scan() {
var m wire
if json.Unmarshal(scanner.Bytes(), &m) != nil {
continue
}
if m.Method == "" {
var key string
_ = json.Unmarshal(m.ID, &key)
p.mu.Lock()
ch := p.pending[key]
delete(p.pending, key)
p.mu.Unlock()
if ch != nil {
ch <- m
}
continue
}
go p.handle(m)
}
}
func (p *player) kv(state string) (jetstream.KeyValue, error) {
return p.js.KeyValue(p.ctx, Module+"_"+state)
}
func (p *player) handle(m wire) {
var a struct {
State string `json:"state"`
Key string `json:"key"`
Value json.RawMessage `json:"value"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &a)
switch m.Method {
case "mesh/state.get":
kv, err := p.kv(a.State)
if err != nil {
p.answer(m.ID, nil, err)
return
}
e, err := kv.Get(p.ctx, a.Key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
p.answer(m.ID, nil, nil)
return
}
if err != nil {
p.answer(m.ID, nil, err)
return
}
p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
case "mesh/state.put":
kv, err := p.kv(a.State)
if err == nil {
var rev uint64
if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil {
p.answer(m.ID, map[string]any{"revision": rev}, nil)
return
}
}
p.answer(m.ID, nil, err)
case "mesh/state.keys":
kv, err := p.kv(a.State)
if err != nil {
p.answer(m.ID, nil, err)
return
}
keys, err := kv.Keys(p.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
keys, err = []string{}, nil
}
p.answer(m.ID, keys, err)
case "mesh/state.delete":
kv, err := p.kv(a.State)
if err == nil {
err = kv.Delete(p.ctx, a.Key)
}
p.answer(m.ID, map[string]any{}, err)
case "mesh/seat.publish":
var e struct {
Subject string `json:"subject"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &e)
p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body))
case "mesh/publish":
var e struct {
Key string `json:"key"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &e)
p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body))
case "mesh/ask":
var ask struct {
Key string `json:"key"`
}
_ = json.Unmarshal(m.Params, &ask)
if ask.Key != StreamsTool {
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
return
}
// The bus module's answer, from the server's own streams.
var detail []map[string]any
for name := range p.js.StreamNames(p.ctx).Name() {
detail = append(detail, map[string]any{"name": name})
}
p.answer(m.ID, map[string]any{"detail": detail}, nil)
default:
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
}
}
func (p *player) call(name string, args map[string]any) (json.RawMessage, error) {
p.mu.Lock()
p.next++
id := fmt.Sprintf("c-%d", p.next)
ch := make(chan wire, 1)
p.pending[id] = ch
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args,
"_meta": map[string]any{"mesh/caller": "person.jochen"}}})
select {
case m := <-ch:
if m.Error != nil {
return nil, errors.New(m.Error.Message)
}
var r struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
IsError bool `json:"isError"`
}
if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 {
return nil, fmt.Errorf("%s answered %s", name, m.Result)
}
if r.IsError {
return nil, errors.New(r.Content[0].Text)
}
return json.RawMessage(r.Content[0].Text), nil
case <-time.After(60 * time.Second):
return nil, fmt.Errorf("%s did not answer in 60s", name)
}
}
// 1952/SC1: one run of the self-check whole ends with checked and a verdict, its findings on the bus.
func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
dir := t.TempDir()
bin := filepath.Join(dir, Module)
build := exec.Command("go", "build", "-o", bin, ".")
build.Stderr = os.Stderr
if err := build.Run(); err != nil {
t.Fatalf("the self-check does not build: %v", err)
}
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
for _, s := range []string{FindingsState, VerdictState, VerdictsState} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: Module + "_" + s}); err != nil {
t.Fatal(err)
}
}
checked, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + CheckedEvent)
if err != nil {
t.Fatal(err)
}
cmd := exec.CommandContext(ctx, bin)
cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module)
stdin, _ := cmd.StdinPipe()
stdout, _ := cmd.StdoutPipe()
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
go p.serve(stdout)
raw, err := p.call(DoctorTool, map[string]any{"run": true})
if err != nil {
t.Fatal(err)
}
var answer struct {
Run Run `json:"run"`
}
if err := json.Unmarshal(raw, &answer); err != nil {
t.Fatal(err)
}
verdicts := map[string]string{}
for _, v := range answer.Run.Probes {
verdicts[v.ID] = v.Verdict
}
if verdicts["D7"] != VerdictFail || verdicts["DW"] != VerdictFail || verdicts["D15"] != VerdictFailedToRun ||
verdicts["D2"] != VerdictDeferred || answer.Run.Controller != "ace" || !strings.Contains(answer.Run.Why, "person.jochen") {
t.Errorf("the run answered %+v", answer.Run)
}
msg, err := checked.NextMsg(10 * time.Second)
if err != nil {
t.Fatalf("checked was not said: %v", err)
}
var said Run
if err := json.Unmarshal(msg.Data, &said); err != nil || said.Run != answer.Run.Run {
t.Errorf("checked said %s", msg.Data)
}
findings, _ := js.KeyValue(ctx, Module+"_"+FindingsState)
e, err := findings.Get(ctx, "bus.EVENTS.stream-missing")
if err != nil {
t.Fatalf("the missing stream is not a finding on the bus: %v", err)
}
f, err := health.DecodeFinding(e.Value())
if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" {
t.Errorf("the finding is %s", e.Value())
}
verdict, _ := js.KeyValue(ctx, Module+"_"+VerdictState)
if _, err := verdict.Get(ctx, LastKey); err != nil {
t.Errorf("the last verdict is not kept: %v", err)
}
if raw, err := p.call(VerdictsTool, map[string]any{}); err != nil || !strings.Contains(string(raw), answer.Run.Run) {
t.Errorf("the week's verdicts answered %s, %v", raw, err)
}
}
+22
View File
@@ -0,0 +1,22 @@
module mesh-self-check
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
+27
View File
@@ -0,0 +1,27 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+64
View File
@@ -0,0 +1,64 @@
{
"module": "mesh-self-check",
"version": "1",
"runs-as": "mesh-self-check",
"claims": [
{
"name": "mesh-self-check",
"scope": "mesh",
"serves": [
"doctor",
"probes"
]
}
],
"invokes": [
"nats.nats_streams",
"nats.nats_closed_connections",
"seat:mesh-watchdogs.signals"
],
"state": [
"findings",
"verdict",
{
"name": "verdicts",
"ttl-seconds": 604800
}
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
"secrets-owner": "mesh-self-check",
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-self-check",
"shell": "/usr/bin/nologin",
"home": "/var/lib/mesh-self-check"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "mesh-self-check",
"place": ".",
"unknown": "program"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-self-check",
"binary": "mesh-self-check",
"loads": [
"mesh-self-check"
]
}
]
}
}