Merge the release branch; checks/health takes its newer sdk pin (issue 1952)
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
# The calls of the controller's hand-act log a manifest of this catalogue may still invoke, each `<module> <verb> <why>`:
|
||||
# the fallback of a recorder (mesh-sdk handacts.Recorder) while the seat mesh-hand-acts has no holder that answers, until
|
||||
# the controller's lease is written to a holder that is no controller. The check of this directory (handacts) refuses any
|
||||
# other, and an entry whose module no longer invokes it: the list only shrinks, and it is empty before the lease write
|
||||
# (novox/hq ADR 0565 §11; issue 1953, issue 774).
|
||||
mesh-config hand-act fallback until the lease write: its sets, clears and held values record through mesh-hand-acts first
|
||||
mesh-config warranted fallback until the lease write: its warrants record through mesh-hand-acts first
|
||||
mesh-healers drill fallback until the lease write: a drill records through mesh-hand-acts first
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/handacts
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,109 @@
|
||||
// Command handacts refuses a manifest that records into the controller's hand-act log (novox/hq ADR 0565 §11): every
|
||||
// recorder records through the seat mesh-hand-acts from its delivery, so no manifest invokes
|
||||
// `seat:mesh-controller.hand-act`, `.warranted` or `.drill`. The recorders' fallbacks to the controller's log while the
|
||||
// seat has no holder are named in checks/handacts/fallbacks, each module and verb with why; that list only shrinks, and
|
||||
// an entry whose module no longer invokes the verb is refused too.
|
||||
//
|
||||
// go run . <catalog root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Verbs are the controller's hand-act verbs, as a manifest invokes them, and Allowed the list of named fallbacks.
|
||||
var Verbs = []string{"hand-act", "warranted", "drill"}
|
||||
|
||||
const (
|
||||
invokePrefix = "seat:mesh-controller."
|
||||
Allowed = "checks/handacts/fallbacks"
|
||||
)
|
||||
|
||||
// Findings is every refusal in the catalog at root, sorted.
|
||||
func Findings(root string) ([]string, error) {
|
||||
allowed := map[string]bool{}
|
||||
raw, err := os.ReadFile(filepath.Join(root, Allowed))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if f := strings.Fields(line); len(f) >= 3 && !strings.HasPrefix(f[0], "#") {
|
||||
allowed[f[0]+" "+f[1]] = true
|
||||
} else if len(f) > 0 && !strings.HasPrefix(f[0], "#") {
|
||||
return nil, fmt.Errorf("%s: %q is not `<module> <verb> <why>`", Allowed, line)
|
||||
}
|
||||
}
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
invoked := map[string]bool{}
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Invokes []string `json:"invokes"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" {
|
||||
continue
|
||||
}
|
||||
for _, inv := range m.Invokes {
|
||||
verb, ok := strings.CutPrefix(strings.TrimSpace(inv), invokePrefix)
|
||||
if !ok || !contains(Verbs, verb) {
|
||||
continue
|
||||
}
|
||||
invoked[m.Module+" "+verb] = true
|
||||
if !allowed[m.Module+" "+verb] {
|
||||
out = append(out, fmt.Sprintf("%s invokes %s%s: record through seat:mesh-hand-acts.%s (novox/hq ADR 0565 §11), "+
|
||||
"or name the fallback in %s", m.Module, invokePrefix, verb, verb, Allowed))
|
||||
}
|
||||
}
|
||||
}
|
||||
for entry := range allowed {
|
||||
if !invoked[entry] {
|
||||
out = append(out, fmt.Sprintf("%s names %q, which is no longer invoked: take it off the list, which only shrinks",
|
||||
Allowed, entry))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: handacts <catalog root>")
|
||||
os.Exit(2)
|
||||
}
|
||||
found, err := Findings(os.Args[1])
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Println(f)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("no manifest records into the controller's hand-act log but the fallbacks named")
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// No manifest invokes the controller's hand-act verbs but a named fallback; a named fallback no longer invoked is
|
||||
// refused (novox/hq ADR 0565 §11's catalog test).
|
||||
func TestFindings(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
want string // empty: clean
|
||||
}{
|
||||
{"clean", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""},
|
||||
{"an unnamed call", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.warranted"]}`},
|
||||
"a invokes seat:mesh-controller.warranted"},
|
||||
{"a named fallback, and the controller's own", map[string]string{
|
||||
"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.drill"]}`,
|
||||
"modules/mesh-controller/module.json": `{"module":"mesh-controller","invokes":["seat:mesh-controller.hand-act"]}`,
|
||||
Allowed: "# why\na drill fallback until the lease write\n"}, ""},
|
||||
{"a fallback no longer invoked", map[string]string{Allowed: "gone hand-act fallback\n"}, `names "gone hand-act"`},
|
||||
{"another verb of the controller", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.conditions"]}`}, ""},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for path, text := range row.files {
|
||||
p := filepath.Join(root, path)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := Findings(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) {
|
||||
t.Fatalf("findings = %q, want one containing %q", all, row.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The run of this check over the real tree.
|
||||
func TestTheCatalogRecordsOnlyThroughTheSeat(t *testing.T) {
|
||||
got, err := Findings(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range got {
|
||||
t.Error(f)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (issue 1936).
|
||||
# every manifest, for what it invokes.
|
||||
modules/*/module.json
|
||||
@@ -2,4 +2,4 @@ module checks/health
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268 h1:+jH77IJmxUiFDdbJljhynQRY0Cc5/5jtwmMV7UUqRI4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
package main
|
||||
|
||||
// **Built dark, compared, then switched by the operator** (novox/hq ADR 0564 clause 9; issue 1952 C5, C8): `compare`
|
||||
// lists the open keys of the keeper and of the controller, each difference with its reason; the kinds that end with the
|
||||
// controller, the kinds only the keeper raises and the keys held while their module is delivered are listed apart and
|
||||
// not counted. `take-over` raises an approve ask under the keeper's own name (ADR 0259), refused while compare shows a
|
||||
// difference, and on the warrant writes the take-over key naming the keeper. `hand-back` writes it back to the
|
||||
// controller until the lease write, and is refused after it.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// Difference is one key the two sides do not agree on, or one listed apart.
|
||||
type Difference struct {
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
// Comparison is compare's answer.
|
||||
type Comparison struct {
|
||||
Differences []Difference `json:"differences"`
|
||||
// Apart are the keys not counted: of kinds that end with the controller, raised only by the keeper, or held.
|
||||
Apart []Difference `json:"apart"`
|
||||
Same int `json:"same"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// Empty says nothing differs.
|
||||
func (c Comparison) Empty() bool { return len(c.Differences) == 0 }
|
||||
|
||||
// Compare is the keeper's open conditions against the controller's.
|
||||
func (k *Keeper) Compare() (Comparison, error) {
|
||||
mine, err := k.Open()
|
||||
if err != nil {
|
||||
return Comparison{}, err
|
||||
}
|
||||
k.mu.Lock()
|
||||
k.init()
|
||||
theirs := map[string]health.Condition{}
|
||||
for key, c := range k.ctrl {
|
||||
theirs[key] = c
|
||||
}
|
||||
heldNow := map[string]held{}
|
||||
for key, h := range k.held {
|
||||
heldNow[key] = h
|
||||
}
|
||||
k.mu.Unlock()
|
||||
out := Comparison{Differences: []Difference{}, Apart: []Difference{}, At: k.Now().UTC()}
|
||||
open := map[string]health.Condition{}
|
||||
for _, c := range mine {
|
||||
open[c.Key] = c
|
||||
}
|
||||
keys := map[string]bool{}
|
||||
for key := range open {
|
||||
keys[key] = true
|
||||
}
|
||||
for key := range theirs {
|
||||
keys[key] = true
|
||||
}
|
||||
for key := range heldNow {
|
||||
keys[key] = true
|
||||
}
|
||||
sorted := make([]string, 0, len(keys))
|
||||
for key := range keys {
|
||||
sorted = append(sorted, key)
|
||||
}
|
||||
sort.Strings(sorted)
|
||||
for _, key := range sorted {
|
||||
c, inMine := open[key]
|
||||
t, inTheirs := theirs[key]
|
||||
kind := c.Kind
|
||||
if !inMine {
|
||||
kind = t.Kind
|
||||
}
|
||||
home, homed := k.homeOf(kind)
|
||||
if h, ok := heldNow[key]; ok {
|
||||
out.Apart = append(out.Apart, Difference{Key: key, Kind: kind,
|
||||
Reason: fmt.Sprintf("held while %s is delivered on %s (version %s)", h.Module, h.Node, short(h.Version))})
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case homed && (home.Carried || home.Replaced):
|
||||
why := "ends with the controller: carried from its bucket until the lease write"
|
||||
if home.Replaced {
|
||||
why = "ends with the controller: its fault is observed otherwise"
|
||||
}
|
||||
out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: why})
|
||||
case inMine && !inTheirs && (!homed || !home.Controller):
|
||||
out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: "keeper only: the controller never raises this kind"})
|
||||
case inMine && !inTheirs:
|
||||
out.Differences = append(out.Differences, Difference{Key: key, Kind: kind,
|
||||
Reason: "open at the keeper, not at the controller"})
|
||||
case !inMine && !homed:
|
||||
out.Differences = append(out.Differences, Difference{Key: key, Kind: kind,
|
||||
Reason: fmt.Sprintf("open at the controller; no home is decided for the kind %s (ADR 0564's gap)", kind)})
|
||||
case !inMine:
|
||||
out.Differences = append(out.Differences, Difference{Key: key, Kind: kind,
|
||||
Reason: fmt.Sprintf("open at the controller; %s has not written its finding", home.Observer)})
|
||||
case c.Severity != t.Severity:
|
||||
out.Differences = append(out.Differences, Difference{Key: key, Kind: kind,
|
||||
Reason: fmt.Sprintf("%s at the keeper, %s at the controller", c.Severity, t.Severity)})
|
||||
default:
|
||||
out.Same++
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Asks is the operator channel as the keeper asks it: an approve ask under its own name, and where one stands.
|
||||
type Asks interface {
|
||||
Ask(a asks.Ask) error
|
||||
}
|
||||
|
||||
// TakeOver raises the approve ask of a switch to the keeper, refused while compare differs or after the lease write.
|
||||
func (k *Keeper) TakeOver(a Asks, by, why string) (map[string]any, error) {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("a take-over says why")
|
||||
}
|
||||
if k.LeaseAway() {
|
||||
return nil, errors.New("the controller's lease is handed away: the keeper is the one source already")
|
||||
}
|
||||
cmp, err := k.Compare()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !cmp.Empty() {
|
||||
return nil, fmt.Errorf("refused: compare shows %d difference(s), the first %s (%s); the take-over waits for none",
|
||||
len(cmp.Differences), cmp.Differences[0].Key, cmp.Differences[0].Reason)
|
||||
}
|
||||
now := k.Now().UTC()
|
||||
id := fmt.Sprintf("take-over-%d", now.UnixNano())
|
||||
act := asks.Act{"verb": Seat + ".take-over", "module": Module, "ask": id}
|
||||
bind, err := asks.ActDigest(act)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ask := asks.Ask{ID: id, Headline: "Switch the mesh's alarms to the new keeper?", Who: asks.Operator,
|
||||
Explanation: fmt.Sprintf("The new condition keeper holds the same %d open alarms as the controller. Your yes makes "+
|
||||
"the alarm messages come from the keeper, with the same words. It can be handed back until the controller "+
|
||||
"gives up its lease.", cmp.Same),
|
||||
Options: []asks.Option{{ID: "yes", Label: "Yes, switch", Does: "Alarms come from the keeper.", Level: asks.Approve, Binds: bind},
|
||||
{ID: "no", Label: "No, leave it", Does: "Alarms keep coming from the controller.", Level: asks.Acknowledge}},
|
||||
Expires: now.Add(asks.ApproveLasts - time.Minute), OnExpiry: "Alarms keep coming from the controller.",
|
||||
About: Module + "-take-over", Details: "Asked by " + by + ": " + why}
|
||||
if err := a.Ask(ask); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"asked": id, "same": cmp.Same, "apart": len(cmp.Apart)}, nil
|
||||
}
|
||||
|
||||
// Warranted is the act on the operator's answer: a yes, compare still empty, writes the take-over key naming the keeper.
|
||||
func (k *Keeper) Warranted(o asks.Option, w asks.Warrant) error {
|
||||
if o.ID != "yes" {
|
||||
k.Logf("the take-over %s was answered %s: alarms stay with the controller", w.Ask, o.ID)
|
||||
return nil
|
||||
}
|
||||
cmp, err := k.Compare()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !cmp.Empty() {
|
||||
return fmt.Errorf("the take-over %s was approved, and compare now shows %d difference(s): not switched", w.Ask,
|
||||
len(cmp.Differences))
|
||||
}
|
||||
return k.writeTakeover(health.Takeover{Follow: health.FollowKeeper, At: k.Now().UTC(), By: w.Says(), Ask: w.Ask})
|
||||
}
|
||||
|
||||
// HandBack switches the readers back to the controller, until the lease write.
|
||||
func (k *Keeper) HandBack(by, why string) (map[string]any, error) {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("a hand-back says why")
|
||||
}
|
||||
if k.LeaseAway() {
|
||||
return nil, errors.New("refused: the controller's lease is handed away, so there is no controller to hand back to")
|
||||
}
|
||||
t := health.Takeover{Follow: health.FollowController, At: k.Now().UTC(), By: by, Why: why}
|
||||
if err := k.writeTakeover(t); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"follow": t.Follow, "at": t.At}, nil
|
||||
}
|
||||
|
||||
// writeTakeover writes the take-over key, and keeps it in history as the record of the switch.
|
||||
func (k *Keeper) writeTakeover(t health.Takeover) error {
|
||||
raw, err := health.EncodeTakeover(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, rev, found, err := k.Store.Get(TakeoverState, health.TakeoverKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found {
|
||||
_, err = k.Store.Update(TakeoverState, health.TakeoverKey, json.RawMessage(raw), rev)
|
||||
} else {
|
||||
_, err = k.Store.Create(TakeoverState, health.TakeoverKey, json.RawMessage(raw))
|
||||
}
|
||||
if errors.Is(err, stdio.ErrChanged) {
|
||||
return fmt.Errorf("the take-over key moved under this write: read it again and ask again")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
k.Takeover(t)
|
||||
k.Logf("the readers of conditions follow the %s from now (%s)", t.Follow, t.By)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
package main
|
||||
|
||||
// **A module being delivered raises no condition until its delivery ends** (novox/hq ADR 0564 clause 13 and Limits 12;
|
||||
// issue 1952 C7, C8). The keeper reads the assigner's wants and the nodes' haves, and judges a module's delivery on a
|
||||
// node with mesh-sdk's delivery.MotionOf, as the view does. While it is delivered there, a finding about it there, or
|
||||
// about what it serves, and a condition carried from the controller's bucket on those subjects, is kept in the state
|
||||
// `held` and raises nothing. When the delivery ends, healthy at the want or failed, what still stands is raised once,
|
||||
// saying it happened during the delivery; what went is dropped with a log line. The hold lasts at most the delivery's
|
||||
// bound, a key of the state `bounds` whose time to live is the bound: its expiry raises what is held, saying the delivery
|
||||
// did not end. A finding about another module or about the machine itself is never held. A condition already open is
|
||||
// not held: it is no news.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The modules whose delivery holds what they serve beyond their own findings (Limits 12).
|
||||
const (
|
||||
controllerModule = "mesh-controller"
|
||||
runtimeModule = "node-tools"
|
||||
)
|
||||
|
||||
// controllerServes are the sources of what the controller serves while it is delivered: its listening, its lease, its
|
||||
// writes and its facts (ADR 0505 §2 (a)), matched to mesh-controller whatever its key names.
|
||||
var controllerServes = map[string]bool{"S4": true, "S7": true, "S12": true, "S13": true, "S14": true, "S25": true}
|
||||
|
||||
// about is a node and a module a finding or a condition is about, for the hold.
|
||||
type about struct{ Node, Module string }
|
||||
|
||||
// aboutOf are the modules on nodes a condition's subject is about (Limits 12): a module on a machine, a provider on its
|
||||
// node, a seat of the same name as its module on a machine, the controller's own work wherever mesh-controller is
|
||||
// delivered, and every module's tools on a machine whose tool runner is delivered. Never the machine itself.
|
||||
func (k *Keeper) aboutOf(kind, source string, s health.Subject) []about {
|
||||
var out []about
|
||||
parts := strings.Split(s.ID, ".")
|
||||
switch s.Scope {
|
||||
case health.ScopeModule:
|
||||
if len(parts) >= 2 {
|
||||
out = append(out, about{Node: parts[len(parts)-1], Module: strings.Join(parts[:len(parts)-1], ".")})
|
||||
}
|
||||
case health.ScopeProvider:
|
||||
if len(parts) >= 2 {
|
||||
out = append(out, about{Node: parts[1], Module: parts[0]})
|
||||
}
|
||||
case health.ScopeSeat:
|
||||
// The module the placement says holds the seat on that node, or the seat's own name where none is read.
|
||||
if len(parts) >= 2 {
|
||||
node, seat := parts[len(parts)-1], strings.Join(parts[:len(parts)-1], ".")
|
||||
module := k.holders[node+"."+seat]
|
||||
if module == "" {
|
||||
module = seat
|
||||
}
|
||||
out = append(out, about{Node: node, Module: module})
|
||||
}
|
||||
case health.ScopeMachine:
|
||||
// The machine itself (presence, network, resolver, disk) is never held; its tools are, while its tool runner is
|
||||
// delivered.
|
||||
if source == "S11" || kind == "tools-silent" {
|
||||
out = append(out, about{Node: s.Machine, Module: runtimeModule})
|
||||
if s.Machine == "" && len(parts) > 0 {
|
||||
out[len(out)-1].Node = parts[0]
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
if controllerServes[source] || (s.Scope == health.ScopeCore && strings.HasPrefix(s.ID, "controller")) {
|
||||
for hk := range k.wants {
|
||||
if node, module := splitNodeModule(hk); module == controllerModule {
|
||||
out = append(out, about{Node: node, Module: module})
|
||||
}
|
||||
}
|
||||
}
|
||||
if s.Machine != "" && s.Scope != health.ScopeMachine {
|
||||
out = append(out, about{Node: s.Machine, Module: runtimeModule})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func splitNodeModule(hk string) (string, string) {
|
||||
node, module, _ := strings.Cut(hk, ".")
|
||||
return node, module
|
||||
}
|
||||
|
||||
// motion is the delivery of module on node, from its want and have.
|
||||
func (k *Keeper) motion(node, module string) (delivery.Motion, delivery.Want) {
|
||||
w, wanted := k.wants[node+"."+module]
|
||||
h, has := k.haves[node+"."+module]
|
||||
return delivery.MotionOf(w, wanted, h, has, "", false), w
|
||||
}
|
||||
|
||||
// holdFor says whether key is held now, and as what. Called with mu held.
|
||||
func (k *Keeper) holdFor(key string, f *health.Finding, carried *health.Condition, open bool) (held, bool) {
|
||||
if open || (f == nil && carried == nil) {
|
||||
return held{}, false
|
||||
}
|
||||
var kind, source string
|
||||
var subject health.Subject
|
||||
if f != nil {
|
||||
kind, source, subject = f.Kind, f.Source, f.Subject
|
||||
} else {
|
||||
kind, source, subject = carried.Kind, carried.Source, carried.Subject
|
||||
}
|
||||
if _, raisedOut := k.outOfHold[key]; raisedOut {
|
||||
return held{}, false
|
||||
}
|
||||
for _, a := range k.aboutOf(kind, source, subject) {
|
||||
m, w := k.motion(a.Node, a.Module)
|
||||
if !m.Delivering || m.Ended() || k.passed[a.Node+"."+a.Module] == w.FP {
|
||||
continue
|
||||
}
|
||||
since := k.Now().UTC()
|
||||
if was, ok := k.held[key]; ok {
|
||||
since = was.Since
|
||||
}
|
||||
return held{Key: key, Node: a.Node, Module: a.Module, Since: since, Version: w.FP}, true
|
||||
}
|
||||
return held{}, false
|
||||
}
|
||||
|
||||
// keepHeld keeps a held key in the state `held`, and starts its delivery's bound once.
|
||||
func (k *Keeper) keepHeld(h held) error {
|
||||
k.mu.Lock()
|
||||
_, was := k.held[h.Key]
|
||||
k.held[h.Key] = h
|
||||
bound := k.bound(h.Module)
|
||||
k.mu.Unlock()
|
||||
if was {
|
||||
return nil
|
||||
}
|
||||
if _, err := k.Store.Create(HeldState, h.Key, h); err != nil && !errors.Is(err, stdio.ErrChanged) {
|
||||
return fmt.Errorf("keeping %s held: %w", h.Key, err)
|
||||
}
|
||||
b := boundKey(h.Node, h.Module)
|
||||
err := k.Store.CreateTTL(BoundsState, b, map[string]any{"node": h.Node, "module": h.Module, "version": h.Version,
|
||||
"bound-seconds": int(bound / time.Second)}, bound)
|
||||
if err != nil && !errors.Is(err, stdio.ErrChanged) {
|
||||
return fmt.Errorf("starting the bound of %s's delivery on %s: %w", h.Module, h.Node, err)
|
||||
}
|
||||
k.Logf("held %s while %s is delivered on %s (version %s), at most %s", h.Key, h.Module, h.Node, h.Version, bound)
|
||||
return nil
|
||||
}
|
||||
|
||||
// bound is a module's delivery bound: its manifest names none yet, so the default (clause 13 (d)). Each key of the state
|
||||
// `bounds` is written with it as its own time to live, and its expiry is the bound passing.
|
||||
func (k *Keeper) bound(string) time.Duration {
|
||||
if k.Bound > 0 {
|
||||
return k.Bound
|
||||
}
|
||||
return delivery.DefaultDeliveryBound
|
||||
}
|
||||
|
||||
func boundKey(node, module string) string { return node + "." + module }
|
||||
|
||||
// dropHeld lets a held key go: raised, or gone with why said.
|
||||
func (k *Keeper) dropHeld(key, why string) {
|
||||
k.mu.Lock()
|
||||
h, ok := k.held[key]
|
||||
delete(k.held, key)
|
||||
k.mu.Unlock()
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := k.Store.Delete(HeldState, key); err != nil {
|
||||
k.Logf("the held %s could not be removed from held: %v", key, err)
|
||||
}
|
||||
if why != "" {
|
||||
k.Logf("dropped %s, held while %s was delivered on %s: %s", key, h.Module, h.Node, why)
|
||||
}
|
||||
}
|
||||
|
||||
// Want takes one change of the assigner's wants; Have one of the nodes' haves. Each may end a delivery, which raises or
|
||||
// drops what it held.
|
||||
func (k *Keeper) Want(key string, value []byte, deleted bool) error {
|
||||
node, module, err := delivery.ParseWantKey(key)
|
||||
if err != nil {
|
||||
return nil // not a want of a module on a node (a held fingerprint's key)
|
||||
}
|
||||
k.mu.Lock()
|
||||
k.init()
|
||||
if deleted {
|
||||
delete(k.wants, node+"."+module)
|
||||
} else if w, err := delivery.DecodeWant(value); err == nil {
|
||||
k.wants[node+"."+module] = w
|
||||
} else {
|
||||
k.Logf("the want %s is not readable: %v", key, err)
|
||||
}
|
||||
k.mu.Unlock()
|
||||
return k.judge(node, module, "")
|
||||
}
|
||||
|
||||
// Have takes one change of a node's have.
|
||||
func (k *Keeper) Have(key string, value []byte, deleted bool) error {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
return nil
|
||||
}
|
||||
k.mu.Lock()
|
||||
k.init()
|
||||
if deleted {
|
||||
delete(k.haves, node+"."+module)
|
||||
} else if h, err := delivery.DecodeHave(value); err == nil {
|
||||
k.haves[node+"."+module] = h
|
||||
} else {
|
||||
k.Logf("the have %s is not readable: %v", key, err)
|
||||
}
|
||||
k.mu.Unlock()
|
||||
return k.judge(node, module, "")
|
||||
}
|
||||
|
||||
// Placement takes one change of the placement: which module holds which seat on a node.
|
||||
func (k *Keeper) Placement(key string, value []byte, deleted bool) {
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
k.init()
|
||||
for hk := range k.holders {
|
||||
if strings.HasPrefix(hk, node+".") {
|
||||
delete(k.holders, hk)
|
||||
}
|
||||
}
|
||||
if deleted {
|
||||
return
|
||||
}
|
||||
p, err := delivery.DecodePlacement(value)
|
||||
if err != nil {
|
||||
k.Logf("the placement of %s is not readable, so its seats' holders are not known: %v", node, err)
|
||||
return
|
||||
}
|
||||
for _, h := range p.Holds {
|
||||
k.holders[node+"."+h.Claim] = h.Module
|
||||
}
|
||||
}
|
||||
|
||||
// BoundPassed is a delivery's bound key expiring: what it holds is raised, saying the delivery did not end.
|
||||
func (k *Keeper) BoundPassed(key string) error {
|
||||
node, module := splitNodeModule(key)
|
||||
k.mu.Lock()
|
||||
k.init()
|
||||
_, w := k.motion(node, module)
|
||||
holding := false
|
||||
for _, h := range k.held {
|
||||
holding = holding || (h.Node == node && h.Module == module)
|
||||
}
|
||||
if !holding {
|
||||
k.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
k.passed[key] = w.FP
|
||||
k.mu.Unlock()
|
||||
return k.judge(node, module, fmt.Sprintf("Its delivery of version %s on %s did not end within %s.", short(w.FP), node,
|
||||
k.bound(module)))
|
||||
}
|
||||
|
||||
// judge raises or drops what node and module hold when their delivery ended, or with line past its bound.
|
||||
func (k *Keeper) judge(node, module, line string) error {
|
||||
k.mu.Lock()
|
||||
m, w := k.motion(node, module)
|
||||
var keys []string
|
||||
for key, h := range k.held {
|
||||
if h.Node == node && h.Module == module {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
if line == "" && !m.Ended() && m.Delivering {
|
||||
k.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
if line == "" {
|
||||
line = fmt.Sprintf("It happened during the delivery of version %s of %s on %s.", short(w.FP), module, node)
|
||||
if m.Failed {
|
||||
line = fmt.Sprintf("It happened as the delivery of version %s of %s on %s failed.", short(w.FP), module, node)
|
||||
}
|
||||
delete(k.passed, node+"."+module)
|
||||
}
|
||||
for _, key := range keys {
|
||||
k.outOfHold[key] = line
|
||||
}
|
||||
k.mu.Unlock()
|
||||
sort.Strings(keys)
|
||||
var problems []string
|
||||
for _, key := range keys {
|
||||
k.dropHeld(key, "")
|
||||
if err := k.Settle(key); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
k.mu.Lock()
|
||||
delete(k.outOfHold, key)
|
||||
k.mu.Unlock()
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return errors.New(strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func short(fp string) string {
|
||||
if _, rest, ok := strings.Cut(fp, ":"); ok {
|
||||
fp = rest
|
||||
}
|
||||
if len(fp) > 12 {
|
||||
return fp[:12]
|
||||
}
|
||||
return fp
|
||||
}
|
||||
|
||||
// Held is every key held now, for the read tool and compare.
|
||||
func (k *Keeper) Held() []held {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
out := make([]held, 0, len(k.held))
|
||||
for _, h := range k.held {
|
||||
out = append(out, h)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
|
||||
return out
|
||||
}
|
||||
|
||||
// restoreHeld takes the state `held` as read at the start.
|
||||
func (k *Keeper) restoreHeld(key string, value []byte) {
|
||||
var h held
|
||||
if json.Unmarshal(value, &h) != nil {
|
||||
return
|
||||
}
|
||||
k.mu.Lock()
|
||||
k.init()
|
||||
k.held[key] = h
|
||||
k.mu.Unlock()
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The fingerprints and composition of mesh-controller on novox, copied from its have (mesh-delivery-view.node_haves,
|
||||
// 2026-10-29): the delivery replayed moves it from the running fingerprint to the next, through the node-engine's phases
|
||||
// (delivery.go Phase*). No tool reads a have's or a want's history yet, so the sequence is the node-engine's documented
|
||||
// order, not a capture; the values are captured.
|
||||
const (
|
||||
ctrlRunning = "src2:fcaa7d7ccb35ea2beed5b9e4b87f4c24c99452244d49f4786d4022e5c543f39c"
|
||||
ctrlComposed = "sha256:e9a4de75c9ca70a32cef3137cb0e6692b0f9558b699013caaee399d88dd1bac0"
|
||||
ctrlNext = "src2:0d1e2f3a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9001a2b3c4d5e6"
|
||||
ctrlNextComp = "sha256:1f2e3d4c5b6a79881726354453627180918a7b6c5d4e3f2a1b0c9d8e7f605142"
|
||||
)
|
||||
|
||||
// ADR 0564 clause 13 and Limits 12 (issue 1952 C7, C8): a delivery of mesh-controller on novox replayed want by have.
|
||||
// Its own findings and the carried controller-deaf raise nothing during it and none after when they went; one still
|
||||
// standing at the end is raised once with its during-the-delivery line; one past the bound with its did-not-end line; a
|
||||
// failed delivery raises what stands at once; another module's and the machine's own are raised at once; compare lists a
|
||||
// held key as held.
|
||||
func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) {
|
||||
want := func(fp, composed string) []byte {
|
||||
raw, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp, Composed: composed})
|
||||
return raw
|
||||
}
|
||||
have := func(fp, composed string, phase delivery.Phase, h delivery.Health) []byte {
|
||||
raw, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp, Composed: composed, Phase: phase, Health: h})
|
||||
return raw
|
||||
}
|
||||
deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4",
|
||||
Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self"})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
// end is the have the delivery ends with, or nil for a delivery past its bound.
|
||||
end []byte
|
||||
standing bool
|
||||
said string
|
||||
line string
|
||||
}{
|
||||
{name: "gone by its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK)},
|
||||
{name: "still standing at its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK),
|
||||
standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "during the delivery of version 0d1e2f3a4b5c"},
|
||||
{name: "past its bound", standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "did not end within"},
|
||||
{name: "failed", end: have(ctrlNext, ctrlNextComp, delivery.PhaseFailed, delivery.HealthFailed), standing: true,
|
||||
said: "raised module.mesh-controller.novox.unhealthy", line: "as the delivery of version 0d1e2f3a4b5c"},
|
||||
} {
|
||||
r := newRig()
|
||||
_ = r.k.Want("want.novox.mesh-controller", want(ctrlRunning, ctrlComposed), false)
|
||||
_ = r.k.Have("novox.mesh-controller", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false)
|
||||
// The delivery starts: the assigner wants the next fingerprint; the node applies it.
|
||||
_ = r.k.Want("want.novox.mesh-controller", want(ctrlNext, ctrlNextComp), false)
|
||||
_ = r.k.Have("novox.mesh-controller", have(ctrlNext, ctrlNextComp, delivery.PhaseApplying, delivery.HealthUnknown), false)
|
||||
own, ownRaw := finding(t, watchman, health.ScopeModule, "mesh-controller.novox", "unhealthy", "health", health.Warning, r.now)
|
||||
other, otherRaw := finding(t, watchman, health.ScopeModule, "postgres.novox", "unhealthy", "health", health.Warning, r.now)
|
||||
machine, machineRaw := finding(t, watchman, health.ScopeMachine, "novox", "silent", "S1", health.Urgent, r.now)
|
||||
for _, f := range []struct {
|
||||
key string
|
||||
raw []byte
|
||||
}{{own, ownRaw}, {other, otherRaw}, {machine, machineRaw}} {
|
||||
if err := r.k.Finding(watchman, f.key, f.raw, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The delivery's bound is its own key's time to live, the module's bound (the default where it names none).
|
||||
if r.st.ttls["bounds/novox.mesh-controller"] != delivery.DefaultDeliveryBound {
|
||||
t.Errorf("%s: the bound key lives %s", c.name, r.st.ttls["bounds/novox.mesh-controller"])
|
||||
}
|
||||
if got := r.changes(); got != "raised "+other+", raised "+machine {
|
||||
t.Errorf("%s: during the delivery said %q: another module's and the machine's at once, nothing of its own", c.name, got)
|
||||
}
|
||||
// C8: compare lists the held keys as held, never as a difference.
|
||||
cmp, _ := r.k.Compare()
|
||||
held := 0
|
||||
for _, d := range cmp.Apart {
|
||||
if strings.HasPrefix(d.Reason, "held while mesh-controller is delivered on novox") {
|
||||
held++
|
||||
}
|
||||
}
|
||||
if held != 2 {
|
||||
t.Errorf("%s: compare lists %d held, %+v", c.name, held, cmp)
|
||||
}
|
||||
// The carried condition about the controller goes within the delivery; its own finding goes unless it stands.
|
||||
_ = r.k.Controller("core.controller.deaf", nil, true)
|
||||
if !c.standing {
|
||||
_ = r.k.Finding(watchman, own, nil, true)
|
||||
}
|
||||
if c.end != nil {
|
||||
_ = r.k.Have("novox.mesh-controller", c.end, false)
|
||||
} else {
|
||||
r.now = r.now.Add(delivery.DefaultDeliveryBound)
|
||||
_ = r.k.BoundPassed("novox.mesh-controller")
|
||||
}
|
||||
if got := r.changes(); got != c.said {
|
||||
t.Errorf("%s: at its end said %q, want %q", c.name, got, c.said)
|
||||
}
|
||||
if co, open := r.open(own); c.standing != open || (open && !strings.Contains(co.Explanation, c.line)) {
|
||||
t.Errorf("%s: its own condition is %v: %q", c.name, open, co.Explanation)
|
||||
}
|
||||
if _, open := r.open("core.controller.deaf"); open || len(r.k.Held()) != 0 {
|
||||
t.Errorf("%s: the carried deaf condition was raised, or something is still held: %+v", c.name, r.k.Held())
|
||||
}
|
||||
}
|
||||
// A seat's finding is about the module the placement says holds the seat on that node (the lead, 2026-10-29): held
|
||||
// while the assigner is delivered on ace, though the seat is named mesh-placement.
|
||||
r := newRig()
|
||||
placed, err := delivery.EncodePlacement(delivery.Placement{Node: "ace", Modules: []delivery.PlacedModule{{Module: "mesh-delivery-assigner"}},
|
||||
Holds: []delivery.Holding{{Claim: "mesh-placement", Scope: "mesh", Module: "mesh-delivery-assigner"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.k.Placement("placement.ace", placed, false)
|
||||
_ = r.k.Want("want.ace.mesh-delivery-assigner", want(ctrlNext, ctrlNextComp), false)
|
||||
_ = r.k.Have("ace.mesh-delivery-assigner", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false)
|
||||
seat, seatRaw := finding(t, selfCheck, health.ScopeSeat, "mesh-placement.ace", "holder-silent", "D3", health.Warning, r.now)
|
||||
_ = r.k.Finding(selfCheck, seat, seatRaw, false)
|
||||
if got := r.changes(); got != "" || len(r.k.Held()) != 1 || r.k.Held()[0].Module != "mesh-delivery-assigner" {
|
||||
t.Errorf("a seat's finding while its holder is delivered: said %q, held %+v", got, r.k.Held())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
// **Where each condition kind lives after the take-over** (novox/hq ADR 0564 clauses 4, 6, 7 and 9, the Context's probe
|
||||
// table; issue 1952 C3, C5): data, so a kind's home is a row and never code. `compare` reads it to tell a difference from
|
||||
// a kind that ends with the controller or that the controller never raises, and the keeper reads it to carry, until the
|
||||
// lease write, the controller's conditions of the kinds that end with it.
|
||||
//
|
||||
// The rows below are what ADR 0564 decides, by the controller's source of each kind as the live controller says it
|
||||
// (mesh-controller.conditions, read 2026-10-29). **About twenty-five kinds the controller raises have no home in the
|
||||
// record** (L4.md, "Missing decision"): they have no row here on purpose, and `compare` names each as "no home decided"
|
||||
// until the lead's decision adds its row.
|
||||
|
||||
// Home is where one condition kind lives.
|
||||
type Home struct {
|
||||
// Source is the controller's row or probe that raises it today, for whoever reads the table.
|
||||
Source string
|
||||
// Controller says the controller raises this kind today: a kind it never raises is the keeper's alone and is no
|
||||
// difference in compare.
|
||||
Controller bool
|
||||
// Observer is the module whose findings raise it after the take-over; empty for a kind that ends.
|
||||
Observer string
|
||||
// Carried says it ends with the controller's lease and is carried from the controller's bucket until then
|
||||
// (clause 9: S4, S7, S12, S13, S14, S25, D5, D9, H-controller).
|
||||
Carried bool
|
||||
// Replaced says the kind itself ends and another observes its fault (D1, D10, the stuck probe): listed apart in
|
||||
// compare and not carried.
|
||||
Replaced bool
|
||||
}
|
||||
|
||||
// The kinds' homes, by kind.
|
||||
var Homes = map[string]Home{
|
||||
// The watchman's watchdogs (clause 4).
|
||||
"silent": {Source: "S1", Controller: true, Observer: watchman},
|
||||
"max-deliveries": {Source: "S9", Controller: true, Observer: watchman},
|
||||
"slow-consumer": {Source: "S9", Controller: true, Observer: watchman},
|
||||
"tools-silent": {Source: "S11", Controller: true, Observer: watchman},
|
||||
"apply-failing": {Source: "S21", Controller: true, Observer: watchman},
|
||||
"module-unhealthy": {Source: "health", Controller: true, Observer: watchman},
|
||||
"stuck": {Source: "DM", Controller: true, Observer: watchman},
|
||||
// The self-check's probes (clause 6, the Context's table).
|
||||
"holder-silent": {Source: "D3", Controller: true, Observer: selfCheck},
|
||||
"archives-missing": {Source: "D4", Controller: true, Observer: selfCheck},
|
||||
"consumer-wrong": {Source: "D6", Controller: true, Observer: selfCheck},
|
||||
"data-unmeasured": {Source: "D13", Controller: true, Observer: selfCheck},
|
||||
"agent-can-become-root": {Source: "DA", Controller: true, Observer: selfCheck},
|
||||
"root-not-free": {Source: "D-root", Controller: true, Observer: selfCheck},
|
||||
"probe-failed": {Source: "doctor", Controller: true, Observer: selfCheck},
|
||||
// The healers' own (clause 7, Limits 2).
|
||||
"healer-wanted": {Source: "S15", Controller: true, Observer: healers},
|
||||
"healers-braked": {Source: "healers", Controller: true, Observer: healers},
|
||||
// The kinds that end with the controller and are carried until its lease write (clauses 4, 9).
|
||||
"controller-deaf": {Source: "S4", Controller: true, Carried: true},
|
||||
"lease-lost": {Source: "S12", Controller: true, Carried: true},
|
||||
// The kinds that end, their faults observed otherwise (the Context's table).
|
||||
"core-behind": {Source: "D10", Controller: true, Replaced: true},
|
||||
}
|
||||
|
||||
// The observers, as the table names them.
|
||||
const (
|
||||
watchman = "mesh-delivery-watchman"
|
||||
selfCheck = "mesh-self-check"
|
||||
healers = "mesh-healers"
|
||||
)
|
||||
|
||||
// homeOf is the home of a condition's kind, and whether the table has one.
|
||||
func (k *Keeper) homeOf(kind string) (Home, bool) {
|
||||
homes := k.Homes
|
||||
if homes == nil {
|
||||
homes = Homes
|
||||
}
|
||||
h, ok := homes[kind]
|
||||
return h, ok
|
||||
}
|
||||
@@ -0,0 +1,665 @@
|
||||
package main
|
||||
|
||||
// The condition keeper (novox/hq ADR 0564 clauses 2, 3, 7, 8, 9 and 13; issue 1952 C2, C3, C7): the one writer of the
|
||||
// mesh's open conditions. It watches every observer's `findings`, the healers' `attempts`, the controller's lease and,
|
||||
// until the lease write, the controller's `conditions`, and the assigner's wants and the nodes' haves for the hold. From
|
||||
// all of them it derives, per key, the condition that should be open, and writes the difference by compare-and-set,
|
||||
// saying each transition as the controller says it (go/health) and keeping it in `history`. Nothing clears a condition
|
||||
// but its finding going (ADR 0231 §14).
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The module, its seat, and the states it keeps.
|
||||
const (
|
||||
Module = health.KeeperModule
|
||||
Seat = "mesh-conditions"
|
||||
|
||||
ConditionsState = health.ConditionsState
|
||||
TakeoverState = health.TakeoverState
|
||||
HistoryState = "history"
|
||||
HeldState = "held"
|
||||
BoundsState = "bounds"
|
||||
// AskedState keeps the keeper's asks of the operator (go/asker).
|
||||
AskedState = "asked"
|
||||
)
|
||||
|
||||
// HistoryKeptFor is how long a transition is kept (clause 3).
|
||||
const HistoryKeptFor = 90 * 24 * time.Hour
|
||||
|
||||
// Observers are the modules whose `findings` the keeper reads (clause 2; ADR 0565 §8 for the assigner's).
|
||||
var Observers = []string{health.WatchmanModule, health.SelfCheckModule, health.HealersModule, delivery.AssignerModule}
|
||||
|
||||
// The other states it reads, as `<module>.<name>`.
|
||||
const (
|
||||
AttemptsRead = health.HealersModule + "." + health.AttemptsState
|
||||
ControllerConditions = "mesh-controller.conditions"
|
||||
ControllerLease = "mesh-controller.lease"
|
||||
ControllerLeaseKey = "holder"
|
||||
WantsRead = delivery.AssignerModule + ".wants"
|
||||
// PlacementRead is the mesh-placement seat's placement, read for which module holds a seat on a node (hold.go; the
|
||||
// lead, 2026-10-29, for ADR 0564 clause 2's inputs).
|
||||
PlacementRead = delivery.PlacementRead
|
||||
HavesRead = "node-engine.have"
|
||||
ConditionWrites = 8
|
||||
)
|
||||
|
||||
// Store is the bus as the keeper uses it: its own states, written by compare-and-set, and a key with its own time to
|
||||
// live for a hold's bound.
|
||||
type Store interface {
|
||||
Get(state, key string) (value []byte, revision uint64, found bool, err error)
|
||||
Create(state, key string, value any) (uint64, error)
|
||||
Update(state, key string, value any, revision uint64) (uint64, error)
|
||||
Delete(state, key string) error
|
||||
Keys(state string) ([]string, error)
|
||||
CreateTTL(state, key string, value any, ttl time.Duration) error
|
||||
}
|
||||
|
||||
// held is a finding or a carried condition kept while its module is delivered (clause 13 (b)).
|
||||
type held struct {
|
||||
Key string `json:"key"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Since time.Time `json:"since"`
|
||||
Version string `json:"version"`
|
||||
Wanted json.RawMessage `json:"-"`
|
||||
}
|
||||
|
||||
// clearedMemo is what a cleared condition leaves for ReopenWithin: the same fault again is a reopening.
|
||||
type clearedMemo struct {
|
||||
at time.Time
|
||||
count int
|
||||
began time.Time
|
||||
gaps []health.Gap
|
||||
silenced *health.Silence
|
||||
tried []health.Tried
|
||||
}
|
||||
|
||||
// Keeper is the condition keeper's state and its writes.
|
||||
type Keeper struct {
|
||||
Store Store
|
||||
// Say publishes one event on the seat; History keeps one transition.
|
||||
Say func(e health.Event) error
|
||||
Now func() time.Time
|
||||
Logf func(format string, a ...any)
|
||||
// Homes says, by kind, where each controller kind lives after the take-over (homes.go); Homes when nil.
|
||||
Homes map[string]Home
|
||||
// Bound is a delivery's hold bound; delivery.DefaultDeliveryBound when zero (hold.go).
|
||||
Bound time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
findings map[string]map[string]health.Finding
|
||||
blind map[string]string
|
||||
attempts map[string]health.Attempt
|
||||
ctrl map[string]health.Condition
|
||||
leaseOut bool
|
||||
takeover health.Takeover
|
||||
wants map[string]delivery.Want
|
||||
haves map[string]delivery.Have
|
||||
// holders are, by `<node>.<seat>`, the module the placement says holds the seat there.
|
||||
holders map[string]string
|
||||
stuck map[string]string
|
||||
held map[string]held
|
||||
// passed are the deliveries whose bound passed, by node and module, at the fingerprint wanted then; outOfHold the
|
||||
// line a key raised out of a hold says (hold.go).
|
||||
passed map[string]string
|
||||
outOfHold map[string]string
|
||||
cleared map[string]clearedMemo
|
||||
seq uint64
|
||||
}
|
||||
|
||||
func (k *Keeper) init() {
|
||||
if k.findings == nil {
|
||||
k.findings, k.blind, k.attempts, k.ctrl = map[string]map[string]health.Finding{}, map[string]string{},
|
||||
map[string]health.Attempt{}, map[string]health.Condition{}
|
||||
k.wants, k.haves, k.stuck, k.held, k.cleared = map[string]delivery.Want{}, map[string]delivery.Have{},
|
||||
map[string]string{}, map[string]held{}, map[string]clearedMemo{}
|
||||
k.takeover = health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController}
|
||||
k.passed, k.outOfHold, k.holders = map[string]string{}, map[string]string{}, map[string]string{}
|
||||
}
|
||||
if k.Now == nil {
|
||||
k.Now = time.Now
|
||||
}
|
||||
if k.Logf == nil {
|
||||
k.Logf = func(string, ...any) {}
|
||||
}
|
||||
}
|
||||
|
||||
// Finding takes one change of an observer's finding: a key put raises or changes its condition, a key deleted clears
|
||||
// it (clause 2). A value that is not a finding is the observer's fault, said and left as it was.
|
||||
func (k *Keeper) Finding(observer, key string, value []byte, deleted bool) error {
|
||||
k.mu.Lock()
|
||||
k.init()
|
||||
if k.findings[observer] == nil {
|
||||
k.findings[observer] = map[string]health.Finding{}
|
||||
}
|
||||
if deleted {
|
||||
delete(k.findings[observer], key)
|
||||
} else {
|
||||
f, err := health.DecodeFinding(value)
|
||||
if err != nil {
|
||||
k.mu.Unlock()
|
||||
k.Logf("%s's finding %s is not one the keeper can raise, so it is left as it stood: %v", observer, key, err)
|
||||
return nil
|
||||
}
|
||||
if f.Key != key {
|
||||
k.mu.Unlock()
|
||||
k.Logf("%s's finding under %s names the key %s, so it is left as it stood", observer, key, f.Key)
|
||||
return nil
|
||||
}
|
||||
k.findings[observer][key] = f
|
||||
}
|
||||
k.mu.Unlock()
|
||||
return k.Settle(key)
|
||||
}
|
||||
|
||||
// Blind says an observer's findings cannot be read: its keys stay as they were, and nothing is cleared for not seeing
|
||||
// (ADR 0227 §4). Empty why says it is read again.
|
||||
func (k *Keeper) Blind(observer, why string) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
k.init()
|
||||
if why == "" {
|
||||
delete(k.blind, observer)
|
||||
} else {
|
||||
k.blind[observer] = why
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt takes one change of the healers' `attempts`: the condition it names gets its tried, resolver and escalation
|
||||
// (clause 7).
|
||||
func (k *Keeper) Attempt(key string, value []byte, deleted bool) error {
|
||||
k.mu.Lock()
|
||||
k.init()
|
||||
var condition string
|
||||
if old, ok := k.attempts[key]; ok {
|
||||
condition = old.Condition
|
||||
}
|
||||
if deleted {
|
||||
delete(k.attempts, key)
|
||||
} else {
|
||||
var a health.Attempt
|
||||
if err := json.Unmarshal(value, &a); err != nil || a.Condition == "" {
|
||||
k.mu.Unlock()
|
||||
k.Logf("the healers' attempt %s is not readable, so it is left out: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
k.attempts[key], condition = a, a.Condition
|
||||
}
|
||||
k.mu.Unlock()
|
||||
if condition == "" {
|
||||
return nil
|
||||
}
|
||||
return k.Settle(condition)
|
||||
}
|
||||
|
||||
// Controller takes one change of the controller's `conditions` bucket (clauses 8, 9): read until the lease write.
|
||||
func (k *Keeper) Controller(key string, value []byte, deleted bool) error {
|
||||
k.mu.Lock()
|
||||
k.init()
|
||||
if deleted {
|
||||
delete(k.ctrl, key)
|
||||
} else {
|
||||
var c health.Condition
|
||||
if err := json.Unmarshal(value, &c); err != nil {
|
||||
k.mu.Unlock()
|
||||
k.Logf("the controller's condition %s is not readable, so nothing is carried from it: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
k.ctrl[key] = c
|
||||
}
|
||||
k.mu.Unlock()
|
||||
return k.Settle(key)
|
||||
}
|
||||
|
||||
// Lease takes the controller's lease key as read (clause 8, Limits 8): once it is handed to a holder that is no
|
||||
// controller, nothing more is carried from the controller's bucket, and the kinds that ended with it clear.
|
||||
func (k *Keeper) Lease(away bool) error {
|
||||
k.mu.Lock()
|
||||
k.init()
|
||||
was := k.leaseOut
|
||||
k.leaseOut = away
|
||||
keys := make([]string, 0, len(k.ctrl))
|
||||
for key := range k.ctrl {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
k.mu.Unlock()
|
||||
if was == away {
|
||||
return nil
|
||||
}
|
||||
return k.settleAll(keys)
|
||||
}
|
||||
|
||||
// LeaseAway says whether the controller's lease is handed away.
|
||||
func (k *Keeper) LeaseAway() bool {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.leaseOut
|
||||
}
|
||||
|
||||
// Takeover takes the keeper's own take-over key as read.
|
||||
func (k *Keeper) Takeover(t health.Takeover) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
k.init()
|
||||
k.takeover = t
|
||||
}
|
||||
|
||||
// Settle derives the condition key should be and writes the difference (clause 2), retried while another write moves
|
||||
// it under this one.
|
||||
func (k *Keeper) Settle(key string) error {
|
||||
var err error
|
||||
for i := 0; i < ConditionWrites; i++ {
|
||||
if err = k.settleOnce(key); !errors.Is(err, stdio.ErrChanged) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the condition %s kept moving under this write, %d tries: %w", key, ConditionWrites, err)
|
||||
}
|
||||
|
||||
func (k *Keeper) settleAll(keys []string) error {
|
||||
sort.Strings(keys)
|
||||
var problems []string
|
||||
for _, key := range keys {
|
||||
if err := k.Settle(key); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return errors.New(strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// wanted is what the inputs say key should be: a finding of an observer, or a condition carried from the controller's
|
||||
// bucket of a kind that ends with the controller, until the lease write (clause 9). Nil when nothing says it.
|
||||
func (k *Keeper) wanted(key string) (*health.Finding, *health.Condition) {
|
||||
observers := make([]string, 0, len(k.findings))
|
||||
for o := range k.findings {
|
||||
observers = append(observers, o)
|
||||
}
|
||||
sort.Strings(observers)
|
||||
for _, o := range observers {
|
||||
if f, ok := k.findings[o][key]; ok {
|
||||
return &f, nil
|
||||
}
|
||||
}
|
||||
if c, ok := k.ctrl[key]; ok && !k.leaseOut && k.endsWithController(c) {
|
||||
return nil, &c
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (k *Keeper) endsWithController(c health.Condition) bool {
|
||||
h, ok := k.homeOf(c.Kind)
|
||||
return ok && h.Carried
|
||||
}
|
||||
|
||||
// settleOnce is one try of Settle.
|
||||
func (k *Keeper) settleOnce(key string) error {
|
||||
now := k.Now().UTC()
|
||||
raw, rev, open, err := k.Store.Get(ConditionsState, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the condition %s: %w", key, err)
|
||||
}
|
||||
var was health.Condition
|
||||
if open {
|
||||
if err := json.Unmarshal(raw, &was); err != nil {
|
||||
return fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||
}
|
||||
}
|
||||
k.mu.Lock()
|
||||
k.init()
|
||||
f, carried := k.wanted(key)
|
||||
hold, holding := k.holdFor(key, f, carried, open)
|
||||
k.mu.Unlock()
|
||||
|
||||
if f == nil && carried == nil {
|
||||
k.dropHeld(key, "its finding went while its module was delivered")
|
||||
if !open {
|
||||
return nil
|
||||
}
|
||||
return k.clear(key, was, rev, now, "its finding went")
|
||||
}
|
||||
if holding {
|
||||
return k.keepHeld(hold)
|
||||
}
|
||||
k.dropHeld(key, "")
|
||||
next := k.derive(key, was, open, f, carried, now)
|
||||
events := transitions(was, next, open)
|
||||
if len(events) == 0 && open && sameBody(was, next) {
|
||||
return nil
|
||||
}
|
||||
if open {
|
||||
_, err = k.Store.Update(ConditionsState, key, next, rev)
|
||||
} else {
|
||||
_, err = k.Store.Create(ConditionsState, key, next)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
k.mu.Lock()
|
||||
delete(k.cleared, key)
|
||||
k.mu.Unlock()
|
||||
for _, e := range events {
|
||||
e.Condition, e.At, e.Event = next, now, health.EventFor(e.Change)
|
||||
e.Show = k.show(next, carried != nil)
|
||||
k.tell(e)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *Keeper) show(c health.Condition, carried bool) string {
|
||||
if carried {
|
||||
return health.ShowController(c.Key)
|
||||
}
|
||||
return health.ShowKeeper(c.Key)
|
||||
}
|
||||
|
||||
// derive is the condition key should be now, from what was open and what the inputs say.
|
||||
func (k *Keeper) derive(key string, was health.Condition, open bool, f *health.Finding, carried *health.Condition, now time.Time) health.Condition {
|
||||
var c health.Condition
|
||||
switch {
|
||||
case carried != nil:
|
||||
// Raised and cleared as the controller's bucket says, with its show and actions (Limits 9).
|
||||
c = *carried
|
||||
c.Epoch = 0
|
||||
c.LooksUnread = ""
|
||||
case open:
|
||||
c = was
|
||||
k.fromFinding(&c, *f, now)
|
||||
default:
|
||||
c = health.Condition{Key: key, Raised: now, Count: 1, Resolver: health.ResolverSelf}
|
||||
k.fromFinding(&c, *f, now)
|
||||
k.mu.Lock()
|
||||
if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= health.ReopenWithin {
|
||||
c.Count = before.count + 1
|
||||
c.First = before.began
|
||||
c.Gaps = append(append([]health.Gap(nil), before.gaps...), health.Gap{Cleared: before.at, Reopened: now})
|
||||
if over := len(c.Gaps) - health.KeptGaps; over > 0 {
|
||||
c.First, c.Gaps = c.Gaps[over-1].Reopened, c.Gaps[over:]
|
||||
}
|
||||
if before.silenced != nil && now.Before(before.silenced.Until) {
|
||||
c.Silenced = before.silenced
|
||||
}
|
||||
c.Tried = before.tried
|
||||
}
|
||||
k.mu.Unlock()
|
||||
}
|
||||
if carried == nil {
|
||||
k.overlay(&c, now)
|
||||
}
|
||||
if !open {
|
||||
k.mu.Lock()
|
||||
if line := k.outOfHold[key]; line != "" {
|
||||
c.Explanation = strings.TrimSpace(c.Explanation + " " + line)
|
||||
}
|
||||
k.mu.Unlock()
|
||||
}
|
||||
if c.Silenced != nil && !now.Before(c.Silenced.Until) {
|
||||
c.Silenced = nil
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// fromFinding is what the condition says of its finding: its words, subject, severity and source, and its evidence
|
||||
// newest first (clause 2). A healer's escalation is not undone by the finding saying a warning again.
|
||||
func (k *Keeper) fromFinding(c *health.Condition, f health.Finding, now time.Time) {
|
||||
escalated := c.Escalated()
|
||||
c.Kind, c.Subject, c.Summary, c.Source = f.Kind, f.Subject, f.Summary, f.Source
|
||||
c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = f.Headline, f.Explanation, f.Resolved, f.Needs, f.Actions
|
||||
if !(escalated && f.Severity != health.Urgent) {
|
||||
c.Severity = f.Severity
|
||||
}
|
||||
if !escalated {
|
||||
c.Resolver = f.Resolver
|
||||
if c.Resolver == "" {
|
||||
c.Resolver = health.ResolverSelf
|
||||
}
|
||||
}
|
||||
if len(f.Evidence) > 0 {
|
||||
c.Evidence = append([]health.Evidence(nil), f.Evidence...)
|
||||
c.LastObserved = f.Evidence[0].At
|
||||
} else if c.Evidence == nil {
|
||||
c.Evidence = []health.Evidence{{At: f.At, Said: f.Summary}}
|
||||
c.LastObserved = f.At
|
||||
}
|
||||
c.Observations = len(c.Evidence)
|
||||
if c.Actions == nil {
|
||||
c.Actions = []health.Action{}
|
||||
}
|
||||
_ = now
|
||||
}
|
||||
|
||||
// overlay writes the healers' attempts into the condition (clause 7) and, until the lease write, what the controller's
|
||||
// bucket holds for the same key (clause 8): its tried, its resolver and escalation, and its silence where the keeper has
|
||||
// none of its own.
|
||||
func (k *Keeper) overlay(c *health.Condition, now time.Time) {
|
||||
k.mu.Lock()
|
||||
var mine []health.Attempt
|
||||
for _, a := range k.attempts {
|
||||
if a.Condition == c.Key {
|
||||
mine = append(mine, a)
|
||||
}
|
||||
}
|
||||
ctrl, carry := k.ctrl[c.Key]
|
||||
carry = carry && !k.leaseOut
|
||||
k.mu.Unlock()
|
||||
sort.Slice(mine, func(i, j int) bool { return mine[i].Begun.Before(mine[j].Begun) })
|
||||
if carry && len(ctrl.Tried) > 0 {
|
||||
c.Tried = append([]health.Tried(nil), ctrl.Tried...)
|
||||
if ctrl.Escalated() {
|
||||
c.Resolver, c.Severity = health.ResolverOperator, health.Urgent
|
||||
} else if ctrl.Resolver != "" && ctrl.Resolver != health.ResolverSelf {
|
||||
c.Resolver = ctrl.Resolver
|
||||
}
|
||||
}
|
||||
for _, a := range mine {
|
||||
t := a.Tried()
|
||||
if !containsTried(c.Tried, t) {
|
||||
c.Tried = append(c.Tried, t)
|
||||
}
|
||||
switch a.Outcome {
|
||||
case health.OutcomeEscalated:
|
||||
c.Resolver, c.Severity = health.ResolverOperator, health.Urgent
|
||||
default:
|
||||
if c.Resolver != health.ResolverOperator {
|
||||
c.Resolver = health.ResolverHealer(a.Healer)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(c.Tried) > health.KeptTried {
|
||||
c.Tried = c.Tried[len(c.Tried)-health.KeptTried:]
|
||||
}
|
||||
if carry && c.Silenced == nil && ctrl.Silenced != nil && now.Before(ctrl.Silenced.Until) {
|
||||
c.Silenced = ctrl.Silenced
|
||||
}
|
||||
}
|
||||
|
||||
func containsTried(list []health.Tried, t health.Tried) bool {
|
||||
for _, x := range list {
|
||||
if x.At.Equal(t.At) && x.By == t.By && x.What == t.What {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// transitions are the events between what was open and what is now, in the controller's order: raised or reopened;
|
||||
// else severity, resolver, silence and words changes. An observation that changes none of these says nothing.
|
||||
func transitions(was, now health.Condition, open bool) []health.Event {
|
||||
if !open {
|
||||
change := health.ChangeRaised
|
||||
if now.Count > 1 {
|
||||
change = health.ChangeReopened
|
||||
}
|
||||
return []health.Event{{Change: change}}
|
||||
}
|
||||
var out []health.Event
|
||||
if was.Severity != now.Severity {
|
||||
out = append(out, health.Event{Change: health.ChangeSeverity, Was: string(was.Severity)})
|
||||
}
|
||||
if was.Resolver != now.Resolver {
|
||||
out = append(out, health.Event{Change: health.ChangeResolver, Was: was.Resolver})
|
||||
}
|
||||
switch {
|
||||
case was.Silenced == nil && now.Silenced != nil:
|
||||
out = append(out, health.Event{Change: health.ChangeSilenced, Why: now.Silenced.Why})
|
||||
case was.Silenced != nil && now.Silenced == nil:
|
||||
out = append(out, health.Event{Change: health.ChangeUnsilenced})
|
||||
}
|
||||
if was.Needs != now.Needs || !sameActions(was.Actions, now.Actions) {
|
||||
out = append(out, health.Event{Change: health.ChangeWords, Was: was.Needs})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sameActions(a, b []health.Action) bool {
|
||||
x, _ := json.Marshal(a)
|
||||
y, _ := json.Marshal(b)
|
||||
return string(x) == string(y) || (len(a) == 0 && len(b) == 0)
|
||||
}
|
||||
|
||||
func sameBody(a, b health.Condition) bool {
|
||||
x, _ := json.Marshal(a)
|
||||
y, _ := json.Marshal(b)
|
||||
return string(x) == string(y)
|
||||
}
|
||||
|
||||
// clear removes an open condition whose finding went, and says it (clause 2).
|
||||
func (k *Keeper) clear(key string, was health.Condition, rev uint64, now time.Time, why string) error {
|
||||
// A delete at the revision read: a write that moved it meanwhile is read again.
|
||||
if _, err := k.Store.Update(ConditionsState, key, json.RawMessage("null"), rev); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.Store.Delete(ConditionsState, key); err != nil {
|
||||
return err
|
||||
}
|
||||
k.mu.Lock()
|
||||
k.cleared[key] = clearedMemo{at: now, count: was.Count, began: was.Began(), gaps: was.Gaps, silenced: was.Silenced, tried: was.Tried}
|
||||
k.mu.Unlock()
|
||||
cleared := now
|
||||
k.tell(health.Event{Condition: was, Event: health.EventCleared, At: now, Change: health.ChangeCleared, Why: why,
|
||||
Cleared: &cleared, Show: k.show(was, false)})
|
||||
return nil
|
||||
}
|
||||
|
||||
// tell keeps the transition in history and says it on the seat. A transition not said is said in the log: the condition
|
||||
// and its history still hold it.
|
||||
func (k *Keeper) tell(e health.Event) {
|
||||
k.mu.Lock()
|
||||
k.seq++
|
||||
hk := fmt.Sprintf("%020d-%06d", e.At.UnixNano(), k.seq)
|
||||
k.mu.Unlock()
|
||||
if _, err := k.Store.Create(HistoryState, hk, e); err != nil {
|
||||
k.Logf("the %s of %s could not be kept in history: %v", e.Change, e.Key, err)
|
||||
}
|
||||
if k.Say == nil {
|
||||
return
|
||||
}
|
||||
if err := k.Say(e); err != nil {
|
||||
k.Logf("the %s of %s could NOT be said on the seat: %v", e.Change, e.Key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Open is every open condition, ordered as `conditions` says them.
|
||||
func (k *Keeper) Open() ([]health.Condition, error) {
|
||||
keys, err := k.Store.Keys(ConditionsState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []health.Condition
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := k.Store.Get(ConditionsState, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found || string(raw) == "null" {
|
||||
continue
|
||||
}
|
||||
var c health.Condition
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return nil, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err)
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
health.Order(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Resync settles every key any input names or any open condition holds (clause 12): what differs is raised or cleared.
|
||||
func (k *Keeper) Resync() (int, error) {
|
||||
open, err := k.Open()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, c := range open {
|
||||
seen[c.Key] = true
|
||||
}
|
||||
k.mu.Lock()
|
||||
k.init()
|
||||
for o, fs := range k.findings {
|
||||
if k.blind[o] != "" {
|
||||
continue
|
||||
}
|
||||
for key := range fs {
|
||||
seen[key] = true
|
||||
}
|
||||
}
|
||||
for key := range k.ctrl {
|
||||
seen[key] = true
|
||||
}
|
||||
k.mu.Unlock()
|
||||
keys := make([]string, 0, len(seen))
|
||||
for key := range seen {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
return len(keys), k.settleAll(keys)
|
||||
}
|
||||
|
||||
// Silence stops a condition's messages for a while, a hand act with why (clause 3, Limits 1): kept as its transition in
|
||||
// history, naming who and why.
|
||||
func (k *Keeper) Silence(key string, d time.Duration, by, why string) (health.Condition, error) {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return health.Condition{}, errors.New("a silence says why")
|
||||
}
|
||||
if d <= 0 || d > health.MaxSilence {
|
||||
return health.Condition{}, fmt.Errorf("a silence lasts more than nothing and at most %s, not %s", health.MaxSilence, d)
|
||||
}
|
||||
for i := 0; i < ConditionWrites; i++ {
|
||||
raw, rev, found, err := k.Store.Get(ConditionsState, key)
|
||||
if err != nil {
|
||||
return health.Condition{}, err
|
||||
}
|
||||
if !found || string(raw) == "null" {
|
||||
return health.Condition{}, fmt.Errorf("no condition %s is open", key)
|
||||
}
|
||||
var c health.Condition
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return health.Condition{}, err
|
||||
}
|
||||
now := k.Now().UTC()
|
||||
c.Silenced = &health.Silence{Until: now.Add(d), By: by, Why: why, Since: now}
|
||||
if _, err := k.Store.Update(ConditionsState, key, c, rev); errors.Is(err, stdio.ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return health.Condition{}, err
|
||||
}
|
||||
k.tell(health.Event{Condition: c, Event: health.EventChanged, At: now, Change: health.ChangeSilenced,
|
||||
Why: why + " (" + by + ")", Show: health.ShowKeeper(key)})
|
||||
return c, nil
|
||||
}
|
||||
return health.Condition{}, fmt.Errorf("the condition %s kept moving under the silence", key)
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// fakeStore is the bus's states in memory, with revisions, as the runtime answers them.
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
next uint64
|
||||
// ttls are the times to live the keys were created with.
|
||||
ttls map[string]time.Duration
|
||||
}
|
||||
|
||||
func newFakeStore() *fakeStore {
|
||||
return &fakeStore{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
}
|
||||
|
||||
func (f *fakeStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
v, ok := f.vals[state+"/"+key]
|
||||
return v, f.revs[state+"/"+key], ok, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) put(state, key string, value any) uint64 {
|
||||
raw, _ := json.Marshal(value)
|
||||
f.next++
|
||||
f.vals[state+"/"+key], f.revs[state+"/"+key] = raw, f.next
|
||||
return f.next
|
||||
}
|
||||
|
||||
func (f *fakeStore) Create(state, key string, value any) (uint64, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if _, ok := f.vals[state+"/"+key]; ok {
|
||||
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
|
||||
}
|
||||
return f.put(state, key, value), nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) Update(state, key string, value any, rev uint64) (uint64, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.revs[state+"/"+key] != rev {
|
||||
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
|
||||
}
|
||||
return f.put(state, key, value), nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) Delete(state, key string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.vals, state+"/"+key)
|
||||
delete(f.revs, state+"/"+key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) Keys(state string) ([]string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []string
|
||||
for k := range f.vals {
|
||||
if s, key, _ := strings.Cut(k, "/"); s == state {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
|
||||
f.mu.Lock()
|
||||
if f.ttls == nil {
|
||||
f.ttls = map[string]time.Duration{}
|
||||
}
|
||||
f.ttls[state+"/"+key] = ttl
|
||||
f.mu.Unlock()
|
||||
_, err := f.Create(state, key, value)
|
||||
return err
|
||||
}
|
||||
|
||||
// rig is a keeper over a fake store, its clock and what it said.
|
||||
type rig struct {
|
||||
k *Keeper
|
||||
st *fakeStore
|
||||
now time.Time
|
||||
said []health.Event
|
||||
}
|
||||
|
||||
func newRig() *rig {
|
||||
r := &rig{st: newFakeStore(), now: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)}
|
||||
r.k = &Keeper{Store: r.st, Now: func() time.Time { return r.now }, Say: func(e health.Event) error {
|
||||
r.said = append(r.said, e)
|
||||
return nil
|
||||
}}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *rig) changes() string {
|
||||
var out []string
|
||||
for _, e := range r.said {
|
||||
out = append(out, e.Change+" "+e.Key)
|
||||
}
|
||||
r.said = nil
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
func (r *rig) open(key string) (health.Condition, bool) {
|
||||
raw, _, ok, _ := r.st.Get(ConditionsState, key)
|
||||
var c health.Condition
|
||||
if ok {
|
||||
_ = json.Unmarshal(raw, &c)
|
||||
}
|
||||
return c, ok
|
||||
}
|
||||
|
||||
// finding is an observer's finding as mesh-sdk's layout writes it (health.EncodeFinding).
|
||||
func finding(t *testing.T, observer, scope, id, kind, source string, sev health.Severity, at time.Time) (string, []byte) {
|
||||
t.Helper()
|
||||
f := health.Finding{Key: health.Key(scope, id, kind), Kind: kind, Subject: health.Subject{Scope: scope, ID: id},
|
||||
Severity: sev, Summary: kind + " of " + id, Headline: kind, Explanation: "It is " + kind + ".", Source: source,
|
||||
Observer: observer, At: at, Evidence: []health.Evidence{{At: at, Said: kind}}}
|
||||
raw, err := health.EncodeFinding(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f.Key, raw
|
||||
}
|
||||
|
||||
// ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer
|
||||
// clears nothing; the healers' attempts and the controller's escalation reach the condition; a reopening within ten
|
||||
// minutes is the same fault; a silence is a transition in history.
|
||||
func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) {
|
||||
r := newRig()
|
||||
key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now)
|
||||
steps := []struct {
|
||||
name string
|
||||
do func() error
|
||||
said string
|
||||
check func(c health.Condition, open bool) string
|
||||
}{
|
||||
{name: "a finding written raises", do: func() error { return r.k.Finding(watchman, key, raw, false) },
|
||||
said: "raised machine.g14.silent", check: func(c health.Condition, open bool) string {
|
||||
if !open || c.Severity != health.Warning || c.Resolver != health.ResolverSelf || c.Epoch != 0 || c.Count != 1 {
|
||||
return fmt.Sprintf("%+v", c)
|
||||
}
|
||||
return ""
|
||||
}},
|
||||
{name: "the same finding again says nothing", do: func() error { return r.k.Finding(watchman, key, raw, false) }},
|
||||
{name: "an observer gone blind clears nothing", do: func() error { r.k.Blind(watchman, "refused"); _, err := r.k.Resync(); return err },
|
||||
check: func(c health.Condition, open bool) string {
|
||||
if !open {
|
||||
return "cleared by blindness"
|
||||
}
|
||||
return ""
|
||||
}},
|
||||
{name: "a healer's attempt reaches tried and the resolver", do: func() error {
|
||||
a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "asserted", Outcome: health.OutcomeActed, Begun: r.now})
|
||||
return r.k.Attempt("h3-1", a, false)
|
||||
}, said: "resolver machine.g14.silent", check: func(c health.Condition, _ bool) string {
|
||||
if c.Resolver != "healer:H3" || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" {
|
||||
return fmt.Sprintf("%+v", c)
|
||||
}
|
||||
return ""
|
||||
}},
|
||||
{name: "an escalation makes it urgent, the operator's", do: func() error {
|
||||
a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "handed over", Outcome: health.OutcomeEscalated,
|
||||
Begun: r.now.Add(time.Minute)})
|
||||
return r.k.Attempt("h3-2", a, false)
|
||||
}, said: "severity machine.g14.silent, resolver machine.g14.silent", check: func(c health.Condition, _ bool) string {
|
||||
if c.Severity != health.Urgent || c.Resolver != health.ResolverOperator || len(c.Tried) != 2 {
|
||||
return fmt.Sprintf("%+v", c)
|
||||
}
|
||||
return ""
|
||||
}},
|
||||
{name: "a silence is a transition with who and why", do: func() error {
|
||||
_, err := r.k.Silence(key, time.Hour, "jochen", "known, a cable")
|
||||
return err
|
||||
}, said: "silenced machine.g14.silent"},
|
||||
{name: "the finding deleted clears", do: func() error { return r.k.Finding(watchman, key, nil, true) },
|
||||
said: "cleared machine.g14.silent", check: func(_ health.Condition, open bool) string {
|
||||
if open {
|
||||
return "still open"
|
||||
}
|
||||
return ""
|
||||
}},
|
||||
{name: "raised again within ten minutes is a reopening", do: func() error {
|
||||
r.now = r.now.Add(5 * time.Minute)
|
||||
return r.k.Finding(watchman, key, raw, false)
|
||||
}, said: "reopened machine.g14.silent", check: func(c health.Condition, _ bool) string {
|
||||
if c.Count != 2 || len(c.Gaps) != 1 {
|
||||
return fmt.Sprintf("%+v", c)
|
||||
}
|
||||
return ""
|
||||
}},
|
||||
// Until the lease write the controller's escalation for the same key reaches the condition (clause 8).
|
||||
{name: "the controller's escalation is carried", do: func() error {
|
||||
ctrl, _ := json.Marshal(health.Condition{Key: key, Kind: "silent", Severity: health.Urgent, Resolver: health.ResolverOperator,
|
||||
Tried: []health.Tried{{At: r.now, What: "x", Outcome: "escalated", By: "healer H1"}}})
|
||||
_ = r.k.Attempt("h3-1", nil, true)
|
||||
_ = r.k.Attempt("h3-2", nil, true)
|
||||
return r.k.Controller(key, ctrl, false)
|
||||
}, check: func(c health.Condition, _ bool) string {
|
||||
if c.Resolver != health.ResolverOperator || c.Severity != health.Urgent {
|
||||
return fmt.Sprintf("%+v", c)
|
||||
}
|
||||
return ""
|
||||
}},
|
||||
}
|
||||
for _, s := range steps {
|
||||
if err := s.do(); err != nil {
|
||||
t.Fatalf("%s: %v", s.name, err)
|
||||
}
|
||||
if got := r.changes(); s.said != "" && got != s.said || s.said == "" && got != "" && s.name != "the controller's escalation is carried" {
|
||||
t.Errorf("%s: said %q, want %q", s.name, got, s.said)
|
||||
}
|
||||
if s.check != nil {
|
||||
c, open := r.open(key)
|
||||
if why := s.check(c, open); why != "" {
|
||||
t.Errorf("%s: %s", s.name, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Every transition is kept in history, and each event carries the keeper's show.
|
||||
keys, _ := r.st.Keys(HistoryState)
|
||||
if len(keys) < 7 {
|
||||
t.Errorf("history keeps %d transitions", len(keys))
|
||||
}
|
||||
raw0, _, _, _ := r.st.Get(HistoryState, keys[0])
|
||||
if !strings.Contains(string(raw0), `"show":"mesh-conditions.conditions key=machine.g14.silent"`) {
|
||||
t.Errorf("the first transition is %s", raw0)
|
||||
}
|
||||
// A carried kind (it ends with the controller) is raised and cleared as the controller's bucket says, until the
|
||||
// lease is handed away, with the controller's show (clause 9, Limits 9).
|
||||
deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4",
|
||||
Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self", Epoch: 9})
|
||||
if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, open := r.open("core.controller.deaf"); !open || c.Epoch != 0 || r.said[0].Show != "mesh-controller.conditions key=core.controller.deaf" {
|
||||
t.Errorf("a carried kind is %+v, said %+v", c, r.said)
|
||||
}
|
||||
r.said = nil
|
||||
if err := r.k.Lease(true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, open := r.open("core.controller.deaf"); open || r.changes() != "cleared core.controller.deaf" {
|
||||
t.Error("a carried kind outlived the lease write")
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0564 clause 9 (issue 1952 C5, C8; the lead 2026-10-29): compare counts only what both sides can raise. A key held
|
||||
// while its module is delivered, a kind that ends with the controller and a kind the controller never raises are listed
|
||||
// apart; a kind with no home decided is a difference naming the gap; take-over is refused while compare differs, and
|
||||
// hand-back after the lease write.
|
||||
func TestCompareCountsOnlyWhatBothSidesRaise(t *testing.T) {
|
||||
r := newRig()
|
||||
put := func(key, kind string) []byte {
|
||||
raw, _ := json.Marshal(health.Condition{Key: key, Kind: kind, Severity: health.Warning, Source: "x"})
|
||||
return raw
|
||||
}
|
||||
key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now)
|
||||
_ = r.k.Finding(watchman, key, raw, false)
|
||||
_ = r.k.Controller(key, put(key, "silent"), false)
|
||||
only, onlyRaw := finding(t, delivery.AssignerModule, health.ScopeSeat, "mesh-placement.ace", "states-unbound", "say", health.Warning, r.now)
|
||||
_ = r.k.Finding(delivery.AssignerModule, only, onlyRaw, false)
|
||||
_ = r.k.Controller("core.controller.deaf", put("core.controller.deaf", "controller-deaf"), false)
|
||||
_ = r.k.Controller("delivery.x.stalled", put("delivery.x.stalled", "stalled"), false)
|
||||
_ = r.k.Controller("seat.vault.ace.holder-silent", put("seat.vault.ace.holder-silent", "holder-silent"), false)
|
||||
cmp, err := r.k.Compare()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reasons := map[string]string{}
|
||||
for _, d := range append(append([]Difference(nil), cmp.Differences...), cmp.Apart...) {
|
||||
reasons[d.Key] = d.Reason
|
||||
}
|
||||
for key, says := range map[string]string{
|
||||
only: "keeper only",
|
||||
"core.controller.deaf": "carried",
|
||||
"delivery.x.stalled": "no home is decided for the kind stalled",
|
||||
"seat.vault.ace.holder-silent": "mesh-self-check has not written its finding",
|
||||
} {
|
||||
if !strings.Contains(reasons[key], says) {
|
||||
t.Errorf("%s: %q, want it to say %q", key, reasons[key], says)
|
||||
}
|
||||
}
|
||||
if cmp.Same != 1 || len(cmp.Differences) != 2 {
|
||||
t.Errorf("compare: %+v", cmp)
|
||||
}
|
||||
if _, err := r.k.TakeOver(nil, "jochen", "now"); err == nil || !strings.Contains(err.Error(), "2 difference") {
|
||||
t.Errorf("take-over while compare differs: %v", err)
|
||||
}
|
||||
if _, err := r.k.HandBack("jochen", "back"); err != nil {
|
||||
t.Errorf("hand-back before the lease write: %v", err)
|
||||
}
|
||||
_ = r.k.Lease(true)
|
||||
if _, err := r.k.HandBack("jochen", "back"); err == nil {
|
||||
t.Error("hand-back after the lease write was taken")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
// mesh-conditions: the condition keeper, holder of the seat mesh-conditions (novox/hq ADR 0564 clauses 1-3, 9, 13). A
|
||||
// Go bundle served by a runtime of its own, on its own bus account (`runs-as`), so that its account alone writes its
|
||||
// states: `conditions` (one key per open condition), `history` (every transition, ninety days), `takeover` (the key the
|
||||
// readers follow), `held` and `bounds` (what a delivery holds, and for how long). It watches every observer's
|
||||
// `findings`, the healers' `attempts`, the assigner's wants, the nodes' haves and the controller's lease and, until the
|
||||
// lease write, its `conditions`; it says every transition on its seat with the controller's body. stdout is the MCP
|
||||
// channel; what this module says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
|
||||
// runtimeStore is the bus, through the runtime that launched this bundle.
|
||||
type runtimeStore struct{}
|
||||
|
||||
func (runtimeStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value, e.Revision, true, nil
|
||||
}
|
||||
|
||||
func (runtimeStore) Create(state, key string, value any) (uint64, error) {
|
||||
return stdio.State(state).Create(key, value)
|
||||
}
|
||||
|
||||
func (runtimeStore) Update(state, key string, value any, revision uint64) (uint64, error) {
|
||||
return stdio.State(state).Update(key, value, revision)
|
||||
}
|
||||
|
||||
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
|
||||
|
||||
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
// CreateTTL creates the key with its own time to live (`bounds` is key-ttl); its expiry is handed over as OpExpire.
|
||||
func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
|
||||
_, err := stdio.State(state).CreateTTL(key, value, ttl)
|
||||
return err
|
||||
}
|
||||
|
||||
// sayOnSeat says a transition on the seat, as the controller says it on its own (ADR 0564 clause 3).
|
||||
func sayOnSeat(e health.Event) error {
|
||||
return stdio.SeatPublish("mesh.seat."+Seat+".event."+e.Event, e, "")
|
||||
}
|
||||
|
||||
// NewKeeper is the keeper over the runtime.
|
||||
func NewKeeper() *Keeper {
|
||||
return &Keeper{Store: runtimeStore{}, Say: sayOnSeat, Now: time.Now, Logf: logf}
|
||||
}
|
||||
|
||||
// watch keeps a watch of one state, said blind while it cannot be read: a state not on the bus yet (an observer not
|
||||
// delivered) is read again when it is.
|
||||
func watch(k *Keeper, state, observer string, handle func(stdio.StateChange) error) {
|
||||
go func() {
|
||||
for wait := 5 * time.Second; ; wait = min(wait*2, 5*time.Minute) {
|
||||
err := stdio.State(state).Watch("", handle)
|
||||
if err == nil {
|
||||
if observer != "" {
|
||||
k.Blind(observer, "")
|
||||
}
|
||||
return
|
||||
}
|
||||
if observer != "" {
|
||||
k.Blind(observer, err.Error())
|
||||
}
|
||||
logf("%s cannot be watched (%v); watched again in %s", state, err, wait)
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Keep starts every watch the keeper reads by (clauses 2, 8, 9, 13).
|
||||
func Keep(k *Keeper) {
|
||||
for _, s := range []string{HeldState} {
|
||||
keys, _ := stdio.State(s).Keys()
|
||||
for _, key := range keys {
|
||||
if e, err := stdio.State(s).Get(key); err == nil && e != nil {
|
||||
k.restoreHeld(key, e.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
if e, err := stdio.State(TakeoverState).Get(health.TakeoverKey); err == nil {
|
||||
var raw []byte
|
||||
if e != nil {
|
||||
raw = e.Value
|
||||
}
|
||||
if t, err := health.DecodeTakeover(raw, e != nil); err == nil {
|
||||
k.Takeover(t)
|
||||
}
|
||||
}
|
||||
said := func(err error) error {
|
||||
if err != nil {
|
||||
logf("%v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
watch(k, ControllerLease, "", func(c stdio.StateChange) error {
|
||||
if c.Key != ControllerLeaseKey {
|
||||
return nil
|
||||
}
|
||||
away, why := buscompose.LeaseHandedAway(c.Value, c.Op == "put", nil)
|
||||
logf("the controller's lease: %s", why)
|
||||
return said(k.Lease(away))
|
||||
})
|
||||
watch(k, WantsRead, "", func(c stdio.StateChange) error { return said(k.Want(c.Key, c.Value, c.Op != "put")) })
|
||||
watch(k, HavesRead, "", func(c stdio.StateChange) error { return said(k.Have(c.Key, c.Value, c.Op != "put")) })
|
||||
watch(k, AttemptsRead, health.HealersModule+" attempts", func(c stdio.StateChange) error {
|
||||
return said(k.Attempt(c.Key, c.Value, c.Op != "put"))
|
||||
})
|
||||
watch(k, ControllerConditions, "", func(c stdio.StateChange) error {
|
||||
if k.LeaseAway() {
|
||||
return nil
|
||||
}
|
||||
return said(k.Controller(c.Key, c.Value, c.Op != "put"))
|
||||
})
|
||||
for _, o := range Observers {
|
||||
watch(k, o+"."+health.FindingsState, o, func(c stdio.StateChange) error {
|
||||
return said(k.Finding(o, c.Key, c.Value, c.Op != "put"))
|
||||
})
|
||||
}
|
||||
watch(k, BoundsState, "", func(c stdio.StateChange) error {
|
||||
if c.Op == stdio.OpExpire {
|
||||
return said(k.BoundPassed(c.Key))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
watch(k, PlacementRead, "", func(c stdio.StateChange) error {
|
||||
k.Placement(c.Key, c.Value, c.Op != stdio.OpPut)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func main() {
|
||||
k := NewKeeper()
|
||||
client := asker.New(AskedState)
|
||||
client.Module = Module
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
Keep(k)
|
||||
if err := client.Listen(k.Warranted, func(h asker.Handled, w asks.Warrant, err error) {
|
||||
if err != nil {
|
||||
logf("the answer to %s (%s): %v", w.Ask, h, err)
|
||||
}
|
||||
}); err != nil {
|
||||
logf("the operator's answers cannot be heard: %v", err)
|
||||
}
|
||||
}()
|
||||
if err := stdio.Serve("", Tools(k, client)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// ADR 0564 clauses 1, 2, 3 and 12 (issue 1952 C1): the manifest says what the code does. It claims the seat
|
||||
// mesh-conditions once for the mesh with the verbs mesh-sdk defines for it; reads every observer's findings, the healers'
|
||||
// attempts, the controller's conditions and lease, the wants and the haves, and never the healers' would; keeps its
|
||||
// states, history ninety days and each delivery's bound as long as the bound; and every state has a tool that reads it,
|
||||
// every act a tool that starts it.
|
||||
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("the manifest is refused: %v", err)
|
||||
}
|
||||
seat, _ := catalogue.SeatNamed(Seat)
|
||||
if m.Module != Module || m.RunsAs != Module || len(m.Claims) != 1 || m.Claims[0].Name != Seat ||
|
||||
!slices.Equal(Verbs, catalogue.VerbNames(seat.Serves)) {
|
||||
t.Errorf("module %s runs as %s and claims %+v; the seat serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(seat.Serves))
|
||||
}
|
||||
reads := []string{ControllerConditions, ControllerLease, WantsRead, HavesRead, AttemptsRead, PlacementRead}
|
||||
for _, o := range Observers {
|
||||
reads = append(reads, o+"."+health.FindingsState)
|
||||
}
|
||||
slices.Sort(reads)
|
||||
got := slices.Clone(m.Reads)
|
||||
slices.Sort(got)
|
||||
if !slices.Equal(got, reads) || slices.Contains(m.Reads, health.HealersModule+"."+health.WouldState) {
|
||||
t.Errorf("reads %v, want %v and never the healers' would", got, reads)
|
||||
}
|
||||
ttl, keyTTL := map[string]int{}, map[string]bool{}
|
||||
for _, s := range m.State {
|
||||
ttl[s.Name], keyTTL[s.Name] = s.TTLSeconds, s.KeyTTL
|
||||
}
|
||||
// Each bound is its own key's time to live (the runtime's per-key TTL), never the bucket's age.
|
||||
if ttl[HistoryState] != int(HistoryKeptFor.Seconds()) || ttl[BoundsState] != 0 || !keyTTL[BoundsState] {
|
||||
t.Errorf("history is kept %d s, a bound's bucket ages %d s, its keys have their own time to live: %v",
|
||||
ttl[HistoryState], ttl[BoundsState], keyTTL[BoundsState])
|
||||
}
|
||||
// Clause 12: each state with the tool that reads it, each act with the tool that starts it.
|
||||
readBy := map[string]string{ConditionsState: Seat + ".conditions", HistoryState: HistoryTool, TakeoverState: TakeoverTool,
|
||||
HeldState: HeldTool, BoundsState: HeldTool, AskedState: Seat + ".take-over", "a resync": Seat + ".resync",
|
||||
"what it reads": InputsTool}
|
||||
tools := map[string]bool{}
|
||||
for _, tool := range Tools(&Keeper{}, nil) {
|
||||
tools[tool.Name] = true
|
||||
}
|
||||
for name := range ttl {
|
||||
if readBy[name] == "" {
|
||||
t.Errorf("the state %s has no tool that reads it", name)
|
||||
}
|
||||
}
|
||||
for what, tool := range readBy {
|
||||
if !tools[tool] {
|
||||
t.Errorf("%s is read or started by %s, which is not served", what, tool)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (issue 1936).
|
||||
modules/mesh-conditions/module.json
|
||||
@@ -0,0 +1,252 @@
|
||||
package main
|
||||
|
||||
// The seat's verbs and the keeper's own tools (novox/hq ADR 0564 clauses 3, 9 and 12; issue 1952 C4): `conditions`
|
||||
// answers as the controller's verb answers, so a caller changes only the seat it asks; and a read of every state the
|
||||
// keeper keeps and of what it reads, so a person sees all it holds and a recovery is a call.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// Verbs are the seat's verbs, as the manifest claims them.
|
||||
var Verbs = []string{"conditions", "silence", "compare", "take-over", "hand-back", "resync"}
|
||||
|
||||
// The keeper's own read tools.
|
||||
const (
|
||||
HistoryTool = "conditions_history"
|
||||
HeldTool = "conditions_held"
|
||||
TakeoverTool = "conditions_takeover"
|
||||
InputsTool = "conditions_inputs"
|
||||
)
|
||||
|
||||
// briefEvidence is how many observations a condition carries in a list: its newest (the controller's, issue 314).
|
||||
const briefEvidence = 1
|
||||
|
||||
// inBrief is a condition in a list: its newest evidence, with how many it keeps and how to read it whole.
|
||||
type inBrief struct {
|
||||
health.Condition
|
||||
EvidenceKept int `json:"evidence-kept,omitempty"`
|
||||
More string `json:"more,omitempty"`
|
||||
}
|
||||
|
||||
func obj(props map[string]any, required ...string) map[string]any {
|
||||
out := map[string]any{"type": "object", "properties": props}
|
||||
if len(required) > 0 {
|
||||
out["required"] = required
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func str(description string) map[string]any {
|
||||
return map[string]any{"type": "string", "description": description}
|
||||
}
|
||||
|
||||
// Tools are what the runtime serves.
|
||||
func Tools(k *Keeper, a Asks) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: Seat + ".conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, then " +
|
||||
"oldest, raised from the observers' findings and cleared when the finding goes, never by hand. Given a key, that " +
|
||||
"one whole; with history, every transition lately.",
|
||||
Input: obj(map[string]any{"key": str("a condition's key"), "scope": str("only this scope"),
|
||||
"severity": str("urgent or warning"), "machine": str("only those about this machine"),
|
||||
"history": str("\"true\": every transition lately"), "days": str("with history: days back (7, at most 90)")}),
|
||||
Run: func(args map[string]any) (any, error) { return k.ConditionsVerb(args) }},
|
||||
{Name: Seat + ".silence", Description: "Stop one condition's messages for a while, a hand act with why, kept as its " +
|
||||
"transition in history; it stays open.",
|
||||
Input: obj(map[string]any{"key": str("the condition's key"), "for": str("30m, 4h, 2d; at most 7d"),
|
||||
"why": str("why"), "cause": str("the cause in a word")}, "key", "for", "why"),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
d, err := duration(strArg(args, "for"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return k.Silence(strArg(args, "key"), d, stdio.CallerOf(args), strArg(args, "why"))
|
||||
}},
|
||||
{Name: Seat + ".compare", Description: "The keeper's open conditions against the controller's, each difference with " +
|
||||
"its reason; what ends with the controller, what only the keeper raises and what is held listed apart.",
|
||||
Run: func(map[string]any) (any, error) { return k.Compare() }},
|
||||
{Name: Seat + ".take-over", Description: "Ask the operator's Approve to switch the readers of conditions to the " +
|
||||
"keeper; refused while compare differs.",
|
||||
Input: obj(map[string]any{"why": str("why now")}, "why"),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return k.TakeOver(a, stdio.CallerOf(args), strArg(args, "why"))
|
||||
}},
|
||||
{Name: Seat + ".hand-back", Description: "Switch the readers of conditions back to the controller, until its lease " +
|
||||
"is handed away.",
|
||||
Input: obj(map[string]any{"why": str("why")}, "why"),
|
||||
Run: func(args map[string]any) (any, error) { return k.HandBack(stdio.CallerOf(args), strArg(args, "why")) }},
|
||||
{Name: Seat + ".resync", Description: "Read every finding again and raise and clear what differs.",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
n, err := k.Resync()
|
||||
return map[string]any{"settled": n}, err
|
||||
}},
|
||||
{Name: HistoryTool, Description: "The keeper's history: every transition it said, newest first, at most limit.",
|
||||
Input: obj(map[string]any{"limit": str("how many (default 50)"), "key": str("only this condition")}),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return k.History(strArg(args, "key"), intArg(args, "limit", 50))
|
||||
}},
|
||||
{Name: HeldTool, Description: "What the keeper holds while its module is delivered (ADR 0564 clause 13): each key " +
|
||||
"with its module, node, version and since when, and the deliveries' bounds.",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
bounds, err := k.Store.Keys(BoundsState)
|
||||
return map[string]any{"held": k.Held(), "bounds": bounds}, err
|
||||
}},
|
||||
{Name: TakeoverTool, Description: "The take-over key: which source the router, the grants part and mesh-watcher hear.",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
raw, _, found, err := k.Store.Get(TakeoverState, health.TakeoverKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return health.DecodeTakeover(raw, found)
|
||||
}},
|
||||
{Name: InputsTool, Description: "What the keeper reads, as it stands: each observer's findings counted, an observer it " +
|
||||
"cannot read and why, the healers' attempts, the controller's conditions and whether its lease is handed away.",
|
||||
Run: func(map[string]any) (any, error) { return k.Inputs(), nil }},
|
||||
}
|
||||
}
|
||||
|
||||
// ConditionsVerb is `conditions`: the list, one whole, or the history.
|
||||
func (k *Keeper) ConditionsVerb(args map[string]any) (any, error) {
|
||||
key := strArg(args, "key")
|
||||
if strArg(args, "history") == "true" {
|
||||
days := intArg(args, "days", 7)
|
||||
if days < 1 || days > 90 {
|
||||
return nil, fmt.Errorf("days is from 1 to 90, not %d", days)
|
||||
}
|
||||
return k.HistorySince(key, k.Now().Add(-time.Duration(days)*24*time.Hour), days)
|
||||
}
|
||||
if key != "" {
|
||||
raw, _, found, err := k.Store.Get(ConditionsState, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found || string(raw) == "null" {
|
||||
return nil, fmt.Errorf("no condition %s is open", key)
|
||||
}
|
||||
var c health.Condition
|
||||
return c, json.Unmarshal(raw, &c)
|
||||
}
|
||||
scope, severity, machine := strArg(args, "scope"), strArg(args, "severity"), strArg(args, "machine")
|
||||
if severity != "" && severity != string(health.Urgent) && severity != string(health.Warning) {
|
||||
return nil, fmt.Errorf("a severity is urgent or warning, not %q", severity)
|
||||
}
|
||||
open, err := k.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
list := []inBrief{}
|
||||
counted := map[string]int{}
|
||||
for _, c := range open {
|
||||
if (scope != "" && c.Subject.Scope != scope) || (severity != "" && string(c.Severity) != severity) ||
|
||||
(machine != "" && !concerns(c, machine)) {
|
||||
continue
|
||||
}
|
||||
b := inBrief{Condition: c}
|
||||
if len(c.Evidence) > briefEvidence {
|
||||
b.EvidenceKept, b.Evidence, b.More = len(c.Evidence), c.Evidence[:briefEvidence:briefEvidence], health.ShowKeeper(c.Key)
|
||||
}
|
||||
list = append(list, b)
|
||||
counted[c.Source+" "+c.Kind]++
|
||||
}
|
||||
return map[string]any{"conditions": list, "open": len(open), "counted": counted,
|
||||
"note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; each with " +
|
||||
"its newest evidence — `conditions key=<key>` gives one whole"}, nil
|
||||
}
|
||||
|
||||
// concerns says whether a condition is about a machine: it names it, or its key does.
|
||||
func concerns(c health.Condition, machine string) bool {
|
||||
return c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
|
||||
slices.Contains(strings.Split(c.Key, "."), machine)
|
||||
}
|
||||
|
||||
// History is the newest transitions, at most limit, of one key or all.
|
||||
func (k *Keeper) History(key string, limit int) ([]health.Event, error) {
|
||||
all, err := k.HistorySince(key, time.Time{}, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
events := all["history"].([]health.Event)
|
||||
slices.Reverse(events)
|
||||
if len(events) > limit {
|
||||
events = events[:limit]
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
|
||||
// HistorySince is every transition since a moment, oldest first, as the controller's verb answers it.
|
||||
func (k *Keeper) HistorySince(key string, since time.Time, days int) (map[string]any, error) {
|
||||
keys, err := k.Store.Keys(HistoryState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
events := []health.Event{}
|
||||
for _, hk := range keys {
|
||||
raw, _, found, err := k.Store.Get(HistoryState, hk)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
var e health.Event
|
||||
if json.Unmarshal(raw, &e) != nil || e.At.Before(since) || (key != "" && e.Key != key) {
|
||||
continue
|
||||
}
|
||||
events = append(events, e)
|
||||
}
|
||||
return map[string]any{"history": events, "days": days}, nil
|
||||
}
|
||||
|
||||
// Inputs is what the keeper reads, as it stands.
|
||||
func (k *Keeper) Inputs() map[string]any {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
k.init()
|
||||
findings := map[string]int{}
|
||||
for o, fs := range k.findings {
|
||||
findings[o] = len(fs)
|
||||
}
|
||||
return map[string]any{"findings": findings, "blind": k.blind, "attempts": len(k.attempts),
|
||||
"controller-conditions": len(k.ctrl), "lease-handed-away": k.leaseOut, "follow": k.takeover.Follow,
|
||||
"wants": len(k.wants), "haves": len(k.haves)}
|
||||
}
|
||||
|
||||
func strArg(a map[string]any, name string) string {
|
||||
switch v := a[name].(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(v)
|
||||
case bool:
|
||||
return strconv.FormatBool(v)
|
||||
case float64:
|
||||
return strconv.FormatFloat(v, 'f', -1, 64)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func intArg(a map[string]any, name string, def int) int {
|
||||
if n, err := strconv.Atoi(strArg(a, name)); err == nil {
|
||||
return n
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// duration reads 30m, 4h or 2d.
|
||||
func duration(s string) (time.Duration, error) {
|
||||
if n, ok := strings.CutSuffix(s, "d"); ok {
|
||||
days, err := strconv.Atoi(n)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
|
||||
}
|
||||
return time.Duration(days) * 24 * time.Hour, nil
|
||||
}
|
||||
d, err := time.ParseDuration(s)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
// The condition keeper whole against a real bus (novox/hq ADR 0564 clauses 2 and 3, issue 1952 C6): the built binary,
|
||||
// played over its real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a
|
||||
// real nats-server's key-value buckets, watches handed over as the runtime hands them, mesh/seat.publish onto its
|
||||
// subjects). An observer's finding written to its bucket raises a condition the keeper keeps in its bucket and says as
|
||||
// condition-raised with the controller's body; the seat's verb lists it; the finding deleted clears it, said as
|
||||
// condition-cleared.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
type wire struct {
|
||||
ID json.RawMessage `json:"id,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Params json.RawMessage `json:"params,omitempty"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// player is the node's tool runtime, as far as the keeper needs it, over a real bus.
|
||||
type player struct {
|
||||
ctx context.Context
|
||||
nc *nats.Conn
|
||||
js jetstream.JetStream
|
||||
mu sync.Mutex
|
||||
in io.Writer
|
||||
next int
|
||||
pending map[string]chan wire
|
||||
}
|
||||
|
||||
func (p *player) send(m map[string]any) {
|
||||
raw, _ := json.Marshal(m)
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
_, _ = p.in.Write(append(raw, '\n'))
|
||||
}
|
||||
|
||||
func (p *player) answer(id json.RawMessage, result any, err error) {
|
||||
if err != nil {
|
||||
p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
|
||||
return
|
||||
}
|
||||
p.send(map[string]any{"id": id, "result": result})
|
||||
}
|
||||
|
||||
func (p *player) serve(out io.Reader) {
|
||||
scanner := bufio.NewScanner(out)
|
||||
scanner.Buffer(make([]byte, 1<<20), 1<<24)
|
||||
for scanner.Scan() {
|
||||
var m wire
|
||||
if json.Unmarshal(scanner.Bytes(), &m) != nil {
|
||||
continue
|
||||
}
|
||||
if m.Method == "" {
|
||||
var key string
|
||||
_ = json.Unmarshal(m.ID, &key)
|
||||
p.mu.Lock()
|
||||
ch := p.pending[key]
|
||||
delete(p.pending, key)
|
||||
p.mu.Unlock()
|
||||
if ch != nil {
|
||||
ch <- m
|
||||
}
|
||||
continue
|
||||
}
|
||||
go p.handle(m)
|
||||
}
|
||||
}
|
||||
|
||||
// kv is a state's bucket: the keeper's own by its local name, another's by `<module>.<name>`.
|
||||
func (p *player) kv(state string) (jetstream.KeyValue, error) {
|
||||
if owner, name, read := strings.Cut(state, "."); read {
|
||||
return p.js.KeyValue(p.ctx, owner+"_"+name)
|
||||
}
|
||||
return p.js.KeyValue(p.ctx, Module+"_"+state)
|
||||
}
|
||||
|
||||
func (p *player) handle(m wire) {
|
||||
var a struct {
|
||||
State string `json:"state"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Subject string `json:"subject"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &a)
|
||||
kv, kvErr := p.kv(a.State)
|
||||
switch m.Method {
|
||||
case "mesh/state.get":
|
||||
if kvErr != nil {
|
||||
p.answer(m.ID, nil, kvErr)
|
||||
return
|
||||
}
|
||||
e, err := kv.Get(p.ctx, a.Key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
p.answer(m.ID, nil, nil)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
|
||||
case "mesh/state.create", "mesh/state.update", "mesh/state.put":
|
||||
if kvErr != nil {
|
||||
p.answer(m.ID, nil, kvErr)
|
||||
return
|
||||
}
|
||||
var rev uint64
|
||||
var err error
|
||||
switch m.Method {
|
||||
case "mesh/state.create":
|
||||
rev, err = kv.Create(p.ctx, a.Key, a.Value)
|
||||
case "mesh/state.update":
|
||||
rev, err = kv.Update(p.ctx, a.Key, a.Value, a.Revision)
|
||||
default:
|
||||
rev, err = kv.Put(p.ctx, a.Key, a.Value)
|
||||
}
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32010, "message": "changed: " + err.Error()}})
|
||||
return
|
||||
}
|
||||
p.answer(m.ID, map[string]any{"revision": rev}, err)
|
||||
case "mesh/state.keys":
|
||||
if kvErr != nil {
|
||||
p.answer(m.ID, nil, kvErr)
|
||||
return
|
||||
}
|
||||
keys, err := kv.Keys(p.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
keys, err = []string{}, nil
|
||||
}
|
||||
p.answer(m.ID, keys, err)
|
||||
case "mesh/state.delete":
|
||||
err := kvErr
|
||||
if err == nil {
|
||||
err = kv.Delete(p.ctx, a.Key)
|
||||
}
|
||||
p.answer(m.ID, map[string]any{}, err)
|
||||
case "mesh/state.watch":
|
||||
if kvErr != nil {
|
||||
p.answer(m.ID, nil, kvErr)
|
||||
return
|
||||
}
|
||||
w, err := kv.WatchAll(p.ctx)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
var watchID string
|
||||
_ = json.Unmarshal(m.ID, &watchID)
|
||||
current := true
|
||||
for e := range w.Updates() {
|
||||
if e == nil {
|
||||
current = false
|
||||
p.answer(m.ID, map[string]any{}, nil)
|
||||
continue
|
||||
}
|
||||
op := "put"
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
op = "delete"
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.next++
|
||||
id := fmt.Sprintf("w-%d", p.next)
|
||||
p.mu.Unlock()
|
||||
p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watchID,
|
||||
"change": map[string]any{"state": a.State, "key": e.Key(), "op": op, "value": json.RawMessage(nullIfEmpty(e.Value())),
|
||||
"revision": e.Revision(), "current": current}}})
|
||||
}
|
||||
case "mesh/seat.publish":
|
||||
p.answer(m.ID, map[string]any{"sequence": 1}, p.nc.Publish(a.Subject, a.Body))
|
||||
case "mesh/subscribe":
|
||||
p.answer(m.ID, map[string]any{}, nil)
|
||||
default:
|
||||
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
|
||||
}
|
||||
}
|
||||
|
||||
func nullIfEmpty(b []byte) []byte {
|
||||
if len(b) == 0 {
|
||||
return []byte("null")
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (p *player) call(name string, args map[string]any) (json.RawMessage, error) {
|
||||
p.mu.Lock()
|
||||
p.next++
|
||||
id := fmt.Sprintf("c-%d", p.next)
|
||||
ch := make(chan wire, 1)
|
||||
p.pending[id] = ch
|
||||
p.mu.Unlock()
|
||||
p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args,
|
||||
"_meta": map[string]any{"mesh/caller": "person.jochen"}}})
|
||||
select {
|
||||
case m := <-ch:
|
||||
if m.Error != nil {
|
||||
return nil, errors.New(m.Error.Message)
|
||||
}
|
||||
var r struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
}
|
||||
if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 {
|
||||
return nil, fmt.Errorf("%s answered %s", name, m.Result)
|
||||
}
|
||||
if r.IsError {
|
||||
return nil, errors.New(r.Content[0].Text)
|
||||
}
|
||||
return json.RawMessage(r.Content[0].Text), nil
|
||||
case <-time.After(60 * time.Second):
|
||||
return nil, fmt.Errorf("%s did not answer in 60s", name)
|
||||
}
|
||||
}
|
||||
|
||||
// 1952/C6: an observer's finding raises a condition on the bus, said with the controller's body, and its deletion clears
|
||||
// it.
|
||||
func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
bin := filepath.Join(t.TempDir(), Module)
|
||||
build := exec.Command("go", "build", "-o", bin, ".")
|
||||
build.Stderr = os.Stderr
|
||||
if err := build.Run(); err != nil {
|
||||
t.Fatalf("the keeper does not build: %v", err)
|
||||
}
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buckets := []string{"mesh-controller_conditions", "mesh-controller_lease", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_placement",
|
||||
"mesh-healers_attempts"}
|
||||
for _, o := range Observers {
|
||||
buckets = append(buckets, o+"_findings")
|
||||
}
|
||||
for _, s := range []string{ConditionsState, HistoryState, TakeoverState, HeldState, BoundsState, AskedState} {
|
||||
buckets = append(buckets, Module+"_"+s)
|
||||
}
|
||||
for _, b := range buckets {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
events, err := nc.SubscribeSync("mesh.seat." + Seat + ".event.>")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, bin)
|
||||
cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module)
|
||||
stdin, _ := cmd.StdinPipe()
|
||||
stdout, _ := cmd.StdoutPipe()
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
|
||||
p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
|
||||
go p.serve(stdout)
|
||||
|
||||
// The first call serves; the watches start after it.
|
||||
if _, err := p.call(InputsTool, map[string]any{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
at := time.Now().UTC()
|
||||
f := health.Finding{Key: "machine.g14.silent", Kind: "silent", Subject: health.Subject{Scope: health.ScopeMachine, ID: "g14",
|
||||
Machine: "g14"}, Severity: health.Warning, Summary: "g14 is silent", Headline: "g14 is offline",
|
||||
Explanation: "g14 said nothing for three minutes.", Resolved: "g14 is back", Source: "S1",
|
||||
Observer: health.WatchmanModule, At: at, Evidence: []health.Evidence{{At: at, Said: "no presence for 3m"}}}
|
||||
raw, err := health.EncodeFinding(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
findings, _ := js.KeyValue(ctx, health.WatchmanModule+"_findings")
|
||||
if _, err := findings.Put(ctx, f.Key, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
msg, err := events.NextMsg(10 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("no condition-raised was said: %v", err)
|
||||
}
|
||||
var e health.Event
|
||||
if err := json.Unmarshal(msg.Data, &e); err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-raised" ||
|
||||
e.Key != f.Key || e.Change != health.ChangeRaised || e.Show != health.ShowKeeper(f.Key) || e.Headline != f.Headline {
|
||||
t.Fatalf("said %s on %s", msg.Data, msg.Subject)
|
||||
}
|
||||
listed, err := p.call(Seat+".conditions", map[string]any{})
|
||||
if err != nil || !strings.Contains(string(listed), `"key":"machine.g14.silent"`) {
|
||||
t.Fatalf("conditions answered %s, %v", listed, err)
|
||||
}
|
||||
if err := findings.Delete(ctx, f.Key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
msg, err = events.NextMsg(10 * time.Second)
|
||||
if err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-cleared" {
|
||||
t.Fatalf("no condition-cleared was said: %v", err)
|
||||
}
|
||||
conditions, _ := js.KeyValue(ctx, Module+"_"+ConditionsState)
|
||||
if _, err := conditions.Get(ctx, f.Key); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
t.Errorf("the cleared condition is still kept: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
module mesh-conditions
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"module": "mesh-conditions",
|
||||
"version": "1",
|
||||
"runs-as": "mesh-conditions",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-conditions",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"conditions",
|
||||
"silence",
|
||||
"compare",
|
||||
"take-over",
|
||||
"hand-back",
|
||||
"resync"
|
||||
]
|
||||
}
|
||||
],
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-watchman.findings",
|
||||
"mesh-self-check.findings",
|
||||
"mesh-healers.findings",
|
||||
"mesh-delivery-assigner.findings",
|
||||
"mesh-healers.attempts",
|
||||
"mesh-controller.conditions",
|
||||
"mesh-controller.lease",
|
||||
"mesh-delivery-assigner.wants",
|
||||
"node-engine.have",
|
||||
"mesh-delivery-assigner.placement"
|
||||
],
|
||||
"state": [
|
||||
"conditions",
|
||||
{
|
||||
"name": "history",
|
||||
"ttl-seconds": 7776000
|
||||
},
|
||||
"takeover",
|
||||
"held",
|
||||
{
|
||||
"name": "bounds",
|
||||
"key-ttl": true
|
||||
},
|
||||
"asked"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-conditions",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-conditions",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-conditions"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-conditions",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-conditions",
|
||||
"binary": "mesh-conditions",
|
||||
"loads": [
|
||||
"mesh-conditions"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -61,3 +61,21 @@ them, newest first; with `proposal`, one read whole, with whether the router's a
|
||||
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
|
||||
|
||||
- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
|
||||
|
||||
## The switch from the controller's records (novox/hq ADR 0565 §3, §4, §13)
|
||||
|
||||
- `mesh-config_declared`: each placed module's declarations at the trunk, with the manifest and commit they come
|
||||
from, written only when they differ.
|
||||
- `mesh-config_takeover`: the key `switch` names the source that holds: none (the interim since mesh-catalog #558:
|
||||
declarations from the modules' own manifests, the list of predefined keys and pull requests' config from the
|
||||
controller), `indexer` after the operator's take-over (the list and pull requests' config from the indexer's
|
||||
`predefined` and `config-candidates`; the controller's `pull-declares` refused), `controller` after a hand-back
|
||||
(declarations, list and pull requests' config from the controller's records). Each take-over ask is kept beside it.
|
||||
- `config_compare` lists every difference between the own sources and the controller's records: each module's
|
||||
declarations, the list, each open pull request's config, and every placed module's value on every node resolved
|
||||
both ways. `config_take_over` asks the operator's Approve, refused while any difference stands and switched on the
|
||||
Approve only if none stands then. `config_hand_back` returns to the controller's records until its lease is handed
|
||||
away. `config_declared` and `config_resync` read and write again what it keeps.
|
||||
- From the controller's lease write a trusted value is set only through propose and the operator's Approve.
|
||||
- Its sets, clears and warrants are recorded in the hand-act log of the seat `mesh-hand-acts`, falling back to the
|
||||
controller's while that seat has no holder and the controller holds its lease.
|
||||
|
||||
@@ -275,7 +275,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) {
|
||||
return nil, fmt.Errorf("%s is refused for %s at %s: %s", show(value), call.Key, level,
|
||||
strings.Join(problems, "; "))
|
||||
}
|
||||
if trusted && call.Caller != ControllerCaller && call.approval == nil {
|
||||
if trusted && call.approval == nil && !c.atTheTerminal(call) {
|
||||
return nil, fmt.Errorf("%s is trusted (it decides what runs or who can reach what), so it is set only at "+
|
||||
"the controller's terminal or on the operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: "+
|
||||
"propose it instead, with config's action propose", call.Key, call.Caller)
|
||||
@@ -293,7 +293,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) {
|
||||
if !had {
|
||||
return nil, fmt.Errorf("nothing is set for %s at %s, so there is nothing to clear", call.Key, level)
|
||||
}
|
||||
if trusted && call.Caller != ControllerCaller && call.approval == nil {
|
||||
if trusted && call.approval == nil && !c.atTheTerminal(call) {
|
||||
return nil, fmt.Errorf("%s is trusted, so it is cleared only at the controller's terminal or on the "+
|
||||
"operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: propose the clear instead, with "+
|
||||
"config's action propose and clear", call.Key, call.Caller)
|
||||
@@ -326,6 +326,13 @@ func (c *Config) change(call Call) (*SetAnswer, error) {
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// atTheTerminal says a call is the controller's relay of its terminal, which holds only until the controller's lease is
|
||||
// handed away: from the lease write a trusted value is set only through propose and the operator's Approve, whoever
|
||||
// calls (novox/hq ADR 0565 §13).
|
||||
func (c *Config) atTheTerminal(call Call) bool {
|
||||
return call.Caller == ControllerCaller && !c.leaseAway()
|
||||
}
|
||||
|
||||
// declared is the rule a value at a level is held to, and whether it is trusted; refused, naming the declared
|
||||
// keys, for a key the module or the list of predefined keys does not declare (ADR 0348 §10).
|
||||
func (c *Config) declared(s Sources, level Level, key string) (rule, bool, error) {
|
||||
|
||||
@@ -19,6 +19,8 @@ import (
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
)
|
||||
|
||||
// Seat is the role this module holds, once for the mesh, and Verb its one verb.
|
||||
@@ -73,9 +75,21 @@ func (runtimeStore) Read(log string) ([]LogEntry, error) {
|
||||
|
||||
func (runtimeStore) Emit(event string, body any) error { return stdio.Emit(event, body) }
|
||||
|
||||
// HandAct records a change in the controller's hand-act log (to-be 45 §7) through its seat's verb.
|
||||
// recorder records into the hand-act log of the seat mesh-hand-acts, falling back to the controller's while the seat
|
||||
// has no holder that answers and the controller still holds its lease (novox/hq ADR 0565 §11).
|
||||
var recorder = handacts.Recorder{Ask: stdio.Ask, LeaseAway: func() bool {
|
||||
e, err := stdio.State(LeaseRead).Get(buscompose.ControllerLeaseKey)
|
||||
var raw []byte
|
||||
if e != nil {
|
||||
raw = e.Value
|
||||
}
|
||||
away, _ := buscompose.LeaseHandedAway(raw, e != nil, err)
|
||||
return away
|
||||
}}
|
||||
|
||||
// HandAct records a change in the hand-act log (to-be 45 §7).
|
||||
func (runtimeStore) HandAct(what, why, cause string) error {
|
||||
_, err := stdio.Ask("seat:mesh-controller.hand-act", map[string]any{"what": what, "why": why, "cause": cause})
|
||||
_, err := recorder.Record(handacts.Request{What: what, Why: why, Cause: cause})
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -90,7 +104,7 @@ func (runtimeAsks) Publish(subject string, body any, id string) error {
|
||||
func (runtimeAsks) Record(id string) (*stdio.StateEntry, error) { return stdio.SeatRecord("", id) }
|
||||
func (runtimeAsks) Kinds() ([]stdio.KindHeld, error) { return stdio.SeatKinds() }
|
||||
func (runtimeAsks) Warranted(ask string) error {
|
||||
_, err := stdio.Ask("seat:mesh-controller.warranted", map[string]any{"asker": Module, "ask": ask})
|
||||
_, err := recorder.RecordWarrant(Module, ask)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -118,7 +132,8 @@ func resolving(c *Config) {
|
||||
}
|
||||
// A build's manifest is written before a want names the build, so a want, a trunk module or a placement moving is
|
||||
// every change of what a module declares where (issue 1953); the builds themselves are read, not watched.
|
||||
for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead} {
|
||||
for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead, IndexerPredefinedRead,
|
||||
CandidatesRead, MirrorDeclaredRead, TakeoverState} {
|
||||
go func(read string) {
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
time.Sleep(wait)
|
||||
@@ -221,9 +236,9 @@ func hearing(c *Config) {
|
||||
}
|
||||
}
|
||||
|
||||
// Tools is the seat's verb as the runtime serves it.
|
||||
// Tools is the seat's verb and the module's own tools of the switch, as the runtime serves them.
|
||||
func Tools(c *Config) []stdio.Tool {
|
||||
return []stdio.Tool{{
|
||||
return append(switchTools(c), stdio.Tool{
|
||||
Name: Seat + "." + Verb,
|
||||
Description: "Set, clear, list or read the history of config values (novox/hq ADR 0348). A value is set at " +
|
||||
"one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " +
|
||||
@@ -268,7 +283,7 @@ func Tools(c *Config) []stdio.Tool {
|
||||
call.Caller = stdio.CallerOf(args)
|
||||
return c.Do(call)
|
||||
},
|
||||
}}
|
||||
})
|
||||
}
|
||||
|
||||
// callOf reads a call's arguments strictly: an argument it does not know is refused, not ignored.
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Every state, log and read the manifest declares has a tool that reads it, and every act the module takes by itself
|
||||
// a tool that starts it (novox/hq ADR 0565 §15): the verb config's actions, and the switch's tools.
|
||||
func TestEveryStateAndActHasItsTool(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
State []string `json:"state"`
|
||||
Logs []string `json:"logs"`
|
||||
Reads []string `json:"reads"`
|
||||
Tools []string `json:"tools"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
via := map[string]string{
|
||||
ValuesState: "config list", ResolvedState: "config list", PredefinedState: "config list",
|
||||
ProposalsState: "config proposals", PullsState: "config list", KeptState: "config list", HistoryLog: "config history",
|
||||
DeclaredState: DeclaredTool, TakeoverState: DeclaredTool,
|
||||
PlacementRead: CompareTool, WantsRead: CompareTool, BuildsRead: CompareTool, ModulesRead: DeclaredTool,
|
||||
ManifestsRead: DeclaredTool, PredefinedRead: CompareTool, MirrorDeclaredRead: CompareTool,
|
||||
IndexerPredefinedRead: CompareTool, CandidatesRead: CompareTool, LeaseRead: HandBackTool,
|
||||
"a resolution": ResyncTool, "the indexer's candidates taken": ResyncTool, "a take-over decided": TakeOverTool,
|
||||
}
|
||||
served := map[string]bool{}
|
||||
for _, tool := range Tools(&Config{}) {
|
||||
served[tool.Name] = true
|
||||
}
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || strings.Join(m.Claims[0].Serves, ",") != strings.Join(SeatVerbs, ",") {
|
||||
t.Errorf("claims %+v, want %s serving %v", m.Claims, Seat, SeatVerbs)
|
||||
}
|
||||
for _, verb := range SeatVerbs {
|
||||
if !served[Seat+"."+verb] {
|
||||
t.Errorf("the seat's verb %s is not served", verb)
|
||||
}
|
||||
}
|
||||
for _, name := range m.Tools {
|
||||
if !served[name] {
|
||||
t.Errorf("the manifest lists the tool %s, which is not served", name)
|
||||
}
|
||||
}
|
||||
for _, list := range [][]string{m.State, m.Logs, m.Reads} {
|
||||
for _, s := range list {
|
||||
if _, ok := via[s]; !ok {
|
||||
t.Errorf("%s has no tool that reads it", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
for what, tool := range via {
|
||||
name, _, _ := strings.Cut(tool, " ")
|
||||
if name == "config" {
|
||||
name = Seat + "." + Verb
|
||||
}
|
||||
if !served[name] {
|
||||
t.Errorf("%s is read or started by %s, which is not served", what, tool)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -758,6 +758,7 @@ func (c *Config) Decide() ([]string, error) {
|
||||
for _, r := range ready {
|
||||
said = append(said, c.decide(r.p, r.w))
|
||||
}
|
||||
said = append(said, c.decideTakeovers()...)
|
||||
c.recordWarrants(now)
|
||||
return said, nil
|
||||
}
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A config value held for the open pull request that introduces its key (novox/hq ADR 0502, issue 1826).
|
||||
@@ -61,31 +63,22 @@ type PullDecl struct {
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// KeptValue is an approved held value, kept apart from the values mesh-config resolves.
|
||||
type KeptValue struct {
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head"`
|
||||
Digest string `json:"declaration"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
By string `json:"by"`
|
||||
Via string `json:"via"`
|
||||
Why string `json:"why"`
|
||||
// Approved is who approved it, as the proposal kept it.
|
||||
Approved string `json:"approved"`
|
||||
Proposal string `json:"proposal"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
// KeptValue is an approved held value, kept apart from the values mesh-config resolves: mesh-sdk's layout, which the
|
||||
// checker and the assigner's composition check read (novox/hq ADR 0565 §5).
|
||||
type KeptValue = delivery.KeptValue
|
||||
|
||||
// keptKey is a held value's key in `kept` (delivery.HeldValueKey), the pull request's key form kept for one that is
|
||||
// not one.
|
||||
func keptKey(pull, node, module, key string) string {
|
||||
if k, err := delivery.HeldValueKey(pull, node, module, key); err == nil {
|
||||
return k
|
||||
}
|
||||
return pullKey(pull) + "." + node + "." + module + "." + key
|
||||
}
|
||||
|
||||
func (k KeptValue) stateKey() string { return keptKey(k.Pull, k.Node, k.Module, k.Key) }
|
||||
func keptStateKey(k KeptValue) string { return keptKey(k.Pull, k.Node, k.Module, k.Key) }
|
||||
|
||||
func (k KeptValue) level() Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} }
|
||||
func keptLevel(k KeptValue) Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} }
|
||||
|
||||
// declDigest is the digest of what a declaration holds a value to: a held value stays only while it is the same.
|
||||
func declDigest(d Declaration) string {
|
||||
@@ -151,6 +144,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) {
|
||||
return nil, fmt.Errorf("pull-declares is the controller's: it reads a pull request's manifests and tells "+
|
||||
"mesh-config what the head declares; %s is not the controller", call.Caller)
|
||||
}
|
||||
if sw, err := c.source(); err != nil {
|
||||
return nil, err
|
||||
} else if sw.Source == SourceIndexer {
|
||||
return nil, errors.New("pull-declares is refused: since the take-over mesh-config reads a pull request's config " +
|
||||
"declarations from the indexer's config-candidates (novox/hq ADR 0565 §4)")
|
||||
}
|
||||
if _, _, _, err := parsePull(call.Pull); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -185,6 +184,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) {
|
||||
pull := strings.TrimSpace(call.Pull)
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.declarePull(pull, head, declares)
|
||||
}
|
||||
|
||||
// declarePull keeps what a pull request's head declares and judges again the values held for it; from the
|
||||
// controller's pull-declares, or after the take-over from the indexer's config candidates. The caller holds the lock.
|
||||
func (c *Config) declarePull(pull, head string, declares map[string]map[string]Declaration) (*PullDeclaresAnswer, error) {
|
||||
if err := c.Store.Put(PullsState, pullKey(pull), PullDecl{Pull: pull, Head: head, Declares: declares,
|
||||
At: c.now()}); err != nil {
|
||||
return nil, fmt.Errorf("the declarations were not kept: %v", err)
|
||||
@@ -235,7 +240,7 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) {
|
||||
// heldChange records a hold, a drop or a merge's set of a held value in the history, the hand-act log and
|
||||
// config-changed; what could not be done is answered, not dropped.
|
||||
func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string {
|
||||
level := kv.level()
|
||||
level := keptLevel(kv)
|
||||
ch := Change{Action: action, Level: &level, Key: kv.Key, Value: kv.Value, By: kv.By, Via: kv.Via, Why: why,
|
||||
Approved: approved, Pull: kv.Pull, Head: kv.Head, At: c.now(), Moved: []Moved{}}
|
||||
unsaid := c.record(level.ValueKey(kv.Key), ch)
|
||||
@@ -252,8 +257,8 @@ func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string
|
||||
|
||||
// dropKept removes a held value, saying why.
|
||||
func (c *Config) dropKept(kv KeptValue, why string) []string {
|
||||
if err := c.Store.Delete(KeptState, kv.stateKey()); err != nil {
|
||||
return []string{"the held value " + kv.stateKey() + " was not dropped: " + err.Error()}
|
||||
if err := c.Store.Delete(KeptState, keptStateKey(kv)); err != nil {
|
||||
return []string{"the held value " + keptStateKey(kv) + " was not dropped: " + err.Error()}
|
||||
}
|
||||
c.logf("[mesh-config] dropped %s of %s on %s held for %s: %s", kv.Key, kv.Module, kv.Node, kv.Pull, why)
|
||||
return c.heldChange(kv, ActionDrop, "dropped, nothing was set: "+why, "")
|
||||
@@ -408,7 +413,7 @@ func (c *Config) keep(p Proposal) (string, error) {
|
||||
kv := KeptValue{Pull: p.Pull, Head: pd.Head, Digest: p.DeclDigest, Node: p.Level.Node, Module: p.Level.Module,
|
||||
Key: p.Key, Value: p.Value, By: "the operator", Via: p.From, Why: p.Why, Approved: p.Approved, Proposal: p.ID,
|
||||
At: c.now()}
|
||||
if err := c.Store.Put(KeptState, kv.stateKey(), kv); err != nil {
|
||||
if err := c.Store.Put(KeptState, keptStateKey(kv), kv); err != nil {
|
||||
return "", fmt.Errorf("the value was not kept: %v", err)
|
||||
}
|
||||
unsaid := c.heldChange(kv, ActionHold, "held for "+p.Pull+" until it merges: "+p.Why, p.Approved)
|
||||
@@ -471,7 +476,7 @@ func (c *Config) settlePull(pull string, merged bool, head string) []string {
|
||||
said = append(said, c.dropKept(kv, kv.Pull+" merged, but "+why)...)
|
||||
continue
|
||||
}
|
||||
vk := kv.level().ValueKey(kv.Key)
|
||||
vk := keptLevel(kv).ValueKey(kv.Key)
|
||||
v := Value{Value: compact(kv.Value), By: kv.By, Via: kv.Via, Why: kv.Why + " (held for " + kv.Pull +
|
||||
", merged)", Approved: kv.Approved, At: c.now()}
|
||||
if err := c.Store.Put(ValuesState, vk, v); err != nil {
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (issue 1936).
|
||||
modules/mesh-config/module.json
|
||||
@@ -12,10 +12,19 @@ import (
|
||||
|
||||
// Reading the sources, writing the resolved values, and publishing the list of predefined keys.
|
||||
|
||||
// sources reads everything a resolution reads, from the bus as it is now: which modules are placed where from the
|
||||
// seat mesh-placement, and each placed module's declarations from its own manifests as built, never from the
|
||||
// controller (issue 1953, ADR 0565 §3).
|
||||
// sources reads everything a resolution reads, from the bus as it is now, from the source the switch names
|
||||
// (takeover.go): which modules are placed where from the seat mesh-placement, and each placed module's declarations
|
||||
// from its own manifests as built (issue 1953, ADR 0565 §3), or after a hand-back from the controller's mirror.
|
||||
func (c *Config) sources() (Sources, error) {
|
||||
sw, err := c.source()
|
||||
if err != nil {
|
||||
return Sources{}, err
|
||||
}
|
||||
return c.sourcesFrom(sw.Source)
|
||||
}
|
||||
|
||||
// sourcesFrom reads the sources with declarations from one source: the own (interim and indexer) or the mirror.
|
||||
func (c *Config) sourcesFrom(source string) (Sources, error) {
|
||||
s := Sources{Declared: map[string]ModuleDeclarations{}, OnNode: map[string]ModuleDeclarations{},
|
||||
Predefined: map[string]Published{}, Values: map[string]Value{}}
|
||||
var problems []string
|
||||
@@ -25,6 +34,9 @@ func (c *Config) sources() (Sources, error) {
|
||||
if err := c.readDeclarations(&s); err != nil {
|
||||
return s, fmt.Errorf("the modules' declarations cannot be read: %w", err)
|
||||
}
|
||||
if source == SourceController {
|
||||
c.mirrorDeclarations(&s)
|
||||
}
|
||||
for _, u := range s.Unresolved {
|
||||
problems = append(problems, fmt.Sprintf("%s on %s: %s", u.Module, u.Node, u.Why))
|
||||
}
|
||||
@@ -350,6 +362,10 @@ type Resolution struct {
|
||||
Moved []Moved `json:"moved"`
|
||||
Published string `json:"published,omitempty"`
|
||||
Withheld string `json:"withheld,omitempty"`
|
||||
// Declared are the modules whose `declared` key was written, and Candidates what the indexer's config candidates
|
||||
// did after the take-over.
|
||||
Declared []string `json:"declared,omitempty"`
|
||||
Candidates []string `json:"candidates,omitempty"`
|
||||
}
|
||||
|
||||
// Reconcile publishes the newest list of predefined keys and brings mesh-config_resolved to what the sources
|
||||
@@ -374,26 +390,39 @@ func (c *Config) Reconcile() (Resolution, error) {
|
||||
c.record("resolved", Change{Action: "resolve", Why: "an assignment, a declaration or the list of predefined " +
|
||||
"keys changed", At: c.now(), Moved: moved})
|
||||
}
|
||||
if written, derr := c.writeDeclared(s); derr != nil {
|
||||
c.logf("[mesh-config] declared is not all written: %v", derr)
|
||||
} else if len(written) > 0 {
|
||||
r.Declared = written
|
||||
}
|
||||
if sw, serr := c.source(); serr == nil && sw.Source == SourceIndexer {
|
||||
r.Candidates = c.takeCandidates()
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
|
||||
// publish makes mesh-config_predefined hold the list the controller kept, when it is newer than the one published
|
||||
// (ADR 0348 §7: the newest trunk build wins). **A list dropping a key that still holds a value is withheld**: the
|
||||
// publish makes mesh-config_predefined hold the list its source keeps — the indexer's after the take-over, the
|
||||
// controller's mirror otherwise — when it is newer than the one published (ADR 0348 §7: the newest trunk build wins;
|
||||
// the indexer never keeps an older merge's, so its list is newer when its commit differs). **A list dropping a key that still holds a value is withheld**: the
|
||||
// list published stays, and why is said in every listing and in the log, because a value for a key no longer on
|
||||
// the list could neither be resolved nor cleared by name (§7: removing a key still holding a value is refused).
|
||||
// It answers what it published, empty when nothing changed.
|
||||
func (c *Config) publish() (string, error) {
|
||||
e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey)
|
||||
sw, err := c.source()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the controller's list cannot be read: %w", err)
|
||||
return "", err
|
||||
}
|
||||
if e == nil {
|
||||
read, whose := c.mirrorList, "the controller's list"
|
||||
if sw.Source == SourceIndexer {
|
||||
read, whose = c.indexerList, "the indexer's list"
|
||||
}
|
||||
list, found, err := read()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s cannot be read: %w", whose, err)
|
||||
}
|
||||
if !found {
|
||||
return "", nil // no list kept yet
|
||||
}
|
||||
var list PredefinedList
|
||||
if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil {
|
||||
return "", fmt.Errorf("the controller's list is not one: %s", e.Value)
|
||||
}
|
||||
current := map[string]Published{}
|
||||
if err := c.each(PredefinedState, func(key string, raw []byte) error {
|
||||
var p Published
|
||||
@@ -405,11 +434,12 @@ func (c *Config) publish() (string, error) {
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
generation := 0
|
||||
generation, sameCommit := 0, len(current) > 0
|
||||
for _, p := range current {
|
||||
generation = max(generation, p.Generation)
|
||||
sameCommit = sameCommit && p.Commit == list.Commit
|
||||
}
|
||||
if len(current) > 0 && generation >= list.Generation {
|
||||
if len(current) > 0 && (sameCommit || (list.Generation > 0 && generation >= list.Generation)) {
|
||||
c.withheld = ""
|
||||
return "", nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package main
|
||||
|
||||
// The tools of the switch (novox/hq ADR 0565 §4, §15): compare, take-over and hand-back, served as the seat's verbs
|
||||
// and as the module's own tools beside them, the read of `declared` and resync.
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// The switch's tools.
|
||||
const (
|
||||
CompareTool = "config_compare"
|
||||
TakeOverTool = "config_take_over"
|
||||
HandBackTool = "config_hand_back"
|
||||
DeclaredTool = "config_declared"
|
||||
ResyncTool = "config_resync"
|
||||
)
|
||||
|
||||
// SeatVerbs are the seat's verbs, as the manifest claims them.
|
||||
var SeatVerbs = []string{Verb, CompareVerb, TakeOverVerb, HandBackVerb}
|
||||
|
||||
func switchTools(c *Config) []stdio.Tool {
|
||||
own := ownSwitchTools(c)
|
||||
byName := map[string]stdio.Tool{}
|
||||
for _, t := range own {
|
||||
byName[t.Name] = t
|
||||
}
|
||||
for verb, tool := range map[string]string{CompareVerb: CompareTool, TakeOverVerb: TakeOverTool, HandBackVerb: HandBackTool} {
|
||||
t := byName[tool]
|
||||
t.Name = Seat + "." + verb
|
||||
own = append(own, t)
|
||||
}
|
||||
return own
|
||||
}
|
||||
|
||||
func ownSwitchTools(c *Config) []stdio.Tool {
|
||||
why := map[string]any{"type": "object", "properties": map[string]any{
|
||||
"why": map[string]any{"type": "string", "description": "why, in a sentence"}}, "required": []string{"why"}}
|
||||
whyOf := func(a map[string]any) string { s, _ := a["why"].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: CompareTool,
|
||||
Description: "Every difference between config's own sources and the controller's records (novox/hq ADR 0565 §4): " +
|
||||
"each placed module's declarations (declared against the mirror declared-config), the list of predefined " +
|
||||
"keys (the indexer's against the mirror's), each open pull request's config (the indexer's candidate " +
|
||||
"against the controller's pull-declares), and every placed module's value on every node resolved both " +
|
||||
"ways; each with its reason. The take-over is refused while any stands.",
|
||||
Run: func(map[string]any) (any, error) { return c.Compare() }},
|
||||
{Name: TakeOverTool,
|
||||
Description: "Ask the operator's Approve, under mesh-config's own name, to switch config to its own sources: " +
|
||||
"the indexer's list of predefined keys and pull request config, refusing the controller's pull-declares. " +
|
||||
"Refused while compare shows a difference; on Approve switched if nothing differs then.",
|
||||
Input: why,
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return c.TakeOver(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)})
|
||||
}},
|
||||
{Name: HandBackTool,
|
||||
Description: "Return config to the controller's records (its mirrors declared-config and predefined, and " +
|
||||
"pull-declares) until the controller hands its lease away; gone after it. Recorded as a hand act.",
|
||||
Input: why,
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return c.HandBack(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)})
|
||||
}},
|
||||
{Name: DeclaredTool,
|
||||
Description: "Each placed module's declarations as config keeps them (its state declared): the manifest and " +
|
||||
"commit they come from and how many keys, or one module whole; and which source holds now.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "one module, whole"}}},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
m, _ := a["module"].(string)
|
||||
return c.Declared(strings.TrimSpace(m))
|
||||
}},
|
||||
{Name: ResyncTool,
|
||||
Description: "Read every source again and write what differs: the published list of predefined keys, the " +
|
||||
"resolved values, declared, and after the take-over the indexer's config candidates.",
|
||||
Run: func(map[string]any) (any, error) { return c.Resync() }},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,771 @@
|
||||
package main
|
||||
|
||||
// Config's records out of the controller (novox/hq ADR 0565 §3, §4, §13, §15, and the progressive insight of
|
||||
// 2026-10-29): mesh-config keeps each placed module's declarations in its own state `declared`; `compare` lists every
|
||||
// difference between its own sources and the controller's mirrors — each module's declarations, the predefined list,
|
||||
// each open pull request's config declarations, and every placed module's resolved value on every node both ways;
|
||||
// `take-over` asks the operator's Approve under mesh-config's own name and, refused while anything differs, writes the
|
||||
// key `takeover.switch` naming the indexer; `hand-back` returns to the mirrors until the lease write.
|
||||
//
|
||||
// **Which source holds** is the key `switch` alone, read by mesh-config alone:
|
||||
// - none written (the interim of mesh-catalog #558): declarations from each placed module's own manifests, the list
|
||||
// of predefined keys from the controller's mirror, a pull request's declarations from the controller's
|
||||
// pull-declares;
|
||||
// - `indexer` (taken over): declarations as before, the list from the indexer's `predefined`, a pull request's from
|
||||
// the indexer's `config-candidates`; pull-declares refused;
|
||||
// - `controller` (handed back): declarations from the mirror `declared-config`, the list from the mirror,
|
||||
// pull-declares taken.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The states and reads of the switch.
|
||||
const (
|
||||
// DeclaredState is each placed module's declarations at the trunk, one key per module (ADR 0565 §3).
|
||||
DeclaredState = "declared"
|
||||
// TakeoverState holds the key SwitchKey and each take-over ask, `ask.<id>`.
|
||||
TakeoverState = "takeover"
|
||||
SwitchKey = "switch"
|
||||
|
||||
// MirrorDeclaredRead is the controller's mirror of every registered module's declarations, one key per module:
|
||||
// what `compare` holds the own source to, and the source after a hand-back.
|
||||
MirrorDeclaredRead = "mesh-controller.declared-config"
|
||||
// IndexerPredefinedRead and CandidatesRead are the indexer's states (ADR 0565 §2).
|
||||
IndexerPredefinedRead = delivery.IndexerModule + "." + delivery.IndexerPredefinedState
|
||||
CandidatesRead = delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState
|
||||
// LeaseRead is the controller's lease: after its write hand-back is gone and the terminal sets nothing (§13).
|
||||
LeaseRead = buscompose.ControllerLease
|
||||
)
|
||||
|
||||
// The sources the switch names.
|
||||
const (
|
||||
SourceInterim = "interim"
|
||||
SourceIndexer = "indexer"
|
||||
SourceController = "controller"
|
||||
)
|
||||
|
||||
// TakeOverVerb and HandBackVerb are the seat's verbs.
|
||||
const (
|
||||
TakeOverVerb = "take-over"
|
||||
HandBackVerb = "hand-back"
|
||||
CompareVerb = "compare"
|
||||
)
|
||||
|
||||
// Switch is the key `takeover.switch`: the source that holds, who chose it, why and on which ask.
|
||||
type Switch struct {
|
||||
Source string `json:"source"`
|
||||
At time.Time `json:"at"`
|
||||
By string `json:"by"`
|
||||
Why string `json:"why"`
|
||||
Ask string `json:"ask,omitempty"`
|
||||
}
|
||||
|
||||
// source is the source that holds now.
|
||||
func (c *Config) source() (Switch, error) {
|
||||
e, err := c.Store.Get(TakeoverState, SwitchKey)
|
||||
if err != nil {
|
||||
return Switch{}, fmt.Errorf("the take-over's switch cannot be read: %w", err)
|
||||
}
|
||||
if e == nil {
|
||||
return Switch{Source: SourceInterim}, nil
|
||||
}
|
||||
var s Switch
|
||||
if err := json.Unmarshal(e.Value, &s); err != nil || (s.Source != SourceIndexer && s.Source != SourceController) {
|
||||
return Switch{}, fmt.Errorf("the take-over's switch is not one: %s", e.Value)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// leaseAway says the controller's lease is handed away (ADR 0561 §3's reading).
|
||||
func (c *Config) leaseAway() bool {
|
||||
e, err := c.Store.Get(LeaseRead, buscompose.ControllerLeaseKey)
|
||||
var raw []byte
|
||||
if e != nil {
|
||||
raw = e.Value
|
||||
}
|
||||
away, _ := buscompose.LeaseHandedAway(raw, e != nil, err)
|
||||
return away
|
||||
}
|
||||
|
||||
// ---- declared --------------------------------------------------------------------------------------------
|
||||
|
||||
// DeclaredKept is one key of `declared`: a module's declarations as the manifest the indexer's `modules.<module>`
|
||||
// names holds them, with that manifest's digest and commit (ADR 0565 §3).
|
||||
type DeclaredKept struct {
|
||||
Module string `json:"module"`
|
||||
Manifest string `json:"manifest"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Config map[string]Declaration `json:"settings"`
|
||||
}
|
||||
|
||||
// trunkKept is a module's declarations at the trunk with the manifest they come from.
|
||||
func (c *Config) trunkKept(module string) (DeclaredKept, error) {
|
||||
key, err := delivery.ModulesKey(module)
|
||||
if err != nil {
|
||||
return DeclaredKept{}, err
|
||||
}
|
||||
e, err := c.Store.Get(ModulesRead, key)
|
||||
if err != nil || e == nil {
|
||||
return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s cannot be read (%v)", module, err)
|
||||
}
|
||||
var m struct {
|
||||
Manifest string `json:"manifest"`
|
||||
Commit string `json:"commit"`
|
||||
}
|
||||
if err := json.Unmarshal(e.Value, &m); err != nil || m.Manifest == "" {
|
||||
return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s names no manifest", module)
|
||||
}
|
||||
d, err := c.manifestDeclarations(module, m.Manifest)
|
||||
if err != nil {
|
||||
return DeclaredKept{}, err
|
||||
}
|
||||
return DeclaredKept{Module: module, Manifest: m.Manifest, Commit: m.Commit, Config: d.Config}, nil
|
||||
}
|
||||
|
||||
// writeDeclared makes `declared` hold each placed module's trunk declarations, writing a key only when it differs
|
||||
// (ADR 0289 §1) and removing one of a module placed nowhere. It answers the modules written.
|
||||
func (c *Config) writeDeclared(s Sources) ([]string, error) {
|
||||
held := map[string][]byte{}
|
||||
if err := c.each(DeclaredState, func(key string, raw []byte) error {
|
||||
held[key] = raw
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var written []string
|
||||
var problems []error
|
||||
placed := map[string]bool{}
|
||||
for _, module := range modulesOf(s.Assignments) {
|
||||
placed[module] = true
|
||||
k, err := c.trunkKept(module)
|
||||
if err != nil {
|
||||
continue // unresolved, said by the resolution; its key stays as it was
|
||||
}
|
||||
raw, _ := json.Marshal(k)
|
||||
if bytes.Equal(raw, held[module]) {
|
||||
continue
|
||||
}
|
||||
if err := c.Store.Put(DeclaredState, module, k); err != nil {
|
||||
problems = append(problems, fmt.Errorf("%s: %w", module, err))
|
||||
continue
|
||||
}
|
||||
written = append(written, module)
|
||||
}
|
||||
for module := range held {
|
||||
if !placed[module] {
|
||||
if err := c.Store.Delete(DeclaredState, module); err != nil {
|
||||
problems = append(problems, fmt.Errorf("%s: %w", module, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
return written, errors.Join(problems...)
|
||||
}
|
||||
|
||||
// mirrorDeclarations makes the sources' declarations the controller's mirror's, for every placed module: the source
|
||||
// after a hand-back, and the other side of compare.
|
||||
func (c *Config) mirrorDeclarations(s *Sources) {
|
||||
s.Declared, s.OnNode, s.Unresolved = map[string]ModuleDeclarations{}, map[string]ModuleDeclarations{}, nil
|
||||
for _, module := range modulesOf(s.Assignments) {
|
||||
e, err := c.Store.Get(MirrorDeclaredRead, module)
|
||||
var d ModuleDeclarations
|
||||
if err == nil && e != nil {
|
||||
d, err = ParseModuleDeclarations(e.Value)
|
||||
} else if err == nil {
|
||||
err = errors.New("the mirror holds no key for it")
|
||||
}
|
||||
if err != nil {
|
||||
for _, a := range s.Assignments {
|
||||
if a.Module == module {
|
||||
s.Unresolved = append(s.Unresolved, Unresolved{Node: a.Node, Module: module,
|
||||
Why: "the controller's mirror declared-config: " + err.Error()})
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
s.Declared[module] = d
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the predefined list and the pull requests from the indexer ------------------------------------------
|
||||
|
||||
// indexerList is the indexer's predefined list as the mirror's list holds one: the keys, and the commit.
|
||||
func (c *Config) indexerList() (PredefinedList, bool, error) {
|
||||
e, err := c.Store.Get(IndexerPredefinedRead, delivery.PredefinedListKey)
|
||||
if err != nil || e == nil {
|
||||
return PredefinedList{}, false, err
|
||||
}
|
||||
var kept delivery.PredefinedKept
|
||||
if err := json.Unmarshal(e.Value, &kept); err != nil {
|
||||
return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not one: %w", err)
|
||||
}
|
||||
list := PredefinedList{Keys: map[string]PredefinedKey{}, Commit: kept.Commit}
|
||||
if err := json.Unmarshal(kept.List, &list.Keys); err != nil {
|
||||
return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not predefined.json: %w", err)
|
||||
}
|
||||
return list, true, nil
|
||||
}
|
||||
|
||||
// mirrorList is the controller's mirror's list.
|
||||
func (c *Config) mirrorList() (PredefinedList, bool, error) {
|
||||
e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey)
|
||||
if err != nil || e == nil {
|
||||
return PredefinedList{}, false, err
|
||||
}
|
||||
var list PredefinedList
|
||||
if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil {
|
||||
return PredefinedList{}, false, fmt.Errorf("the controller's list is not one: %s", e.Value)
|
||||
}
|
||||
return list, true, nil
|
||||
}
|
||||
|
||||
// candidates is every open pull request's config declarations as the indexer keeps them, by pull key.
|
||||
func (c *Config) candidates() (map[string]PullDecl, error) {
|
||||
out := map[string]PullDecl{}
|
||||
err := c.each(CandidatesRead, func(key string, raw []byte) error {
|
||||
var cand delivery.ConfigCandidate
|
||||
if err := json.Unmarshal(raw, &cand); err != nil || cand.Pull == "" {
|
||||
return fmt.Errorf("the indexer's config candidate %s is not one", key)
|
||||
}
|
||||
p := PullDecl{Pull: cand.Pull, Head: cand.Head, Declares: map[string]map[string]Declaration{}, At: cand.At}
|
||||
for module, keys := range cand.Declares {
|
||||
raw, _ := json.Marshal(keys)
|
||||
var d map[string]Declaration
|
||||
if err := json.Unmarshal(raw, &d); err != nil {
|
||||
return fmt.Errorf("the indexer's config candidate %s declares %s unreadably", key, module)
|
||||
}
|
||||
for k, decl := range d {
|
||||
if !decl.HasDefault() {
|
||||
decl.Default = nil
|
||||
}
|
||||
d[k] = decl
|
||||
}
|
||||
p.Declares[module] = d
|
||||
}
|
||||
out[pullKey(cand.Pull)] = p
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// takeCandidates is the indexer's config candidates taken in place of pull-declares, after the take-over: each one
|
||||
// whose head or declarations differ from what `pulls` holds is declared again, its held values judged again (ADR 0502
|
||||
// §6). The caller holds the lock.
|
||||
func (c *Config) takeCandidates() []string {
|
||||
cands, err := c.candidates()
|
||||
if err != nil {
|
||||
return []string{"the indexer's config candidates cannot be read: " + err.Error()}
|
||||
}
|
||||
var s Sources
|
||||
if err := c.readPulls(&s); err != nil {
|
||||
return []string{err.Error()}
|
||||
}
|
||||
var said []string
|
||||
for _, k := range sortedKeys(cands) {
|
||||
p := cands[k]
|
||||
if have, ok := s.Pulls[k]; ok && have.Head == p.Head && declaresDigest(have.Declares) == declaresDigest(p.Declares) {
|
||||
continue
|
||||
}
|
||||
a, err := c.declarePull(p.Pull, p.Head, p.Declares)
|
||||
if err != nil {
|
||||
said = append(said, fmt.Sprintf("%s from the indexer: %v", p.Pull, err))
|
||||
continue
|
||||
}
|
||||
said = append(said, a.Said+" (from the indexer)")
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
// declaresDigest is what a pull request's declarations hold a value to, in one encoding.
|
||||
func declaresDigest(d map[string]map[string]Declaration) string {
|
||||
flat := map[string]string{}
|
||||
for m, keys := range d {
|
||||
for k, decl := range keys {
|
||||
flat[m+"."+k] = declDigest(decl)
|
||||
}
|
||||
}
|
||||
raw, _ := json.Marshal(flat)
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
// ---- compare ----------------------------------------------------------------------------------------------
|
||||
|
||||
// Difference is one thing the own sources and the controller's mirrors say differently, and why.
|
||||
type Difference struct {
|
||||
What string `json:"what"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
Pull string `json:"pull,omitempty"`
|
||||
Own json.RawMessage `json:"own,omitempty"`
|
||||
Mirror json.RawMessage `json:"mirror,omitempty"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// The kinds of difference.
|
||||
const (
|
||||
DiffDeclared = "declared"
|
||||
DiffPredefined = "predefined"
|
||||
DiffPull = "pull"
|
||||
DiffValue = "value"
|
||||
)
|
||||
|
||||
// Comparison is what compare answers.
|
||||
type Comparison struct {
|
||||
Source string `json:"source"`
|
||||
Differences []Difference `json:"differences"`
|
||||
Count int `json:"count"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
// Compare lists every difference (ADR 0565 §4 and its insight). The caller holds no lock.
|
||||
func (c *Config) Compare() (*Comparison, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.compare()
|
||||
}
|
||||
|
||||
// compare is Compare; the caller holds the lock.
|
||||
func (c *Config) compare() (*Comparison, error) {
|
||||
sw, err := c.source()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
own, err := c.sourcesFrom(SourceInterim)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mesh-config's own sources cannot be read: %w", err)
|
||||
}
|
||||
mirror := own
|
||||
mirror.Declared, mirror.OnNode, mirror.Unresolved = nil, nil, nil
|
||||
c.mirrorDeclarations(&mirror)
|
||||
var diffs []Difference
|
||||
|
||||
// Each placed module's declarations: `declared` against the mirror's key for it.
|
||||
for _, module := range modulesOf(own.Assignments) {
|
||||
var ownRaw []byte
|
||||
if e, err := c.Store.Get(DeclaredState, module); err == nil && e != nil {
|
||||
var k DeclaredKept
|
||||
if json.Unmarshal(e.Value, &k) == nil {
|
||||
ownRaw, _ = json.Marshal(normal(k.Config))
|
||||
}
|
||||
}
|
||||
var mirrorRaw []byte
|
||||
if d, ok := mirror.Declared[module]; ok {
|
||||
mirrorRaw, _ = json.Marshal(normal(d.Config))
|
||||
}
|
||||
switch {
|
||||
case ownRaw == nil && mirrorRaw == nil:
|
||||
diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Why: "neither side holds its declarations"})
|
||||
case ownRaw == nil:
|
||||
diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Mirror: mirrorRaw,
|
||||
Why: "the mirror holds its declarations and declared does not"})
|
||||
case mirrorRaw == nil:
|
||||
diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw,
|
||||
Why: "declared holds its declarations and the mirror does not"})
|
||||
case !bytes.Equal(ownRaw, mirrorRaw):
|
||||
diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw, Mirror: mirrorRaw,
|
||||
Why: "the trunk's manifest declares it otherwise than the mirror"})
|
||||
}
|
||||
}
|
||||
|
||||
// The predefined list: the indexer's against the mirror's.
|
||||
il, iok, ierr := c.indexerList()
|
||||
ml, mok, merr := c.mirrorList()
|
||||
switch {
|
||||
case ierr != nil || merr != nil:
|
||||
diffs = append(diffs, Difference{What: DiffPredefined, Why: "a list cannot be read: " + errors.Join(ierr, merr).Error()})
|
||||
case !iok || !mok:
|
||||
if iok != mok {
|
||||
diffs = append(diffs, Difference{What: DiffPredefined, Why: fmt.Sprintf("the indexer holds a list: %v; the "+
|
||||
"mirror holds one: %v", iok, mok)})
|
||||
}
|
||||
default:
|
||||
for _, k := range unionKeys(il.Keys, ml.Keys) {
|
||||
a, aok := il.Keys[k]
|
||||
b, bok := ml.Keys[k]
|
||||
ar, _ := json.Marshal(a)
|
||||
br, _ := json.Marshal(b)
|
||||
switch {
|
||||
case !aok:
|
||||
diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Mirror: br, Why: "only the mirror's list holds it"})
|
||||
case !bok:
|
||||
diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Why: "only the indexer's list holds it"})
|
||||
case !bytes.Equal(ar, br):
|
||||
diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Mirror: br,
|
||||
Why: "the two lists declare it otherwise"})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Each open pull request: the indexer's candidate against what the controller told by pull-declares.
|
||||
cands, cerr := c.candidates()
|
||||
if cerr != nil {
|
||||
diffs = append(diffs, Difference{What: DiffPull, Why: "the indexer's config candidates cannot be read: " + cerr.Error()})
|
||||
}
|
||||
for _, k := range unionKeys(cands, own.Pulls) {
|
||||
a, aok := cands[k]
|
||||
b, bok := own.Pulls[k]
|
||||
switch {
|
||||
case !aok:
|
||||
diffs = append(diffs, Difference{What: DiffPull, Pull: b.Pull, Why: "the controller told its declarations and the indexer holds no candidate"})
|
||||
case !bok:
|
||||
diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the indexer holds a candidate and the controller told nothing"})
|
||||
case a.Head != b.Head:
|
||||
diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: fmt.Sprintf("the indexer's candidate is at %s, "+
|
||||
"the controller told %s", short(a.Head), short(b.Head))})
|
||||
case declaresDigest(a.Declares) != declaresDigest(b.Declares):
|
||||
diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the two declare its config otherwise"})
|
||||
}
|
||||
}
|
||||
|
||||
// Every placed module's value on every node, resolved both ways.
|
||||
ownValues, mirrorValues := Resolve(own), Resolve(mirror)
|
||||
for _, k := range unionKeys(ownValues, mirrorValues) {
|
||||
a, aok := ownValues[k]
|
||||
b, bok := mirrorValues[k]
|
||||
ar, _ := json.Marshal(a)
|
||||
br, _ := json.Marshal(b)
|
||||
node, module, key := a.Node, a.Module, a.Key
|
||||
if !aok {
|
||||
node, module, key = b.Node, b.Module, b.Key
|
||||
}
|
||||
d := Difference{What: DiffValue, Node: node, Module: module, Key: key}
|
||||
switch {
|
||||
case !aok:
|
||||
d.Mirror, d.Why = br, "only the mirror's declarations resolve it on this node"
|
||||
case !bok:
|
||||
d.Own, d.Why = ar, "only mesh-config's own declarations (with this node's build) resolve it"
|
||||
case !bytes.Equal(ar, br):
|
||||
d.Own, d.Mirror, d.Why = ar, br, "it resolves to another value or source"
|
||||
default:
|
||||
continue
|
||||
}
|
||||
diffs = append(diffs, d)
|
||||
}
|
||||
for _, u := range mirror.Unresolved {
|
||||
diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from the mirror: " + u.Why})
|
||||
}
|
||||
for _, u := range own.Unresolved {
|
||||
diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from its own source: " + u.Why})
|
||||
}
|
||||
sort.SliceStable(diffs, func(i, j int) bool {
|
||||
a, b := diffs[i], diffs[j]
|
||||
return a.What+a.Module+a.Node+a.Key+a.Pull < b.What+b.Module+b.Node+b.Key+b.Pull
|
||||
})
|
||||
if diffs == nil {
|
||||
diffs = []Difference{}
|
||||
}
|
||||
cmp := &Comparison{Source: sw.Source, Differences: diffs, Count: len(diffs)}
|
||||
if len(diffs) == 0 {
|
||||
cmp.Said = "nothing differs: the take-over may be asked"
|
||||
} else {
|
||||
cmp.Said = fmt.Sprintf("%d difference(s): the take-over is refused while any stands", len(diffs))
|
||||
}
|
||||
return cmp, nil
|
||||
}
|
||||
|
||||
// normal is declarations in one encoding: a null default as none.
|
||||
func normal(d map[string]Declaration) map[string]Declaration {
|
||||
out := map[string]Declaration{}
|
||||
for k, decl := range d {
|
||||
if !decl.HasDefault() {
|
||||
decl.Default = nil
|
||||
} else {
|
||||
decl.Default = compact(decl.Default)
|
||||
}
|
||||
out[k] = decl
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func unionKeys[A, B any](a map[string]A, b map[string]B) []string {
|
||||
seen := map[string]bool{}
|
||||
for k := range a {
|
||||
seen[k] = true
|
||||
}
|
||||
for k := range b {
|
||||
seen[k] = true
|
||||
}
|
||||
return sortedKeys(seen)
|
||||
}
|
||||
|
||||
// ---- take-over and hand-back ---------------------------------------------------------------------------------
|
||||
|
||||
// TakeoverAsk is one take-over asked of the operator, `takeover.ask.<id>`: what the ask is composed from again, and
|
||||
// where it stands.
|
||||
type TakeoverAsk struct {
|
||||
ID string `json:"id"`
|
||||
From string `json:"from"`
|
||||
Why string `json:"why"`
|
||||
At time.Time `json:"at"`
|
||||
State string `json:"state"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
Decided time.Time `json:"decided,omitzero"`
|
||||
Answered string `json:"answered,omitempty"`
|
||||
Recorded bool `json:"recorded,omitempty"`
|
||||
}
|
||||
|
||||
func askKey(id string) string { return "ask." + id }
|
||||
|
||||
// act is what an answer to the take-over binds.
|
||||
func (t TakeoverAsk) act(option string) asks.Act {
|
||||
return asks.Act{"verb": Seat + "." + TakeOverVerb, "option": option, "ask": t.ID, "from": t.From,
|
||||
"at": t.At.UTC().Format(time.RFC3339Nano)}
|
||||
}
|
||||
|
||||
// ask is the take-over's ask, composed from it alone.
|
||||
func (t TakeoverAsk) ask() asks.Ask {
|
||||
why, _ := sayable("Why: " + t.Why)
|
||||
a := asks.Ask{ID: t.ID, Headline: "Switch config to its own sources",
|
||||
Explanation: strings.Join([]string{
|
||||
"mesh-config would read the list of predefined keys and each pull request's config from the delivery's " +
|
||||
"indexer, no longer from the controller. Nothing any node resolves changes: compare showed no difference.",
|
||||
point(why)}, "\n"),
|
||||
Who: asks.Operator, Expires: t.At.Add(ProposalLasts),
|
||||
OnExpiry: "nothing changes; config keeps reading the controller's records", About: "config.takeover",
|
||||
Details: "Read what was compared with mesh-config's compare verb, through the mesh MCP server. hand-back returns " +
|
||||
"to the controller's records until the controller's lease is handed away."}
|
||||
for _, o := range []struct{ id, label, does string }{
|
||||
{OptionApprove, "Approve", "config switches to its own sources at once, if nothing differs then"},
|
||||
{OptionDecline, "Decline", "nothing changes"},
|
||||
} {
|
||||
binds, _ := asks.ActDigest(t.act(o.id))
|
||||
a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds})
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// TakeoverAnswer is what take-over and hand-back answer.
|
||||
type TakeoverAnswer struct {
|
||||
Said string `json:"said"`
|
||||
Ask string `json:"ask,omitempty"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
// TakeOver asks the operator's Approve to switch to the own sources, refused while compare shows a difference.
|
||||
func (c *Config) TakeOver(call Call) (*TakeoverAnswer, error) {
|
||||
if strings.TrimSpace(call.Why) == "" {
|
||||
return nil, errors.New("say why the take-over is asked now: it is shown to the operator")
|
||||
}
|
||||
if call.Caller == "" || call.Caller == Unproven {
|
||||
return nil, errors.New("the call names no proven caller")
|
||||
}
|
||||
if c.Asks == nil {
|
||||
return nil, errors.New("the operator cannot be asked from here, so nothing is switched")
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
sw, err := c.source()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if sw.Source == SourceIndexer {
|
||||
return &TakeoverAnswer{Source: sw.Source, Said: "config reads its own sources already, since " +
|
||||
sw.At.UTC().Format(time.RFC3339) + " (" + sw.By + ")"}, nil
|
||||
}
|
||||
cmp, err := c.compare()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cmp.Count > 0 {
|
||||
first := cmp.Differences[0]
|
||||
return nil, fmt.Errorf("the take-over is refused: compare shows %d difference(s), the first %s %s%s%s%s: %s. "+
|
||||
"Nothing was asked", cmp.Count, first.What, first.Module, first.Node, first.Key, first.Pull, first.Why)
|
||||
}
|
||||
now := c.now()
|
||||
var open []string
|
||||
_ = c.each(TakeoverState, func(key string, raw []byte) error {
|
||||
var t TakeoverAsk
|
||||
if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting &&
|
||||
now.Before(t.At.Add(ProposalLasts)) {
|
||||
open = append(open, t.ID)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if len(open) > 0 {
|
||||
return nil, fmt.Errorf("the take-over is asked already, as %s, and waits for the operator's answer", open[0])
|
||||
}
|
||||
t := TakeoverAsk{ID: "takeover-" + newProposalID(), From: call.Caller, Why: strings.TrimSpace(call.Why), At: now.UTC(),
|
||||
State: ProposalWaiting}
|
||||
ask := t.ask()
|
||||
if err := ask.Check(now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := c.Store.Put(TakeoverState, askKey(t.ID), t); err != nil {
|
||||
return nil, fmt.Errorf("the take-over could not be kept, so the operator was not asked: %w", err)
|
||||
}
|
||||
if err := c.Asks.Publish(asks.AskSubject(Module), ask, ask.ID); err != nil {
|
||||
t.State, t.Outcome = ProposalUnasked, "the ask could not be published: "+err.Error()
|
||||
_ = c.Store.Put(TakeoverState, askKey(t.ID), t)
|
||||
return nil, fmt.Errorf("the operator could not be asked: %w", err)
|
||||
}
|
||||
return &TakeoverAnswer{Ask: t.ID, Source: sw.Source, Said: fmt.Sprintf("the operator is asked %q under "+
|
||||
"mesh-config's own name; nothing changes until they approve, and on Approve config switches if nothing differs "+
|
||||
"then", ask.Headline)}, nil
|
||||
}
|
||||
|
||||
// decideTakeovers acts on each take-over ask the router has closed. The caller holds the lock.
|
||||
func (c *Config) decideTakeovers() []string {
|
||||
var said []string
|
||||
var waiting []TakeoverAsk
|
||||
_ = c.each(TakeoverState, func(key string, raw []byte) error {
|
||||
var t TakeoverAsk
|
||||
if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting {
|
||||
waiting = append(waiting, t)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
now := c.now()
|
||||
for _, t := range waiting {
|
||||
keep := func(state, outcome string) string {
|
||||
t.State, t.Outcome, t.Decided = state, outcome, now
|
||||
_ = c.Store.Put(TakeoverState, askKey(t.ID), t)
|
||||
return "take-over " + t.ID + ": " + outcome
|
||||
}
|
||||
e, err := c.Asks.Record(t.ID)
|
||||
if err != nil || e == nil {
|
||||
if err == nil && now.After(t.At.Add(ProposalLasts+time.Hour)) {
|
||||
said = append(said, keep(ProposalEnded, "the router keeps no record of its ask; nothing was switched"))
|
||||
}
|
||||
continue
|
||||
}
|
||||
var rec routerRecord
|
||||
if json.Unmarshal(e.Value, &rec) != nil || rec.State == "open" || rec.Warrant == nil {
|
||||
continue
|
||||
}
|
||||
w := *rec.Warrant
|
||||
if w.Asker != Module || w.Ask != t.ID {
|
||||
said = append(said, keep(ProposalRefused, "the router's record is for another ask; nothing was switched"))
|
||||
continue
|
||||
}
|
||||
if w.Outcome != asks.OutcomeChosen {
|
||||
said = append(said, keep(ProposalEnded, "the ask "+string(w.Outcome)+"; nothing was switched"))
|
||||
continue
|
||||
}
|
||||
option, err := w.For(Module, t.ask())
|
||||
if err == nil {
|
||||
err = option.Performs(t.act(option.ID))
|
||||
}
|
||||
if err != nil {
|
||||
said = append(said, keep(ProposalRefused, "the answer is refused: "+err.Error()))
|
||||
continue
|
||||
}
|
||||
t.Answered = w.Says()
|
||||
if option.ID != OptionApprove {
|
||||
said = append(said, keep(ProposalDeclined, "declined by the operator; nothing was switched"))
|
||||
continue
|
||||
}
|
||||
cmp, err := c.compare()
|
||||
switch {
|
||||
case err != nil:
|
||||
said = append(said, keep(ProposalRefused, "approved, and compare could not be read, so nothing was switched: "+err.Error()))
|
||||
continue
|
||||
case cmp.Count > 0:
|
||||
said = append(said, keep(ProposalRefused, fmt.Sprintf("approved, and refused: compare shows %d difference(s) "+
|
||||
"now; nothing was switched, ask again once it is empty", cmp.Count)))
|
||||
continue
|
||||
}
|
||||
sw := Switch{Source: SourceIndexer, At: now.UTC(), By: approvedTakeover(w), Why: t.Why, Ask: t.ID}
|
||||
if err := c.Store.Put(TakeoverState, SwitchKey, sw); err != nil {
|
||||
said = append(said, fmt.Sprintf("take-over %s: approved, and the switch could not be written (%v); read again "+
|
||||
"on the next look", t.ID, err))
|
||||
continue
|
||||
}
|
||||
if err := c.Asks.Warranted(t.ID); err == nil {
|
||||
t.Recorded = true
|
||||
}
|
||||
said = append(said, keep(ProposalApproved, "approved by the operator: config reads its own sources from now"))
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
func approvedTakeover(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return "the operator"
|
||||
}
|
||||
return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity)
|
||||
}
|
||||
|
||||
// HandBack returns to the controller's records until the controller's lease is handed away.
|
||||
func (c *Config) HandBack(call Call) (*TakeoverAnswer, error) {
|
||||
if strings.TrimSpace(call.Why) == "" {
|
||||
return nil, errors.New("say why config hands back: it is kept with the switch")
|
||||
}
|
||||
if call.Caller == "" || call.Caller == Unproven {
|
||||
return nil, errors.New("the call names no proven caller")
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.leaseAway() {
|
||||
return nil, errors.New("hand-back is gone: the controller handed its lease away, and its records end with it " +
|
||||
"(novox/hq ADR 0565 §4)")
|
||||
}
|
||||
sw, err := c.source()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if sw.Source == SourceController {
|
||||
return &TakeoverAnswer{Source: sw.Source, Said: "config reads the controller's records already"}, nil
|
||||
}
|
||||
now := c.now()
|
||||
back := Switch{Source: SourceController, At: now.UTC(), By: call.Caller, Why: strings.TrimSpace(call.Why)}
|
||||
if err := c.Store.Put(TakeoverState, SwitchKey, back); err != nil {
|
||||
return nil, fmt.Errorf("the switch could not be written: %w", err)
|
||||
}
|
||||
answer := &TakeoverAnswer{Source: back.Source, Said: "config reads the controller's records again: its mirrors " +
|
||||
"declared-config and predefined, and pull-declares"}
|
||||
if err := c.Store.HandAct("handed config back to the controller's records", back.Why, "config-hand-back"); err != nil {
|
||||
answer.Said += "; the hand-act log does not hold it: " + err.Error()
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// Declared is the read of `declared`: every module's key, or one whole.
|
||||
func (c *Config) Declared(module string) (any, error) {
|
||||
if module != "" {
|
||||
e, err := c.Store.Get(DeclaredState, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e == nil {
|
||||
return nil, fmt.Errorf("declared holds no %s: it is placed nowhere, or its manifest cannot be read", module)
|
||||
}
|
||||
return json.RawMessage(e.Value), nil
|
||||
}
|
||||
rows := map[string]any{}
|
||||
err := c.each(DeclaredState, func(key string, raw []byte) error {
|
||||
var k DeclaredKept
|
||||
if json.Unmarshal(raw, &k) == nil {
|
||||
rows[key] = map[string]any{"manifest": k.Manifest, "commit": k.Commit, "keys": len(k.Config)}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sw, _ := c.source()
|
||||
return map[string]any{"declared": rows, "count": len(rows), "source": sw}, err
|
||||
}
|
||||
|
||||
// Resync reads every source again and writes what differs: the published list, the resolved values, `declared`, and
|
||||
// after the take-over the indexer's config candidates (ADR 0565 §15).
|
||||
func (c *Config) Resync() (any, error) {
|
||||
r, err := c.Reconcile()
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
out := map[string]any{"resolution": r}
|
||||
if sw, err := c.source(); err == nil && sw.Source == SourceIndexer {
|
||||
out["candidates"] = c.takeCandidates()
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The trunk's manifests aMesh declares, as the controller's mirror declared-config holds the same declarations.
|
||||
const (
|
||||
claudeDeclared = `{"module":"claude-code","settings":{
|
||||
"managed":{"kind":"","default":{},"why":"","type":"object","description":"managed settings","trusted":true,"predefined":"agent-managed-settings"}}}`
|
||||
oldDeclared = `{"module":"old","settings":{"workers":{"kind":"preference","default":4,"why":"two per core"}}}`
|
||||
)
|
||||
|
||||
// anAgreeingMesh is aProposingMesh whose controller mirrors say what mesh-config's own sources say: the mirror's
|
||||
// declarations of every placed module, and the indexer's predefined list equal to the mirror's.
|
||||
func anAgreeingMesh(t *testing.T) (*Config, *memStore, *standIn) {
|
||||
t.Helper()
|
||||
c, m, r := aProposingMesh(t)
|
||||
m.mirror(t, MirrorDeclaredRead, "dunst", dunstDeclared)
|
||||
m.mirror(t, MirrorDeclaredRead, "claude-code", claudeDeclared)
|
||||
m.mirror(t, MirrorDeclaredRead, "old", oldDeclared)
|
||||
indexerList(t, m, "aaaaaaaaaaaa", `{"agent-managed-settings":{"type":"object","description":"managed settings","trusted":true},
|
||||
"font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`)
|
||||
if _, err := c.Reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c, m, r
|
||||
}
|
||||
|
||||
// indexerList is the indexer's `predefined.list` at a commit.
|
||||
func indexerList(t *testing.T, m *memStore, commit, list string) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(delivery.PredefinedKept{V: 1, Commit: commit, List: json.RawMessage(list)})
|
||||
m.mirror(t, IndexerPredefinedRead, delivery.PredefinedListKey, string(raw))
|
||||
}
|
||||
|
||||
// candidate is the indexer's config candidate of a pull request.
|
||||
func candidate(t *testing.T, m *memStore, pull, head, declares string) {
|
||||
t.Helper()
|
||||
key, err := delivery.ConfigCandidateKey(pull)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.mirror(t, CandidatesRead, key, `{"v":1,"pull":"`+pull+`","head":"`+head+`","declares":`+declares+`}`)
|
||||
}
|
||||
|
||||
func kinds(cmp *Comparison) []string {
|
||||
var out []string
|
||||
for _, d := range cmp.Differences {
|
||||
k := d.What + ":" + d.Module + d.Pull + d.Key
|
||||
if d.Node != "" {
|
||||
k += "@" + d.Node
|
||||
}
|
||||
if !slices.Contains(out, k) {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// compare lists every difference with its reason — a module's declarations, the predefined list, a pull request's
|
||||
// config, and each value a node resolves both ways — and the take-over is refused while one stands (ADR 0565 §4 and
|
||||
// its insight of 2026-10-29; issue 1953 L5-C4).
|
||||
func TestCompareListsEveryDifferenceAndTheTakeOverWaitsForNone(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
change func(t *testing.T, m *memStore)
|
||||
want []string
|
||||
}{
|
||||
{"all agree", func(*testing.T, *memStore) {}, nil},
|
||||
{"the mirror declares a default otherwise", func(t *testing.T, m *memStore) {
|
||||
m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1))
|
||||
}, []string{"declared:dunst", "value:dunstwidth@g14", "value:dunstwidth@novox"}},
|
||||
{"the mirror lacks a placed module", func(t *testing.T, m *memStore) { _ = m.Delete(MirrorDeclaredRead, "old") },
|
||||
[]string{"declared:old", "value:old@g14", "value:oldworkers@g14"}},
|
||||
{"a node's build adds a key the mirror lacks", func(t *testing.T, m *memStore) {
|
||||
m.want(t, "g14", "dunst", "src2:"+strings.Repeat("ab", 32), strings.Replace(dunstDeclared, `"width"`, `"height":{"type":"number","default":5},"width"`, 1))
|
||||
}, []string{"value:dunstheight@g14", "value:dunstheight@novox"}}, // a key only a build declares resolves on every node (#558)
|
||||
{"the lists differ", func(t *testing.T, m *memStore) {
|
||||
indexerList(t, m, "bbbbbbbbbbbb", `{"font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`)
|
||||
}, []string{"predefined:agent-managed-settings"}},
|
||||
{"a pull request only the indexer holds", func(t *testing.T, m *memStore) {
|
||||
candidate(t, m, "novox/mesh-catalog#9", "h1", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`)
|
||||
}, []string{"pull:novox/mesh-catalog#9"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
c, m, r := anAgreeingMesh(t)
|
||||
tc.change(t, m)
|
||||
cmp, err := c.Compare()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", tc.name, err)
|
||||
}
|
||||
if got := kinds(cmp); !slices.Equal(got, tc.want) {
|
||||
t.Errorf("%s: compare lists %v, want %v (%+v)", tc.name, got, tc.want, cmp.Differences)
|
||||
}
|
||||
for _, d := range cmp.Differences {
|
||||
if d.Why == "" {
|
||||
t.Errorf("%s: a difference without its reason: %+v", tc.name, d)
|
||||
}
|
||||
}
|
||||
a, err := c.TakeOver(Call{Why: "compare is empty", Caller: "person.jochen"})
|
||||
if len(tc.want) > 0 {
|
||||
if err == nil || !strings.Contains(err.Error(), "refused") || len(r.subjects) != 0 {
|
||||
t.Errorf("%s: the take-over was asked while compare differs: %v", tc.name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil || a.Ask == "" || len(r.subjects) != 1 {
|
||||
t.Errorf("%s: the take-over was not asked: %v", tc.name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The operator's Approve switches config to the indexer: pull-declares is refused, the list and the pull requests'
|
||||
// config come from the indexer; hand-back returns to the mirrors until the lease write and is gone after it; from the
|
||||
// lease write a trusted value is set only on an Approve; declared is written once per manifest (ADR 0565 §3, §4, §13).
|
||||
func TestTheTakeOverAndTheHandBack(t *testing.T) {
|
||||
c, m, r := anAgreeingMesh(t)
|
||||
steps := []struct {
|
||||
name string
|
||||
do func() (string, error)
|
||||
want string // what the answer or the error says
|
||||
}{
|
||||
{"declared is written when it differs, and once", func() (string, error) {
|
||||
_ = m.Delete(DeclaredState, "dunst")
|
||||
first, err := c.Reconcile()
|
||||
again, _ := c.Reconcile()
|
||||
return strings.Join(first.Declared, ",") + "|" + strings.Join(again.Declared, ","), err
|
||||
}, "dunst|"},
|
||||
{"pull-declares taken before the take-over", func() (string, error) {
|
||||
a, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#7", Head: "h0",
|
||||
Declares: map[string]any{}})
|
||||
return answered(a), err
|
||||
}, "keeps what"},
|
||||
{"the take-over is asked", func() (string, error) {
|
||||
_ = m.Delete(PullsState, "novox.mesh-catalog.7") // compare holds the told pull request against no candidate
|
||||
a, err := c.TakeOver(Call{Why: "the release is delivered", Caller: "person.jochen"})
|
||||
return answered(a), err
|
||||
}, "is asked"},
|
||||
{"the operator approves: switched to the indexer", func() (string, error) {
|
||||
r.choose(t, r.lastAsk(t), OptionApprove, nil)
|
||||
out, err := c.Decide()
|
||||
return strings.Join(out, "; "), err
|
||||
}, "reads its own sources from now"},
|
||||
{"pull-declares refused after it, naming the indexer", func() (string, error) {
|
||||
_, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#8", Head: "h1",
|
||||
Declares: map[string]any{}})
|
||||
return "", err
|
||||
}, "config-candidates"},
|
||||
{"the list comes from the indexer", func() (string, error) {
|
||||
indexerList(t, m, "cccccccccccc", `{"font-size":{"type":"number","description":"text size","minimum":6,"maximum":72},
|
||||
"agent-managed-settings":{"type":"object","description":"managed settings","trusted":true},
|
||||
"theme":{"type":"string","description":"the desktop's theme"}}`)
|
||||
res, err := c.Reconcile()
|
||||
return res.Published, err
|
||||
}, "theme"},
|
||||
{"a pull request's config comes from the indexer", func() (string, error) {
|
||||
candidate(t, m, "novox/mesh-catalog#9", "h9", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`)
|
||||
res, err := c.Reconcile()
|
||||
return strings.Join(res.Candidates, "; "), err
|
||||
}, "from the indexer"},
|
||||
{"hand-back returns to the mirrors", func() (string, error) {
|
||||
a, err := c.HandBack(Call{Why: "a difference was found after the switch", Caller: "person.jochen"})
|
||||
return answered(a), err
|
||||
}, "controller's records again"},
|
||||
{"after it, declarations resolve from the mirror", func() (string, error) {
|
||||
m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1))
|
||||
_, err := c.Reconcile()
|
||||
return string(m.resolved(t, "g14", "dunst", "width").Value), err
|
||||
}, "400"},
|
||||
{"hand-back is gone after the lease write", func() (string, error) {
|
||||
m.mirror(t, LeaseRead, "holder", `{"handed-to":"the operator's hand-over"}`)
|
||||
_, err := c.HandBack(Call{Why: "again", Caller: "person.jochen"})
|
||||
return "", err
|
||||
}, "gone"},
|
||||
{"the terminal sets no trusted value after the lease write", func() (string, error) {
|
||||
_, err := c.Change(Call{Action: "set", Key: "managed", Module: "claude-code", Node: "g14", Value: map[string]any{},
|
||||
Why: "x", Caller: ControllerCaller, By: "person.jochen"})
|
||||
return "", err
|
||||
}, "propose"},
|
||||
}
|
||||
for _, s := range steps {
|
||||
got, err := s.do()
|
||||
text := got
|
||||
if err != nil {
|
||||
text = err.Error()
|
||||
}
|
||||
if !strings.Contains(text, s.want) {
|
||||
t.Errorf("%s: said %q, want %q", s.name, text, s.want)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(m.handActs, "handed config back to the controller's records | a difference was found after the switch | config-hand-back") {
|
||||
t.Errorf("the hand-back is not recorded: %v", m.handActs)
|
||||
}
|
||||
if e, _ := m.Get(DeclaredState, "dunst"); e == nil || !strings.Contains(string(e.Value), `"manifest":"`) || !strings.Contains(string(e.Value), `"commit":"c1"`) {
|
||||
t.Errorf("declared holds no dunst with its manifest: %v", e)
|
||||
}
|
||||
// An Approve that finds compare differing at the answer switches nothing.
|
||||
c2, m2, r2 := anAgreeingMesh(t)
|
||||
if _, err := c2.TakeOver(Call{Why: "now", Caller: "person.jochen"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m2.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1))
|
||||
r2.choose(t, r2.lastAsk(t), OptionApprove, nil)
|
||||
out, _ := c2.Decide()
|
||||
if sw, _ := c2.source(); sw.Source != SourceInterim || !strings.Contains(strings.Join(out, ""), "difference") {
|
||||
t.Errorf("an Approve while compare differs switched: %s, %v", sw.Source, out)
|
||||
}
|
||||
}
|
||||
|
||||
// lastAsk is the newest ask the router took.
|
||||
func (r *standIn) lastAsk(t *testing.T) string {
|
||||
t.Helper()
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var newest string
|
||||
var at time.Time
|
||||
for id, a := range r.asks {
|
||||
if newest == "" || a.Expires.After(at) {
|
||||
newest, at = id, a.Expires
|
||||
}
|
||||
}
|
||||
if newest == "" {
|
||||
t.Fatal("the router took no ask")
|
||||
}
|
||||
return newest
|
||||
}
|
||||
|
||||
func answered(a any) string {
|
||||
raw, _ := json.Marshal(a)
|
||||
return string(raw)
|
||||
}
|
||||
@@ -2,4 +2,13 @@ module mesh-config
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791
|
||||
|
||||
require (
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,2 +1,26 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
|
||||
@@ -7,7 +7,10 @@
|
||||
"name": "mesh-config",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"config"
|
||||
"config",
|
||||
"compare",
|
||||
"take-over",
|
||||
"hand-back"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -23,6 +26,8 @@
|
||||
"gitea.forge.pull-request.closed"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-hand-acts.warranted",
|
||||
"seat:mesh-controller.hand-act",
|
||||
"seat:mesh-controller.warranted",
|
||||
"gitea.gitea_get_pull_request"
|
||||
@@ -33,7 +38,18 @@
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-controller.predefined"
|
||||
"mesh-controller.predefined",
|
||||
"mesh-controller.declared-config",
|
||||
"mesh-delivery-indexer.predefined",
|
||||
"mesh-delivery-indexer.config-candidates",
|
||||
"mesh-controller.lease"
|
||||
],
|
||||
"tools": [
|
||||
"config_compare",
|
||||
"config_take_over",
|
||||
"config_hand_back",
|
||||
"config_declared",
|
||||
"config_resync"
|
||||
],
|
||||
"state": [
|
||||
"values",
|
||||
@@ -41,7 +57,9 @@
|
||||
"predefined",
|
||||
"proposals",
|
||||
"pulls",
|
||||
"kept"
|
||||
"kept",
|
||||
"declared",
|
||||
"takeover"
|
||||
],
|
||||
"logs": [
|
||||
"history"
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
/mesh-hand-acts
|
||||
/cmd/mesh-hand-acts/mesh-hand-acts
|
||||
@@ -0,0 +1,26 @@
|
||||
# mesh-hand-acts
|
||||
|
||||
Holds the seat `mesh-hand-acts` (novox/hq ADR 0565 §10, §11): the hand-act log, moved out of the controller. Go, one
|
||||
binary (`cmd/mesh-hand-acts`), served by a runtime of its own on its own bus account (`runs-as`), so that its account
|
||||
alone writes the log.
|
||||
|
||||
## What it keeps
|
||||
|
||||
- the log `acts` (the stream `LOG_mesh-hand-acts_acts`, a module log of ADR 0297, as ADR 0565 §10 has it a stream):
|
||||
every act appended in its order under its key `act-<ns>-<n>` (`warrant-<asker>-<ask>` for a warrant), in the layout
|
||||
of the controller's log. The log keeps each act 400 days (`max-age-days`) and refuses a new act when
|
||||
full rather than drop one; `hand-acts` says how full it is against its cap. Each act is said as `act-recorded`.
|
||||
- `mesh-hand-acts_warrants`: one key `<asker>.<ask>` per warrant, made once by create before its act is appended, so of
|
||||
several instances one records it; a warrant whose act was lost is written again from the router's record.
|
||||
|
||||
## Its verbs
|
||||
|
||||
- `mesh-hand-acts.hand-act`: what, why, cause, the condition addressed, and the act's verb (a repair when not given;
|
||||
`hand-act drill` for a drill, whose cause is always `drill`). Who did it is the caller the runtime proves.
|
||||
- `mesh-hand-acts.warranted`: asker and ask. Read from the router's own record of that ask (`messenger.asks`), never
|
||||
from the caller, and recorded once per ask; an ask still open, ended without a choice or another asker's is refused.
|
||||
- `mesh-hand-acts.hand-acts`: both logs, this one and the controller's `mesh-controller_hand-acts` until its lease
|
||||
ends, newest first, with the causes repaired by hand twice within a fortnight; `warrants=true` lists the warrants.
|
||||
|
||||
No act is written to both logs: every recorder calls this seat from its delivery, and the controller keeps its own
|
||||
acts in its own bucket until it loses its lease.
|
||||
@@ -0,0 +1,375 @@
|
||||
package main
|
||||
|
||||
// The hand-act log (novox/hq ADR 0565 §10, §11): every act done by hand, with who, why and its cause, so a repair a
|
||||
// person keeps making is found (S15) and a person's decision is read back. mesh-hand-acts holds the seat
|
||||
// mesh-hand-acts, once for the mesh, and is the one writer of its log `acts` — a stream, as ADR 0565 §10 has it:
|
||||
// the mesh's module log (ADR 0297), append-only, every act kept in its order under its key, refusing a new act when
|
||||
// full rather than dropping an old one, so every act is held 400 days and longer — and of its bucket `warrants` (one
|
||||
// key `<asker>.<ask>` made once by create). Each act is said as `act-recorded`.
|
||||
//
|
||||
// **Who did it is the runtime's word, never the caller's**: `by` is the bus user the runtime proved. A warrant is the
|
||||
// router's word, never the caller's: `warranted` reads the router's own record of the asker's ask and records who
|
||||
// chose, through which channel, with which proofs and which answer, once per ask. The controller's bucket
|
||||
// `mesh-controller_hand-acts` stays its own until it loses its lease and is read, never written: `hand-acts` answers
|
||||
// over both logs, and no act is written to both (ADR 0565 §11).
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
)
|
||||
|
||||
// The seat, the log, the states and the event, as mesh-sdk's handacts names them.
|
||||
const (
|
||||
Module = handacts.Module
|
||||
Seat = handacts.Seat
|
||||
ActsLog = handacts.ActsLog
|
||||
WarrantsState = handacts.WarrantsState
|
||||
ControllerActs = handacts.ControllerActs
|
||||
// RouterRecords is the router's record of every ask, `<asker>.<ask>` (ADR 0259): the warrant is read there.
|
||||
RouterRecords = "messenger.asks"
|
||||
RecordedEvent = handacts.EventRecorded
|
||||
// RecordedSubject is where the seat says it: a seat's events are on mesh.seat.<seat>.event.<name>, which its
|
||||
// consumers hear (catalogue.SeatSays).
|
||||
RecordedSubject = "mesh.seat." + Seat + ".event." + RecordedEvent
|
||||
|
||||
VerbRecord = handacts.VerbRecord
|
||||
VerbDrill = handacts.VerbDrill
|
||||
VerbWarrant = handacts.VerbWarrant
|
||||
CauseDrill = handacts.CauseDrill
|
||||
CauseOperatorAnswer = handacts.CauseOperatorAnswer
|
||||
)
|
||||
|
||||
// RepeatedWithin is how far back a cause counts as repeated (S15).
|
||||
const RepeatedWithin = 14 * 24 * time.Hour
|
||||
|
||||
// Act is one entry of the log: mesh-sdk's handacts.Act, the controller's HandAct unchanged.
|
||||
type Act = handacts.Act
|
||||
|
||||
// Store is the bus's buckets and this module's log as it reaches them, through its runtime. Create writes a key only
|
||||
// when it has none, and says ErrExists otherwise.
|
||||
type Store interface {
|
||||
Get(state, key string) ([]byte, bool, error)
|
||||
Create(state, key string, value any) error
|
||||
Keys(state string) ([]string, error)
|
||||
// Append adds an entry to the end of a log; Last is a key's newest entry; Entries every entry in order; Fill how
|
||||
// many entries and bytes the log holds against its cap.
|
||||
Append(log, key string, value any) error
|
||||
Last(log, key string) ([]byte, bool, error)
|
||||
Entries(log string) ([][]byte, error)
|
||||
Fill(log string) (Fill, error)
|
||||
}
|
||||
|
||||
// Fill is how full a log is: its entries, its bytes and its cap in bytes.
|
||||
type Fill struct {
|
||||
Entries uint64 `json:"entries"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
MaxBytes uint64 `json:"max-bytes"`
|
||||
}
|
||||
|
||||
// ErrExists is a create of a key another writer made first.
|
||||
var ErrExists = errors.New("the key was made by another writer first")
|
||||
|
||||
// Log is mesh-hand-acts: the log and what records into it.
|
||||
type Log struct {
|
||||
Store Store
|
||||
// Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish).
|
||||
Publish func(subject string, body any, id string) error
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
var seq atomic.Uint64
|
||||
|
||||
// actKey is an act's key: its time and a sequence, so the keys list in order, as the controller's do.
|
||||
func actKey(at time.Time) string {
|
||||
return "act-" + strconv.FormatInt(at.UnixNano(), 10) + "-" + strconv.FormatUint(seq.Add(1), 10)
|
||||
}
|
||||
|
||||
// Recorded is what a recorder is answered: the act as kept.
|
||||
type Recorded struct {
|
||||
Act Act `json:"act"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
// HandAct records one act a caller says it did, by hand or on purpose: what, why, its cause and the condition it
|
||||
// addresses. by is the caller the runtime proved. verb is the act's verb as the log names it: a repair by default, a
|
||||
// drill with VerbDrill, or a recorder's own (`conditions silence`, `config set`), which S15 reads as it reads the
|
||||
// controller's.
|
||||
func (l *Log) HandAct(by, verb, what, why, cause, condition string) (Recorded, error) {
|
||||
what, why, cause, verb = strings.TrimSpace(what), strings.TrimSpace(why), strings.TrimSpace(cause), strings.TrimSpace(verb)
|
||||
switch {
|
||||
case !proven(by):
|
||||
return Recorded{}, fmt.Errorf("who acts is the caller the runtime proves, and it proved none (%q): nothing was recorded", by)
|
||||
case what == "":
|
||||
return Recorded{}, errors.New("a hand act says what was done: nothing was recorded")
|
||||
case why == "":
|
||||
return Recorded{}, errors.New("a hand act says why: nothing was recorded")
|
||||
}
|
||||
if verb == "" {
|
||||
verb = VerbRecord
|
||||
}
|
||||
if verb == VerbDrill {
|
||||
cause = CauseDrill
|
||||
}
|
||||
switch {
|
||||
case cause == "":
|
||||
return Recorded{}, errors.New("a hand act says its cause, in a word or a condition's kind: the word a second act " +
|
||||
"for the same reason uses, which is how a repair done twice is found. Nothing was recorded")
|
||||
case strings.EqualFold(cause, CauseDrill) && verb != VerbDrill:
|
||||
return Recorded{}, errors.New("a drill is not recorded as a repair: record it with verb " + VerbDrill +
|
||||
", which no healer is wanted for. Nothing was recorded")
|
||||
}
|
||||
at := l.Now().UTC()
|
||||
act := Act{ID: actKey(at), At: at, By: by, Verb: verb, Args: []string{what}, Why: why, Cause: cause,
|
||||
Condition: strings.TrimSpace(condition)}
|
||||
if err := l.record(act); err != nil {
|
||||
return Recorded{}, err
|
||||
}
|
||||
return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s did %q, because %q (cause: %s)", act.ID, by, what,
|
||||
why, cause)}, nil
|
||||
}
|
||||
|
||||
// record writes one act and says it. An act not said is still kept, and the log says so.
|
||||
func (l *Log) record(act Act) error {
|
||||
if err := l.Store.Append(ActsLog, act.ID, act); err != nil {
|
||||
return fmt.Errorf("the act could not be recorded: %w", err)
|
||||
}
|
||||
if l.Publish != nil {
|
||||
if err := l.Publish(RecordedSubject, act, act.ID); err != nil {
|
||||
l.Logf("[%s] %s is recorded and could NOT be said as %s: %v", Module, act.ID, RecordedEvent, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// routerRecord is what warranted reads of the router's record of an ask.
|
||||
type routerRecord struct {
|
||||
State string `json:"state"`
|
||||
Warrant *asks.Warrant `json:"warrant"`
|
||||
Ask map[string]any `json:"ask"`
|
||||
}
|
||||
|
||||
var askerModule = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
|
||||
|
||||
// Warrant is one ask's warrant as recorded: the key `<asker>.<ask>` of `warrants`, naming the act it recorded.
|
||||
type Warrant struct {
|
||||
Act string `json:"act"`
|
||||
At time.Time `json:"at"`
|
||||
By string `json:"recorded-at-the-word-of"`
|
||||
}
|
||||
|
||||
// Warranted records an asker's act on the operator's warrant for its ask, read from the router's record, once per ask
|
||||
// however often it is asked (ADR 0274, ADR 0259). The caller gives which ask and nothing else.
|
||||
func (l *Log) Warranted(caller, asker, ask string) (Recorded, error) {
|
||||
switch {
|
||||
case !proven(caller):
|
||||
return Recorded{}, fmt.Errorf("who asks is the caller the runtime proves, and it proved none (%q): nothing was recorded", caller)
|
||||
case !askerModule.MatchString(asker):
|
||||
return Recorded{}, fmt.Errorf("%q is not a module's name", asker)
|
||||
case !asks.UsableID(ask):
|
||||
return Recorded{}, fmt.Errorf("%q is not an ask's id", ask)
|
||||
}
|
||||
key := asker + "." + ask
|
||||
claimed, found, err := l.Store.Get(WarrantsState, key)
|
||||
if err != nil {
|
||||
return Recorded{}, err
|
||||
}
|
||||
if found {
|
||||
if r, ok, err := l.already(key, claimed); err != nil || ok {
|
||||
return r, err
|
||||
}
|
||||
}
|
||||
raw, found, err := l.Store.Get(RouterRecords, key)
|
||||
if err != nil {
|
||||
return Recorded{}, fmt.Errorf("the router's record of %s's ask %s cannot be read: %w", asker, ask, err)
|
||||
}
|
||||
var rec routerRecord
|
||||
if found {
|
||||
if err := json.Unmarshal(raw, &rec); err != nil {
|
||||
return Recorded{}, fmt.Errorf("the router's record of %s's ask %s is not readable: %w", asker, ask, err)
|
||||
}
|
||||
}
|
||||
w := rec.Warrant
|
||||
switch {
|
||||
case rec.State == "open" || rec.State == "held":
|
||||
return Recorded{}, fmt.Errorf("%s's ask %s is still %s: nobody has answered it. Nothing was recorded", asker, ask, rec.State)
|
||||
case !found || rec.State == "" || w == nil:
|
||||
return Recorded{}, fmt.Errorf("the router holds no closed record of %s's ask %s. Nothing was recorded", asker, ask)
|
||||
case w.Asker != asker || w.Ask != ask:
|
||||
return Recorded{}, fmt.Errorf("the router's record is for %s's ask %s. Nothing was recorded", w.Asker, w.Ask)
|
||||
case w.Outcome != asks.OutcomeChosen || w.By == nil:
|
||||
return Recorded{}, fmt.Errorf("%s's ask %s ended %s: no person chose, so there is no warrant to record. Nothing was "+
|
||||
"recorded", asker, ask, w.Outcome)
|
||||
case w.AskDigest == "":
|
||||
return Recorded{}, fmt.Errorf("the router's warrant for %s's ask %s names no ask digest. Nothing was recorded", asker, ask)
|
||||
}
|
||||
at := l.Now().UTC()
|
||||
act := Act{ID: "warrant-" + asker + "-" + ask, At: w.At.UTC(), Verb: VerbWarrant,
|
||||
Args: []string{fmt.Sprintf("the operator chose %s on %s's ask %s", w.Label, asker, ask)},
|
||||
Why: fmt.Sprintf("%s (ask %s of %s)", w.Says(), ask, asker),
|
||||
By: fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity),
|
||||
Cause: CauseOperatorAnswer, Via: via(*w), Ask: ask, Proofs: w.Proofs,
|
||||
RequestedBy: asker + ", recorded at the word of " + caller,
|
||||
Outcome: "chosen; what " + asker + " did with it is in its own record"}
|
||||
if claimed == nil {
|
||||
// The claim first: of any number of instances asking at once, the one whose create lands records the act.
|
||||
if err := l.Store.Create(WarrantsState, key, Warrant{Act: act.ID, At: at, By: caller}); errors.Is(err, ErrExists) {
|
||||
return Recorded{Said: "another instance records " + act.ID + " now"}, nil
|
||||
} else if err != nil {
|
||||
return Recorded{}, fmt.Errorf("the warrant could not be recorded: %w", err)
|
||||
}
|
||||
}
|
||||
if err := l.record(act); err != nil {
|
||||
return Recorded{}, err
|
||||
}
|
||||
return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s, through %s", act.ID, act.Why, act.Via)}, nil
|
||||
}
|
||||
|
||||
// already answers a warrant recorded before, when its act is in the log; false when the act was lost between the
|
||||
// claim and the log, so it is read from the router again and written.
|
||||
func (l *Log) already(key string, raw []byte) (Recorded, bool, error) {
|
||||
var w Warrant
|
||||
if err := json.Unmarshal(raw, &w); err != nil {
|
||||
return Recorded{}, false, fmt.Errorf("the warrant %s is not readable: %w", key, err)
|
||||
}
|
||||
actRaw, found, err := l.Store.Last(ActsLog, w.Act)
|
||||
if err != nil || !found {
|
||||
return Recorded{}, false, err
|
||||
}
|
||||
var act Act
|
||||
_ = json.Unmarshal(actRaw, &act)
|
||||
return Recorded{Act: act, Said: "already recorded as " + act.ID + ": " + act.Why}, true, nil
|
||||
}
|
||||
|
||||
func via(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return w.Channel
|
||||
}
|
||||
v := w.Channel + " (" + w.By.Kind + ")"
|
||||
if w.By.Verified != "" {
|
||||
v += ", " + w.By.Verified
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// proven is a bus user the runtime proved: never empty, never the SDK's unproven word.
|
||||
func proven(by string) bool {
|
||||
by = strings.TrimSpace(by)
|
||||
return by != "" && !strings.EqualFold(by, "unproven") && !strings.HasPrefix(by, "(")
|
||||
}
|
||||
|
||||
// Listing is what hand-acts answers: every act of both logs within the days asked, newest first, narrowed by cause
|
||||
// and who, and the causes repaired by hand twice or more within a fortnight.
|
||||
type Listing struct {
|
||||
Acts []Act `json:"acts"`
|
||||
Repeated map[string]int `json:"repeated"`
|
||||
Days int `json:"days"`
|
||||
// Unread says which log could not be read, so what is missing is said and never read as nothing.
|
||||
Unread map[string]string `json:"unread,omitempty"`
|
||||
Kept map[string]int `json:"kept"`
|
||||
// Fill is how full this seat's log is against its cap: full, it refuses the next act.
|
||||
Fill *Fill `json:"fill,omitempty"`
|
||||
}
|
||||
|
||||
// HandActs reads both logs.
|
||||
func (l *Log) HandActs(days int, cause, who string) Listing {
|
||||
now := l.Now()
|
||||
since := now.Add(-time.Duration(days) * 24 * time.Hour)
|
||||
out := Listing{Acts: []Act{}, Repeated: map[string]int{}, Days: days, Kept: map[string]int{}}
|
||||
var all []Act
|
||||
for _, log := range []string{ActsLog, ControllerActs} {
|
||||
acts, kept, err := l.read(log, now.Add(-max(time.Duration(days)*24*time.Hour, RepeatedWithin)))
|
||||
if err != nil {
|
||||
if out.Unread == nil {
|
||||
out.Unread = map[string]string{}
|
||||
}
|
||||
out.Unread[log] = err.Error()
|
||||
continue
|
||||
}
|
||||
out.Kept[log] = kept
|
||||
all = append(all, acts...)
|
||||
}
|
||||
for _, a := range all {
|
||||
if now.Sub(a.At) <= RepeatedWithin && a.Repair() {
|
||||
out.Repeated[a.Cause]++
|
||||
}
|
||||
if a.At.Before(since) || (cause != "" && a.Cause != cause) || (who != "" && !strings.Contains(a.By, who)) {
|
||||
continue
|
||||
}
|
||||
out.Acts = append(out.Acts, a)
|
||||
}
|
||||
for c, n := range out.Repeated {
|
||||
if n < 2 {
|
||||
delete(out.Repeated, c)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out.Acts, func(i, j int) bool { return out.Acts[i].At.After(out.Acts[j].At) })
|
||||
if f, err := l.Store.Fill(ActsLog); err == nil {
|
||||
out.Fill = &f
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// read is every act of one log since a moment, and how many it keeps: this seat's log entry by entry, the
|
||||
// controller's bucket by its keys, each of which names its time, so only those within are read whole.
|
||||
func (l *Log) read(log string, since time.Time) ([]Act, int, error) {
|
||||
var out []Act
|
||||
keep := func(raw []byte, from string) {
|
||||
var a Act
|
||||
if json.Unmarshal(raw, &a) != nil || a.At.Before(since) {
|
||||
return
|
||||
}
|
||||
a.Log = from
|
||||
out = append(out, a)
|
||||
}
|
||||
if log == ActsLog {
|
||||
entries, err := l.Store.Entries(ActsLog)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
for _, raw := range entries {
|
||||
keep(raw, handacts.LogSeat)
|
||||
}
|
||||
return out, len(entries), nil
|
||||
}
|
||||
keys, err := l.Store.Keys(log)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
for _, k := range keys {
|
||||
if at, ok := keyTime(k); ok && at.Before(since) {
|
||||
continue
|
||||
}
|
||||
raw, found, err := l.Store.Get(log, k)
|
||||
if err != nil {
|
||||
return nil, len(keys), err
|
||||
}
|
||||
if found {
|
||||
keep(raw, handacts.LogController)
|
||||
}
|
||||
}
|
||||
return out, len(keys), nil
|
||||
}
|
||||
|
||||
// keyTime is the time an act's key names (`act-<ns>-<n>`), when it names one.
|
||||
func keyTime(key string) (time.Time, bool) {
|
||||
parts := strings.Split(key, "-")
|
||||
if len(parts) != 3 || parts[0] != "act" {
|
||||
return time.Time{}, false
|
||||
}
|
||||
ns, err := strconv.ParseInt(parts[1], 10, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.Unix(0, ns), true
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type entry struct {
|
||||
key string
|
||||
raw []byte
|
||||
}
|
||||
|
||||
type memStore struct {
|
||||
mu sync.Mutex
|
||||
kv map[string]map[string][]byte
|
||||
logs map[string][]entry
|
||||
keysErr map[string]error
|
||||
}
|
||||
|
||||
func newStore() *memStore {
|
||||
return &memStore{kv: map[string]map[string][]byte{}, logs: map[string][]entry{}, keysErr: map[string]error{}}
|
||||
}
|
||||
|
||||
func (s *memStore) Append(log, key string, value any) error {
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := s.keysErr[log]; err != nil {
|
||||
return err
|
||||
}
|
||||
s.logs[log] = append(s.logs[log], entry{key, raw})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) Last(log, key string) ([]byte, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for i := len(s.logs[log]) - 1; i >= 0; i-- {
|
||||
if s.logs[log][i].key == key {
|
||||
return s.logs[log][i].raw, true, nil
|
||||
}
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
func (s *memStore) Entries(log string) ([][]byte, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := s.keysErr[log]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out [][]byte
|
||||
for _, e := range s.logs[log] {
|
||||
out = append(out, e.raw)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *memStore) Fill(log string) (Fill, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return Fill{Entries: uint64(len(s.logs[log])), MaxBytes: 1 << 30}, nil
|
||||
}
|
||||
|
||||
// logKeys are a log's keys in order.
|
||||
func (s *memStore) logKeys(log string) []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []string
|
||||
for _, e := range s.logs[log] {
|
||||
out = append(out, e.key)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *memStore) Get(state, key string) ([]byte, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v, ok := s.kv[state][key]
|
||||
return v, ok, nil
|
||||
}
|
||||
|
||||
func (s *memStore) put(state, key string, value any, create bool) error {
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.kv[state] == nil {
|
||||
s.kv[state] = map[string][]byte{}
|
||||
}
|
||||
if _, ok := s.kv[state][key]; ok && create {
|
||||
return ErrExists
|
||||
}
|
||||
s.kv[state][key] = raw
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) }
|
||||
func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) }
|
||||
|
||||
func (s *memStore) Keys(state string) ([]string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := s.keysErr[state]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for k := range s.kv[state] {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func newLog() (*Log, *memStore, *[]Act, *time.Time) {
|
||||
store := newStore()
|
||||
var said []Act
|
||||
now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
|
||||
l := &Log{Store: store, Logf: func(string, ...any) {}, Now: func() time.Time { return now },
|
||||
Publish: func(subject string, body any, id string) error {
|
||||
if a, ok := body.(Act); ok && subject == "mesh.seat.mesh-hand-acts.event.act-recorded" && id == a.ID {
|
||||
said = append(said, a)
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
return l, store, &said, &now
|
||||
}
|
||||
|
||||
// hand-act records what the proven caller says it did, once, and says it; it refuses what cannot be read back, a
|
||||
// caller the runtime did not prove, and a drill recorded as a repair (ADR 0565 §10; issue 1953 L5-C1).
|
||||
func TestHandActRecordsWhatTheProvenCallerDid(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, by, verb, what, why, cause string
|
||||
wantVerb, wantCause, err string
|
||||
}{
|
||||
{"a repair", "ace.mesh-config", "", "set dns", "it was wrong", "config", VerbRecord, "config", ""},
|
||||
{"a recorder's own verb", "novox.mesh-conditions", "conditions silence", "silenced x", "known", "operator-answer",
|
||||
"conditions silence", "operator-answer", ""},
|
||||
{"a drill takes its cause", "novox.mesh-healers", VerbDrill, "stopped records", "does it raise", "", VerbDrill, CauseDrill, ""},
|
||||
{"no proven caller", "unproven", "", "x", "y", "z", "", "", "proves"},
|
||||
{"by said empty", "", "", "x", "y", "z", "", "", "proves"},
|
||||
{"no why", "person.jochen", "", "x", "", "z", "", "", "says why"},
|
||||
{"no what", "person.jochen", "", "", "y", "z", "", "", "says what"},
|
||||
{"no cause", "person.jochen", "", "x", "y", "", "", "", "cause"},
|
||||
{"a drill as a repair", "person.jochen", "", "x", "y", "drill", "", "", "not recorded as a repair"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
l, store, said, _ := newLog()
|
||||
r, err := l.HandAct(tc.by, tc.verb, tc.what, tc.why, tc.cause, "bus.x.missing")
|
||||
keys := store.logKeys(ActsLog)
|
||||
if tc.err != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 || len(*said) != 0 {
|
||||
t.Errorf("%s: %v, kept %v", tc.name, err, keys)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil || len(keys) != 1 || keys[0] != r.Act.ID || len(*said) != 1 {
|
||||
t.Fatalf("%s: %v, kept %v, said %d", tc.name, err, keys, len(*said))
|
||||
}
|
||||
var kept Act
|
||||
raw, _, _ := store.Last(ActsLog, keys[0])
|
||||
_ = json.Unmarshal(raw, &kept)
|
||||
if kept.By != tc.by || kept.Verb != tc.wantVerb || kept.Cause != tc.wantCause || kept.Condition != "bus.x.missing" ||
|
||||
kept.Args[0] != tc.what || kept.Log != "" {
|
||||
t.Errorf("%s: kept %+v", tc.name, kept)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// warranted records the router's word once per ask, never the caller's; it refuses an ask still open, one ended
|
||||
// without a choice, another asker's, and one the router holds no record of (ADR 0565 §10, ADR 0259).
|
||||
func TestWarrantedRecordsTheRoutersWordOnce(t *testing.T) {
|
||||
chosen := `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"chosen","label":"Approve",
|
||||
"by":{"who":"operator","kind":"telegram","identity":"42","verified":"P2"},"channel":"telegram","proofs":["P1","P2"],
|
||||
"at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}`
|
||||
cases := []struct {
|
||||
name, record, asker, ask, err string
|
||||
// claimed is a warrant claimed by an instance that stopped before its act reached the log.
|
||||
claimed bool
|
||||
}{
|
||||
{"chosen", chosen, "claude-code", "a1", "", false},
|
||||
{"claimed, its act lost: written now", chosen, "claude-code", "a1", "", true},
|
||||
{"still open", `{"state":"open"}`, "claude-code", "a1", "still open", false},
|
||||
{"no record", "", "claude-code", "a1", "no closed record", false},
|
||||
{"expired", `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"expired","at":"2026-10-29T21:00:00Z"}}`,
|
||||
"claude-code", "a1", "no person chose", false},
|
||||
{"another asker's", strings.Replace(chosen, `"asker":"claude-code"`, `"asker":"mesh-config"`, 1), "claude-code", "a1",
|
||||
"record is for mesh-config", false},
|
||||
{"no digest", strings.Replace(chosen, `,"ask-digest":"sha256:x"`, "", 1), "claude-code", "a1", "no ask digest", false},
|
||||
{"not a module", chosen, "Claude Code", "a1", "not a module", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
l, store, said, _ := newLog()
|
||||
if tc.record != "" {
|
||||
_ = store.Put(RouterRecords, "claude-code.a1", json.RawMessage(tc.record))
|
||||
}
|
||||
if tc.claimed {
|
||||
_ = store.Create(WarrantsState, "claude-code.a1", Warrant{Act: "warrant-claude-code-a1"})
|
||||
}
|
||||
r, err := l.Warranted("ace.claude-code", tc.asker, tc.ask)
|
||||
if tc.err != "" {
|
||||
if keys := store.logKeys(ActsLog); err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 {
|
||||
t.Errorf("%s: %v, kept %v", tc.name, err, keys)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil || r.Act.Verb != VerbWarrant || r.Act.Cause != CauseOperatorAnswer ||
|
||||
!strings.Contains(r.Act.By, "operator, as telegram identity 42") || r.Act.Via != "telegram (telegram), P2" ||
|
||||
!strings.Contains(r.Act.RequestedBy, "ace.claude-code") || len(r.Act.Proofs) != 2 {
|
||||
t.Fatalf("%s: %+v, %v", tc.name, r.Act, err)
|
||||
}
|
||||
// Asked again, by another instance: nothing more is recorded.
|
||||
again, err := l.Warranted("g14.claude-code", "claude-code", "a1")
|
||||
keys := store.logKeys(ActsLog)
|
||||
if err != nil || !strings.HasPrefix(again.Said, "already recorded") || len(keys) != 1 || len(*said) != 1 {
|
||||
t.Errorf("%s: a second warranted: %q, %v, kept %v, said %d", tc.name, again.Said, err, keys, len(*said))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hand-acts answers over both logs, newest first, narrowed by days, cause and who, and counts each cause repaired by
|
||||
// hand twice in a fortnight across both, never a person's decision; a log it cannot read is said, never read as empty.
|
||||
func TestHandActsReadsBothLogs(t *testing.T) {
|
||||
l, store, _, now := newLog()
|
||||
put := func(log, verb, cause, by string, ago time.Duration) {
|
||||
at := now.Add(-ago)
|
||||
a := Act{At: at, Verb: verb, Cause: cause, By: by, Why: "x"}
|
||||
if log == ActsLog {
|
||||
_ = store.Append(log, actKey(at), a)
|
||||
return
|
||||
}
|
||||
_ = store.Put(log, actKey(at), a)
|
||||
}
|
||||
put(ActsLog, VerbRecord, "relogin", "person.jochen", time.Hour)
|
||||
put(ControllerActs, VerbRecord, "relogin", "shanks.claude-code", 3*24*time.Hour)
|
||||
put(ControllerActs, VerbDrill, CauseDrill, "person.jochen", time.Hour)
|
||||
put(ActsLog, VerbDrill, CauseDrill, "person.jochen", 2*time.Hour)
|
||||
put(ActsLog, VerbWarrant, CauseOperatorAnswer, "the operator", time.Hour)
|
||||
put(ActsLog, VerbWarrant, CauseOperatorAnswer, "the operator", 2*time.Hour)
|
||||
put(ControllerActs, VerbRecord, "config", "ace.mesh-config", 30*24*time.Hour)
|
||||
cases := []struct {
|
||||
name string
|
||||
days int
|
||||
cause, who string
|
||||
want int
|
||||
}{
|
||||
{"a fortnight", 14, "", "", 6},
|
||||
{"a day", 1, "", "", 5},
|
||||
{"forty days", 40, "", "", 7},
|
||||
{"by cause", 14, "relogin", "", 2},
|
||||
{"by who", 14, "", "claude-code", 1},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := l.HandActs(tc.days, tc.cause, tc.who)
|
||||
if len(got.Acts) != tc.want || len(got.Repeated) != 1 || got.Repeated["relogin"] != 2 || got.Unread != nil {
|
||||
t.Errorf("%s: %d acts, repeated %v, unread %v", tc.name, len(got.Acts), got.Repeated, got.Unread)
|
||||
}
|
||||
for i := 1; i < len(got.Acts); i++ {
|
||||
if got.Acts[i].At.After(got.Acts[i-1].At) {
|
||||
t.Errorf("%s: not newest first", tc.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if got := l.HandActs(14, "", ""); got.Acts[0].Log != Seat || got.Kept[ControllerActs] != 3 || got.Fill == nil ||
|
||||
got.Fill.Entries != 4 {
|
||||
t.Errorf("the log of an act or the kept counts are not said: %+v", got.Kept)
|
||||
}
|
||||
store.keysErr[ControllerActs] = errors.New("the bus is away")
|
||||
if got := l.HandActs(14, "", ""); got.Unread[ControllerActs] == "" || len(got.Acts) != 4 {
|
||||
t.Errorf("an unreadable log: %+v", got.Unread)
|
||||
}
|
||||
w, err := l.Warrants()
|
||||
if err != nil || w.(map[string]any)["count"] != 0 {
|
||||
t.Errorf("warrants answered %v, %v", w, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
// mesh-hand-acts: the holder of the seat mesh-hand-acts (novox/hq ADR 0565 §10). A Go bundle served by a runtime of
|
||||
// its own, on its own bus account (`runs-as`), so that its account alone writes the hand-act log (`acts`, its module
|
||||
// log: a stream that keeps every act in order) and `warrants`. It reads the router's record of each ask for a warrant and the controller's log for
|
||||
// `hand-acts`. stdout is the MCP channel; what this module says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
|
||||
// runtimeStore is the bus, through the runtime that launched this bundle.
|
||||
type runtimeStore struct{}
|
||||
|
||||
func (runtimeStore) Get(state, key string) ([]byte, bool, error) {
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}
|
||||
|
||||
func (runtimeStore) Create(state, key string, value any) error {
|
||||
_, err := stdio.State(state).Create(key, value)
|
||||
if errors.Is(err, stdio.ErrChanged) {
|
||||
return ErrExists
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
func (runtimeStore) Append(log, key string, value any) error {
|
||||
_, err := stdio.Log(log).Append(key, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (runtimeStore) Last(log, key string) ([]byte, bool, error) {
|
||||
e, err := stdio.Log(log).Last(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}
|
||||
|
||||
func (runtimeStore) Entries(log string) ([][]byte, error) {
|
||||
var out [][]byte
|
||||
for from := uint64(0); ; {
|
||||
page, err := stdio.Log(log).Read(from, "", 1000)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range page.Entries {
|
||||
out = append(out, e.Value)
|
||||
}
|
||||
if page.Next == 0 {
|
||||
return out, nil
|
||||
}
|
||||
from = page.Next
|
||||
}
|
||||
}
|
||||
|
||||
func (runtimeStore) Fill(log string) (Fill, error) {
|
||||
i, err := stdio.Log(log).Info()
|
||||
if err != nil {
|
||||
return Fill{}, err
|
||||
}
|
||||
return Fill{Entries: i.Entries, Bytes: i.Bytes, MaxBytes: i.MaxBytes}, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
l := &Log{Store: runtimeStore{}, Now: time.Now, Logf: logf,
|
||||
Publish: stdio.SeatPublish}
|
||||
if err := stdio.Serve("", Tools(l)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The manifest says what the code does: it claims the seat mesh-hand-acts once for the mesh with its verbs (ADR 0565
|
||||
// §1, §10), runs on its own account, says act-recorded, reads the controller's log and the router's records, keeps its
|
||||
// acts as its log (a stream, §10) and warrants as a bucket that never ages, so a warrant is recorded once. Every state
|
||||
// and log it keeps and reads has a tool that reads it (ADR 0565 §15).
|
||||
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Runs string `json:"runs-as"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
Emits []string `json:"emits"`
|
||||
Reads []string `json:"reads"`
|
||||
Logs []struct {
|
||||
Name string `json:"name"`
|
||||
MaxAgeDays int `json:"max-age-days"`
|
||||
} `json:"logs"`
|
||||
State []json.RawMessage `json:"state"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Module != Module || m.Runs != Module {
|
||||
t.Errorf("module %q runs as %q", m.Module, m.Runs)
|
||||
}
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) {
|
||||
t.Errorf("claims %+v", m.Claims)
|
||||
}
|
||||
// act-recorded is the seat's event, said on the seat's subject: a module lists no event of a seat it claims.
|
||||
if len(m.Emits) != 0 || !slices.Equal(m.Reads, []string{ControllerActs, RouterRecords}) {
|
||||
t.Errorf("emits %v, reads %v", m.Emits, m.Reads)
|
||||
}
|
||||
readBy := map[string]string{ActsLog: HandActsTool, WarrantsState: HandActsTool, ControllerActs: HandActsTool,
|
||||
RouterRecords: WarrantedTool}
|
||||
tools := map[string]bool{}
|
||||
for _, tool := range Tools(&Log{}) {
|
||||
tools[tool.Name] = true
|
||||
}
|
||||
// The acts are a log kept 400 days (ADR 0565 §10).
|
||||
if len(m.Logs) != 1 || m.Logs[0].Name != ActsLog || m.Logs[0].MaxAgeDays != 400 {
|
||||
t.Errorf("logs %+v, want the acts as a log kept 400 days", m.Logs)
|
||||
}
|
||||
for _, raw := range m.State {
|
||||
var name string
|
||||
if json.Unmarshal(raw, &name) != nil || name != WarrantsState {
|
||||
t.Errorf("the state %s is not the warrants, or ages", raw)
|
||||
}
|
||||
if !tools[readBy[name]] {
|
||||
t.Errorf("the state %s has no tool that reads it", name)
|
||||
}
|
||||
}
|
||||
if !tools[readBy[ActsLog]] {
|
||||
t.Errorf("the log %s has no tool that reads it", ActsLog)
|
||||
}
|
||||
for _, r := range m.Reads {
|
||||
if !tools[readBy[r]] {
|
||||
t.Errorf("the read %s has no tool that answers it", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (issue 1936).
|
||||
modules/mesh-hand-acts/module.json
|
||||
@@ -0,0 +1,110 @@
|
||||
package main
|
||||
|
||||
// The seat's verbs (novox/hq ADR 0565 §10, §15): hand-act and warranted record, hand-acts reads both logs and, with
|
||||
// warrants, every warrant recorded. A recovery is a call: a warrant whose act was lost is written again by asking
|
||||
// warranted again, from the router's record.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// Verbs are the seat's verbs, as the manifest claims them.
|
||||
var Verbs = []string{"hand-act", "hand-acts", "warranted"}
|
||||
|
||||
// Tool names.
|
||||
const (
|
||||
HandActTool = Seat + ".hand-act"
|
||||
HandActsTool = Seat + ".hand-acts"
|
||||
WarrantedTool = Seat + ".warranted"
|
||||
)
|
||||
|
||||
func obj(props map[string]any, required ...string) map[string]any {
|
||||
m := map[string]any{"type": "object", "properties": props}
|
||||
if len(required) > 0 {
|
||||
m["required"] = required
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} }
|
||||
|
||||
// Tools are what the runtime serves.
|
||||
func Tools(l *Log) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: HandActTool,
|
||||
Description: "Record one act done by hand (novox/hq ADR 0565 §10): what was done, why, its cause in a word or a " +
|
||||
"condition's kind (the word a second act for the same reason uses, which is how a repair done twice is " +
|
||||
"found) and the condition it addresses. Who did it is the caller the runtime proves. verb is the act's " +
|
||||
"verb as the log names it: a repair when not given, `hand-act drill` for a drill, or a recorder's own " +
|
||||
"(`conditions silence`, `config set`). Said as act-recorded.",
|
||||
Input: obj(map[string]any{"what": str("what was done"), "why": str("why, in a sentence"),
|
||||
"cause": str("the cause, in a word or a condition's kind"), "condition": str("the condition's key, if any"),
|
||||
"verb": str("the act's verb in the log; a repair when not given")}, "what", "why"),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return l.HandAct(stdio.CallerOf(a), arg(a, "verb"), arg(a, "what"), arg(a, "why"), arg(a, "cause"), arg(a, "condition"))
|
||||
}},
|
||||
{Name: HandActsTool,
|
||||
Description: "What was done by hand lately, over both logs (this seat's and the controller's until its lease " +
|
||||
"ends), newest first: each act with who, why, its cause and which log holds it, and the causes repaired by " +
|
||||
"hand twice or more within a fortnight (a healer is wanted). Narrowed by cause and who; warrants=true lists " +
|
||||
"every warrant recorded instead.",
|
||||
Input: obj(map[string]any{"days": map[string]any{"type": "number", "description": "how many days back (default 14)"},
|
||||
"cause": str("only this cause"), "who": str("only acts whose who contains this"),
|
||||
"warrants": str("\"true\": every warrant recorded, `<asker>.<ask>` with its act")}),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
if strings.EqualFold(arg(a, "warrants"), "true") || a["warrants"] == true {
|
||||
return l.Warrants()
|
||||
}
|
||||
return l.HandActs(days(a), arg(a, "cause"), arg(a, "who")), nil
|
||||
}},
|
||||
{Name: WarrantedTool,
|
||||
Description: "Record an asker's act on the operator's warrant (ADR 0274, ADR 0259): read from the router's own " +
|
||||
"record of that ask — who chose, through which channel, with which proofs and which answer — never from the " +
|
||||
"caller, once per ask; refused for an ask still open, ended without a choice, or another asker's.",
|
||||
Input: obj(map[string]any{"asker": str("the module that asked"), "ask": str("its ask's id")}, "asker", "ask"),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return l.Warranted(stdio.CallerOf(a), arg(a, "asker"), arg(a, "ask"))
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// Warrants is every warrant recorded, by key.
|
||||
func (l *Log) Warrants() (any, error) {
|
||||
keys, err := l.Store.Keys(WarrantsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the warrants cannot be read: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
rows := map[string]json.RawMessage{}
|
||||
for _, k := range keys {
|
||||
if raw, found, err := l.Store.Get(WarrantsState, k); err == nil && found {
|
||||
rows[k] = raw
|
||||
}
|
||||
}
|
||||
return map[string]any{"warrants": rows, "count": len(rows)}, nil
|
||||
}
|
||||
|
||||
func arg(a map[string]any, k string) string {
|
||||
s, _ := a[k].(string)
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
func days(a map[string]any) int {
|
||||
switch v := a["days"].(type) {
|
||||
case float64:
|
||||
if v > 0 {
|
||||
return int(v)
|
||||
}
|
||||
case string:
|
||||
var n int
|
||||
if _, err := fmt.Sscan(v, &n); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 14
|
||||
}
|
||||
@@ -0,0 +1,405 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
// mesh-hand-acts whole against a real bus (novox/hq ADR 0565 §10; issue 1953 L5-C2): the built binary, played over its
|
||||
// real stdin and stdout as the node's tool runtime plays it (tools/call with the caller in _meta; mesh/state.* answered
|
||||
// from a real nats-server's key-value buckets, mesh/log.* from the log's real stream, mesh/publish onto its subjects). A recorder's hand act is kept in the
|
||||
// log on the bus under the proven caller and heard as act-recorded; a warrant the router recorded is recorded once.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
type wire struct {
|
||||
ID json.RawMessage `json:"id,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Params json.RawMessage `json:"params,omitempty"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// player is the node's tool runtime, as far as the self-check needs it, over a real bus.
|
||||
type player struct {
|
||||
ctx context.Context
|
||||
nc *nats.Conn
|
||||
js jetstream.JetStream
|
||||
mu sync.Mutex
|
||||
in io.Writer
|
||||
next int
|
||||
pending map[string]chan wire
|
||||
}
|
||||
|
||||
func (p *player) send(m map[string]any) {
|
||||
raw, _ := json.Marshal(m)
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
_, _ = p.in.Write(append(raw, '\n'))
|
||||
}
|
||||
|
||||
func (p *player) answer(id json.RawMessage, result any, err error) {
|
||||
if err != nil {
|
||||
p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
|
||||
return
|
||||
}
|
||||
p.send(map[string]any{"id": id, "result": result})
|
||||
}
|
||||
|
||||
func (p *player) serve(out io.Reader) {
|
||||
scanner := bufio.NewScanner(out)
|
||||
scanner.Buffer(make([]byte, 1<<20), 1<<24)
|
||||
for scanner.Scan() {
|
||||
var m wire
|
||||
if json.Unmarshal(scanner.Bytes(), &m) != nil {
|
||||
continue
|
||||
}
|
||||
if m.Method == "" {
|
||||
var key string
|
||||
_ = json.Unmarshal(m.ID, &key)
|
||||
p.mu.Lock()
|
||||
ch := p.pending[key]
|
||||
delete(p.pending, key)
|
||||
p.mu.Unlock()
|
||||
if ch != nil {
|
||||
ch <- m
|
||||
}
|
||||
continue
|
||||
}
|
||||
go p.handle(m)
|
||||
}
|
||||
}
|
||||
|
||||
// kv is the bucket a state names: this module's own by its local name, another's as `<owner>.<name>`.
|
||||
func (p *player) kv(state string) (jetstream.KeyValue, error) {
|
||||
if owner, name, ok := strings.Cut(state, "."); ok {
|
||||
return p.js.KeyValue(p.ctx, owner+"_"+name)
|
||||
}
|
||||
return p.js.KeyValue(p.ctx, Module+"_"+state)
|
||||
}
|
||||
|
||||
func (p *player) handle(m wire) {
|
||||
var a struct {
|
||||
State string `json:"state"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &a)
|
||||
switch m.Method {
|
||||
case "mesh/state.get":
|
||||
kv, err := p.kv(a.State)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
e, err := kv.Get(p.ctx, a.Key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
p.answer(m.ID, nil, nil)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
|
||||
case "mesh/state.put":
|
||||
kv, err := p.kv(a.State)
|
||||
if err == nil {
|
||||
var rev uint64
|
||||
if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil {
|
||||
p.answer(m.ID, map[string]any{"revision": rev}, nil)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.answer(m.ID, nil, err)
|
||||
case "mesh/state.create":
|
||||
kv, err := p.kv(a.State)
|
||||
if err == nil {
|
||||
var rev uint64
|
||||
if rev, err = kv.Create(p.ctx, a.Key, a.Value); err == nil {
|
||||
p.answer(m.ID, map[string]any{"revision": rev}, nil)
|
||||
return
|
||||
}
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32001, "message": "the key was written by another writer first"}})
|
||||
return
|
||||
}
|
||||
}
|
||||
p.answer(m.ID, nil, err)
|
||||
case "mesh/state.watch":
|
||||
kv, err := p.kv(a.State)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
key := a.Key
|
||||
if key == "" {
|
||||
key = ">"
|
||||
}
|
||||
w, err := kv.Watch(p.ctx, key)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
var watch string
|
||||
_ = json.Unmarshal(m.ID, &watch)
|
||||
current := true
|
||||
for e := range w.Updates() {
|
||||
if e == nil {
|
||||
current = false
|
||||
p.answer(m.ID, map[string]any{}, nil)
|
||||
continue
|
||||
}
|
||||
op := "put"
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
op = "delete"
|
||||
}
|
||||
change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current}
|
||||
if op == "put" {
|
||||
change["value"] = json.RawMessage(e.Value())
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.next++
|
||||
id := fmt.Sprintf("w-%d", p.next)
|
||||
p.mu.Unlock()
|
||||
p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}})
|
||||
}
|
||||
case "mesh/subscribe":
|
||||
p.answer(m.ID, map[string]any{}, nil)
|
||||
case "mesh/log.append", "mesh/log.read", "mesh/log.last", "mesh/log.info":
|
||||
p.log(m)
|
||||
case "mesh/state.keys":
|
||||
kv, err := p.kv(a.State)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
keys, err := kv.Keys(p.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
keys, err = []string{}, nil
|
||||
}
|
||||
p.answer(m.ID, keys, err)
|
||||
case "mesh/state.delete":
|
||||
kv, err := p.kv(a.State)
|
||||
if err == nil {
|
||||
err = kv.Delete(p.ctx, a.Key)
|
||||
}
|
||||
p.answer(m.ID, map[string]any{}, err)
|
||||
case "mesh/seat.publish":
|
||||
var e struct {
|
||||
Subject string `json:"subject"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &e)
|
||||
p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body))
|
||||
case "mesh/publish":
|
||||
var e struct {
|
||||
Key string `json:"key"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &e)
|
||||
p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body))
|
||||
default:
|
||||
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
|
||||
}
|
||||
}
|
||||
|
||||
// log answers the runtime's log verbs from the log's real stream, LOG_<module>_<name> on mesh.log.<module>.<name>.<key>.
|
||||
func (p *player) log(m wire) {
|
||||
var a struct {
|
||||
Log string `json:"log"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
From uint64 `json:"from"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &a)
|
||||
stream := "LOG_" + Module + "_" + a.Log
|
||||
subject := "mesh.log." + Module + "." + a.Log
|
||||
s, err := p.js.Stream(p.ctx, stream)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
entry := func(r *jetstream.RawStreamMsg) map[string]any {
|
||||
return map[string]any{"sequence": r.Sequence, "key": strings.TrimPrefix(r.Subject, subject+"."),
|
||||
"value": json.RawMessage(r.Data), "at": r.Time}
|
||||
}
|
||||
switch m.Method {
|
||||
case "mesh/log.append":
|
||||
ack, err := p.js.Publish(p.ctx, subject+"."+a.Key, a.Value)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
p.answer(m.ID, map[string]any{"sequence": ack.Sequence}, nil)
|
||||
case "mesh/log.last":
|
||||
r, err := s.GetLastMsgForSubject(p.ctx, subject+"."+a.Key)
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
p.answer(m.ID, nil, nil)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
p.answer(m.ID, entry(r), nil)
|
||||
case "mesh/log.read":
|
||||
info, err := s.Info(p.ctx)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
entries := []map[string]any{}
|
||||
for seq := max(a.From, 1); seq <= info.State.LastSeq; seq++ {
|
||||
if r, err := s.GetMsg(p.ctx, seq); err == nil {
|
||||
entries = append(entries, entry(r))
|
||||
}
|
||||
}
|
||||
p.answer(m.ID, map[string]any{"entries": entries, "next": 0, "last": info.State.LastSeq}, nil)
|
||||
case "mesh/log.info":
|
||||
info, err := s.Info(p.ctx)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
p.answer(m.ID, map[string]any{"entries": info.State.Msgs, "bytes": info.State.Bytes,
|
||||
"max_bytes": info.Config.MaxBytes, "first": info.State.FirstSeq, "last": info.State.LastSeq}, nil)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *player) call(name, caller string, args map[string]any) (json.RawMessage, error) {
|
||||
p.mu.Lock()
|
||||
p.next++
|
||||
id := fmt.Sprintf("c-%d", p.next)
|
||||
ch := make(chan wire, 1)
|
||||
p.pending[id] = ch
|
||||
p.mu.Unlock()
|
||||
p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args,
|
||||
"_meta": map[string]any{"mesh/caller": caller}}})
|
||||
select {
|
||||
case m := <-ch:
|
||||
if m.Error != nil {
|
||||
return nil, errors.New(m.Error.Message)
|
||||
}
|
||||
var r struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
}
|
||||
if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 {
|
||||
return nil, fmt.Errorf("%s answered %s", name, m.Result)
|
||||
}
|
||||
if r.IsError {
|
||||
return nil, errors.New(r.Content[0].Text)
|
||||
}
|
||||
return json.RawMessage(r.Content[0].Text), nil
|
||||
case <-time.After(60 * time.Second):
|
||||
return nil, fmt.Errorf("%s did not answer in 60s", name)
|
||||
}
|
||||
}
|
||||
|
||||
// 1953/L5-C2: an act recorded through the seat is kept on the bus and heard as act-recorded.
|
||||
func TestTheHandActLogWholeRecordsOnTheBus(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
dir := t.TempDir()
|
||||
bin := filepath.Join(dir, Module)
|
||||
build := exec.Command("go", "build", "-o", bin, ".")
|
||||
build.Stderr = os.Stderr
|
||||
if err := build.Run(); err != nil {
|
||||
t.Fatalf("mesh-hand-acts does not build: %v", err)
|
||||
}
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, b := range []string{Module + "_" + WarrantsState, "mesh-controller_hand-acts", "messenger_asks"} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The log as the mesh makes it (ADR 0297): a stream kept by limits, refusing when full, never deleting.
|
||||
logStream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "LOG_" + Module + "_" + ActsLog,
|
||||
Subjects: []string{"mesh.log." + Module + "." + ActsLog + ".>"}, MaxBytes: 64 << 20, Discard: jetstream.DiscardNew,
|
||||
DenyDelete: true, DenyPurge: true, AllowDirect: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recorded, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + RecordedEvent)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, bin)
|
||||
cmd.Env = append(os.Environ(), "MESH_NODE=novox", "MESH_MODULE="+Module)
|
||||
stdin, _ := cmd.StdinPipe()
|
||||
stdout, _ := cmd.StdoutPipe()
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
|
||||
p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
|
||||
go p.serve(stdout)
|
||||
|
||||
raw, err := p.call(HandActTool, "ace.mesh-config", map[string]any{"what": "set dns on g14", "why": "it was wrong",
|
||||
"cause": "config", "by": "someone else"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var r Recorded
|
||||
if err := json.Unmarshal(raw, &r); err != nil || r.Act.By != "ace.mesh-config" {
|
||||
t.Fatalf("hand-act answered %s", raw)
|
||||
}
|
||||
e, err := logStream.GetLastMsgForSubject(ctx, "mesh.log."+Module+"."+ActsLog+"."+r.Act.ID)
|
||||
if err != nil || !strings.Contains(string(e.Data), `"by":"ace.mesh-config"`) {
|
||||
t.Fatalf("the act is not kept in the log on the bus: %v", err)
|
||||
}
|
||||
msg, err := recorded.NextMsg(10 * time.Second)
|
||||
if err != nil || !strings.Contains(string(msg.Data), r.Act.ID) {
|
||||
t.Fatalf("act-recorded was not said: %v", err)
|
||||
}
|
||||
// A warrant the router recorded, asked twice: one act.
|
||||
asks, _ := js.KeyValue(ctx, "messenger_asks")
|
||||
if _, err := asks.Put(ctx, "claude-code.a1", []byte(`{"state":"closed","warrant":{"ask":"a1","asker":"claude-code",
|
||||
"outcome":"chosen","label":"Approve","by":{"who":"operator","kind":"telegram","identity":"42"},"channel":"telegram",
|
||||
"at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, caller := range []string{"ace.claude-code", "g14.claude-code"} {
|
||||
if _, err := p.call(WarrantedTool, caller, map[string]any{"asker": "claude-code", "ask": "a1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if info, err := logStream.Info(ctx); err != nil || info.State.Msgs != 2 {
|
||||
t.Errorf("the log keeps %v, want the act and one warrant (%v)", info, err)
|
||||
}
|
||||
listed, err := p.call(HandActsTool, "person.jochen", map[string]any{})
|
||||
if err != nil || !strings.Contains(string(listed), "warrant-claude-code-a1") || !strings.Contains(string(listed), r.Act.ID) {
|
||||
t.Errorf("hand-acts answered %s, %v", listed, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
module mesh-hand-acts
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,65 @@
|
||||
{
|
||||
"module": "mesh-hand-acts",
|
||||
"version": "1",
|
||||
"runs-as": "mesh-hand-acts",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-hand-acts",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"hand-act",
|
||||
"hand-acts",
|
||||
"warranted"
|
||||
]
|
||||
}
|
||||
],
|
||||
"reads": [
|
||||
"mesh-controller.hand-acts",
|
||||
"messenger.asks"
|
||||
],
|
||||
"logs": [
|
||||
{
|
||||
"name": "acts",
|
||||
"max-age-days": 400
|
||||
}
|
||||
],
|
||||
"state": [
|
||||
"warrants"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-hand-acts",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-hand-acts",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-hand-acts"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-hand-acts",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-hand-acts",
|
||||
"binary": "mesh-hand-acts",
|
||||
"loads": [
|
||||
"mesh-hand-acts"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
/mesh-healers
|
||||
/cmd/mesh-healers/mesh-healers
|
||||
@@ -0,0 +1,46 @@
|
||||
# mesh-healers
|
||||
|
||||
Holds the seat `mesh-healers` (novox/hq ADR 0564 §7, §8): the mesh's healers, moved out of the controller. Go, one
|
||||
binary (`cmd/mesh-healers`), served by a runtime of its own on its own bus account (`runs-as`).
|
||||
|
||||
## What it does
|
||||
|
||||
It hears the condition keeper's `condition-raised` and `condition-changed` (seat `mesh-conditions`) on a durable
|
||||
consumer. A condition whose kind a healer answers is acted on within the healer's budget, its settle and the mesh-wide
|
||||
brake (twelve acts an hour). A repair is a call of the seat or tool that owns the act: H3 asks the grants part's
|
||||
`grants_assert` for the module whose consumer is gone. H4 (the controller's own consumer) and H5 (the provider's own
|
||||
repair) are registered and run elsewhere.
|
||||
|
||||
Each act is begun by a create of its key in `attempts` before it is made and finished after, so a holder that dies
|
||||
mid-act has spent it and a second instance cannot begin it again. The condition keeper writes the condition's
|
||||
`tried`, resolver and escalation from `attempts`; a healer writes no condition.
|
||||
|
||||
**Only once the controller's lease names no controller.** Until the write that hands the lease away, each act it would
|
||||
take is written to `would`, never to `attempts`: it counts for no budget, settle or brake and reaches no condition.
|
||||
|
||||
Each key of `attempts` and `would` lives a week from its last write, its own time to live (the buckets are declared
|
||||
`key-ttl`): nothing else deletes `would`, and once the lease is handed away nothing is written there. The healers' waits
|
||||
are keys of `wakes` whose time to live is the wait: `settle.<condition key>` ends with a settle and judges the condition
|
||||
again, `fortnight` ends when S15's oldest counted act leaves the fortnight and counts again. The watch of `wakes` hears
|
||||
each expiry; no timer runs in the process. A wait still running is on the bus across a restart; one that expired while
|
||||
no instance watched is judged at the start, once the lease is first read.
|
||||
|
||||
S15: it hears the seat mesh-hand-acts' `act-recorded` and watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a
|
||||
fortnight; a drill and a person's decision by design never count.
|
||||
|
||||
## What it keeps
|
||||
|
||||
- `mesh-healers_attempts`: every act and escalation, each key kept a week.
|
||||
- `mesh-healers_would`: the newest act it would take per healer and condition while the controller holds the lease,
|
||||
each key kept a week.
|
||||
- `mesh-healers_findings`: the brake, its own blindness, and S15.
|
||||
- `mesh-healers_wakes`: the waits, each key living as long as it waits.
|
||||
|
||||
## Its verbs and tools
|
||||
|
||||
- `mesh-healers.healers`: the registry, the acts of the last days, the would-acts, the brake, whether it acts and why;
|
||||
`recount=true` counts the hand-act log again.
|
||||
- `mesh-healers.heal`: one act now on an open condition, by key and why.
|
||||
- `mesh-healers.drill`: records a drill in the hand-act log of the seat `mesh-hand-acts` as `hand-act drill` (the
|
||||
controller's `drill` while that seat has no holder and the controller holds its lease). S15 never counts a drill.
|
||||
- `healers_findings`, `healers_would`, `healers_wakes`: the reads of its buckets.
|
||||
@@ -0,0 +1,663 @@
|
||||
package main
|
||||
|
||||
// The healers (novox/hq ADR 0564 §7, §8; ADR 0231): a known failure heals itself, under a brake, and every repair is
|
||||
// said. The registry is compiled in; mesh-healers hears `condition-raised` and `condition-changed` of the seat
|
||||
// mesh-conditions on a durable consumer, instead of looking every half minute. A repair is a call of the seat or tool
|
||||
// that owns the act (H3: the grants part's grants_assert). Each act is begun by a create of its key in the bucket
|
||||
// `attempts` before it is made and finished after, so a holder that dies mid-act has spent it and a second instance
|
||||
// cannot begin the same act; budgets, settles and the mesh-wide brake count from there. Success is never the
|
||||
// healer's to say: the condition clears when its observer no longer sees it, and the condition keeper writes `tried`,
|
||||
// the resolver and the escalation into the condition from `attempts`.
|
||||
//
|
||||
// **Only once the controller's lease names no controller** (ADR 0564 §8): until then each act it would take is
|
||||
// written to the bucket `would`, never to `attempts`, so a would-act counts for no budget, settle or brake, reaches
|
||||
// no `tried` and escalates nothing. `would` keeps each key a week (its bucket's age): nothing else deletes it, and once
|
||||
// the lease is handed away nothing is written there again.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// Module and Seat are this module and the seat it holds once for the mesh.
|
||||
const (
|
||||
Module = "mesh-healers"
|
||||
Seat = "mesh-healers"
|
||||
)
|
||||
|
||||
// The states it keeps and reads, and the event it says.
|
||||
const (
|
||||
AttemptsState = health.AttemptsState
|
||||
WouldState = health.WouldState
|
||||
FindingsState = health.FindingsState
|
||||
LeaseRead = buscompose.ControllerLease
|
||||
LeaseKey = buscompose.ControllerLeaseKey
|
||||
ConditionsRead = "mesh-conditions.conditions"
|
||||
HandActsRead = "mesh-controller.hand-acts"
|
||||
ActedEvent = health.EventHealerActed
|
||||
// ActedSubject is where the seat says it, which its consumers hear (catalogue.SeatSays).
|
||||
ActedSubject = "mesh.seat." + Seat + ".event." + ActedEvent
|
||||
// RecordedEvent is the hand-act log's seat event, each act recorded there (ADR 0565 §11).
|
||||
RecordedEvent = handacts.Seat + "." + handacts.EventRecorded
|
||||
// The events it hears: the keeper's, on the seat mesh-conditions.
|
||||
RaisedEvent = "mesh-conditions.condition-raised"
|
||||
ChangedEvent = "mesh-conditions.condition-changed"
|
||||
// AssertTool is the grants part's tool that asserts a module's buckets, logs and streams again.
|
||||
AssertTool = "mesh-grants.grants_assert"
|
||||
)
|
||||
|
||||
// The mesh-wide brake (to-be 45 §7): how many acts all healers together may take in an hour.
|
||||
const (
|
||||
BrakeLimit = 12
|
||||
BrakeWindow = time.Hour
|
||||
)
|
||||
|
||||
// The outcomes an attempt has.
|
||||
const (
|
||||
OutcomeActing = health.OutcomeActing
|
||||
OutcomeActed = health.OutcomeActed
|
||||
OutcomeFailed = health.OutcomeFailed
|
||||
OutcomeEscalated = health.OutcomeEscalated
|
||||
)
|
||||
|
||||
// Store is the bus's buckets as this module reaches them, through its runtime. CreateTTL writes a key only when it has
|
||||
// none, and says ErrExists otherwise; it and PutTTL give the key its own time to live, at whose end the bus removes it
|
||||
// and a watch of the bucket hears it expire (a bucket declared key-ttl).
|
||||
type Store interface {
|
||||
Get(state, key string) ([]byte, bool, error)
|
||||
Put(state, key string, value any) error
|
||||
CreateTTL(state, key string, value any, ttl time.Duration) error
|
||||
PutTTL(state, key string, value any, ttl time.Duration) error
|
||||
Delete(state, key string) error
|
||||
Keys(state string) ([]string, error)
|
||||
}
|
||||
|
||||
// KeptFor is how long an attempt and a would-act are kept: each key a week from its last write (ADR 0564 Limits).
|
||||
const KeptFor = 7 * 24 * time.Hour
|
||||
|
||||
// WakesState holds the healers' waits as keys whose time to live is the wait: `settle.<condition key>` ends with a
|
||||
// settle, `fortnight` when S15's oldest counted act leaves the fortnight. Its expiry, heard on the bucket's watch, is
|
||||
// the wake: no timer runs in the process, and a restart loses nothing a key on the bus holds.
|
||||
const WakesState = "wakes"
|
||||
|
||||
// The wakes' keys.
|
||||
const (
|
||||
settlePrefix = "settle."
|
||||
fortnightKey = "fortnight"
|
||||
)
|
||||
|
||||
// ErrExists is a create of a key another writer made first.
|
||||
var ErrExists = errors.New("the key was made by another writer first")
|
||||
|
||||
// Condition is what a healer reads of a condition the keeper said: the controller's condition body, its looks aside.
|
||||
type Condition struct {
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
Subject Subject `json:"subject"`
|
||||
Severity string `json:"severity"`
|
||||
Resolver string `json:"resolver"`
|
||||
Tried []json.RawMessage `json:"tried,omitempty"`
|
||||
}
|
||||
|
||||
// Escalated says a healer's budget was spent: the operator resolves it now, and no healer touches it again until
|
||||
// observation clears it.
|
||||
func (c Condition) Escalated() bool { return c.Resolver == "operator" && len(c.Tried) > 0 }
|
||||
|
||||
// Attempt, Would and Acted are mesh-sdk's health layouts: an act begun in `attempts` (its key
|
||||
// `<healer>.<condition key>.<begun ns>` is the healers' own), an act written to `would` while the lease names a
|
||||
// controller (one key per healer and condition, `<healer>.<condition key>`, its newest), and the body of healer-acted,
|
||||
// epoch left out (ADR 0564 Limits 10).
|
||||
type (
|
||||
Attempt = health.Attempt
|
||||
Would = health.Would
|
||||
Acted = health.HealerActed
|
||||
)
|
||||
|
||||
// Healer is one row of the registry.
|
||||
type Healer struct {
|
||||
ID string
|
||||
Kinds []string
|
||||
Condition string
|
||||
Repair string
|
||||
Then string
|
||||
From string
|
||||
Budget int
|
||||
Window time.Duration
|
||||
Settle time.Duration
|
||||
// ActsIn is where the repair runs: here, or a module that repairs its own (H5), or the controller (H4).
|
||||
ActsIn string
|
||||
// applies says whether this condition is one it may act on, and what its budget counts against; why when not.
|
||||
applies func(c Condition) (budget string, ok bool, why string)
|
||||
// repair is the act: what it did, what came of it, or why it could not.
|
||||
repair func(h *Healers, c Condition) (act, said string, err error)
|
||||
}
|
||||
|
||||
// actsHere marks a healer whose repair mesh-healers makes.
|
||||
const actsHere = "mesh-healers"
|
||||
|
||||
// Registry is to-be 45 §7's table, compiled in: every healer the mesh has, the ones mesh-healers runs with their act.
|
||||
func Registry() []Healer {
|
||||
return []Healer{
|
||||
{ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"},
|
||||
Condition: "a durable consumer of a module the bus does not hold (D6), or a seat's holder that does not answer (D3)",
|
||||
Repair: "assert the module's buckets, logs and streams again through the grants part (grants_assert)",
|
||||
Then: "resolver operator, urgent", From: "a controller restarted to make a missing object again (208, 248)",
|
||||
Budget: 1, Window: time.Hour, Settle: 6 * time.Minute, ActsIn: actsHere,
|
||||
applies: appliesToAModulesObject, repair: assertAgain},
|
||||
{ID: "H4", Kinds: []string{"consumer-behind"},
|
||||
Condition: "a durable consumer far behind its stream's head (D6) that the stream table marks resettable",
|
||||
Repair: "re-make the consumer to deliver from now; the one marked resettable is the controller's own",
|
||||
Then: "resolver operator, urgent", From: "a consumer re-made from now by hand (248)",
|
||||
Budget: 1, Window: 24 * time.Hour, Settle: 6 * time.Minute,
|
||||
ActsIn: "the controller, until its lease: the one consumer marked resettable is its own (ADR 0564 §7)"},
|
||||
{ID: "H5", Kinds: []string{"provider-failing"},
|
||||
Condition: "the identity provider's administrator refusing the mesh's secret (provider-failing)",
|
||||
Repair: "the provider repairs it itself through the server's own bootstrap command (ADR 0224 §5)",
|
||||
Then: "announced failing by the provider, braked from ten minutes doubling to six hours",
|
||||
From: "the identity provider's admin reset through its bootstrap command (179)",
|
||||
Budget: 1, Window: 10 * time.Minute, Settle: 10 * time.Minute,
|
||||
ActsIn: "the provider module (keycloak), ADR 0224 §5"},
|
||||
}
|
||||
}
|
||||
|
||||
// Healers is mesh-healers: the registry, what its acts reach, and what it holds.
|
||||
type Healers struct {
|
||||
Store Store
|
||||
Ask func(key string, body any) (json.RawMessage, error)
|
||||
// Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish).
|
||||
Publish func(subject string, body any, id string) error
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
Registry []Healer
|
||||
|
||||
mu sync.Mutex
|
||||
// lease is the controller's lease as last read: acting is on only while it is handed away.
|
||||
acting bool
|
||||
leaseWhy string
|
||||
// declined is why each condition was last passed over, said once.
|
||||
declined map[string]string
|
||||
// acts are the hand-act log as watched, for S15.
|
||||
acts map[string]HandAct
|
||||
}
|
||||
|
||||
// LeaseHeard records what the lease says: acting is on only while it names a holder that is no controller (ADR 0564
|
||||
// §8, Limits 8, as ADR 0561 §3's reader reads it).
|
||||
func (h *Healers) LeaseHeard(raw []byte, found bool, readErr error) {
|
||||
on, why := buscompose.LeaseHandedAway(raw, found, readErr)
|
||||
h.mu.Lock()
|
||||
flipped := on != h.acting || h.leaseWhy == ""
|
||||
h.acting, h.leaseWhy = on, leaseWords(on, why)
|
||||
h.mu.Unlock()
|
||||
if flipped {
|
||||
h.Logf("[%s] the healers %s: %s", Module, map[bool]string{true: "ACT", false: "act in shadow"}[on], h.leaseWhy)
|
||||
}
|
||||
}
|
||||
|
||||
// leaseWords says the lease's state as the healers read it; the SDK's words speak of minting.
|
||||
func leaseWords(on bool, why string) string {
|
||||
if on {
|
||||
return "the controller's lease is handed away: the healers act"
|
||||
}
|
||||
if strings.Contains(why, "cannot be read") || strings.Contains(why, "not readable") {
|
||||
return "the controller's lease cannot be read, so it is taken to name a controller: every act is written as would"
|
||||
}
|
||||
return "the controller's lease names a controller (or none is written, so one may take it): every act is written as would"
|
||||
}
|
||||
|
||||
// Acting says whether the healers act, and why.
|
||||
func (h *Healers) Acting() (bool, string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.leaseWhy == "" {
|
||||
return false, "the controller's lease has not been read yet: every act is written as would"
|
||||
}
|
||||
return h.acting, h.leaseWhy
|
||||
}
|
||||
|
||||
// healerFor is the healer that acts here on a kind.
|
||||
func (h *Healers) healerFor(kind string) (Healer, bool) {
|
||||
for _, r := range h.Registry {
|
||||
if r.repair != nil && slices.Contains(r.Kinds, kind) {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return Healer{}, false
|
||||
}
|
||||
|
||||
// Heard is a condition the keeper raised or changed: judged, and acted on, written as would, or passed over.
|
||||
func (h *Healers) Heard(body []byte) error {
|
||||
var c Condition
|
||||
if err := json.Unmarshal(body, &c); err != nil {
|
||||
h.Logf("[%s] a condition event is not readable, and is passed over: %v", Module, err)
|
||||
return nil
|
||||
}
|
||||
_, err := h.Judge(c, "")
|
||||
return err
|
||||
}
|
||||
|
||||
// Judge is one condition judged by the healer registered for its kind. by names a person's `heal`, empty for an
|
||||
// event. It answers what was done, in words.
|
||||
func (h *Healers) Judge(c Condition, by string) (string, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
row, ok := h.healerFor(c.Kind)
|
||||
if !ok {
|
||||
return "no healer here answers the kind " + c.Kind, nil
|
||||
}
|
||||
if c.Escalated() {
|
||||
return "it is handed to the operator already: no healer touches it until observation clears it", nil
|
||||
}
|
||||
budget, applies, why := row.applies(c)
|
||||
if !applies {
|
||||
h.decline(c.Key, row.ID, why)
|
||||
return fmt.Sprintf("healer %s leaves it alone: %s", row.ID, why), nil
|
||||
}
|
||||
delete(h.declined, c.Key)
|
||||
now := h.Now()
|
||||
if !h.acting {
|
||||
w := Would{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: row.Repair, At: now.UTC(), Why: h.whyNot()}
|
||||
if err := h.Store.PutTTL(WouldState, row.ID+"."+c.Key, w, KeptFor); err != nil {
|
||||
return "", fmt.Errorf("the would-act of %s on %s is not written: %w", row.ID, c.Key, err)
|
||||
}
|
||||
return fmt.Sprintf("healer %s would act on %s: %s", row.ID, c.Key, w.Why), nil
|
||||
}
|
||||
all, err := h.attempts()
|
||||
if err != nil {
|
||||
h.blind(err)
|
||||
return "", fmt.Errorf("the healers cannot read what they did, so they count no budget and act on nothing: %w", err)
|
||||
}
|
||||
h.see("blind", nil)
|
||||
acts := actsWithin(all, now.Add(-BrakeWindow))
|
||||
if braked, said := h.brakeHolds(acts, now); braked {
|
||||
h.see("brake", []Observation{brakeFinding(acts, said)})
|
||||
return "every healer has stopped: " + said, nil
|
||||
}
|
||||
h.see("brake", nil)
|
||||
spent := spentOn(all, row, budget, now)
|
||||
if n := len(spent); n > 0 && now.Sub(spent[n-1].Begun) < row.Settle {
|
||||
h.wake(budget, spent[n-1].Begun.Add(row.Settle), c.Key)
|
||||
return fmt.Sprintf("healer %s waits: its last act on it is still the observation's to judge until %s", row.ID,
|
||||
spent[n-1].Begun.Add(row.Settle).UTC().Format(time.RFC3339)), nil
|
||||
}
|
||||
if len(spent) >= row.Budget {
|
||||
return h.escalate(row, c, budget, spent)
|
||||
}
|
||||
if len(acts) >= BrakeLimit {
|
||||
return "the brake holds from the next act on", nil
|
||||
}
|
||||
return h.act(row, c, budget, len(spent), by)
|
||||
}
|
||||
|
||||
func (h *Healers) whyNot() string {
|
||||
if h.leaseWhy == "" {
|
||||
return "the controller's lease has not been read yet"
|
||||
}
|
||||
return h.leaseWhy
|
||||
}
|
||||
|
||||
// act is one healer's act: begun by a create in attempts, made, finished, and said.
|
||||
func (h *Healers) act(row Healer, c Condition, budget string, spent int, by string) (string, error) {
|
||||
begun := h.Now().UTC()
|
||||
key := attemptKey(row.ID, c.Key, begun)
|
||||
if by == "" {
|
||||
by = "healer " + row.ID
|
||||
}
|
||||
window := fmt.Sprintf("act %d of %d within %s", spent+1, row.Budget, row.Window)
|
||||
a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: row.Repair, Outcome: OutcomeActing,
|
||||
Begun: begun}
|
||||
if by != "healer "+row.ID {
|
||||
a.Said = "asked by " + by
|
||||
}
|
||||
if err := h.Store.CreateTTL(AttemptsState, key, a, KeptFor); err != nil {
|
||||
if errors.Is(err, ErrExists) {
|
||||
return "another instance began this act first", nil
|
||||
}
|
||||
return "", fmt.Errorf("healer %s did not act on %s: its attempt could not be begun: %w", row.ID, c.Key, err)
|
||||
}
|
||||
act, said, err := row.repair(h, c)
|
||||
a.Outcome = OutcomeActed
|
||||
if err != nil {
|
||||
a.Outcome, said = OutcomeFailed, err.Error()
|
||||
}
|
||||
if act != "" {
|
||||
a.Act = act
|
||||
}
|
||||
finished := h.Now().UTC()
|
||||
if a.Said != "" {
|
||||
said = a.Said + "; " + said
|
||||
}
|
||||
a.Said, a.Finished = said+" ("+window+")", &finished
|
||||
if err := h.Store.PutTTL(AttemptsState, key, a, KeptFor); err != nil {
|
||||
h.Logf("[%s] healer %s %s %s and could not record what came of it: %v", Module, row.ID, a.Outcome, c.Key, err)
|
||||
}
|
||||
h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: a.Act, Outcome: a.Outcome, Said: said, Budget: window})
|
||||
h.wake(budget, begun.Add(row.Settle), c.Key)
|
||||
return fmt.Sprintf("healer %s %s %s: %s — %s (%s)", row.ID, a.Outcome, c.Key, a.Act, said, window), nil
|
||||
}
|
||||
|
||||
// escalate is a spent budget: the condition is the operator's now. Written once as an attempt the keeper turns into
|
||||
// the condition's resolver and urgency (ADR 0564 §7); an escalated condition is passed over until it clears.
|
||||
func (h *Healers) escalate(row Healer, c Condition, budget string, spent []Attempt) (string, error) {
|
||||
var tried []string
|
||||
for _, s := range spent {
|
||||
tried = append(tried, fmt.Sprintf("%s at %s (%s)", s.Outcome, s.Begun.UTC().Format("15:04 MST"), firstLine(s.Said)))
|
||||
}
|
||||
said := fmt.Sprintf("its budget of %d act(s) within %s is spent and %s is still open: %s", row.Budget, row.Window,
|
||||
c.Key, strings.Join(tried, "; "))
|
||||
now := h.Now().UTC()
|
||||
window := fmt.Sprintf("%d of %d within %s", len(spent), row.Budget, row.Window)
|
||||
a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: "handed the condition to the operator",
|
||||
Outcome: OutcomeEscalated, Said: said, Begun: now, Finished: &now}
|
||||
if err := h.Store.CreateTTL(AttemptsState, attemptKey(row.ID, c.Key, now), a, KeptFor); err != nil && !errors.Is(err, ErrExists) {
|
||||
return "", fmt.Errorf("healer %s could not record handing %s to the operator, and does not: %w", row.ID, c.Key, err)
|
||||
}
|
||||
h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: "handed to the operator", Outcome: OutcomeEscalated,
|
||||
Said: said, Budget: window})
|
||||
return fmt.Sprintf("healer %s handed %s to the operator: %s", row.ID, c.Key, said), nil
|
||||
}
|
||||
|
||||
// tell says a heal as healer-acted. Not said is said in the log: the attempt holds it.
|
||||
func (h *Healers) tell(e Acted) {
|
||||
e.Event, e.At, e.By = ActedEvent, h.Now().UTC(), "healer "+e.Healer
|
||||
e.Show = "mesh-conditions.conditions key=" + e.Condition
|
||||
if h.Publish == nil {
|
||||
return
|
||||
}
|
||||
if err := h.Publish(ActedSubject, e, e.Healer+"."+e.Condition+"."+e.Outcome+"."+strconv.FormatInt(e.At.UnixNano(), 10)); err != nil {
|
||||
h.Logf("[%s] healer %s %s %s, and that could NOT be said on the bus: %v", Module, e.Healer, e.Outcome, e.Condition, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Rearm is the healers' start, once the lease is first read: a settle whose key expired while no instance watched has
|
||||
// no expiry left to hear, so each newest act whose settle ended within its window is judged at once. A settle still
|
||||
// running keeps its key on the bus and needs nothing.
|
||||
func (h *Healers) Rearm() {
|
||||
all, err := h.attempts()
|
||||
if err != nil {
|
||||
h.Logf("[%s] what the healers did cannot be read, so no ended settle is judged: %v", Module, err)
|
||||
return
|
||||
}
|
||||
now := h.Now()
|
||||
newest := map[string]Attempt{} // the newest act per healer and budget; all is oldest first
|
||||
for _, a := range all {
|
||||
if a.Outcome != OutcomeEscalated {
|
||||
newest[a.Healer+" "+a.BudgetKey] = a
|
||||
}
|
||||
}
|
||||
var due []string
|
||||
for _, a := range newest {
|
||||
for _, r := range h.Registry {
|
||||
if r.ID == a.Healer && r.repair != nil && !now.Before(a.Begun.Add(r.Settle)) && now.Sub(a.Begun) <= r.Window {
|
||||
due = append(due, a.Condition)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(due)
|
||||
for _, key := range due {
|
||||
h.again(key)
|
||||
}
|
||||
}
|
||||
|
||||
// wake writes the wait for a settle's end as a key whose time to live is the wait; its expiry judges the condition
|
||||
// again (WakeHeard). Written again with the same end, it is the same wait.
|
||||
func (h *Healers) wake(_ string, at time.Time, key string) {
|
||||
ttl := at.Sub(h.Now()).Round(time.Second) + time.Second
|
||||
if err := h.Store.PutTTL(WakesState, settlePrefix+key, map[string]any{"condition": key, "at": at.UTC()}, ttl); err != nil {
|
||||
h.Logf("[%s] the settle of %s is not written as a wait, so its end is judged at the next event: %v", Module, key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// WakeHeard is a change of the wakes bucket: an expiry is a wait ended — a settle's, whose condition is judged again,
|
||||
// or S15's fortnight, counted again.
|
||||
func (h *Healers) WakeHeard(key, op string) {
|
||||
if op != OpExpire {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case key == fortnightKey:
|
||||
h.Wanted()
|
||||
case strings.HasPrefix(key, settlePrefix):
|
||||
h.again(strings.TrimPrefix(key, settlePrefix))
|
||||
}
|
||||
}
|
||||
|
||||
// OpExpire is the watch's word for a key whose own time to live ended.
|
||||
const OpExpire = stdio.OpExpire
|
||||
|
||||
// again judges a condition again from the keeper's bucket, if it is still open.
|
||||
func (h *Healers) again(key string) {
|
||||
raw, found, err := h.Store.Get(ConditionsRead, key)
|
||||
if err != nil || !found {
|
||||
return
|
||||
}
|
||||
if _, err := h.Judge(conditionOf(raw), ""); err != nil {
|
||||
h.Logf("[%s] %s judged again at its settle's end: %v", Module, key, err)
|
||||
}
|
||||
}
|
||||
|
||||
func conditionOf(raw []byte) Condition {
|
||||
var c Condition
|
||||
_ = json.Unmarshal(raw, &c)
|
||||
return c
|
||||
}
|
||||
|
||||
// decline remembers why a healer passed a condition over, said once.
|
||||
func (h *Healers) decline(key, healer, why string) {
|
||||
if h.declined == nil {
|
||||
h.declined = map[string]string{}
|
||||
}
|
||||
if h.declined[key] != why {
|
||||
h.Logf("[%s] healer %s leaves %s alone: %s", Module, healer, key, why)
|
||||
}
|
||||
h.declined[key] = why
|
||||
}
|
||||
|
||||
func attemptKey(healer, condition string, begun time.Time) string {
|
||||
return healer + "." + condition + "." + strconv.FormatInt(begun.UnixNano(), 10)
|
||||
}
|
||||
|
||||
// attempts are every attempt kept, oldest first.
|
||||
func (h *Healers) attempts() ([]Attempt, error) {
|
||||
keys, err := h.Store.Keys(AttemptsState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []Attempt
|
||||
for _, k := range keys {
|
||||
raw, found, err := h.Store.Get(AttemptsState, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
var a Attempt
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the attempt %s is not readable: %w", k, err)
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Begun.Before(out[j].Begun) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// actsWithin are the attempts since a moment that were acts: an escalation is a saying, and the brake does not count it.
|
||||
func actsWithin(all []Attempt, since time.Time) []Attempt {
|
||||
var out []Attempt
|
||||
for _, a := range all {
|
||||
if !a.Begun.Before(since) && a.Outcome != OutcomeEscalated {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// spentOn is one healer's acts against one budget key within its window, oldest first.
|
||||
func spentOn(all []Attempt, row Healer, budget string, now time.Time) []Attempt {
|
||||
var out []Attempt
|
||||
for _, a := range actsWithin(all, now.Add(-row.Window)) {
|
||||
if a.Healer == row.ID && a.BudgetKey == budget {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// brakeHolds says whether the mesh-wide brake holds: the limit reached within the hour, or the brake already written
|
||||
// and an act within the hour still — it lets go an hour after the last act, not the first.
|
||||
func (h *Healers) brakeHolds(acts []Attempt, now time.Time) (bool, string) {
|
||||
said := fmt.Sprintf("%d act(s) by the healers in the last %s, the limit %d", len(acts), BrakeWindow, BrakeLimit)
|
||||
if len(acts) >= BrakeLimit {
|
||||
return true, said
|
||||
}
|
||||
_, held, _ := h.Store.Get(FindingsState, brakeKey)
|
||||
if held && len(acts) > 0 {
|
||||
last := acts[len(acts)-1].Begun
|
||||
return true, fmt.Sprintf("%s; held until an hour after the last, at %s", said,
|
||||
last.Add(BrakeWindow).UTC().Format("15:04 MST"))
|
||||
}
|
||||
return false, said
|
||||
}
|
||||
|
||||
var brakeKey = health.Key(ScopeMesh, "healers", "braked")
|
||||
|
||||
// brakeFinding is the brake, as the urgent finding that says it.
|
||||
func brakeFinding(acts []Attempt, said string) Observation {
|
||||
counts := map[string]int{}
|
||||
for _, a := range acts {
|
||||
counts[a.Healer+" on "+a.Condition]++
|
||||
}
|
||||
var most []string
|
||||
for what, n := range counts {
|
||||
most = append(most, fmt.Sprintf("%s ×%d", what, n))
|
||||
}
|
||||
sort.Strings(most)
|
||||
return Observation{Kind: "healers-braked", Token: "braked", Severity: Urgent, Subject: Subject{Scope: ScopeMesh, ID: "healers"},
|
||||
Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " +
|
||||
"healer looping, not repairing — " + said,
|
||||
Said: strings.Join(most, ", "),
|
||||
Headline: "All automatic repairs stopped",
|
||||
Needs: "read the details and decide whether the repairs may run again.",
|
||||
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which looks " +
|
||||
"like a loop, so all of them stopped.",
|
||||
Resolved: "Automatic repairs run again"}
|
||||
}
|
||||
|
||||
// blind writes the healers' own blindness: they cannot read what they did.
|
||||
func (h *Healers) blind(err error) {
|
||||
h.see("blind", []Observation{{Kind: "probe-failed", Token: "failed", Severity: Warning,
|
||||
Subject: Subject{Scope: ScopeProbe, ID: "healers"},
|
||||
Summary: "the healers cannot read what they did, so they count no budget and act on nothing",
|
||||
Said: firstLine(err.Error()), Headline: "A check of the mesh could not run",
|
||||
Explanation: "The automatic repairs cannot read what they did, so they act on nothing until they can.",
|
||||
Resolved: "The automatic repairs read what they did again"}})
|
||||
}
|
||||
|
||||
// see writes the healers' own findings of one source, each only when it changes, and deletes those of the source it
|
||||
// no longer sees (ADR 0564 §2).
|
||||
func (h *Healers) see(source string, found []Observation) {
|
||||
now := h.Now()
|
||||
want := map[string]health.Finding{}
|
||||
for _, o := range found {
|
||||
o.Source = source
|
||||
want[o.Key()] = o.Finding("", now)
|
||||
}
|
||||
keys, err := h.Store.Keys(FindingsState)
|
||||
if err != nil {
|
||||
h.Logf("[%s] its own findings cannot be read, so none is written or cleared: %v", Module, err)
|
||||
return
|
||||
}
|
||||
for _, k := range keys {
|
||||
raw, found, err := h.Store.Get(FindingsState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
was, err := health.DecodeFinding(raw)
|
||||
if err != nil || was.Source != source {
|
||||
continue
|
||||
}
|
||||
if f, still := want[k]; still {
|
||||
if says(was) == says(f) {
|
||||
delete(want, k)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := h.Store.Delete(FindingsState, k); err != nil {
|
||||
h.Logf("[%s] the finding %s no longer seen is not deleted: %v", Module, k, err)
|
||||
}
|
||||
}
|
||||
for k, f := range want {
|
||||
raw, err := health.EncodeFinding(f)
|
||||
if err == nil {
|
||||
err = h.Store.Put(FindingsState, k, json.RawMessage(raw))
|
||||
}
|
||||
if err != nil {
|
||||
h.Logf("[%s] the finding %s is not written: %v", Module, k, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- H3 --------------------------------------------------------------------------------------------------------
|
||||
|
||||
// moduleOf is the module whose objects a condition names: a consumer of a module's own (`<stream>.<module>`). The
|
||||
// controller's own consumers are not a module's: the controller asserts them itself until its lease.
|
||||
func moduleOf(c Condition) (string, string) {
|
||||
stream, consumer, ok := strings.Cut(c.Subject.ID, ".")
|
||||
if c.Kind != "consumer-lost" || !ok || stream == "" || consumer == "" {
|
||||
return "", "it names no consumer of a module whose objects the grants part asserts"
|
||||
}
|
||||
if consumer == buscompose.ControllerName || strings.HasPrefix(consumer, buscompose.ControllerName+"-") {
|
||||
return "", "the consumer is the controller's own, which the controller asserts itself while it holds its lease"
|
||||
}
|
||||
return consumer, ""
|
||||
}
|
||||
|
||||
// appliesToAModulesObject is H3's: a module's consumer the bus does not hold, its budget per condition.
|
||||
func appliesToAModulesObject(c Condition) (string, bool, string) {
|
||||
if _, why := moduleOf(c); why != "" {
|
||||
return "", false, why
|
||||
}
|
||||
return c.Key, true, ""
|
||||
}
|
||||
|
||||
// assertAgain is H3: the grants part asserts the module's buckets, logs and streams again.
|
||||
func assertAgain(h *Healers, c Condition) (string, string, error) {
|
||||
module, why := moduleOf(c)
|
||||
if module == "" {
|
||||
return "", "", errors.New(why)
|
||||
}
|
||||
raw, err := h.Ask(AssertTool, map[string]any{"module": module})
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("the grants part did not assert %s: %w", module, err)
|
||||
}
|
||||
var a struct {
|
||||
Says string `json:"says"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &a)
|
||||
return "asserted " + module + "'s buckets, logs and streams again through the grants part",
|
||||
orSaid(a.Says, "the grants part answered") + "; the probe that raised it says on its next run whether it is there", nil
|
||||
}
|
||||
|
||||
func orSaid(s, otherwise string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return otherwise
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
line, _, _ := strings.Cut(s, "\n")
|
||||
return line
|
||||
}
|
||||
@@ -0,0 +1,496 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
type memStore struct {
|
||||
mu sync.Mutex
|
||||
kv map[string]map[string][]byte
|
||||
keysErr map[string]error
|
||||
ttl map[string]time.Duration
|
||||
}
|
||||
|
||||
func newStore() *memStore {
|
||||
return &memStore{kv: map[string]map[string][]byte{}, keysErr: map[string]error{}}
|
||||
}
|
||||
|
||||
func (s *memStore) Get(state, key string) ([]byte, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v, ok := s.kv[state][key]
|
||||
return v, ok, nil
|
||||
}
|
||||
|
||||
func (s *memStore) put(state, key string, value any, create bool) error {
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.kv[state] == nil {
|
||||
s.kv[state] = map[string][]byte{}
|
||||
}
|
||||
if _, ok := s.kv[state][key]; ok && create {
|
||||
return ErrExists
|
||||
}
|
||||
s.kv[state][key] = raw
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) }
|
||||
func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) }
|
||||
|
||||
// CreateTTL and PutTTL keep the key's time to live beside it, for a test to read.
|
||||
func (s *memStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
|
||||
if err := s.put(state, key, value, true); err != nil {
|
||||
return err
|
||||
}
|
||||
s.setTTL(state, key, ttl)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) PutTTL(state, key string, value any, ttl time.Duration) error {
|
||||
if err := s.put(state, key, value, false); err != nil {
|
||||
return err
|
||||
}
|
||||
s.setTTL(state, key, ttl)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) setTTL(state, key string, ttl time.Duration) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.ttl == nil {
|
||||
s.ttl = map[string]time.Duration{}
|
||||
}
|
||||
s.ttl[state+"/"+key] = ttl
|
||||
}
|
||||
|
||||
// ttlOf is a key's time to live as last written; zero for none.
|
||||
func (s *memStore) ttlOf(state, key string) time.Duration {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.ttl[state+"/"+key]
|
||||
}
|
||||
|
||||
func (s *memStore) Delete(state, key string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
delete(s.kv[state], key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) Keys(state string) ([]string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := s.keysErr[state]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for k := range s.kv[state] {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *memStore) keys(state string) []string {
|
||||
k, _ := s.Keys(state)
|
||||
return k
|
||||
}
|
||||
|
||||
func (s *memStore) attempt(t *testing.T, key string) Attempt {
|
||||
t.Helper()
|
||||
raw, _, _ := s.Get(AttemptsState, key)
|
||||
var a Attempt
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
t.Fatalf("%s: %s", key, raw)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// world is mesh-healers over memory, with its clock, its asks, what it said and the wakes it set.
|
||||
type world struct {
|
||||
h *Healers
|
||||
store *memStore
|
||||
now time.Time
|
||||
asked []string
|
||||
said []Acted
|
||||
}
|
||||
|
||||
func newWorld() *world {
|
||||
w := &world{store: newStore(), now: time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)}
|
||||
w.fresh()
|
||||
return w
|
||||
}
|
||||
|
||||
// fresh is the healers started again over the same buckets: nothing of the last one's memory, its wakes forgotten.
|
||||
func (w *world) fresh() {
|
||||
w.h = &Healers{Store: w.store, Registry: Registry(), Logf: func(string, ...any) {},
|
||||
Now: func() time.Time { return w.now },
|
||||
Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
w.asked = append(w.asked, key+" "+string(raw))
|
||||
return json.RawMessage(`{"says":"2 the bus lacks: created at this ask"}`), nil
|
||||
},
|
||||
Publish: func(subject string, body any, _ string) error {
|
||||
if a, ok := body.(Acted); ok && subject == "mesh.seat.mesh-healers.event.healer-acted" {
|
||||
w.said = append(w.said, a)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (w *world) handAway() {
|
||||
w.h.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil)
|
||||
}
|
||||
|
||||
// lost is a module's consumer the bus does not hold, as D6 raises it.
|
||||
func lost(module string) Condition {
|
||||
return Condition{Key: "bus.EVENTS." + module + ".missing", Kind: "consumer-lost", Severity: "warning", Resolver: "self",
|
||||
Subject: Subject{Scope: "bus", ID: "EVENTS." + module}}
|
||||
}
|
||||
|
||||
func body(c Condition) []byte {
|
||||
raw, _ := json.Marshal(map[string]any{"key": c.Key, "kind": c.Kind, "subject": c.Subject, "severity": c.Severity,
|
||||
"resolver": c.Resolver, "tried": c.Tried, "event": "condition-raised", "show": "mesh-conditions.conditions key=" + c.Key})
|
||||
return raw
|
||||
}
|
||||
|
||||
// No act is made while the lease names a controller: each is written to would, never to attempts, so it counts for no
|
||||
// budget or brake and nothing is asked or said; one is made from the write that hands the lease away (ADR 0564 §8).
|
||||
func TestTheHealersActOnlyOnceTheLeaseNamesNoController(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
lease func(h *Healers)
|
||||
acts bool
|
||||
}{
|
||||
{"never read", func(*Healers) {}, false},
|
||||
{"absent", func(h *Healers) { h.LeaseHeard(nil, false, nil) }, false},
|
||||
{"unreadable", func(h *Healers) { h.LeaseHeard(nil, false, errors.New("timeout")) }, false},
|
||||
{"a controller holds it", func(h *Healers) {
|
||||
h.LeaseHeard([]byte(`{"instance":"novox/1/2026-10-27","epoch":57}`), true, nil)
|
||||
}, false},
|
||||
{"handed away", func(h *Healers) { h.LeaseHeard([]byte(`{"handed-to":"the operator"}`), true, nil) }, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
w := newWorld()
|
||||
tc.lease(w.h)
|
||||
if err := w.h.Heard(body(lost("records"))); err != nil {
|
||||
t.Fatalf("%s: %v", tc.name, err)
|
||||
}
|
||||
attempts, would := w.store.keys(AttemptsState), w.store.keys(WouldState)
|
||||
if tc.acts {
|
||||
if len(attempts) != 1 || len(would) != 0 || len(w.asked) != 1 || !strings.Contains(w.asked[0], `"module":"records"`) ||
|
||||
len(w.said) != 1 || w.said[0].Outcome != OutcomeActed {
|
||||
t.Errorf("%s: attempts %v, would %v, asked %v, said %+v", tc.name, attempts, would, w.asked, w.said)
|
||||
continue
|
||||
}
|
||||
a := w.store.attempt(t, attempts[0])
|
||||
if a.Outcome != OutcomeActed || a.Finished == nil || a.Healer != "H3" || a.Condition != "bus.EVENTS.records.missing" {
|
||||
t.Errorf("%s: the attempt is %+v", tc.name, a)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if len(attempts) != 0 || len(w.asked) != 0 || len(w.said) != 0 || !slices.Equal(would, []string{"H3.bus.EVENTS.records.missing"}) {
|
||||
t.Errorf("%s: attempts %v, would %v, asked %v, said %v", tc.name, attempts, would, w.asked, w.said)
|
||||
}
|
||||
if _, braked, _ := w.store.Get(FindingsState, brakeKey); braked {
|
||||
t.Errorf("%s: a would-act moved the brake", tc.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An act waits out its settle, then its spent budget hands the condition to the operator once; an escalated condition
|
||||
// is passed over; a second instance cannot begin an act already begun; the brake stops every healer past twelve acts
|
||||
// in an hour and lets go an hour after the last (ADR 0231 §21, ADR 0564 §7).
|
||||
func TestBudgetSettleEscalationAndBrake(t *testing.T) {
|
||||
w := newWorld()
|
||||
w.handAway()
|
||||
c := lost("records")
|
||||
_ = w.store.Put(ConditionsRead, c.Key, c)
|
||||
settle := settlePrefix + c.Key
|
||||
steps := []struct {
|
||||
name string
|
||||
after time.Duration
|
||||
cond Condition
|
||||
attempts int
|
||||
outcome string // the newest said outcome, empty for none new
|
||||
// how is what happens: an event, a restart over the same buckets, or the settle key's expiry heard.
|
||||
how string
|
||||
// wait is the settle key's time to live after the step, zero for no check.
|
||||
wait time.Duration
|
||||
}{
|
||||
{"acts: its settle is a key that expires", 0, c, 1, OutcomeActed, "event", 6*time.Minute + time.Second},
|
||||
{"within its settle: waits for the same end", 2 * time.Minute, c, 1, "", "event", 4*time.Minute + time.Second},
|
||||
{"a restart within its settle: the key on the bus still waits", time.Minute, c, 1, "", "restart", 4*time.Minute + time.Second},
|
||||
{"its key expires: judged again, budget spent, escalates", 4 * time.Minute, c, 2, OutcomeEscalated, "expire", 0},
|
||||
{"escalated: passed over", time.Minute, Condition{Key: c.Key, Kind: c.Kind, Subject: c.Subject, Resolver: "operator",
|
||||
Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, "", "event", 0},
|
||||
}
|
||||
for _, s := range steps {
|
||||
w.now = w.now.Add(s.after)
|
||||
before := len(w.said)
|
||||
switch s.how {
|
||||
case "restart":
|
||||
w.fresh()
|
||||
w.handAway()
|
||||
w.h.Rearm()
|
||||
case "expire":
|
||||
w.h.WakeHeard(settle, OpExpire)
|
||||
default:
|
||||
if _, err := w.h.Judge(s.cond, ""); err != nil {
|
||||
t.Fatalf("%s: %v", s.name, err)
|
||||
}
|
||||
}
|
||||
if n := len(w.store.keys(AttemptsState)); n != s.attempts {
|
||||
t.Errorf("%s: %d attempts, want %d", s.name, n, s.attempts)
|
||||
}
|
||||
if s.wait != 0 && w.store.ttlOf(WakesState, settle) != s.wait {
|
||||
t.Errorf("%s: the settle waits %s, want %s", s.name, w.store.ttlOf(WakesState, settle), s.wait)
|
||||
}
|
||||
switch {
|
||||
case s.outcome == "" && len(w.said) != before:
|
||||
t.Errorf("%s: said %+v", s.name, w.said[before:])
|
||||
case s.outcome != "" && (len(w.said) != before+1 || w.said[before].Outcome != s.outcome):
|
||||
t.Errorf("%s: said %+v, want %s", s.name, w.said[before:], s.outcome)
|
||||
}
|
||||
}
|
||||
// Each attempt is kept a week from its last write.
|
||||
for _, k := range w.store.keys(AttemptsState) {
|
||||
if ttl := w.store.ttlOf(AttemptsState, k); ttl != KeptFor {
|
||||
t.Errorf("the attempt %s is kept %s, want a week", k, ttl)
|
||||
}
|
||||
}
|
||||
// A settle whose key expired while no instance watched is judged at the start.
|
||||
r := newWorld()
|
||||
r.handAway()
|
||||
_ = r.store.Put(ConditionsRead, c.Key, c)
|
||||
if _, err := r.h.Judge(c, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.now = r.now.Add(7 * time.Minute)
|
||||
r.fresh()
|
||||
r.handAway()
|
||||
r.h.Rearm()
|
||||
if len(r.said) != 2 || r.said[1].Outcome != OutcomeEscalated {
|
||||
t.Errorf("a settle that ended unheard is not judged at the start: %+v", r.said)
|
||||
}
|
||||
// A second instance cannot begin the act an attempt's key already holds.
|
||||
x := newWorld()
|
||||
x.handAway()
|
||||
_ = x.store.CreateTTL(AttemptsState, attemptKey("H3", lost("mail").Key, x.now.UTC()), Attempt{}, KeptFor)
|
||||
if said, _ := x.h.Judge(lost("mail"), ""); !strings.Contains(said, "another instance") || len(x.asked) != 0 {
|
||||
t.Errorf("a second instance acted: %q, asked %v", said, x.asked)
|
||||
}
|
||||
// The brake: twelve acts in the hour stop every healer, said as an urgent finding.
|
||||
b := newWorld()
|
||||
b.handAway()
|
||||
for i := 0; i < BrakeLimit; i++ {
|
||||
b.now = b.now.Add(time.Minute)
|
||||
_ = b.store.CreateTTL(AttemptsState, attemptKey("H3", "x", b.now), Attempt{Healer: "H3", Condition: "x", BudgetKey: "x",
|
||||
Outcome: OutcomeActed, Begun: b.now}, KeptFor)
|
||||
}
|
||||
if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "stopped") || len(b.asked) != 0 {
|
||||
t.Errorf("past the brake: %q, asked %v", said, b.asked)
|
||||
}
|
||||
raw, braked, _ := b.store.Get(FindingsState, brakeKey)
|
||||
f, _ := health.DecodeFinding(raw)
|
||||
if !braked || f.Severity != Urgent || f.Kind != "healers-braked" || f.Observer != Module {
|
||||
t.Errorf("the brake is written as %s", raw)
|
||||
}
|
||||
b.now = b.now.Add(55 * time.Minute) // fewer than twelve within the hour, but the brake is said: held
|
||||
if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "held until") {
|
||||
t.Errorf("the brake let go before an hour after the last act: %q", said)
|
||||
}
|
||||
b.now = b.now.Add(6 * time.Minute)
|
||||
if _, err := b.h.Judge(lost("mail"), ""); err != nil || len(b.asked) != 1 {
|
||||
t.Errorf("an hour after the last act the healers do not act again: %v, asked %v", err, b.asked)
|
||||
}
|
||||
if _, braked, _ := b.store.Get(FindingsState, brakeKey); braked {
|
||||
t.Error("the brake's finding outlived the brake")
|
||||
}
|
||||
// Blind to what they did, they act on nothing and say so.
|
||||
z := newWorld()
|
||||
z.handAway()
|
||||
z.store.keysErr[AttemptsState] = errors.New("the bus is away")
|
||||
if _, err := z.h.Judge(lost("mail"), ""); err == nil || len(z.asked) != 0 {
|
||||
t.Errorf("blind, the healers acted: %v", err)
|
||||
}
|
||||
if keys := z.store.keys(FindingsState); !slices.Equal(keys, []string{"probe.healers.failed"}) {
|
||||
t.Errorf("blind, the findings are %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// H3 answers a module's consumer the bus does not hold, never the controller's own or a condition naming no consumer;
|
||||
// the registry holds every healer the mesh has, H3 run here, H4 and H5 run elsewhere (ADR 0231 §2, ADR 0564 §7).
|
||||
func TestTheRegistryAndWhatH3Answers(t *testing.T) {
|
||||
cases := []struct {
|
||||
c Condition
|
||||
holds bool
|
||||
}{
|
||||
{lost("records"), true},
|
||||
{lost("controller"), false},
|
||||
{lost("controller-merges"), false},
|
||||
{Condition{Key: "seat.x.silent", Kind: "holder-silent", Subject: Subject{Scope: "seat", ID: "issue-tracker"}}, false},
|
||||
{Condition{Key: "bus.EVENTS.records.behind", Kind: "consumer-behind", Subject: Subject{Scope: "bus", ID: "EVENTS.records"}}, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
w := newWorld()
|
||||
w.handAway()
|
||||
said, _ := w.h.Judge(tc.c, "")
|
||||
if acted := len(w.asked) == 1; acted != tc.holds {
|
||||
t.Errorf("%s: acted %v, want %v (%s)", tc.c.Key, acted, tc.holds, said)
|
||||
}
|
||||
}
|
||||
runs := map[string]bool{}
|
||||
for _, r := range Registry() {
|
||||
if len(r.Kinds) == 0 || r.Budget < 1 || r.Window <= 0 || r.Settle <= 0 || r.Repair == "" || r.ActsIn == "" {
|
||||
t.Errorf("%s is not a whole row: %+v", r.ID, r)
|
||||
}
|
||||
runs[r.ID] = r.repair != nil
|
||||
}
|
||||
if !runs["H3"] || runs["H4"] || runs["H5"] || len(runs) != 3 {
|
||||
t.Errorf("the healers run here are %v, want H3 alone of H3, H4, H5", runs)
|
||||
}
|
||||
}
|
||||
|
||||
// S15: a cause repaired by hand twice within a fortnight is a finding; a decision by design never counts; it clears
|
||||
// when fewer than two remain, and the healers wake when the oldest leaves the fortnight (ADR 0564 Limits 2).
|
||||
func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) {
|
||||
w := newWorld()
|
||||
act := func(key, verb, cause string, ago time.Duration) {
|
||||
raw, _ := json.Marshal(HandAct{ID: key, At: w.now.Add(-ago), By: "person.jochen", Verb: verb, Why: "x", Cause: cause})
|
||||
w.h.HandActHeard(key, "put", raw)
|
||||
}
|
||||
act("a1", "broker consumer-reset", "consumer-behind", 10*24*time.Hour)
|
||||
if keys := w.store.keys(FindingsState); len(keys) != 0 {
|
||||
t.Fatalf("one act is a finding: %v", keys)
|
||||
}
|
||||
// The second of the cause is the seat's own act, heard as act-recorded: one in each log counts two (ADR 0565 §11).
|
||||
seatAct, _ := json.Marshal(HandAct{ID: "a2", At: w.now.Add(-24 * time.Hour), By: "person.jochen",
|
||||
Verb: "broker consumer-reset", Why: "x", Cause: "consumer-behind"})
|
||||
w.h.ActRecorded(seatAct)
|
||||
// A drill in each log counts none.
|
||||
act("a3", "hand-act drill", "drill", time.Hour)
|
||||
drill, _ := json.Marshal(HandAct{ID: "a4", At: w.now.Add(-2 * time.Hour), Verb: "hand-act drill", Why: "x", Cause: "drill"})
|
||||
w.h.ActRecorded(drill)
|
||||
act("a5", "conditions silence", "operator-answer", time.Hour)
|
||||
act("a6", "conditions silence", "operator-answer", time.Hour)
|
||||
act("a7", "hand-act record", "relogin", 20*24*time.Hour)
|
||||
act("a8", "hand-act record", "relogin", time.Hour)
|
||||
want := []string{"mesh.hand-acts.consumer-behind.healer-wanted"}
|
||||
if keys := w.store.keys(FindingsState); !slices.Equal(keys, want) {
|
||||
t.Fatalf("findings %v, want %v", keys, want)
|
||||
}
|
||||
raw, _, _ := w.store.Get(FindingsState, want[0])
|
||||
if !strings.Contains(string(raw), "healer H4 answers this cause") {
|
||||
t.Errorf("the finding does not say its healer is not enough: %s", raw)
|
||||
}
|
||||
if ttl := w.store.ttlOf(WakesState, fortnightKey); ttl != 4*24*time.Hour+time.Second {
|
||||
t.Errorf("the fortnight waits %s, want the oldest act's leaving it in 4 days", ttl)
|
||||
}
|
||||
w.now = w.now.Add(4*24*time.Hour + time.Second)
|
||||
w.h.Wanted()
|
||||
if keys := w.store.keys(FindingsState); len(keys) != 0 {
|
||||
t.Errorf("past the fortnight the finding stays: %v", keys)
|
||||
}
|
||||
w.h.HandActHeard("a5", "delete", nil)
|
||||
if len(w.h.acts) != 7 {
|
||||
t.Errorf("a deleted entry is kept: %d", len(w.h.acts))
|
||||
}
|
||||
}
|
||||
|
||||
// The verbs: heal acts on an open condition by key and says why it was asked; drill waits for the log's home; the reads
|
||||
// answer every bucket.
|
||||
func TestTheVerbsAndReads(t *testing.T) {
|
||||
w := newWorld()
|
||||
w.handAway()
|
||||
c := lost("records")
|
||||
_ = w.store.Put(ConditionsRead, c.Key, c)
|
||||
if _, err := w.h.Heal(c.Key, "", "person.jochen"); err == nil {
|
||||
t.Error("heal took no why")
|
||||
}
|
||||
if _, err := w.h.Heal("bus.none", "x", "person.jochen"); err == nil {
|
||||
t.Error("heal acted on no open condition")
|
||||
}
|
||||
if _, err := w.h.Heal(c.Key, "it is gone again", "person.jochen"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keys := w.store.keys(AttemptsState)
|
||||
if len(keys) != 1 || !strings.Contains(w.store.attempt(t, keys[0]).Said, "asked by person.jochen (it is gone again)") {
|
||||
t.Errorf("heal's attempt is %v", keys)
|
||||
}
|
||||
answer, err := w.h.Answer(7, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := answer.(map[string]any)
|
||||
if a["acting"] != true || len(a["acts"].([]Attempt)) != 1 || len(a["healers"].([]map[string]any)) != 3 {
|
||||
t.Errorf("healers answered %v", a)
|
||||
}
|
||||
// drill records into mesh-hand-acts as a drill; while that seat has no holder, into the controller's drill until
|
||||
// the lease is handed away (ADR 0565 §11).
|
||||
drills := []struct {
|
||||
name string
|
||||
seatGone bool
|
||||
leaseAway bool
|
||||
args map[string]any
|
||||
log, err string
|
||||
}{
|
||||
{"into the seat", false, true, map[string]any{"what": "stopped records", "why": "does it raise"}, handacts.LogSeat, ""},
|
||||
{"no holder: the controller's", true, false, map[string]any{"what": "stopped records", "why": "x"}, handacts.LogController, ""},
|
||||
{"no holder, lease away", true, true, map[string]any{"what": "stopped records", "why": "x"}, "", "handed its lease away"},
|
||||
{"says nothing", false, true, map[string]any{}, "", "says what"},
|
||||
}
|
||||
for _, d := range drills {
|
||||
x := newWorld()
|
||||
if d.leaseAway {
|
||||
x.handAway()
|
||||
} else {
|
||||
x.h.LeaseHeard([]byte(`{"instance":"novox/1","epoch":5}`), true, nil)
|
||||
}
|
||||
var asked []string
|
||||
x.h.Ask = func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
asked = append(asked, key+" "+string(raw))
|
||||
if d.seatGone && strings.HasPrefix(key, "seat:mesh-hand-acts.") {
|
||||
return nil, errors.New("nats: no responders available for request")
|
||||
}
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
var got any
|
||||
var err error
|
||||
for _, tool := range Tools(x.h) {
|
||||
if tool.Name == DrillTool {
|
||||
got, err = tool.Run(d.args)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case d.err != "":
|
||||
if err == nil || !strings.Contains(err.Error(), d.err) {
|
||||
t.Errorf("drill %s: %v", d.name, err)
|
||||
}
|
||||
case err != nil || got.(map[string]any)["log"] != d.log:
|
||||
t.Errorf("drill %s: %v, %v", d.name, got, err)
|
||||
case d.log == handacts.LogSeat && !strings.Contains(asked[0], `"verb":"hand-act drill"`):
|
||||
t.Errorf("drill %s asked %v", d.name, asked)
|
||||
}
|
||||
}
|
||||
if _, err := w.h.read(WouldState, ""); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
// mesh-healers: the holder of the seat mesh-healers (novox/hq ADR 0564 §7, §8). A Go bundle served by a runtime of its
|
||||
// own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `attempts` (every act begun,
|
||||
// made and finished, and every escalation, kept a week), `would` (what it would have done while the controller holds
|
||||
// its lease, kept a week) and `findings` (the brake, its blindness, and S15). It hears the condition keeper's
|
||||
// condition-raised and condition-changed on a durable consumer, watches the controller's lease, and watches the
|
||||
// controller's hand-act log for S15. stdout is the MCP channel; what this module says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
|
||||
// runtimeStore is the bus, through the runtime that launched this bundle.
|
||||
type runtimeStore struct{}
|
||||
|
||||
func (runtimeStore) Get(state, key string) ([]byte, bool, error) {
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}
|
||||
|
||||
func (runtimeStore) Put(state, key string, value any) error {
|
||||
_, err := stdio.State(state).Put(key, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error {
|
||||
_, err := stdio.State(state).CreateTTL(key, value, ttl)
|
||||
if errors.Is(err, stdio.ErrChanged) {
|
||||
return ErrExists
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (runtimeStore) PutTTL(state, key string, value any, ttl time.Duration) error {
|
||||
_, err := stdio.State(state).PutTTL(key, value, ttl)
|
||||
return err
|
||||
}
|
||||
|
||||
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
|
||||
|
||||
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs.
|
||||
func watched(state, key string, handle func(stdio.StateChange) error) {
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
time.Sleep(wait)
|
||||
if err := stdio.State(state).Watch(key, handle); err != nil {
|
||||
logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait)
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf,
|
||||
Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) },
|
||||
Publish: stdio.SeatPublish}
|
||||
// The lease first: until it is read the healers write would-acts only. Once it is first read, a settle that ended
|
||||
// while no instance watched is judged (Rearm); the waits still running are keys on the bus, heard on their expiry.
|
||||
var started sync.Once
|
||||
go watched(LeaseRead, LeaseKey, func(c stdio.StateChange) error {
|
||||
h.LeaseHeard(c.Value, c.Op == "put", nil)
|
||||
started.Do(func() { go h.Rearm() })
|
||||
return nil
|
||||
})
|
||||
go watched(WakesState, "", func(c stdio.StateChange) error {
|
||||
h.WakeHeard(c.Key, c.Op)
|
||||
return nil
|
||||
})
|
||||
go watched(HandActsRead, "", func(c stdio.StateChange) error {
|
||||
h.HandActHeard(c.Key, c.Op, c.Value)
|
||||
return nil
|
||||
})
|
||||
// S15 over both logs: the seat's act-recorded on a durable consumer, beside the controller's bucket's watch.
|
||||
go func() {
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
time.Sleep(wait)
|
||||
err := stdio.Subscribe(RecordedEvent, func(e stdio.Envelope) error {
|
||||
h.ActRecorded(e.Body)
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
logf("[%s] does not hear %s yet (%v); asking again in %s", Module, RecordedEvent, err, wait)
|
||||
}
|
||||
}()
|
||||
for _, event := range []string{RaisedEvent, ChangedEvent} {
|
||||
go func(event string) {
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
time.Sleep(wait)
|
||||
err := stdio.Subscribe(event, func(e stdio.Envelope) error { return h.Heard(e.Body) })
|
||||
if err == nil {
|
||||
logf("[%s] hears %s", Module, event)
|
||||
return
|
||||
}
|
||||
logf("[%s] does not hear %s yet (%v); asking again in %s", Module, event, err, wait)
|
||||
}
|
||||
}(event)
|
||||
}
|
||||
if err := stdio.Serve("", Tools(h)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
RunsAs string `json:"runs-as"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
Emits []string `json:"emits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Reads []string `json:"reads"`
|
||||
State []json.RawMessage `json:"state"`
|
||||
}
|
||||
|
||||
// The manifest says what the code does: it claims the seat mesh-healers once for the mesh with its verbs (ADR 0564 §1,
|
||||
// §7), runs on its own account, says healer-acted, hears the keeper's raised and changed, asks only the grants part's
|
||||
// assert, reads the lease, the keeper's conditions and the hand-act log, and keeps attempts and would a week each.
|
||||
// Every state it keeps has a tool that reads it, and every act it takes by itself a tool that starts it (ADR 0564 §12).
|
||||
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Module != Module || m.RunsAs != Module {
|
||||
t.Errorf("module %q runs as %q", m.Module, m.RunsAs)
|
||||
}
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) {
|
||||
t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
got, want []string
|
||||
}{
|
||||
{"emits (healer-acted is the seat's, said on its subject)", m.Emits, nil},
|
||||
{"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent, RecordedEvent}},
|
||||
{"invokes", m.Invokes, []string{AssertTool, "seat:mesh-hand-acts.hand-act", "seat:mesh-controller.drill"}},
|
||||
{"reads", m.Reads, []string{LeaseRead, ConditionsRead, HandActsRead}},
|
||||
} {
|
||||
if !slices.Equal(c.got, c.want) {
|
||||
t.Errorf("%s %v, want %v", c.name, c.got, c.want)
|
||||
}
|
||||
}
|
||||
states := map[string]bool{}
|
||||
for _, s := range m.State {
|
||||
var d struct {
|
||||
Name string `json:"name"`
|
||||
KeyTTL bool `json:"key-ttl"`
|
||||
}
|
||||
if json.Unmarshal(s, &d.Name) != nil {
|
||||
_ = json.Unmarshal(s, &d)
|
||||
}
|
||||
states[d.Name] = d.KeyTTL
|
||||
}
|
||||
// Each key of attempts and would lives a week from its last write (KeptFor, would has no other deleter), and each
|
||||
// wait in wakes as long as it waits: their buckets take a key's own time to live.
|
||||
if !states[AttemptsState] || !states[WouldState] || !states[WakesState] || states[FindingsState] {
|
||||
t.Errorf("the states take a key's own time to live as %v, want attempts, would and wakes alone", states)
|
||||
}
|
||||
readBy := map[string]string{AttemptsState: HealersTool, WouldState: WouldTool, FindingsState: FindingsTool,
|
||||
WakesState: WakesTool, "a settle's end": HealTool,
|
||||
"an act on an event": HealTool, "S15's count": HealersTool + " recount=true", "the lease as read": HealersTool}
|
||||
tools := map[string]bool{}
|
||||
for _, tool := range Tools(&Healers{}) {
|
||||
tools[tool.Name] = true
|
||||
}
|
||||
for state := range states {
|
||||
if _, ok := readBy[state]; !ok {
|
||||
t.Errorf("the state %s has no tool that reads it", state)
|
||||
}
|
||||
}
|
||||
for what, tool := range readBy {
|
||||
if name, _, _ := strings.Cut(tool, " "); !tools[name] {
|
||||
t.Errorf("%s is read or started by %s, which is not served", what, tool)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
// What the healers observe of themselves, and the finding it is written as (novox/hq ADR 0564 §2): the brake, their
|
||||
// blindness and S15, each written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the
|
||||
// controller's key for the same condition.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The severities and scopes, as mesh-sdk's health names them.
|
||||
const (
|
||||
Urgent = health.Urgent
|
||||
Warning = health.Warning
|
||||
ScopeProbe = health.ScopeProbe
|
||||
ScopeMesh = health.ScopeMesh
|
||||
)
|
||||
|
||||
// Subject is what a finding is about.
|
||||
type Subject = health.Subject
|
||||
|
||||
// Observation is one thing a probe sees wrong now.
|
||||
type Observation struct {
|
||||
Kind string
|
||||
Token string
|
||||
Subject Subject
|
||||
Severity health.Severity
|
||||
Summary string
|
||||
Said string
|
||||
Headline string
|
||||
Explanation string
|
||||
Resolved string
|
||||
Needs string
|
||||
// Source is what saw it: the brake, the healers' blindness, or S15.
|
||||
Source string
|
||||
}
|
||||
|
||||
// Key is where it is kept: `<scope>.<id>.<token>`, the controller's key for the same condition.
|
||||
func (o Observation) Key() string {
|
||||
token := o.Token
|
||||
if token == "" {
|
||||
token = o.Kind
|
||||
}
|
||||
return health.Key(o.Subject.Scope, o.Subject.ID, token)
|
||||
}
|
||||
|
||||
// Finding is the observation as written by run at, its evidence the one line it said.
|
||||
func (o Observation) Finding(run string, at time.Time) health.Finding {
|
||||
f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary,
|
||||
Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source,
|
||||
Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}}
|
||||
if o.Token != "" && o.Token != o.Kind {
|
||||
f.Token = o.Token
|
||||
}
|
||||
said := o.Said
|
||||
if said == "" {
|
||||
said = o.Summary
|
||||
}
|
||||
f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said})
|
||||
return f
|
||||
}
|
||||
|
||||
// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write
|
||||
// (ADR 0289 §1).
|
||||
func says(f health.Finding) string {
|
||||
f.At, f.Run, f.V = time.Time{}, "", 0
|
||||
ev := make([]health.Evidence, len(f.Evidence))
|
||||
for i, e := range f.Evidence {
|
||||
ev[i] = health.Evidence{Said: e.Said}
|
||||
}
|
||||
f.Evidence = ev
|
||||
raw, _ := json.Marshal(f)
|
||||
return string(raw)
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (issue 1936).
|
||||
modules/mesh-healers/module.json
|
||||
@@ -0,0 +1,230 @@
|
||||
package main
|
||||
|
||||
// The seat's verbs and the module's own reads (novox/hq ADR 0564 §7, §12): `healers` is the registry, what the healers
|
||||
// did and would do, the brake and whether they act; `heal` takes one act now on a condition, within budget and brake;
|
||||
// `drill` waits for the hand-act log's home; and a read of each bucket it keeps, so a recovery is a call.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
)
|
||||
|
||||
// Verbs are the seat's verbs, as the manifest claims them.
|
||||
var Verbs = []string{"healers", "heal", "drill"}
|
||||
|
||||
// Tool names.
|
||||
const (
|
||||
HealersTool = Seat + ".healers"
|
||||
HealTool = Seat + ".heal"
|
||||
DrillTool = Seat + ".drill"
|
||||
FindingsTool = "healers_findings"
|
||||
WouldTool = "healers_would"
|
||||
WakesTool = "healers_wakes"
|
||||
)
|
||||
|
||||
func obj(props map[string]any) map[string]any {
|
||||
return map[string]any{"type": "object", "properties": props}
|
||||
}
|
||||
|
||||
// Tools are what the runtime serves.
|
||||
func Tools(h *Healers) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: HealersTool,
|
||||
Description: "The mesh's healers (novox/hq ADR 0564 §7, §8): the registry, every act and escalation of the last " +
|
||||
"days (the bucket attempts), whether the mesh-wide brake holds, whether they act or only write what they " +
|
||||
"would do while the controller holds its lease, and the would-acts. recount=true counts the hand-act log " +
|
||||
"again for S15 (a cause repaired by hand twice in a fortnight).",
|
||||
Input: obj(map[string]any{
|
||||
"days": map[string]any{"type": "number", "description": "how many days of acts back (default 7)"},
|
||||
"recount": map[string]any{"type": "boolean", "description": "count the hand-act log again now"}}),
|
||||
Run: func(a map[string]any) (any, error) { return h.Answer(intArg(a, "days", 7), boolArg(a, "recount")) }},
|
||||
{Name: HealTool,
|
||||
Description: "One healer's act now on an open condition, by its key, within its budget and the brake, as an " +
|
||||
"event would have it — or, while the controller holds its lease, the act it would take written as would. " +
|
||||
"Says what was done.",
|
||||
Input: obj(map[string]any{
|
||||
"key": map[string]any{"type": "string", "description": "the condition's key"},
|
||||
"why": map[string]any{"type": "string", "description": "why, in a sentence"}}),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return h.Heal(strArg(a, "key"), strArg(a, "why"), stdio.CallerOf(a))
|
||||
}},
|
||||
{Name: DrillTool,
|
||||
Description: "Record a drill: something broken on purpose to see the mesh raise and clear it, with what it " +
|
||||
"tests, in the hand-act log of the seat mesh-hand-acts (the controller's drill while that seat has no " +
|
||||
"holder and the controller holds its lease). A drill is never a repair: S15 never counts it.",
|
||||
Input: obj(map[string]any{"what": map[string]any{"type": "string", "description": "what was done on purpose"},
|
||||
"why": map[string]any{"type": "string", "description": "what the drill tests"},
|
||||
"condition": map[string]any{"type": "string", "description": "the condition it is meant to raise, if any"}}),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return h.Drill(strArg(a, "what"), strArg(a, "why"), strArg(a, "condition"))
|
||||
}},
|
||||
{Name: FindingsTool,
|
||||
Description: "What the healers find of themselves (their bucket findings): the brake, their blindness, and " +
|
||||
"each cause repaired by hand twice in a fortnight (S15); or one whole by key.",
|
||||
Input: obj(map[string]any{"key": map[string]any{"type": "string"}}),
|
||||
Run: func(a map[string]any) (any, error) { return h.read(FindingsState, strArg(a, "key")) }},
|
||||
{Name: WakesTool,
|
||||
Description: "What the healers wait for (their bucket wakes): each settle still running (`settle.<condition key>`) " +
|
||||
"and S15's fortnight end (`fortnight`), each a key whose time to live is the wait and whose expiry wakes them; " +
|
||||
"or one whole by key. heal acts on a condition now, and healers with recount=true counts S15 now.",
|
||||
Input: obj(map[string]any{"key": map[string]any{"type": "string"}}),
|
||||
Run: func(a map[string]any) (any, error) { return h.read(WakesState, strArg(a, "key")) }},
|
||||
{Name: WouldTool,
|
||||
Description: "What the healers would have done while the controller holds its lease (their bucket would, each " +
|
||||
"key kept a week): one per healer and condition, its newest; or one whole by key.",
|
||||
Input: obj(map[string]any{"key": map[string]any{"type": "string"}}),
|
||||
Run: func(a map[string]any) (any, error) { return h.read(WouldState, strArg(a, "key")) }},
|
||||
}
|
||||
}
|
||||
|
||||
// Drill records a drill (ADR 0565 §11, which ends ADR 0564 Limits 1's refusal): through the seat mesh-hand-acts as
|
||||
// the act `hand-act drill`, and while that seat has no holder that answers and the controller still holds its lease,
|
||||
// through the controller's own `drill`, which records it as a drill too. It answers which log took it.
|
||||
func (h *Healers) Drill(what, why, condition string) (any, error) {
|
||||
if what == "" || why == "" {
|
||||
return nil, errors.New("a drill says what was done on purpose and what it tests: nothing was recorded")
|
||||
}
|
||||
if h.Ask == nil {
|
||||
return nil, errors.New("the hand-act log cannot be reached from here: nothing was recorded")
|
||||
}
|
||||
body := map[string]any{"what": what, "why": why, "verb": handacts.VerbDrill}
|
||||
controller := map[string]any{"what": what, "why": why}
|
||||
if condition != "" {
|
||||
body["condition"], controller["condition"] = condition, condition
|
||||
}
|
||||
answer, err := h.Ask("seat:"+handacts.Seat+".hand-act", body)
|
||||
if err == nil {
|
||||
return map[string]any{"log": handacts.LogSeat, "answer": answer}, nil
|
||||
}
|
||||
if !handacts.NoResponder(err) {
|
||||
return nil, fmt.Errorf("%s.hand-act refused the drill: %w", handacts.Seat, err)
|
||||
}
|
||||
if acting, _ := h.Acting(); acting {
|
||||
return nil, fmt.Errorf("%s has no holder that answers, and the controller handed its lease away: nothing was "+
|
||||
"recorded (%w)", handacts.Seat, err)
|
||||
}
|
||||
answer, cerr := h.Ask("seat:mesh-controller.drill", controller)
|
||||
if cerr != nil {
|
||||
return nil, fmt.Errorf("%s has no holder that answers (%v), and the controller's drill refused too: %w", handacts.Seat,
|
||||
err, cerr)
|
||||
}
|
||||
return map[string]any{"log": handacts.LogController, "answer": answer}, nil
|
||||
}
|
||||
|
||||
// Heal is the verb heal: one condition judged now by its healer.
|
||||
func (h *Healers) Heal(key, why, caller string) (any, error) {
|
||||
if key == "" || why == "" {
|
||||
return nil, errors.New("heal names the condition's key and says why")
|
||||
}
|
||||
raw, found, err := h.Store.Get(ConditionsRead, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the open conditions cannot be read: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("no condition is open under %s", key)
|
||||
}
|
||||
said, err := h.Judge(conditionOf(raw), caller+" ("+why+")")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"said": said}, nil
|
||||
}
|
||||
|
||||
// Answer is the verb healers.
|
||||
func (h *Healers) Answer(days int, recount bool) (any, error) {
|
||||
if recount {
|
||||
h.Wanted()
|
||||
}
|
||||
now := h.Now()
|
||||
all, err := h.attempts()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what the healers did cannot be read: %w", err)
|
||||
}
|
||||
var recent []Attempt
|
||||
for _, a := range all {
|
||||
if now.Sub(a.Begun) <= time.Duration(days)*24*time.Hour {
|
||||
recent = append(recent, a)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(recent, func(i, j int) bool { return recent[i].Begun.After(recent[j].Begun) })
|
||||
braked, said := h.brakeHolds(actsWithin(all, now.Add(-BrakeWindow)), now)
|
||||
var rows []map[string]any
|
||||
for _, r := range h.Registry {
|
||||
rows = append(rows, map[string]any{"id": r.ID, "kinds": r.Kinds, "condition": r.Condition, "repair": r.Repair,
|
||||
"budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle),
|
||||
"then": r.Then, "acts-in": r.ActsIn, "from": r.From})
|
||||
}
|
||||
acting, why := h.Acting()
|
||||
would, err := h.read(WouldState, "")
|
||||
if err != nil {
|
||||
would = map[string]any{"unread": err.Error()}
|
||||
}
|
||||
h.mu.Lock()
|
||||
declined := map[string]string{}
|
||||
for k, v := range h.declined {
|
||||
declined[k] = v
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if recent == nil {
|
||||
recent = []Attempt{}
|
||||
}
|
||||
return map[string]any{"healers": rows, "acts": recent, "days": days, "acting": acting, "why": why, "would": would,
|
||||
"brake": map[string]any{"holds": braked, "said": said, "limit": BrakeLimit, "window": BrakeWindow.String()},
|
||||
"declined": declined,
|
||||
"note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"}, nil
|
||||
}
|
||||
|
||||
// read is every key of a bucket with its value, or one whole.
|
||||
func (h *Healers) read(state, key string) (any, error) {
|
||||
if key != "" {
|
||||
raw, found, err := h.Store.Get(state, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("nothing is kept under %s in %s", key, state)
|
||||
}
|
||||
return json.RawMessage(raw), nil
|
||||
}
|
||||
keys, err := h.Store.Keys(state)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := map[string]json.RawMessage{}
|
||||
for _, k := range keys {
|
||||
if raw, found, err := h.Store.Get(state, k); err == nil && found {
|
||||
rows[k] = raw
|
||||
}
|
||||
}
|
||||
return map[string]any{state: rows, "count": len(rows)}, nil
|
||||
}
|
||||
|
||||
func strArg(a map[string]any, k string) string {
|
||||
s, _ := a[k].(string)
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// boolArg reads a flag as a boolean or as "true".
|
||||
func boolArg(a map[string]any, k string) bool {
|
||||
switch v := a[k].(type) {
|
||||
case bool:
|
||||
return v
|
||||
case string:
|
||||
return strings.EqualFold(strings.TrimSpace(v), "true")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func intArg(a map[string]any, k string, otherwise int) int {
|
||||
if n, ok := a[k].(float64); ok && n > 0 {
|
||||
return int(n)
|
||||
}
|
||||
return otherwise
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package main
|
||||
|
||||
// S15: a cause repaired by hand twice within a fortnight is a healer wanted (ADR 0231 §27, §28; ADR 0564 Limits 2).
|
||||
// The hand-act log is the controller's bucket mesh-controller_hand-acts, a key-value bucket with no events: mesh-healers
|
||||
// watches it, the one of the four holders granted its read, counts each cause again on every put, and writes the
|
||||
// finding `healer-wanted` per cause. **A heal is never a hand act**, and an act that is a person's decision by design
|
||||
// (a drill above all) never counts. The finding clears when fewer than two acts of its cause remain within the
|
||||
// fortnight: the key `fortnight` of the healers' wakes expires when the oldest counted act leaves it. From the log's home (issue 1953) they
|
||||
// watch that instead.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
)
|
||||
|
||||
// Within is how far back a repeated cause counts.
|
||||
const Within = 14 * 24 * time.Hour
|
||||
|
||||
// HandAct is what S15 reads of one entry of the controller's hand-act log (mesh-controller internal/link HandAct).
|
||||
type HandAct struct {
|
||||
ID string `json:"id"`
|
||||
At time.Time `json:"at"`
|
||||
By string `json:"by"`
|
||||
Verb string `json:"verb"`
|
||||
Args []string `json:"arguments,omitempty"`
|
||||
Why string `json:"why"`
|
||||
Cause string `json:"cause"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
}
|
||||
|
||||
// decided are the verbs whose entries are a person's decision by design, with the causes they hold for (every cause
|
||||
// when none): the controller's handActVerbs, read as S15 reads them. A verb not listed counts as a repair.
|
||||
var decided = map[string][]string{
|
||||
"warrant": nil,
|
||||
"seat": {"seat-passed"},
|
||||
"conditions silence": {"operator-answer"},
|
||||
"hand-act record": {"drill"},
|
||||
"secret issued": nil,
|
||||
"hand-act drill": nil,
|
||||
"retire approve": nil,
|
||||
"retire reject": nil,
|
||||
"cleanup delete": nil,
|
||||
"dead-letters deliver": nil,
|
||||
"dead-letters drop": nil,
|
||||
"collect": nil,
|
||||
"mirrors": nil,
|
||||
"move-placement": nil,
|
||||
"move-config": nil,
|
||||
"bus upgrade": nil,
|
||||
"upgrade release-backlog": nil,
|
||||
"secret accept": nil,
|
||||
"secret rotate": {"leaked-in-logs"},
|
||||
"rotate": {"leaked-in-logs"},
|
||||
}
|
||||
|
||||
// personsDecision is whether an act is a person's decision by design, by the verb that recorded it.
|
||||
func personsDecision(a HandAct) bool {
|
||||
causes, ok := decided[a.Verb]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return len(causes) == 0 || slices.Contains(causes, a.Cause)
|
||||
}
|
||||
|
||||
// ActRecorded is one act the seat mesh-hand-acts recorded, heard as act-recorded: counted beside the controller's
|
||||
// log's, under a key of its own log, so an act is counted once whichever log holds it (ADR 0565 §11).
|
||||
func (h *Healers) ActRecorded(body []byte) {
|
||||
var a HandAct
|
||||
if json.Unmarshal(body, &a) != nil || a.ID == "" {
|
||||
return
|
||||
}
|
||||
h.HandActHeard(handacts.LogSeat+":"+a.ID, "put", body)
|
||||
}
|
||||
|
||||
// HandActHeard is one change of the hand-act log: kept or forgotten, and every cause counted again.
|
||||
func (h *Healers) HandActHeard(key, op string, value []byte) {
|
||||
h.mu.Lock()
|
||||
if h.acts == nil {
|
||||
h.acts = map[string]HandAct{}
|
||||
}
|
||||
if op != "put" {
|
||||
delete(h.acts, key)
|
||||
} else {
|
||||
var a HandAct
|
||||
if json.Unmarshal(value, &a) == nil {
|
||||
h.acts[key] = a
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
h.Wanted()
|
||||
}
|
||||
|
||||
// Wanted counts every cause repaired by hand within the fortnight and writes a finding for each counted twice or more,
|
||||
// deleting the rest; its wait `fortnight` expires when the oldest counted act of a wanted cause leaves the fortnight.
|
||||
func (h *Healers) Wanted() []Observation {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
now := h.Now()
|
||||
byCause := map[string][]HandAct{}
|
||||
for _, a := range h.acts {
|
||||
if personsDecision(a) || now.Sub(a.At) > Within {
|
||||
continue
|
||||
}
|
||||
byCause[a.Cause] = append(byCause[a.Cause], a)
|
||||
}
|
||||
var causes []string
|
||||
for c, acts := range byCause {
|
||||
if len(acts) >= 2 {
|
||||
causes = append(causes, c)
|
||||
}
|
||||
}
|
||||
sort.Strings(causes)
|
||||
var found []Observation
|
||||
var next time.Time
|
||||
for _, cause := range causes {
|
||||
acts := byCause[cause]
|
||||
sort.Slice(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) })
|
||||
var lines []string
|
||||
for _, a := range acts {
|
||||
lines = append(lines, fmt.Sprintf("%s %s by %s: %s", a.At.UTC().Format("2006-01-02 15:04"),
|
||||
strings.TrimSpace(a.Verb+" "+strings.Join(a.Args, " ")), a.By, a.Why))
|
||||
}
|
||||
newest := acts[len(acts)-1]
|
||||
wanted := "a healer is wanted for it"
|
||||
for _, r := range h.Registry {
|
||||
if slices.Contains(r.Kinds, cause) {
|
||||
wanted = fmt.Sprintf("healer %s answers this cause and a person still repaired it: its reach or its budget "+
|
||||
"is not enough", r.ID)
|
||||
}
|
||||
}
|
||||
words := strings.NewReplacer(".", " ", "_", " ").Replace(cause)
|
||||
found = append(found, Observation{Kind: "healer-wanted", Token: "healer-wanted", Severity: Warning,
|
||||
Subject: Subject{Scope: ScopeMesh, ID: "hand-acts." + cause},
|
||||
Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause, len(acts),
|
||||
int(Within.Hours()/24), newest.By, wanted),
|
||||
Said: strings.Join(lines, "; "),
|
||||
Headline: capital(words) + " keeps being fixed by hand",
|
||||
Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is wanted "+
|
||||
"for it. Nothing is broken now.", words, len(acts), int(Within.Hours()/24)),
|
||||
Resolved: "Resolved: no more hand repairs of " + words})
|
||||
if leaves := acts[0].At.Add(Within); next.IsZero() || leaves.Before(next) {
|
||||
next = leaves
|
||||
}
|
||||
}
|
||||
h.see("S15", found)
|
||||
if !next.IsZero() {
|
||||
ttl := next.Sub(now).Round(time.Second) + time.Second
|
||||
if err := h.Store.PutTTL(WakesState, fortnightKey, map[string]any{"at": next.UTC()}, ttl); err != nil {
|
||||
h.Logf("[%s] the fortnight's end is not written as a wait, so it is counted at the next act: %v", Module, err)
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
func capital(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
return strings.ToUpper(s[:1]) + s[1:]
|
||||
}
|
||||
@@ -0,0 +1,423 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
// mesh-healers whole against a real bus (novox/hq ADR 0564 §7, §8; issue 1952 H3): the built binary, played over its
|
||||
// real stdin and stdout as the node's tool runtime plays it (mesh/state.* and watches answered from a real nats-server's
|
||||
// key-value buckets, the keeper's events handed over as the runtime hands them, the grants part's assert answered). An
|
||||
// induced failure, a module's consumer gone, is only written as would while the controller holds its lease; from the
|
||||
// write that hands the lease away a healer acts, its attempt on the bus and healer-acted said; past twelve acts in the
|
||||
// hour the brake stops every healer, said as an urgent finding on the bus.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
type wire struct {
|
||||
ID json.RawMessage `json:"id,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Params json.RawMessage `json:"params,omitempty"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// player is the node's tool runtime, as far as the self-check needs it, over a real bus.
|
||||
type player struct {
|
||||
ctx context.Context
|
||||
nc *nats.Conn
|
||||
js jetstream.JetStream
|
||||
mu sync.Mutex
|
||||
in io.Writer
|
||||
next int
|
||||
pending map[string]chan wire
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (p *player) send(m map[string]any) {
|
||||
raw, _ := json.Marshal(m)
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
_, _ = p.in.Write(append(raw, '\n'))
|
||||
}
|
||||
|
||||
func (p *player) answer(id json.RawMessage, result any, err error) {
|
||||
if err != nil {
|
||||
p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
|
||||
return
|
||||
}
|
||||
p.send(map[string]any{"id": id, "result": result})
|
||||
}
|
||||
|
||||
func (p *player) serve(out io.Reader) {
|
||||
scanner := bufio.NewScanner(out)
|
||||
scanner.Buffer(make([]byte, 1<<20), 1<<24)
|
||||
for scanner.Scan() {
|
||||
var m wire
|
||||
if json.Unmarshal(scanner.Bytes(), &m) != nil {
|
||||
continue
|
||||
}
|
||||
if m.Method == "" {
|
||||
var key string
|
||||
_ = json.Unmarshal(m.ID, &key)
|
||||
p.mu.Lock()
|
||||
ch := p.pending[key]
|
||||
delete(p.pending, key)
|
||||
p.mu.Unlock()
|
||||
if ch != nil {
|
||||
ch <- m
|
||||
}
|
||||
continue
|
||||
}
|
||||
go p.handle(m)
|
||||
}
|
||||
}
|
||||
|
||||
// kv is the bucket a state names: this module's own by its local name, another's as `<owner>.<name>`.
|
||||
func (p *player) kv(state string) (jetstream.KeyValue, error) {
|
||||
if owner, name, ok := strings.Cut(state, "."); ok {
|
||||
return p.js.KeyValue(p.ctx, owner+"_"+name)
|
||||
}
|
||||
return p.js.KeyValue(p.ctx, Module+"_"+state)
|
||||
}
|
||||
|
||||
// deliver hands the bundle one event as the runtime does from its consumer.
|
||||
func (p *player) deliver(key string, body []byte) {
|
||||
p.mu.Lock()
|
||||
p.next++
|
||||
id := fmt.Sprintf("e-%d", p.next)
|
||||
p.mu.Unlock()
|
||||
p.send(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": map[string]any{
|
||||
"key": key, "node": "ace", "body": json.RawMessage(body)}}})
|
||||
}
|
||||
|
||||
func (p *player) handle(m wire) {
|
||||
var a struct {
|
||||
State string `json:"state"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &a)
|
||||
switch m.Method {
|
||||
case "mesh/state.get":
|
||||
kv, err := p.kv(a.State)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
e, err := kv.Get(p.ctx, a.Key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
p.answer(m.ID, nil, nil)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
|
||||
case "mesh/state.put":
|
||||
kv, err := p.kv(a.State)
|
||||
if err == nil {
|
||||
var rev uint64
|
||||
if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil {
|
||||
p.answer(m.ID, map[string]any{"revision": rev}, nil)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.answer(m.ID, nil, err)
|
||||
case "mesh/state.create":
|
||||
kv, err := p.kv(a.State)
|
||||
if err == nil {
|
||||
var rev uint64
|
||||
if rev, err = kv.Create(p.ctx, a.Key, a.Value); err == nil {
|
||||
p.answer(m.ID, map[string]any{"revision": rev}, nil)
|
||||
return
|
||||
}
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32001, "message": "the key was written by another writer first"}})
|
||||
return
|
||||
}
|
||||
}
|
||||
p.answer(m.ID, nil, err)
|
||||
case "mesh/state.watch":
|
||||
kv, err := p.kv(a.State)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
key := a.Key
|
||||
if key == "" {
|
||||
key = ">"
|
||||
}
|
||||
w, err := kv.Watch(p.ctx, key)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
var watch string
|
||||
_ = json.Unmarshal(m.ID, &watch)
|
||||
current := true
|
||||
for e := range w.Updates() {
|
||||
if e == nil {
|
||||
current = false
|
||||
p.answer(m.ID, map[string]any{}, nil)
|
||||
continue
|
||||
}
|
||||
op := "put"
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
op = "delete"
|
||||
}
|
||||
change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current}
|
||||
if op == "put" {
|
||||
change["value"] = json.RawMessage(e.Value())
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.next++
|
||||
id := fmt.Sprintf("w-%d", p.next)
|
||||
p.mu.Unlock()
|
||||
p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}})
|
||||
}
|
||||
case "mesh/subscribe":
|
||||
p.answer(m.ID, map[string]any{}, nil)
|
||||
case "mesh/state.keys":
|
||||
kv, err := p.kv(a.State)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
keys, err := kv.Keys(p.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
keys, err = []string{}, nil
|
||||
}
|
||||
p.answer(m.ID, keys, err)
|
||||
case "mesh/state.delete":
|
||||
kv, err := p.kv(a.State)
|
||||
if err == nil {
|
||||
err = kv.Delete(p.ctx, a.Key)
|
||||
}
|
||||
p.answer(m.ID, map[string]any{}, err)
|
||||
case "mesh/seat.publish":
|
||||
var e struct {
|
||||
Subject string `json:"subject"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &e)
|
||||
p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body))
|
||||
case "mesh/publish":
|
||||
var e struct {
|
||||
Key string `json:"key"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &e)
|
||||
p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body))
|
||||
case "mesh/ask":
|
||||
var ask struct {
|
||||
Key string `json:"key"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &ask)
|
||||
p.mu.Lock()
|
||||
p.asked = append(p.asked, ask.Key+" "+string(ask.Body))
|
||||
p.mu.Unlock()
|
||||
if ask.Key != AssertTool {
|
||||
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
|
||||
return
|
||||
}
|
||||
p.answer(m.ID, map[string]any{"says": "1 the bus lacks: created at this ask"}, nil)
|
||||
default:
|
||||
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *player) call(name string, args map[string]any) (json.RawMessage, error) {
|
||||
p.mu.Lock()
|
||||
p.next++
|
||||
id := fmt.Sprintf("c-%d", p.next)
|
||||
ch := make(chan wire, 1)
|
||||
p.pending[id] = ch
|
||||
p.mu.Unlock()
|
||||
p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args,
|
||||
"_meta": map[string]any{"mesh/caller": "person.jochen"}}})
|
||||
select {
|
||||
case m := <-ch:
|
||||
if m.Error != nil {
|
||||
return nil, errors.New(m.Error.Message)
|
||||
}
|
||||
var r struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
}
|
||||
if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 {
|
||||
return nil, fmt.Errorf("%s answered %s", name, m.Result)
|
||||
}
|
||||
if r.IsError {
|
||||
return nil, errors.New(r.Content[0].Text)
|
||||
}
|
||||
return json.RawMessage(r.Content[0].Text), nil
|
||||
case <-time.After(60 * time.Second):
|
||||
return nil, fmt.Errorf("%s did not answer in 60s", name)
|
||||
}
|
||||
}
|
||||
|
||||
func eventually(t *testing.T, what string, ok func() bool) {
|
||||
t.Helper()
|
||||
for end := time.Now().Add(30 * time.Second); time.Now().Before(end); time.Sleep(50 * time.Millisecond) {
|
||||
if ok() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("never: %s", what)
|
||||
}
|
||||
|
||||
func raised(key, module string) []byte {
|
||||
raw, _ := json.Marshal(map[string]any{"key": key, "kind": "consumer-lost", "severity": "warning", "resolver": "self",
|
||||
"subject": map[string]any{"scope": "bus", "id": "EVENTS." + module}, "event": "condition-raised",
|
||||
"show": health.ShowKeeper(key)})
|
||||
return raw
|
||||
}
|
||||
|
||||
// 1952/H3: an induced failure; the healer acts only from the lease write, and brakes.
|
||||
func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
dir := t.TempDir()
|
||||
bin := filepath.Join(dir, Module)
|
||||
build := exec.Command("go", "build", "-o", bin, ".")
|
||||
build.Stderr = os.Stderr
|
||||
if err := build.Run(); err != nil {
|
||||
t.Fatalf("mesh-healers does not build: %v", err)
|
||||
}
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, b := range []string{Module + "_" + AttemptsState, Module + "_" + WouldState, Module + "_" + FindingsState, Module + "_" + WakesState,
|
||||
"mesh-controller_lease", "mesh-conditions_conditions", "mesh-controller_hand-acts"} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
bucket := func(name string) jetstream.KeyValue {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
keys := func(name string) []string {
|
||||
k, _ := bucket(name).Keys(ctx)
|
||||
return k
|
||||
}
|
||||
lease := bucket("mesh-controller_lease")
|
||||
if _, err := lease.Put(ctx, LeaseKey, []byte(`{"instance":"novox/1/2026-10-27","epoch":57}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
acted, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + ActedEvent)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, bin)
|
||||
cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module)
|
||||
stdin, _ := cmd.StdinPipe()
|
||||
stdout, _ := cmd.StdoutPipe()
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
|
||||
p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
|
||||
go p.serve(stdout)
|
||||
healers := func() map[string]any {
|
||||
raw, err := p.call(HealersTool, map[string]any{})
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var a map[string]any
|
||||
_ = json.Unmarshal(raw, &a)
|
||||
return a
|
||||
}
|
||||
eventually(t, "the lease read as a controller's", func() bool {
|
||||
a := healers()
|
||||
return a != nil && strings.Contains(fmt.Sprint(a["why"]), "names a controller")
|
||||
})
|
||||
|
||||
// The controller holds the lease: the induced failure is written as would, never acted on.
|
||||
p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records"))
|
||||
eventually(t, "the would-act written", func() bool { return len(keys(Module+"_"+WouldState)) == 1 })
|
||||
if k := keys(Module + "_" + AttemptsState); len(k) != 0 {
|
||||
t.Fatalf("acted under the controller's lease: %v", k)
|
||||
}
|
||||
// The write that hands the lease away: the same failure is acted on.
|
||||
if _, err := lease.Put(ctx, LeaseKey, []byte(`{"handed-to":"the operator's hand-over"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "the healers acting", func() bool { return healers()["acting"] == true })
|
||||
p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records"))
|
||||
eventually(t, "the act begun and finished on the bus", func() bool {
|
||||
k := keys(Module + "_" + AttemptsState)
|
||||
if len(k) != 1 {
|
||||
return false
|
||||
}
|
||||
e, err := bucket(Module+"_"+AttemptsState).Get(ctx, k[0])
|
||||
return err == nil && strings.Contains(string(e.Value()), `"outcome":"acted"`)
|
||||
})
|
||||
msg, err := acted.NextMsg(10 * time.Second)
|
||||
if err != nil || !strings.Contains(string(msg.Data), `"healer":"H3"`) {
|
||||
t.Fatalf("healer-acted was not said: %v", err)
|
||||
}
|
||||
p.mu.Lock()
|
||||
asked := strings.Join(p.asked, "\n")
|
||||
p.mu.Unlock()
|
||||
if !strings.Contains(asked, AssertTool+` {"module":"records"}`) {
|
||||
t.Errorf("the grants part was asked %s", asked)
|
||||
}
|
||||
// Eleven more acts within the hour: the twelfth was the limit, and the next condition meets the brake.
|
||||
attempts := bucket(Module + "_" + AttemptsState)
|
||||
for i := 0; i < BrakeLimit-1; i++ {
|
||||
begun := time.Now().Add(-time.Duration(i+1) * time.Minute).UTC()
|
||||
raw, _ := json.Marshal(Attempt{Healer: "H3", Condition: fmt.Sprintf("bus.EVENTS.m%d.missing", i), BudgetKey: "x",
|
||||
Outcome: OutcomeActed, Begun: begun})
|
||||
if _, err := attempts.Put(ctx, attemptKey("H3", fmt.Sprintf("bus.EVENTS.m%d.missing", i), begun), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
p.deliver(RaisedEvent, raised("bus.EVENTS.mail.missing", "mail"))
|
||||
eventually(t, "the brake written as a finding on the bus", func() bool {
|
||||
e, err := bucket(Module+"_"+FindingsState).Get(ctx, brakeKey)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
f, err := health.DecodeFinding(e.Value())
|
||||
return err == nil && f.Severity == health.Urgent && f.Kind == "healers-braked"
|
||||
})
|
||||
if n := len(keys(Module + "_" + AttemptsState)); n != BrakeLimit {
|
||||
t.Errorf("past the brake the attempts are %d, want %d", n, BrakeLimit)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
module mesh-healers
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"module": "mesh-healers",
|
||||
"version": "1",
|
||||
"runs-as": "mesh-healers",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-healers",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"healers",
|
||||
"heal",
|
||||
"drill"
|
||||
]
|
||||
}
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-conditions.condition-raised",
|
||||
"mesh-conditions.condition-changed",
|
||||
"mesh-hand-acts.act-recorded"
|
||||
],
|
||||
"invokes": [
|
||||
"mesh-grants.grants_assert",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-controller.drill"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-controller.lease",
|
||||
"mesh-conditions.conditions",
|
||||
"mesh-controller.hand-acts"
|
||||
],
|
||||
"state": [
|
||||
{
|
||||
"name": "attempts",
|
||||
"key-ttl": true
|
||||
},
|
||||
{
|
||||
"name": "would",
|
||||
"key-ttl": true
|
||||
},
|
||||
"findings",
|
||||
{
|
||||
"name": "wakes",
|
||||
"key-ttl": true
|
||||
}
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-healers",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-healers",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-healers"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-healers",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-healers",
|
||||
"binary": "mesh-healers",
|
||||
"loads": [
|
||||
"mesh-healers"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
/mesh-provisioner
|
||||
/cmd/mesh-provisioner/mesh-provisioner
|
||||
@@ -0,0 +1,204 @@
|
||||
package main
|
||||
|
||||
// The bus as the holder composes from it (frombus.Reader), read through this module's runtime with its own `reads`:
|
||||
// placement, each node's record in its own have key, the indexer's modules and manifests, the builder's builds and
|
||||
// mesh-config's resolved values. A plan names no value, so no own secret and no credential is read (compose.Planning).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// busReader reads the bus for frombus through a Store.
|
||||
type busReader struct {
|
||||
s Store
|
||||
fetch func(ctx context.Context, url string) ([]byte, error)
|
||||
}
|
||||
|
||||
func (r busReader) each(read string, visit func(key string, value []byte)) error {
|
||||
keys, err := r.s.Keys(read)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", read, err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
e, err := r.s.Get(read, k)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s %s: %w", read, k, err)
|
||||
}
|
||||
if e != nil {
|
||||
visit(k, e.Value)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r busReader) Placements(context.Context) (map[string]delivery.Placement, error) {
|
||||
out := map[string]delivery.Placement{}
|
||||
err := r.each(PlacementRead, func(k string, v []byte) {
|
||||
node, err := delivery.ParsePlacementKey(k)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if p, err := delivery.DecodePlacement(v); err == nil && p.Node == node {
|
||||
out[node] = p
|
||||
}
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (r busReader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
|
||||
out := map[string]delivery.NodeRecord{}
|
||||
err := r.each(HaveRead, func(k string, v []byte) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
return
|
||||
}
|
||||
if h, err := delivery.DecodeHaveNode(v); err == nil && h.Record != nil {
|
||||
out[node] = *h.Record
|
||||
}
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (r busReader) Modules(context.Context) (map[string]delivery.Module, error) {
|
||||
out := map[string]delivery.Module{}
|
||||
err := r.each(ModulesRead, func(k string, v []byte) {
|
||||
module, err := delivery.ParseModulesKey(k)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if m, err := delivery.DecodeModule(v); err == nil {
|
||||
out[module] = m
|
||||
}
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (r busReader) Manifest(_ context.Context, digest string) ([]byte, error) {
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := r.s.Get(ManifestsRead, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e == nil {
|
||||
return nil, fmt.Errorf("the indexer holds no manifest %s", digest)
|
||||
}
|
||||
m, err := delivery.DecodeManifest(e.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []byte(m.JSON), nil
|
||||
}
|
||||
|
||||
func (r busReader) Values(_ context.Context, node, module string) (map[string]any, error) {
|
||||
var out map[string]any
|
||||
prefix := node + "." + module + "."
|
||||
err := r.each(ResolvedRead, func(k string, v []byte) {
|
||||
name, ok := strings.CutPrefix(k, prefix)
|
||||
if !ok || strings.Contains(name, ".") {
|
||||
return
|
||||
}
|
||||
if value, laid := catalogue.LaidValue(v); laid {
|
||||
if out == nil {
|
||||
out = map[string]any{}
|
||||
}
|
||||
out[name] = value
|
||||
}
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (busReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { return nil, nil }
|
||||
|
||||
func (busReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func (r busReader) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := r.s.Get(BuildsRead, key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(e.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &b, nil
|
||||
}
|
||||
|
||||
// archiveWithin bounds one read of a templates archive.
|
||||
const archiveWithin = 30 * time.Second
|
||||
|
||||
func (r busReader) Archive(ctx context.Context, url, digest string) ([]byte, error) {
|
||||
if r.fetch == nil {
|
||||
return nil, fmt.Errorf("this holder reads no archive (%s)", url)
|
||||
}
|
||||
fetching, cancel := context.WithTimeout(ctx, archiveWithin)
|
||||
defer cancel()
|
||||
body, err := r.fetch(fetching, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
|
||||
return nil, fmt.Errorf("%s was recorded as %s and what arrived is %s", url, digest, got)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (r busReader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
|
||||
all, err := r.Values(ctx, node, module)
|
||||
var out map[string]any
|
||||
for _, k := range keys {
|
||||
if v, ok := all[k]; ok {
|
||||
if out == nil {
|
||||
out = map[string]any{}
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
// httpFetch reads an archive from the artifact store over HTTP.
|
||||
func httpFetch(ctx context.Context, url string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("%s answered %s", url, resp.Status)
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, 256<<20))
|
||||
}
|
||||
|
||||
// FromBus is the holder's composer over the bus: frombus's plans of a node, at the builds keptAt names.
|
||||
func FromBus(s Store) Composer {
|
||||
r := busReader{s: s, fetch: httpFetch}
|
||||
return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) {
|
||||
return frombus.Plans(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,673 @@
|
||||
package main
|
||||
|
||||
// The seat's holder (novox/hq ADR 0570 clauses 1 to 5, 7; issue 1954 H1-H3): mesh-provisioner composes every placed
|
||||
// module's provisioning plan and every provision from bus state, on change events and never on a timer, and keeps them
|
||||
// in its states `plans` (`<node>.<module>`) and `provisions` (`<node>.<consumer>.<provision>[.<local>]`), which it alone
|
||||
// writes.
|
||||
//
|
||||
// - **Composed per node.** A change of a node's placement or of one of its wants composes that node; a change of the
|
||||
// indexer's module composes the nodes the module is placed on. Each node is composed twice: at the builds its wants
|
||||
// name (its plans, what its client applies) and at the newest build of each placed module, so a build that adds a
|
||||
// requirement has its key before its want (clause 2). A provision is needed while either needs it (clause 5).
|
||||
// - **A key's state comes from its provider's outcome alone** (clause 3, 7): the provider's own entry for that
|
||||
// consumer, `<provider>_bound` under the same key: present is provided (the vault's generation carried into the
|
||||
// reference, clause 6), refused is failed with why, waits is waits with what. The holder never sets provided by
|
||||
// itself: a key of a provider that writes no outcome stays asked.
|
||||
// - **Retiring is asked** (clause 5): a key no longer needed is `retiring`, one approve ask under mesh-provisioner's
|
||||
// name says what goes from which provider on which node; the warrant goes into the key and the provider acts on it;
|
||||
// a decline is kept and not asked again while nothing changes; a key needed again returns to its outcome's state.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
// The states it keeps, beside Registry, Needs, Applied and Collect.
|
||||
const (
|
||||
ProvisionsState = provisions.ProvisionsState
|
||||
PlansState = provisions.PlansState
|
||||
)
|
||||
|
||||
// The reads it composes from.
|
||||
const (
|
||||
PlacementRead = delivery.PlacementRead
|
||||
WantsRead = delivery.AssignerModule + ".wants"
|
||||
ModulesRead = delivery.IndexerModule + ".modules"
|
||||
ManifestsRead = delivery.IndexerModule + ".manifests"
|
||||
BuildsRead = delivery.BuilderModule + ".builds"
|
||||
HaveRead = "node-engine.have"
|
||||
ResolvedRead = "mesh-config.resolved"
|
||||
)
|
||||
|
||||
// Providers are the providing modules that keep an entry per consumer, `<module>_bound`, read as `<module>.bound`: the
|
||||
// outcome of each provision they make. A provider not named here keeps none.
|
||||
var Providers = []string{"gitea", "keycloak", "postgres", "route-adapter", "mesh-vault"}
|
||||
|
||||
// boundRead is a provider's entries as this module reads them.
|
||||
func boundRead(module string) string { return module + ".bound" }
|
||||
|
||||
// Composer is each module's provisioning plan on a node, composed from the bus: at the builds keptAt names
|
||||
// (module → fingerprint), the newest build of every other placed module.
|
||||
type Composer func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error)
|
||||
|
||||
// Asker is the operator's channel as this module asks on it: publish an ask under its own name, and read the router's
|
||||
// record of it.
|
||||
type Asker interface {
|
||||
Publish(subject string, body any, id string) error
|
||||
Record(id string) ([]byte, bool, error)
|
||||
}
|
||||
|
||||
// Holder is the seat's holder.
|
||||
type Holder struct {
|
||||
Store Store
|
||||
Compose Composer
|
||||
Asks Asker
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// Composed is what one node's composition did.
|
||||
type Composed struct {
|
||||
Node string `json:"node"`
|
||||
Plans []string `json:"plans-written,omitempty"`
|
||||
Gone []string `json:"plans-removed,omitempty"`
|
||||
Changed []string `json:"provisions-changed,omitempty"`
|
||||
Asked []string `json:"retiring-asked,omitempty"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
}
|
||||
|
||||
// ComposeNode composes one node: its plans and its provisions.
|
||||
func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
out := Composed{Node: node}
|
||||
wanted, err := h.wantedOn(node)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
atWants, err := h.Compose(ctx, node, wanted)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the plans of %s cannot be composed: %w", node, err)
|
||||
}
|
||||
newest, err := h.Compose(ctx, node, nil)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the plans of %s at the newest builds cannot be composed: %w", node, err)
|
||||
}
|
||||
if err := h.writePlans(node, atWants, &out); err != nil {
|
||||
out.Problems = append(out.Problems, err.Error())
|
||||
}
|
||||
needed := map[string]provisions.Provision{}
|
||||
for _, plans := range []map[string]catalogue.Plan{atWants, newest} {
|
||||
for module, plan := range plans {
|
||||
for _, p := range plan.Provisions {
|
||||
pv := provisions.Provision{Provision: p.Provision, Local: p.Local,
|
||||
Consumer: provisions.End{Module: module, Node: node},
|
||||
Provider: provisions.End{Module: p.ProviderModule, Node: p.Provider}}
|
||||
key, err := pv.Key()
|
||||
if err != nil {
|
||||
out.Problems = append(out.Problems, err.Error())
|
||||
continue
|
||||
}
|
||||
if _, seen := needed[key]; !seen {
|
||||
needed[key] = pv
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
h.judgeNode(node, needed, &out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// wantedOn is the fingerprint each want on a node names, by module.
|
||||
func (h *Holder) wantedOn(node string) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
keys, err := h.Store.Keys(WantsRead)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the wants cannot be read: %w", err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
n, module, err := delivery.ParseWantKey(k)
|
||||
if err != nil || n != node {
|
||||
continue
|
||||
}
|
||||
e, err := h.Store.Get(WantsRead, k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
if w, err := delivery.DecodeWant(e.Value); err == nil && !w.Removed && w.FP != "" {
|
||||
out[module] = w.FP
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there.
|
||||
func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *Composed) error {
|
||||
var problems []error
|
||||
for _, module := range sortedKeys(plans) {
|
||||
key, err := provisions.PlanKey(node, module)
|
||||
if err != nil {
|
||||
problems = append(problems, err)
|
||||
continue
|
||||
}
|
||||
raw, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: plans[module]})
|
||||
if err != nil {
|
||||
problems = append(problems, err)
|
||||
continue
|
||||
}
|
||||
if e, err := h.Store.Get(PlansState, key); err == nil && e != nil && bytes.Equal(e.Value, raw) {
|
||||
continue
|
||||
}
|
||||
if err := h.Store.Put(PlansState, key, json.RawMessage(raw)); err != nil {
|
||||
problems = append(problems, fmt.Errorf("the plan %s: %w", key, err))
|
||||
continue
|
||||
}
|
||||
out.Plans = append(out.Plans, key)
|
||||
}
|
||||
keys, err := h.Store.Keys(PlansState)
|
||||
if err != nil {
|
||||
return errors.Join(append(problems, err)...)
|
||||
}
|
||||
for _, k := range keys {
|
||||
n, module, ok := strings.Cut(k, ".")
|
||||
if !ok || n != node {
|
||||
continue
|
||||
}
|
||||
if _, still := plans[module]; still {
|
||||
continue
|
||||
}
|
||||
if err := h.Store.Delete(PlansState, k); err != nil {
|
||||
problems = append(problems, err)
|
||||
continue
|
||||
}
|
||||
out.Gone = append(out.Gone, k)
|
||||
}
|
||||
return errors.Join(problems...)
|
||||
}
|
||||
|
||||
// judgeNode brings the node's provision keys to what is needed and what each provider said.
|
||||
func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, out *Composed) {
|
||||
held := h.held(node)
|
||||
for _, key := range unionKeys(needed, held) {
|
||||
want, isNeeded := needed[key]
|
||||
have, isHeld := held[key]
|
||||
var next provisions.Provision
|
||||
switch {
|
||||
case isNeeded:
|
||||
next = want
|
||||
if isHeld {
|
||||
next.Reference = have.Reference
|
||||
}
|
||||
h.outcome(key, &next)
|
||||
case have.State == provisions.Retiring:
|
||||
next = have
|
||||
if next.Ask == "" && next.Warrant == nil {
|
||||
if err := h.ask(&next, key); err != nil {
|
||||
out.Problems = append(out.Problems, err.Error())
|
||||
} else {
|
||||
out.Asked = append(out.Asked, key)
|
||||
}
|
||||
}
|
||||
if gone, why := h.retired(key, have); gone {
|
||||
if err := h.Store.Delete(ProvisionsState, key); err != nil {
|
||||
out.Problems = append(out.Problems, err.Error())
|
||||
} else {
|
||||
out.Changed = append(out.Changed, key+" (retired: "+why+")")
|
||||
}
|
||||
continue
|
||||
}
|
||||
default:
|
||||
next = have
|
||||
next.State, next.Why = provisions.Retiring, fmt.Sprintf("no build wanted or placed on %s requires %s of %s on %s "+
|
||||
"any more", node, have.Provision, have.Provider.Module, have.Provider.Node)
|
||||
next.Warrant = nil
|
||||
if err := h.ask(&next, key); err != nil {
|
||||
out.Problems = append(out.Problems, err.Error())
|
||||
} else {
|
||||
out.Asked = append(out.Asked, key)
|
||||
}
|
||||
}
|
||||
if isHeld && same(have, next) {
|
||||
continue
|
||||
}
|
||||
next.At = h.Now().UTC()
|
||||
raw, err := provisions.Encode(next)
|
||||
if err != nil {
|
||||
out.Problems = append(out.Problems, err.Error())
|
||||
continue
|
||||
}
|
||||
if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil {
|
||||
out.Problems = append(out.Problems, fmt.Sprintf("%s: %v", key, err))
|
||||
continue
|
||||
}
|
||||
out.Changed = append(out.Changed, key+" "+next.State)
|
||||
}
|
||||
}
|
||||
|
||||
// held is the node's provision keys as kept.
|
||||
func (h *Holder) held(node string) map[string]provisions.Provision {
|
||||
out := map[string]provisions.Provision{}
|
||||
keys, err := h.Store.Keys(ProvisionsState)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
for _, k := range keys {
|
||||
if !strings.HasPrefix(k, node+".") {
|
||||
continue
|
||||
}
|
||||
e, err := h.Store.Get(ProvisionsState, k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
if p, err := provisions.Decode(e.Value); err == nil {
|
||||
out[k] = p
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// same says two provisions say the same, their time aside.
|
||||
func same(a, b provisions.Provision) bool {
|
||||
a.At, b.At, a.V, b.V = time.Time{}, time.Time{}, 0, 0
|
||||
ra, _ := json.Marshal(a)
|
||||
rb, _ := json.Marshal(b)
|
||||
return bytes.Equal(ra, rb)
|
||||
}
|
||||
|
||||
// entry is a provider's outcome for one consumer, as its entry says it: the provider helper's Entry, read for what
|
||||
// the holder needs and nothing more.
|
||||
type entry struct {
|
||||
// State and Why are the provider's outcome (provider.Entry, ADR 0570 clause 3): provided, failed or waits.
|
||||
State string `json:"state,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Waits string `json:"waits,omitempty"`
|
||||
VaultName string `json:"vault-name,omitempty"`
|
||||
Generation int `json:"generation,omitempty"`
|
||||
}
|
||||
|
||||
// outcome sets a needed provision's state from its provider's entry: provided, failed or waits; asked while it has
|
||||
// none. The holder never sets provided by itself (clause 3): a provider whose entries it does not read keeps the key
|
||||
// asked, said so.
|
||||
func (h *Holder) outcome(key string, p *provisions.Provision) {
|
||||
p.Ask, p.Warrant = "", nil
|
||||
if !keepsEntries(p.Provider.Module) {
|
||||
h.haveOutcome(p)
|
||||
return
|
||||
}
|
||||
e, err := h.Store.Get(boundRead(p.Provider.Module), key)
|
||||
if err != nil {
|
||||
p.State, p.Why = provisions.Asked, "the provider's entries cannot be read: "+err.Error()
|
||||
return
|
||||
}
|
||||
if e == nil {
|
||||
p.State, p.Why = provisions.Asked, ""
|
||||
return
|
||||
}
|
||||
var en entry
|
||||
if err := json.Unmarshal(e.Value, &en); err != nil {
|
||||
p.State, p.Why = provisions.Asked, "the provider's entry is not readable"
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case en.State == provisions.Failed:
|
||||
p.State, p.Why = provisions.Failed, orSaid(en.Why, en.Refused, "the provider said it failed")
|
||||
case en.State == provisions.Waits:
|
||||
p.State, p.Why = provisions.Waits, orSaid(en.Why, en.Waits, "the provider waits for a person")
|
||||
case en.State == "" && en.Refused != "":
|
||||
p.State, p.Why = provisions.Failed, en.Refused
|
||||
case en.State == "" && en.Waits != "":
|
||||
p.State, p.Why = provisions.Waits, en.Waits
|
||||
case en.State != "" && en.State != provisions.Provided:
|
||||
p.State, p.Why = provisions.Asked, "the provider's entry says "+en.State+", which no state of a provision is"
|
||||
return
|
||||
default:
|
||||
p.State, p.Why = provisions.Provided, ""
|
||||
if en.VaultName != "" || en.Generation > 0 {
|
||||
p.Reference = &provisions.Reference{ID: en.VaultName, Generation: en.Generation}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// haveOutcome is the outcome of a provider that makes no credential per consumer: its own have on its node (the lead's
|
||||
// reading of ADR 0570 clause 3, recorded as an insight). Running with health ok at the build its want names is
|
||||
// provided; failed is failed with the have's words; anything else is asked, waiting for the provider to run.
|
||||
func (h *Holder) haveOutcome(p *provisions.Provision) {
|
||||
where := p.Provider.Module + " on " + p.Provider.Node
|
||||
key := p.Provider.Node + "." + p.Provider.Module
|
||||
e, err := h.Store.Get(HaveRead, key)
|
||||
if err != nil {
|
||||
p.State, p.Why = provisions.Asked, "the have of "+where+" cannot be read: "+err.Error()
|
||||
return
|
||||
}
|
||||
if e == nil {
|
||||
p.State, p.Why = provisions.Asked, "waits for "+where+" to run: it says nothing yet"
|
||||
return
|
||||
}
|
||||
have, err := delivery.DecodeHave(e.Value)
|
||||
if err != nil {
|
||||
p.State, p.Why = provisions.Asked, "the have of "+where+" is not readable"
|
||||
return
|
||||
}
|
||||
wantFP := ""
|
||||
if we, err := h.Store.Get(WantsRead, "want."+key); err == nil && we != nil {
|
||||
if w, err := delivery.DecodeWant(we.Value); err == nil && !w.Removed {
|
||||
wantFP = w.FP
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case have.Phase == delivery.PhaseFailed || have.Health == delivery.HealthFailed:
|
||||
p.State, p.Why = provisions.Failed, orSaid(have.Detail, where+" failed")
|
||||
case have.Phase == delivery.PhaseRunning && have.Health == delivery.HealthOK && (wantFP == "" || have.FP == wantFP):
|
||||
p.State, p.Why = provisions.Provided, ""
|
||||
default:
|
||||
p.State, p.Why = provisions.Asked, fmt.Sprintf("waits for %s to run healthy at the build its want names (it is %s, "+
|
||||
"health %s)", where, have.Phase, have.Health)
|
||||
}
|
||||
}
|
||||
|
||||
func keepsEntries(module string) bool {
|
||||
for _, m := range Providers {
|
||||
if m == module {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// retired says a retiring key the operator approved is retired: its provider's entry is gone.
|
||||
func (h *Holder) retired(key string, p provisions.Provision) (bool, string) {
|
||||
if p.Warrant == nil || p.Warrant.Option != OptionApprove {
|
||||
return false, ""
|
||||
}
|
||||
if !keepsEntries(p.Provider.Module) {
|
||||
return false, ""
|
||||
}
|
||||
e, err := h.Store.Get(boundRead(p.Provider.Module), key)
|
||||
if err != nil || e != nil {
|
||||
return false, ""
|
||||
}
|
||||
return true, "its provider removed its credential on the operator's warrant"
|
||||
}
|
||||
|
||||
// ---- the retiring ask ----------------------------------------------------------------------------------------------
|
||||
|
||||
// The options a retiring ask offers.
|
||||
const (
|
||||
OptionApprove = "approve"
|
||||
OptionDecline = "decline"
|
||||
)
|
||||
|
||||
// askLasts is how long a retiring ask stays open.
|
||||
const askLasts = asks.ApproveLasts - 10*time.Minute
|
||||
|
||||
// retiringAsk is the ask of one retiring provision, composed from the key alone.
|
||||
func retiringAsk(key string, p provisions.Provision, at time.Time, id string) asks.Ask {
|
||||
what := fmt.Sprintf("%s's access to %s on %s, for %s on %s", p.Consumer.Module, p.Provision, p.Provider.Node,
|
||||
p.Consumer.Module, p.Consumer.Node)
|
||||
a := asks.Ask{ID: id, Headline: "Remove " + p.Consumer.Module + "'s " + p.Provision + " access?",
|
||||
Explanation: fmt.Sprintf("Nothing wanted or placed on %s needs %s any more. On Approve, %s removes %s: what it owns "+
|
||||
"is handed to the resource's owner and its credential goes; the resource and its data stay.", p.Consumer.Node,
|
||||
p.Provision, p.Provider.Module, what),
|
||||
Who: asks.Operator, Expires: at.Add(askLasts), OnExpiry: "nothing is removed; it is asked again when it changes",
|
||||
About: "provision." + key, Details: "The provision " + key + " of mesh-provisioner's state provisions."}
|
||||
for _, o := range []struct{ id, label, does string }{
|
||||
{OptionApprove, "Remove", "the provider removes the credential; the resource and its data stay"},
|
||||
{OptionDecline, "Keep", "nothing is removed, and it is not asked again while nothing changes"},
|
||||
} {
|
||||
binds, _ := asks.ActDigest(retireAct(key, id, o.id))
|
||||
a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds})
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func retireAct(key, id, option string) asks.Act {
|
||||
return asks.Act{"verb": "mesh-provisioner.retire", "provision": key, "ask": id, "option": option}
|
||||
}
|
||||
|
||||
// ask raises the retiring ask of a key, under mesh-provisioner's own name; one already raised stands.
|
||||
func (h *Holder) ask(p *provisions.Provision, key string) error {
|
||||
if p.Ask != "" {
|
||||
return nil
|
||||
}
|
||||
if h.Asks == nil {
|
||||
return errors.New("the operator cannot be asked from here, so " + key + " is retiring and nothing is removed")
|
||||
}
|
||||
now := h.Now().UTC()
|
||||
id := "retire-" + strings.ReplaceAll(key, ".", "-") + "-" + fmt.Sprint(now.Unix())
|
||||
a := retiringAsk(key, *p, now, id)
|
||||
if err := a.Check(now); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := h.Asks.Publish(asks.AskSubject(Module), a, id); err != nil {
|
||||
return fmt.Errorf("the retiring ask of %s could not be published: %w", key, err)
|
||||
}
|
||||
p.Ask = id
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decide reads the router's record of each retiring key's ask and writes the warrant into the key (clause 5): the
|
||||
// provider acts on an Approve; a Decline is kept, and not asked again while nothing changes.
|
||||
func (h *Holder) Decide() []string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.Asks == nil {
|
||||
return nil
|
||||
}
|
||||
keys, err := h.Store.Keys(ProvisionsState)
|
||||
if err != nil {
|
||||
return []string{"the provisions cannot be read: " + err.Error()}
|
||||
}
|
||||
var said []string
|
||||
for _, k := range keys {
|
||||
e, err := h.Store.Get(ProvisionsState, k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
p, err := provisions.Decode(e.Value)
|
||||
if err != nil || p.State != provisions.Retiring || p.Ask == "" || p.Warrant != nil {
|
||||
continue
|
||||
}
|
||||
raw, found, err := h.Asks.Record(p.Ask)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
var rec struct {
|
||||
State string `json:"state"`
|
||||
Ask *asks.Ask `json:"ask"`
|
||||
Warrant *asks.Warrant `json:"warrant"`
|
||||
}
|
||||
if json.Unmarshal(raw, &rec) != nil || rec.State == "open" || rec.Warrant == nil || rec.Ask == nil {
|
||||
continue
|
||||
}
|
||||
w := *rec.Warrant
|
||||
if w.Asker != Module || w.Ask != p.Ask {
|
||||
continue
|
||||
}
|
||||
option := ""
|
||||
if w.Outcome == asks.OutcomeChosen {
|
||||
o, err := w.For(Module, *rec.Ask)
|
||||
if err == nil {
|
||||
err = o.Performs(retireAct(k, p.Ask, o.ID))
|
||||
}
|
||||
if err != nil {
|
||||
said = append(said, k+": the answer is refused: "+err.Error())
|
||||
continue
|
||||
}
|
||||
option = o.ID
|
||||
} else {
|
||||
// Ended with no choice: asked again at the next change.
|
||||
p.Ask = ""
|
||||
h.put(k, p)
|
||||
said = append(said, k+": the ask "+string(w.Outcome)+"; asked again when it changes")
|
||||
continue
|
||||
}
|
||||
by := "the operator"
|
||||
if w.By != nil {
|
||||
by = fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity)
|
||||
}
|
||||
p.Warrant = &provisions.Warrant{Ask: p.Ask, Option: option, By: by, At: w.At.UTC()}
|
||||
h.put(k, p)
|
||||
said = append(said, k+": "+option+" by "+by)
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
func (h *Holder) put(key string, p provisions.Provision) {
|
||||
p.At = h.Now().UTC()
|
||||
if raw, err := provisions.Encode(p); err == nil {
|
||||
if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil {
|
||||
h.Logf("[%s] %s is not written: %v", Module, key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- what a change touches --------------------------------------------------------------------------------------
|
||||
|
||||
// NodesOfPlacement is the node a placement key names.
|
||||
func NodesOfPlacement(key string) []string {
|
||||
if n, err := delivery.ParsePlacementKey(key); err == nil {
|
||||
return []string{n}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NodesOfWant is the node a want key names.
|
||||
func NodesOfWant(key string) []string {
|
||||
if n, _, err := delivery.ParseWantKey(key); err == nil {
|
||||
return []string{n}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NodesOfModule is every node a module is placed on, and every node whose plans name it as a provider: what a change
|
||||
// of the module's manifest touches.
|
||||
func (h *Holder) NodesOfModule(module string) []string {
|
||||
nodes := map[string]bool{}
|
||||
if keys, err := h.Store.Keys(PlacementRead); err == nil {
|
||||
for _, k := range keys {
|
||||
n, err := delivery.ParsePlacementKey(k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
e, err := h.Store.Get(PlacementRead, k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
if p, err := delivery.DecodePlacement(e.Value); err == nil {
|
||||
for _, m := range p.Modules {
|
||||
if m.Module == module {
|
||||
nodes[n] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if keys, err := h.Store.Keys(ProvisionsState); err == nil {
|
||||
for _, k := range keys {
|
||||
if e, err := h.Store.Get(ProvisionsState, k); err == nil && e != nil {
|
||||
if p, err := provisions.Decode(e.Value); err == nil && p.Provider.Module == module {
|
||||
nodes[p.Consumer.Node] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return sortedKeys(nodes)
|
||||
}
|
||||
|
||||
// NodeOfEntry is the consumer's node a provider's entry key names.
|
||||
func NodeOfEntry(key string) []string {
|
||||
if n, _, _, _, err := provisions.ParseKey(key); err == nil {
|
||||
return []string{n}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func unionKeys[A, B any](a map[string]A, b map[string]B) []string {
|
||||
seen := map[string]bool{}
|
||||
for k := range a {
|
||||
seen[k] = true
|
||||
}
|
||||
for k := range b {
|
||||
seen[k] = true
|
||||
}
|
||||
return sortedKeys(seen)
|
||||
}
|
||||
|
||||
// Rejudge sets one key's state again from its provider's outcome: what a change of a provider's entry touches. A
|
||||
// retiring key whose provider removed its entry on the operator's warrant goes.
|
||||
func (h *Holder) Rejudge(key string) (string, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
e, err := h.Store.Get(ProvisionsState, key)
|
||||
if err != nil || e == nil {
|
||||
return "", err
|
||||
}
|
||||
have, err := provisions.Decode(e.Value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if have.State == provisions.Retiring {
|
||||
if gone, why := h.retired(key, have); gone {
|
||||
return key + " retired: " + why, h.Store.Delete(ProvisionsState, key)
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
next := have
|
||||
h.outcome(key, &next)
|
||||
if same(have, next) {
|
||||
return "", nil
|
||||
}
|
||||
h.put(key, next)
|
||||
return key + " " + next.State, nil
|
||||
}
|
||||
|
||||
// RetireNeeds deletes every key of the retired state `needs`, which the clients no longer write (clause 1): run once at
|
||||
// the start.
|
||||
func (h *Holder) RetireNeeds() (int, error) {
|
||||
keys, err := h.Store.Keys(Needs)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n := 0
|
||||
for _, k := range keys {
|
||||
if err := h.Store.Delete(Needs, k); err != nil {
|
||||
return n, err
|
||||
}
|
||||
n++
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// orSaid is the first of the words that says something.
|
||||
func orSaid(words ...string) string {
|
||||
for _, w := range words {
|
||||
if strings.TrimSpace(w) != "" {
|
||||
return w
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
// router is the operator's channel as the holder asks on it.
|
||||
type router struct {
|
||||
asked map[string]asks.Ask
|
||||
records map[string][]byte
|
||||
}
|
||||
|
||||
func (r *router) Publish(_ string, body any, id string) error {
|
||||
r.asked[id] = body.(asks.Ask)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *router) Record(id string) ([]byte, bool, error) {
|
||||
raw, ok := r.records[id]
|
||||
return raw, ok, nil
|
||||
}
|
||||
|
||||
// answer is the operator's answer to an ask.
|
||||
func (r *router) answer(t *testing.T, id, option string) {
|
||||
t.Helper()
|
||||
a := r.asked[id]
|
||||
o, ok := a.Option(option)
|
||||
if !ok {
|
||||
t.Fatalf("the ask %s offers no %s", id, option)
|
||||
}
|
||||
w := asks.Warrant{Ask: id, Asker: Module, About: a.About, Outcome: asks.OutcomeChosen, Option: o.ID, Label: o.Label,
|
||||
Level: o.Level, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42"}, Channel: "telegram",
|
||||
At: time.Date(2026, 10, 29, 23, 0, 0, 0, time.UTC), AskDigest: a.Digest()}
|
||||
r.records[id], _ = json.Marshal(map[string]any{"state": "closed", "ask": a, "warrant": w})
|
||||
}
|
||||
|
||||
// mesh is the holder over memory, its composer answering each node from tables the test sets: the plans at the wanted
|
||||
// builds and at the newest.
|
||||
type mesh struct {
|
||||
h *Holder
|
||||
s *memStore
|
||||
r *router
|
||||
atWant map[string]map[string]catalogue.Plan
|
||||
atNewest map[string]map[string]catalogue.Plan
|
||||
composed []string
|
||||
}
|
||||
|
||||
func newMesh() *mesh {
|
||||
m := &mesh{s: newStore(), r: &router{asked: map[string]asks.Ask{}, records: map[string][]byte{}},
|
||||
atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{}}
|
||||
now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC)
|
||||
m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {},
|
||||
Now: func() time.Time { now = now.Add(time.Second); return now },
|
||||
Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) {
|
||||
m.composed = append(m.composed, node)
|
||||
if keptAt != nil {
|
||||
return m.atWant[node], nil
|
||||
}
|
||||
return m.atNewest[node], nil
|
||||
}}
|
||||
return m
|
||||
}
|
||||
|
||||
// needs makes a module on a node require provisions in its wanted build (and the same in its newest unless set).
|
||||
func planOf(provs ...catalogue.PlanProvision) catalogue.Plan {
|
||||
return catalogue.Plan{Provisions: provs}
|
||||
}
|
||||
|
||||
var (
|
||||
pgDB = catalogue.PlanProvision{Provision: "postgres-database", Provider: "ace", ProviderModule: "postgres"}
|
||||
oidc = catalogue.PlanProvision{Provision: "oidc-client", Provider: "novox", ProviderModule: "keycloak"}
|
||||
split = catalogue.PlanProvision{Provision: "split-dns", Provider: "g14", ProviderModule: "systemd-resolved"}
|
||||
)
|
||||
|
||||
func (m *mesh) state(t *testing.T, key string) provisions.Provision {
|
||||
t.Helper()
|
||||
e, _ := m.s.Get(ProvisionsState, key)
|
||||
if e == nil {
|
||||
return provisions.Provision{}
|
||||
}
|
||||
p, err := provisions.Decode(e.Value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (m *mesh) compose(t *testing.T, node string) Composed {
|
||||
t.Helper()
|
||||
c, err := m.h.ComposeNode(context.Background(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// A node composes its plans and provisions from what is wanted and placed; a key's state follows its provider's entry
|
||||
// alone; the vault's generation reaches the reference; a
|
||||
// quiet pass writes nothing (ADR 0570 clauses 1-3, 6, 7; issue 1954 H1, H2).
|
||||
func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) {
|
||||
m := newMesh()
|
||||
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, split)}
|
||||
m.atNewest["g14"] = m.atWant["g14"]
|
||||
key := "g14.gitea.postgres-database"
|
||||
steps := []struct {
|
||||
name string
|
||||
entry string // the provider's entry written before the step, "-" to delete
|
||||
state string
|
||||
why string
|
||||
}{
|
||||
{"composed: asked", "", provisions.Asked, ""},
|
||||
{"the provider waits for a person", `{"waits":"the database's owner to accept"}`, provisions.Waits, "owner"},
|
||||
{"the provider refused", `{"refused":"no room on ace"}`, provisions.Failed, "no room"},
|
||||
{"the provider says failed in its state", `{"state":"failed","why":"the disk is full"}`, provisions.Failed, "disk"},
|
||||
{"the provider made it", `{"state":"provided","vault-name":"pair/gitea-postgres","generation":3}`, provisions.Provided, ""},
|
||||
}
|
||||
for _, s := range steps {
|
||||
if s.entry != "" {
|
||||
m.s.write(t, "postgres.bound", key, s.entry)
|
||||
}
|
||||
m.compose(t, "g14")
|
||||
p := m.state(t, key)
|
||||
if p.State != s.state || !strings.Contains(p.Why, s.why) || p.Provider.Module != "postgres" || p.Provider.Node != "ace" {
|
||||
t.Errorf("%s: %+v", s.name, p)
|
||||
}
|
||||
}
|
||||
if p := m.state(t, key); p.Reference == nil || p.Reference.ID != "pair/gitea-postgres" || p.Reference.Generation != 3 {
|
||||
t.Errorf("the vault's generation does not reach the reference: %+v", p.Reference)
|
||||
}
|
||||
// A provider making no credential per consumer reports through its own have (clause 3): never provided by the
|
||||
// holder alone.
|
||||
split := "g14.gitea.split-dns"
|
||||
for _, row := range []struct{ have, want, state, why string }{
|
||||
{"", "", provisions.Asked, "says nothing yet"},
|
||||
{`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"applying","health":"unknown"}`, "", provisions.Asked, "applying"},
|
||||
{`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Asked, "build its want"},
|
||||
{`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"failed","health":"failed","detail":"the unit did not start"}`, "", provisions.Failed, "did not start"},
|
||||
{`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Provided, ""},
|
||||
} {
|
||||
if row.have != "" {
|
||||
m.s.write(t, HaveRead, "g14.systemd-resolved", row.have)
|
||||
}
|
||||
if row.want != "" {
|
||||
m.s.write(t, WantsRead, "want.g14.systemd-resolved", row.want)
|
||||
}
|
||||
m.compose(t, "g14")
|
||||
if p := m.state(t, split); p.State != row.state || !strings.Contains(p.Why, row.why) {
|
||||
t.Errorf("the provider's have %s: %+v, want %s", row.have, p, row.state)
|
||||
}
|
||||
}
|
||||
if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil {
|
||||
t.Error("the plan is not written")
|
||||
}
|
||||
if c := m.compose(t, "g14"); len(c.Plans)+len(c.Changed)+len(c.Asked) != 0 {
|
||||
t.Errorf("a quiet pass wrote %+v", c)
|
||||
}
|
||||
// An entry's change judges its key alone, no composition.
|
||||
before := len(m.composed)
|
||||
m.s.write(t, "postgres.bound", key, `{"refused":"the login is locked"}`)
|
||||
if said, err := m.h.Rejudge(key); err != nil || !strings.Contains(said, provisions.Failed) || len(m.composed) != before {
|
||||
t.Errorf("rejudged %q, %v, composed %d more", said, err, len(m.composed)-before)
|
||||
}
|
||||
}
|
||||
|
||||
// A provision no build wanted or placed needs is retiring with one ask naming provider, node and what goes; one composed
|
||||
// ahead of its want raises none; a change removing one requirement and adding another retires only the removed one; the
|
||||
// warrant goes into the key and the key goes once the provider removed its entry; a decline is kept and not asked again;
|
||||
// a provision needed again returns from retiring (ADR 0570 clause 5; issue 1954 H3).
|
||||
func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) {
|
||||
m := newMesh()
|
||||
pg, kc := "g14.gitea.postgres-database", "g14.gitea.oidc-client"
|
||||
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)}
|
||||
m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, oidc)} // the newest build adds oidc ahead of its want
|
||||
m.s.write(t, "postgres.bound", pg, `{}`)
|
||||
steps := []struct {
|
||||
name string
|
||||
change func()
|
||||
asked []string
|
||||
states map[string]string
|
||||
removed bool
|
||||
}{
|
||||
{"a key composed ahead of its want is asked, not retiring", func() {}, nil,
|
||||
map[string]string{pg: provisions.Provided, kc: provisions.Asked}, false},
|
||||
{"one requirement removed and another added: only the removed retires", func() {
|
||||
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
|
||||
m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)}
|
||||
}, []string{pg}, map[string]string{pg: provisions.Retiring, kc: provisions.Asked}, false},
|
||||
{"composed again: no second ask", func() {}, nil, map[string]string{pg: provisions.Retiring}, false},
|
||||
{"needed again before the warrant: back to its outcome", func() {
|
||||
m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc, pgDB)}
|
||||
}, nil, map[string]string{pg: provisions.Provided}, false},
|
||||
{"gone again: asked again", func() { m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} },
|
||||
[]string{pg}, map[string]string{pg: provisions.Retiring}, false},
|
||||
}
|
||||
for _, s := range steps {
|
||||
s.change()
|
||||
c := m.compose(t, "g14")
|
||||
if !slices.Equal(c.Asked, s.asked) {
|
||||
t.Errorf("%s: asked %v, want %v", s.name, c.Asked, s.asked)
|
||||
}
|
||||
for k, st := range s.states {
|
||||
if p := m.state(t, k); p.State != st {
|
||||
t.Errorf("%s: %s is %s, want %s (%s)", s.name, k, p.State, st, p.Why)
|
||||
}
|
||||
}
|
||||
}
|
||||
p := m.state(t, pg)
|
||||
a := m.r.asked[p.Ask]
|
||||
if !strings.Contains(a.Explanation, "postgres") || !strings.Contains(a.Explanation, "ace") ||
|
||||
!strings.Contains(a.Explanation, "data stay") {
|
||||
t.Errorf("the ask does not say what goes from which provider on which node: %+v", a)
|
||||
}
|
||||
// Nothing before the warrant: the provider's entry stands and the key stays.
|
||||
if said := m.h.Decide(); len(said) != 0 || m.state(t, pg).Warrant != nil {
|
||||
t.Errorf("decided before an answer: %v", said)
|
||||
}
|
||||
m.r.answer(t, p.Ask, OptionApprove)
|
||||
m.h.Decide()
|
||||
if w := m.state(t, pg).Warrant; w == nil || w.Option != OptionApprove || !strings.Contains(w.By, "42") {
|
||||
t.Fatalf("the warrant is not in the key: %+v", w)
|
||||
}
|
||||
if said, _ := m.h.Rejudge(pg); said != "" || m.state(t, pg).State != provisions.Retiring {
|
||||
t.Errorf("retired while the provider's entry stands: %q", said)
|
||||
}
|
||||
_ = m.s.Delete("postgres.bound", pg)
|
||||
if said, _ := m.h.Rejudge(pg); !strings.Contains(said, "retired") || m.state(t, pg).State != "" {
|
||||
t.Errorf("not retired once the provider removed its entry: %q", said)
|
||||
}
|
||||
// A decline is kept, and nothing is asked again while nothing changes.
|
||||
d := newMesh()
|
||||
d.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)}
|
||||
d.compose(t, "g14")
|
||||
d.atWant["g14"] = map[string]catalogue.Plan{}
|
||||
d.compose(t, "g14")
|
||||
d.r.answer(t, d.state(t, pg).Ask, OptionDecline)
|
||||
d.h.Decide()
|
||||
if c := d.compose(t, "g14"); len(c.Asked) != 0 || d.state(t, pg).Warrant.Option != OptionDecline {
|
||||
t.Errorf("a decline was asked again: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// The holder holds no ticker: it composes on change events alone (ADR 0570 clause 2; issue 1954 H1).
|
||||
func TestTheHolderHoldsNoTimer(t *testing.T) {
|
||||
files, _ := filepath.Glob("*.go")
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f, "_test.go") {
|
||||
continue
|
||||
}
|
||||
file, err := parser.ParseFile(token.NewFileSet(), f, nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ast.Inspect(file, func(n ast.Node) bool {
|
||||
if sel, ok := n.(*ast.SelectorExpr); ok {
|
||||
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "time" &&
|
||||
slices.Contains([]string{"NewTicker", "Tick", "AfterFunc", "NewTimer"}, sel.Sel.Name) {
|
||||
t.Errorf("%s uses time.%s: the holder composes on events alone", f, sel.Sel.Name)
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package main
|
||||
|
||||
// The holder's tools (novox/hq ADR 0570 Consequences, ADR 0564 §12's rule): a read of every key of `provisions` and
|
||||
// `plans`, one key whole, each state's fill, composing one node now, and reading the operator's answers now.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
)
|
||||
|
||||
// The holder's tools.
|
||||
const (
|
||||
ProvisionsTool = "provisioner_provisions"
|
||||
PlansTool = "provisioner_plans"
|
||||
ComposeTool = "provisioner_compose"
|
||||
DecideTool = "provisioner_decide"
|
||||
)
|
||||
|
||||
func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} }
|
||||
|
||||
func holderTools(h *Holder) []stdio.Tool {
|
||||
arg := func(a map[string]any, k string) string { s, _ := a[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: ProvisionsTool,
|
||||
Description: "Every provision the seat keeps (novox/hq ADR 0570): its key <node>.<consumer>.<provision>[.<local>], " +
|
||||
"provider, consumer and state (asked, provided, waits, failed, retiring, with why), narrowed by node and " +
|
||||
"state, with how many keys and bytes the state holds; or one key whole.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one key, whole"),
|
||||
"node": str("a consumer's node"), "state": str("asked, provided, waits, failed or retiring")}},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return h.read(ProvisionsState, arg(a, "key"), arg(a, "node"), arg(a, "state"))
|
||||
}},
|
||||
{Name: PlansTool,
|
||||
Description: "Every module's provisioning plan the seat composed, <node>.<module>, narrowed by node, with how many " +
|
||||
"keys and bytes the state holds; or one plan whole.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one plan, <node>.<module>"),
|
||||
"node": str("a node")}},
|
||||
Run: func(a map[string]any) (any, error) { return h.read(PlansState, arg(a, "key"), arg(a, "node"), "") }},
|
||||
{Name: ComposeTool,
|
||||
Description: "Compose one node's plans and provisions now from the bus, as a change event would, and say what " +
|
||||
"was written, removed, changed and asked.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{"node": str("the node")},
|
||||
"required": []string{"node"}},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
if arg(a, "node") == "" {
|
||||
return nil, fmt.Errorf("compose names the node")
|
||||
}
|
||||
return h.ComposeNode(context.Background(), arg(a, "node"))
|
||||
}},
|
||||
{Name: DecideTool,
|
||||
Description: "Read the operator's answers to every retiring provision's ask now, writing each warrant into its " +
|
||||
"key, as the router's word on an ask would.",
|
||||
Run: func(map[string]any) (any, error) { return map[string]any{"said": h.Decide()}, nil }},
|
||||
}
|
||||
}
|
||||
|
||||
// read is a state's keys narrowed, or one whole, with the state's fill.
|
||||
func (h *Holder) read(state, key, node, want string) (any, error) {
|
||||
if key != "" {
|
||||
e, err := h.Store.Get(state, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e == nil {
|
||||
return nil, fmt.Errorf("%s holds no %s", state, key)
|
||||
}
|
||||
return json.RawMessage(e.Value), nil
|
||||
}
|
||||
keys, err := h.Store.Keys(state)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := []map[string]any{}
|
||||
bytes := 0
|
||||
for _, k := range keys {
|
||||
e, err := h.Store.Get(state, k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
bytes += len(e.Value)
|
||||
if node != "" && !strings.HasPrefix(k, node+".") {
|
||||
continue
|
||||
}
|
||||
row := map[string]any{"key": k}
|
||||
if state == ProvisionsState {
|
||||
p, err := provisions.Decode(e.Value)
|
||||
if err != nil {
|
||||
row["unreadable"] = err.Error()
|
||||
} else {
|
||||
if want != "" && p.State != want {
|
||||
continue
|
||||
}
|
||||
row["state"], row["provider"], row["why"] = p.State, p.Provider.Module+" on "+p.Provider.Node, p.Why
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return map[string]any{state: rows, "count": len(rows), "fill": map[string]int{"keys": len(keys), "bytes": bytes}}, nil
|
||||
}
|
||||
@@ -1,23 +1,28 @@
|
||||
// mesh-provisioner: the provisioner's server, holder of the mesh-provisioner seat once for the mesh (novox/hq ADR
|
||||
// 0349 clause 3, design 57 P4). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR
|
||||
// 0259 §8), keeping four buckets: mesh-provisioner_registry (which domain answers which kind of placeholder, its
|
||||
// alone), and mesh-provisioner_needs, _applied and _collect, which each machine's provisioner client writes for its
|
||||
// own machine (the controller grants the client those keys alone, issue 670). stdout is the MCP channel; what this
|
||||
// module says, it says on stderr.
|
||||
// 0349 clause 3, ADR 0570). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR 0259
|
||||
// §8), keeping mesh-provisioner_provisions and _plans (every provision and every plan, composed from the bus on change
|
||||
// events, its alone), _registry (which domain answers which kind of placeholder, its alone), and _applied and
|
||||
// _collect, which each machine's provisioner client writes for its own machine; _needs is retired and emptied at its
|
||||
// start. stdout is the MCP channel; what this module says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Seat is the role this module holds, once for the mesh, and Verb its one verb.
|
||||
const (
|
||||
Seat = "mesh-provisioner"
|
||||
Verb = "provisioning"
|
||||
Seat = "mesh-provisioner"
|
||||
Module = "mesh-provisioner"
|
||||
Verb = "provisioning"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
@@ -42,30 +47,149 @@ func (runtimeStore) Delete(state, key string) error { return stdio.State(state).
|
||||
|
||||
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
// runtimeAsks is the operator's channel through the runtime: asks published under this module's own name, and the
|
||||
// router's record of one read under that name.
|
||||
type runtimeAsks struct{}
|
||||
|
||||
func (runtimeAsks) Publish(subject string, body any, id string) error {
|
||||
return stdio.SeatPublish(subject, body, id)
|
||||
}
|
||||
|
||||
func (runtimeAsks) Record(id string) ([]byte, bool, error) {
|
||||
e, err := stdio.SeatRecord("", id)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}
|
||||
|
||||
// work is the nodes a change touched, composed one at a time by one worker: no timer, a change's event is the wake.
|
||||
type work struct {
|
||||
mu sync.Mutex
|
||||
pending map[string]bool
|
||||
wake chan struct{}
|
||||
}
|
||||
|
||||
func (w *work) add(nodes ...string) {
|
||||
if len(nodes) == 0 {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
for _, n := range nodes {
|
||||
w.pending[n] = true
|
||||
}
|
||||
w.mu.Unlock()
|
||||
select {
|
||||
case w.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (w *work) run(h *Holder) {
|
||||
for range w.wake {
|
||||
w.mu.Lock()
|
||||
nodes := sortedKeys(w.pending)
|
||||
w.pending = map[string]bool{}
|
||||
w.mu.Unlock()
|
||||
for _, n := range nodes {
|
||||
c, err := h.ComposeNode(context.Background(), n)
|
||||
switch {
|
||||
case err != nil:
|
||||
logf("[%s] %s is not composed: %v", Module, n, err)
|
||||
case len(c.Plans)+len(c.Gone)+len(c.Changed)+len(c.Asked)+len(c.Problems) > 0:
|
||||
logf("[%s] composed %s: %+v", Module, n, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs.
|
||||
func watched(state string, handle func(stdio.StateChange) error) {
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
time.Sleep(wait)
|
||||
if err := stdio.State(state).Watch("", handle); err != nil {
|
||||
logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait)
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
go publishing(runtimeStore{})
|
||||
if err := stdio.Serve("", Tools(runtimeStore{})); err != nil {
|
||||
s := runtimeStore{}
|
||||
h := &Holder{Store: s, Compose: FromBus(s), Asks: runtimeAsks{}, Now: time.Now, Logf: logf}
|
||||
w := &work{pending: map[string]bool{}, wake: make(chan struct{}, 1)}
|
||||
go w.run(h)
|
||||
go starting(h)
|
||||
go watched(PlacementRead, func(c stdio.StateChange) error { w.add(NodesOfPlacement(c.Key)...); return nil })
|
||||
go watched(WantsRead, func(c stdio.StateChange) error { w.add(NodesOfWant(c.Key)...); return nil })
|
||||
go watched(ModulesRead, func(c stdio.StateChange) error {
|
||||
if module, err := delivery.ParseModulesKey(c.Key); err == nil {
|
||||
w.add(h.NodesOfModule(module)...)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
for _, provider := range Providers {
|
||||
go watched(boundRead(provider), func(c stdio.StateChange) error {
|
||||
if said, err := h.Rejudge(c.Key); err != nil {
|
||||
logf("[%s] %s is not judged again: %v", Module, c.Key, err)
|
||||
} else if said != "" {
|
||||
logf("[%s] %s", Module, said)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
go watched(Registry, func(c stdio.StateChange) error {
|
||||
if c.Op != "put" {
|
||||
if _, _, err := Publish(s); err != nil {
|
||||
logf("[%s] the registry is not written back: %v", Module, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
go func() {
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
time.Sleep(wait)
|
||||
err := stdio.Subscribe(asks.Seat+".decided."+Module, func(stdio.Envelope) error {
|
||||
for _, said := range h.Decide() {
|
||||
logf("[%s] %s", Module, said)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
logf("[%s] does not hear the operator's answers yet (%v); asking again in %s", Module, err, wait)
|
||||
}
|
||||
}()
|
||||
if err := stdio.Serve("", append(Tools(s), holderTools(h)...)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// publishing writes the registry once the runtime answers, and again every hour, so a registry emptied by hand or
|
||||
// a bucket made again is written back without a restart. The runtime answers only once Serve runs, so the first
|
||||
// pass is tried again until it lands.
|
||||
func publishing(s Store) {
|
||||
for wait := 2 * time.Second; ; {
|
||||
time.Sleep(wait)
|
||||
written, deleted, err := Publish(s)
|
||||
switch {
|
||||
case err != nil:
|
||||
logf("[mesh-provisioner] the registry is not written yet (%v); trying again in %s", err, wait)
|
||||
wait = min(wait*2, time.Minute)
|
||||
continue
|
||||
case len(written) > 0 || len(deleted) > 0:
|
||||
logf("[mesh-provisioner] the registry: wrote %v, deleted %v", written, deleted)
|
||||
// starting writes the registry, deletes the retired needs and reads the operator's answers once the runtime answers,
|
||||
// each tried again until it lands.
|
||||
func starting(h *Holder) {
|
||||
<-stdio.Served()
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
written, deleted, err := Publish(h.Store)
|
||||
if err == nil {
|
||||
if len(written)+len(deleted) > 0 {
|
||||
logf("[%s] the registry: wrote %v, deleted %v", Module, written, deleted)
|
||||
}
|
||||
break
|
||||
}
|
||||
wait = time.Hour
|
||||
logf("[%s] the registry is not written yet (%v); trying again in %s", Module, err, wait)
|
||||
time.Sleep(wait)
|
||||
}
|
||||
if n, err := h.RetireNeeds(); err != nil {
|
||||
logf("[%s] the retired needs are not all deleted: %v", Module, err)
|
||||
} else if n > 0 {
|
||||
logf("[%s] deleted %d key(s) of the retired needs", Module, n)
|
||||
}
|
||||
for _, said := range h.Decide() {
|
||||
logf("[%s] %s", Module, said)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -106,9 +106,9 @@ func TestTheRegistryAnswersEveryKindATemplateMayName(t *testing.T) {
|
||||
for kind, bucket := range map[string]string{"config": "mesh-config_resolved", "collected": "mesh-provisioner_" + Collect,
|
||||
"shell": "mesh-provisioner_" + Collect, "environment": "mesh-provisioner_" + Collect,
|
||||
"contribution": "mesh-provisioner_" + Collect,
|
||||
"dir": "mesh-controller_assignments", "access": "mesh-controller_assignments",
|
||||
"port": "mesh-controller_assignments", "seat": "mesh-controller_assignments",
|
||||
"machine": "mesh-controller_assignments", "backup": "mesh-controller_assignments"} {
|
||||
"dir": "mesh-provisioner_plans", "access": "mesh-provisioner_plans",
|
||||
"port": "mesh-provisioner_plans", "seat": "mesh-provisioner_plans",
|
||||
"machine": "mesh-provisioner_plans", "backup": "mesh-provisioner_plans"} {
|
||||
i := slices.IndexFunc(TheRegistry, func(a Answer) bool { return a.Kind == kind })
|
||||
if TheRegistry[i].Bucket != bucket {
|
||||
t.Errorf("%s is found in %s, want %s", kind, TheRegistry[i].Bucket, bucket)
|
||||
@@ -288,6 +288,8 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) {
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
State []string `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
Tools []string `json:"tools"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
@@ -300,7 +302,39 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) {
|
||||
!slices.Equal(m.Claims[0].Serves, []string{Verb}) {
|
||||
t.Errorf("the claim: %+v", m.Claims)
|
||||
}
|
||||
if want := []string{Registry, Needs, Applied, Collect}; !slices.Equal(m.State, want) {
|
||||
// needs stays declared this release so its first start can empty it (ADR 0570 clause 1).
|
||||
if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState}; !slices.Equal(m.State, want) {
|
||||
t.Errorf("the state is %v, want %v", m.State, want)
|
||||
}
|
||||
// Every state and act has its tool (ADR 0570 Consequences): the verb reads the clients' states, the holder's tools
|
||||
// its own and the composition it does on a change.
|
||||
served := map[string]bool{}
|
||||
for _, tool := range append(Tools(newStore()), holderTools(&Holder{})...) {
|
||||
served[tool.Name] = true
|
||||
}
|
||||
for what, tool := range map[string]string{Registry: Seat + "." + Verb, Needs: Seat + "." + Verb,
|
||||
Applied: Seat + "." + Verb, Collect: Seat + "." + Verb, ProvisionsState: ProvisionsTool, PlansState: PlansTool,
|
||||
"a composition on a change": ComposeTool, "a decision on an answer": DecideTool} {
|
||||
if !served[tool] {
|
||||
t.Errorf("%s has no tool: %s is not served", what, tool)
|
||||
}
|
||||
}
|
||||
for _, tool := range m.Tools {
|
||||
if !served[tool] {
|
||||
t.Errorf("the manifest lists %s, which is not served", tool)
|
||||
}
|
||||
}
|
||||
for _, r := range []string{PlacementRead, WantsRead, ModulesRead, ManifestsRead, BuildsRead, HaveRead, ResolvedRead} {
|
||||
if !slices.Contains(m.Reads, r) {
|
||||
t.Errorf("the manifest does not read %s, which the composition reads", r)
|
||||
}
|
||||
}
|
||||
for _, p := range Providers {
|
||||
if !slices.Contains(m.Reads, boundRead(p)) {
|
||||
t.Errorf("the manifest does not read %s's entries", p)
|
||||
}
|
||||
}
|
||||
if slices.Contains(m.Reads, "mesh-controller.assignments") {
|
||||
t.Error("the manifest reads the controller's assignments (ADR 0570 clause 2)")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (issue 1936).
|
||||
modules/mesh-provisioner/module.json
|
||||
@@ -48,16 +48,15 @@ type Answer struct {
|
||||
Decision string `json:"decision"`
|
||||
}
|
||||
|
||||
// placement is the controller's: where a module runs, its directories, accesses, ports, seats and the machine's
|
||||
// facts (ADR 0349 clause 6). On the bus, in the provisioner's plan of each module the controller handed to it, which
|
||||
// its assignment carries (mesh-controller internal/inventory Provisioning, issue 671), or for a module only pulled in
|
||||
// by another, its value of its own under the same key (issue 765): a client never reads the node's declaration,
|
||||
// which the node-engine keeps for root alone.
|
||||
// placement is the seat's: where a module runs, its directories, accesses, ports, seats and the machine's facts (ADR
|
||||
// 0349 clause 6), in the provisioning plan this seat composes of each module on each node from the bus, its state
|
||||
// `plans` (novox/hq ADR 0570 clauses 1, 2): a client never reads the node's declaration, which the node-engine keeps for
|
||||
// root alone, nor the controller's assignments.
|
||||
func placement(kind, what string) Answer {
|
||||
return Answer{Kind: kind, Domain: "mesh-controller", From: FromBucket, Bucket: "mesh-controller_assignments",
|
||||
return Answer{Kind: kind, Domain: Module, From: FromBucket, Bucket: Module + "_" + PlansState,
|
||||
Key: "<node>.<module>",
|
||||
Why: what + ": placement, the controller's, in the provisioner's plan its assignment carries (provisioning." +
|
||||
"placement)", Decision: "novox/hq ADR 0349 clause 6"}
|
||||
Why: what + ": placement, in the provisioning plan the seat mesh-provisioner composes of the module on the node " +
|
||||
"(plans, plan.placement)", Decision: "novox/hq ADR 0570 clause 1"}
|
||||
}
|
||||
|
||||
// oldCollection is a namespace read as a collection of its own name (novox/hq ADR 0349 clause 13, design 57 L1).
|
||||
|
||||
@@ -2,4 +2,4 @@ module mesh-provisioner
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791
|
||||
|
||||
@@ -1,2 +1,14 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
|
||||
@@ -11,11 +11,36 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-assigner.placement",
|
||||
"mesh-delivery-assigner.wants",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-builder.builds",
|
||||
"node-engine.have",
|
||||
"mesh-config.resolved",
|
||||
"gitea.bound",
|
||||
"keycloak.bound",
|
||||
"postgres.bound",
|
||||
"route-adapter.bound",
|
||||
"mesh-vault.bound"
|
||||
],
|
||||
"tools": [
|
||||
"provisioner_provisions",
|
||||
"provisioner_plans",
|
||||
"provisioner_compose",
|
||||
"provisioner_decide"
|
||||
],
|
||||
"state": [
|
||||
"registry",
|
||||
"needs",
|
||||
"applied",
|
||||
"collect"
|
||||
"collect",
|
||||
"provisions",
|
||||
"plans"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
/mesh-self-check
|
||||
/cmd/mesh-self-check/mesh-self-check
|
||||
@@ -0,0 +1,33 @@
|
||||
# mesh-self-check
|
||||
|
||||
Holds the seat `mesh-self-check` (novox/hq ADR 0564 §6): the mesh's self-check, moved out of the controller. Go, one
|
||||
binary (`cmd/mesh-self-check`), served by a runtime of its own on its own bus account (`runs-as`), so that its account
|
||||
alone writes its buckets.
|
||||
|
||||
## What it does
|
||||
|
||||
Every five minutes (the first run a minute after it starts) it runs its registry of probes, each within thirty
|
||||
seconds. A probe is one live invariant of a design, with its assertion as the controller had it, reading its facts on
|
||||
the bus (the bus module's tools for the bus's own state). A probe whose facts have no home on the bus yet answers
|
||||
`deferred`, naming them: never a pass. A probe that cannot run is said in the verdict at once, and written as a finding
|
||||
of its own (`probe.<id>.failed`) when the next run cannot run it either.
|
||||
|
||||
What a probe finds is written as a finding, one key `<scope>.<id>.<kind>` per fault, only when what it says changes, and
|
||||
deleted when a run no longer sees it. The condition keeper (`mesh-conditions`) raises and clears conditions from them.
|
||||
A finding one look can be wrong about (a consumer behind after a burst) is written when two runs in a row see it.
|
||||
|
||||
Every run ends with the event `checked`, the body of the controller's `doctor-heartbeat`, which the watchman (S10) and
|
||||
mesh-watcher on another machine hear: a self-check that stops checking is said through a path that does not pass
|
||||
through it.
|
||||
|
||||
## What it keeps
|
||||
|
||||
- `mesh-self-check_findings`: what it finds wrong now.
|
||||
- `mesh-self-check_verdict`: the key `last`, the last run whole.
|
||||
- `mesh-self-check_verdicts`: every run (each probe's findings at most twenty), kept a week by the bucket's age.
|
||||
|
||||
## Its verbs and tools
|
||||
|
||||
- `mesh-self-check.doctor`: the last verdict; `run=true` runs now; `probe=<id>` one probe whole.
|
||||
- `mesh-self-check.probes`: the registry, with what each deferred probe lacks.
|
||||
- `self_check_findings`, `self_check_verdicts`: the reads of its buckets.
|
||||
@@ -0,0 +1,57 @@
|
||||
// mesh-self-check: the holder of the seat mesh-self-check (novox/hq ADR 0564 §6). A Go bundle served by a runtime of
|
||||
// its own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `findings` (what it finds
|
||||
// wrong now, which the condition keeper raises and clears conditions from), `verdict` (the last run) and `verdicts`
|
||||
// (every run, kept a week). It runs the probes every five minutes and says `checked` at the end of each run. stdout is
|
||||
// the MCP channel; what this module says, it says on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||
|
||||
// runtimeStore is the bus, through the runtime that launched this bundle.
|
||||
type runtimeStore struct{}
|
||||
|
||||
func (runtimeStore) Get(state, key string) ([]byte, bool, error) {
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}
|
||||
|
||||
func (runtimeStore) Put(state, key string, value any) error {
|
||||
_, err := stdio.State(state).Put(key, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
|
||||
|
||||
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
// NewChecker is the self-check over the runtime.
|
||||
func NewChecker() *Checker {
|
||||
return &Checker{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, Host: os.Getenv("MESH_NODE"),
|
||||
Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) },
|
||||
Publish: stdio.SeatPublish}
|
||||
}
|
||||
|
||||
func main() {
|
||||
c := NewChecker()
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
c.Keep(context.Background())
|
||||
}()
|
||||
if err := stdio.Serve("", Tools(c)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
RunsAs string `json:"runs-as"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
Emits []string `json:"emits"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Reads []string `json:"reads"`
|
||||
State []json.RawMessage `json:"state"`
|
||||
}
|
||||
|
||||
func readManifest(t *testing.T) manifest {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// The manifest says what the code does: it claims the seat mesh-self-check once for the mesh with the verbs it serves
|
||||
// (ADR 0564 §1, §6), runs on its own account, says checked, asks only what its probes ask, keeps a week of verdicts,
|
||||
// and reads no bucket of the controller's. Every state it keeps has a tool that reads it, and every act it takes by
|
||||
// itself a tool that starts it by hand (ADR 0564 §12).
|
||||
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
if m.Module != Module || m.RunsAs != Module {
|
||||
t.Errorf("module %q runs as %q", m.Module, m.RunsAs)
|
||||
}
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) {
|
||||
t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs)
|
||||
}
|
||||
// checked is the seat's event, said on the seat's subject: a module lists no event of a seat it claims.
|
||||
if len(m.Emits) != 0 {
|
||||
t.Errorf("emits %v", m.Emits)
|
||||
}
|
||||
if !slices.Equal(m.Invokes, []string{StreamsTool, ClosedTool, SignalsVerb}) {
|
||||
t.Errorf("invokes %v, want what the probes ask", m.Invokes)
|
||||
}
|
||||
if len(m.Reads) != 0 {
|
||||
t.Errorf("reads %v: the self-check reads its facts through the bus module's tools", m.Reads)
|
||||
}
|
||||
states := map[string]int{}
|
||||
for _, s := range m.State {
|
||||
var d struct {
|
||||
Name string `json:"name"`
|
||||
TTL int `json:"ttl-seconds"`
|
||||
}
|
||||
if json.Unmarshal(s, &d.Name) != nil {
|
||||
_ = json.Unmarshal(s, &d)
|
||||
}
|
||||
states[d.Name] = d.TTL
|
||||
}
|
||||
if states[VerdictsState] != 7*24*60*60 {
|
||||
t.Errorf("verdicts are kept %d s, want a week", states[VerdictsState])
|
||||
}
|
||||
// ADR 0564 §12: each state with the tool that reads it, each act with the tool that starts it.
|
||||
readBy := map[string]string{FindingsState: FindingsTool, VerdictState: DoctorTool, VerdictsState: VerdictsTool,
|
||||
"the run": DoctorTool + " run=true"}
|
||||
tools := map[string]bool{}
|
||||
for _, tool := range Tools(&Checker{}) {
|
||||
tools[tool.Name] = true
|
||||
}
|
||||
for state := range states {
|
||||
if _, ok := readBy[state]; !ok {
|
||||
t.Errorf("the state %s has no tool that reads it", state)
|
||||
}
|
||||
}
|
||||
for what, tool := range readBy {
|
||||
name, _, _ := cutSpace(tool)
|
||||
if !tools[name] {
|
||||
t.Errorf("%s is read or started by %s, which is not served", what, tool)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func cutSpace(s string) (string, string, bool) {
|
||||
for i, r := range s {
|
||||
if r == ' ' {
|
||||
return s[:i], s[i+1:], true
|
||||
}
|
||||
}
|
||||
return s, "", false
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package main
|
||||
|
||||
// What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an
|
||||
// Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong
|
||||
// about it. It is written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the controller's key
|
||||
// for the same condition, so the condition the keeper raises from it and the one the controller raised share one key.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// The severities and scopes, as mesh-sdk's health names them.
|
||||
const (
|
||||
Urgent = health.Urgent
|
||||
Warning = health.Warning
|
||||
ScopeBus = health.ScopeBus
|
||||
ScopeCore = health.ScopeCore
|
||||
ScopeProbe = health.ScopeProbe
|
||||
ScopeMesh = health.ScopeMesh
|
||||
)
|
||||
|
||||
// Subject is what a finding is about.
|
||||
type Subject = health.Subject
|
||||
|
||||
// Observation is one thing a probe sees wrong now.
|
||||
type Observation struct {
|
||||
Kind string
|
||||
Token string
|
||||
Subject Subject
|
||||
Severity health.Severity
|
||||
Summary string
|
||||
Said string
|
||||
Headline string
|
||||
Explanation string
|
||||
Resolved string
|
||||
Needs string
|
||||
// Source is the probe that saw it, or the self-check itself for a probe that could not run.
|
||||
Source string
|
||||
// Confirm says one look can be wrong about it: written only when two runs in a row see it (issue 277).
|
||||
Confirm bool
|
||||
}
|
||||
|
||||
// Key is where it is kept: `<scope>.<id>.<token>`, the controller's key for the same condition.
|
||||
func (o Observation) Key() string {
|
||||
token := o.Token
|
||||
if token == "" {
|
||||
token = o.Kind
|
||||
}
|
||||
return health.Key(o.Subject.Scope, o.Subject.ID, token)
|
||||
}
|
||||
|
||||
// Finding is the observation as written by run at, its evidence the one line it said.
|
||||
func (o Observation) Finding(run string, at time.Time) health.Finding {
|
||||
f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary,
|
||||
Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source,
|
||||
Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}}
|
||||
if o.Token != "" && o.Token != o.Kind {
|
||||
f.Token = o.Token
|
||||
}
|
||||
said := o.Said
|
||||
if said == "" {
|
||||
said = o.Summary
|
||||
}
|
||||
f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said})
|
||||
return f
|
||||
}
|
||||
|
||||
// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write
|
||||
// (ADR 0289 §1).
|
||||
func says(f health.Finding) string {
|
||||
f.At, f.Run, f.V = time.Time{}, "", 0
|
||||
ev := make([]health.Evidence, len(f.Evidence))
|
||||
for i, e := range f.Evidence {
|
||||
ev[i] = health.Evidence{Said: e.Said}
|
||||
}
|
||||
f.Evidence = ev
|
||||
raw, _ := json.Marshal(f)
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
// worded gives a finding the operator's words registered for its kind, where it says none of its own.
|
||||
func worded(f Observation) Observation {
|
||||
w, ok := words[f.Kind]
|
||||
if !ok {
|
||||
return f
|
||||
}
|
||||
if f.Headline == "" {
|
||||
f.Headline = w.Headline
|
||||
}
|
||||
if f.Explanation == "" {
|
||||
f.Explanation = w.Explanation
|
||||
}
|
||||
if f.Resolved == "" {
|
||||
f.Resolved = w.Resolved
|
||||
}
|
||||
if f.Needs == "" {
|
||||
f.Needs = w.Needs
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
type plain struct{ Headline, Explanation, Resolved, Needs string }
|
||||
|
||||
// words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the
|
||||
// controller's name for itself replaced by the self-check's where the words spoke of it.
|
||||
var words = map[string]plain{
|
||||
"consumer-lost": {Headline: "A listener on the bus is gone",
|
||||
Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.",
|
||||
Resolved: "The listener is back"},
|
||||
"consumer-wrong": {Headline: "A listener on the bus is set up wrong",
|
||||
Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets it up again.",
|
||||
Resolved: "The listener is set up right again"},
|
||||
"consumer-behind": {Headline: "Messages pile up for a listener",
|
||||
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A healer " +
|
||||
"restarts it if it stays.",
|
||||
Resolved: "The listener caught up"},
|
||||
"stream-wrong": {Headline: "Part of the bus's storage is wrong",
|
||||
Needs: "check the machine the bus runs on; the details say what is missing.",
|
||||
Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.",
|
||||
Resolved: "The bus's storage is right again"},
|
||||
"bus-reconnects": {Headline: "A client keeps losing the bus",
|
||||
Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. While " +
|
||||
"it reconnects, what it says and what it is asked waits.",
|
||||
Resolved: "Resolved: the client stays connected"},
|
||||
"core-unhealthy": {Headline: "The bus is not healthy",
|
||||
Needs: "check the machine the bus runs on; the details say what fails.",
|
||||
Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds and tool " +
|
||||
"calls may fail until it recovers.",
|
||||
Resolved: "The bus is healthy again"},
|
||||
"watchdogs-silent": {Headline: "The mesh's watchdogs stopped",
|
||||
Needs: "restart the watchman if this stays, from the mesh MCP server.",
|
||||
Explanation: "The watchdogs, which notice when something expected does not happen, do not answer, so missed " +
|
||||
"signals are not noticed.",
|
||||
Resolved: "The watchdogs run again"},
|
||||
"probe-failed": {Headline: "A check of the mesh could not run",
|
||||
Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs again " +
|
||||
"every few minutes.",
|
||||
Resolved: "The check runs again"},
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
package main
|
||||
|
||||
// The registry (novox/hq ADR 0564 §6 and the probe table of its Context): the probes that move from the controller,
|
||||
// each with its assertion unchanged, reading its facts where they live on the bus. A probe whose facts have no home
|
||||
// on the bus yet answers deferred, naming them, and is never a pass (tracked by issue 1952). Not registered, as the
|
||||
// table says: D1 (nodes compose their own; a refusal is the watchman's S21), D10 (a want ahead of its have is the
|
||||
// watchman's pulse), D5, D9 and H-controller (they end with the controller), stuck (the watchman's own finding) and
|
||||
// the bus step (its home is issue 1953's).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
)
|
||||
|
||||
// The tools a probe asks: the bus module's reads of its own server, and the watchdogs' seat.
|
||||
const (
|
||||
StreamsTool = "nats.nats_streams"
|
||||
ClosedTool = "nats.nats_closed_connections"
|
||||
SignalsVerb = "seat:mesh-watchdogs.signals"
|
||||
notOnTheBus = "the bus module's tools (nats_streams) answer this only once asked; "
|
||||
storeOnly = "held only in the controller's store today"
|
||||
behindBound = 1000
|
||||
reconnectMax = 12
|
||||
)
|
||||
|
||||
// Registry is the self-check's registry, in to-be 45's order. **The registry is the design's live form**: a probe
|
||||
// that moves here is a row added, and the registry test holds it to ADR 0564's table.
|
||||
func Registry() []Probe {
|
||||
return []Probe{
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong",
|
||||
Deferred: "the machines' names and the resolver holders' addresses to ask: the overlay's addresses are " + storeOnly},
|
||||
{ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every machine that " +
|
||||
"is heard from", From: "issues 208, 218", Kind: "holder-silent",
|
||||
Deferred: "the seats on record with their holders per machine, and which machines are heard from: the seat " +
|
||||
"records are " + storeOnly + ", and the watchman's presence has no read granted to this module"},
|
||||
{ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253", Kind: "archives-unheld",
|
||||
Deferred: "the artifact store's kept archives with the manifests that hold them: the archive index is " + storeOnly},
|
||||
{ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " +
|
||||
"stream's head", From: "issues 248, 266", Kind: "consumer-wrong",
|
||||
Raises: []string{"consumer-lost", "consumer-behind"}, Run: probeConsumers},
|
||||
{ID: "D7", Asserts: "every stream the mesh defines exists with its definition", From: "issue 208",
|
||||
Kind: "stream-wrong", Run: probeStreams},
|
||||
{ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list",
|
||||
From: "issue 238", Kind: "own-address-banned",
|
||||
Deferred: "the machines' private addresses and endpoints: the overlay is " + storeOnly},
|
||||
{ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding its " +
|
||||
"cleanup", From: "ADR 0230", Kind: "cleanup-waiting",
|
||||
Deferred: "the retirement records of providers' consumers: " + storeOnly},
|
||||
{ID: "D12", Asserts: "every consumer of a provision that keeps its data is bound where it was last sent, or moves " +
|
||||
"by a pin", From: "issue 273, ADR 0232", Kind: "binding-moved",
|
||||
Deferred: "the bindings of provisions to their consumers: " + storeOnly},
|
||||
{ID: "D13", Asserts: "every item of data a machine declares is measured, is there, holds what it held, and is " +
|
||||
"backed up within its bound", From: "issue 273, ADR 0233", Kind: "data-shrank",
|
||||
Deferred: "the data items' measurements and their history: " + storeOnly},
|
||||
{ID: "D15", Asserts: "no user of the bus had its connection dropped more than twelve times in the last hour: the " +
|
||||
"bus module's nats_closed_connections", From: "issue 327", Kind: "bus-reconnects", Run: probeReconnects},
|
||||
{ID: "DA", Asserts: "every machine that names an agent account has it judged, on its node-engine's newest " +
|
||||
"statement, unable to become root without a person", From: "ADR 0266, ADR 0259 §8",
|
||||
Kind: "agent-can-become-root",
|
||||
Deferred: "the node-engines' statements of the agent account's judgement: " + storeOnly},
|
||||
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
||||
"proving its sender runs", From: "ADR 0259 §8", Kind: "root-not-free",
|
||||
Deferred: "the node-engines' statements of root and the tools that run as root: " + storeOnly},
|
||||
{ID: "D-fill", Asserts: "every module's bucket and log holds less than three quarters of its cap; one said " +
|
||||
"filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501",
|
||||
Kind: "bucket-or-log-filling", Raises: []string{"bucket-or-log-unread"},
|
||||
Deferred: "each bucket's and log's cap: the bus module's nats_buckets answers bytes and no cap, and the " +
|
||||
"declared caps are " + storeOnly},
|
||||
{ID: "D-heard", Asserts: "no grant of a restricted stream names a module that composes nothing for it",
|
||||
From: "ADR 0376 §2", Kind: "grant-composes-nothing",
|
||||
Deferred: "the grants of restricted streams with what each machine composed for them: " + storeOnly},
|
||||
{ID: "DW", Asserts: "the watchdogs answer: the watchman holds the seat mesh-watchdogs and answers signals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Run: probeWatchdogs},
|
||||
{ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses it " +
|
||||
"to the bus's machine and back", From: "ADR 0236, to-be 45 §8", Kind: "core-unhealthy", Run: probeBusHealth},
|
||||
}
|
||||
}
|
||||
|
||||
// --- the bus's own state, as its module answers it ---------------------------------------------------------------
|
||||
|
||||
// streamsAnswer is what nats_streams answers.
|
||||
type streamsAnswer struct {
|
||||
Detail []streamRow `json:"detail"`
|
||||
}
|
||||
|
||||
type streamRow struct {
|
||||
Name string `json:"name"`
|
||||
Subjects []string `json:"subjects"`
|
||||
Retention string `json:"retention"`
|
||||
MaxPerSubject int64 `json:"max_per_subject"`
|
||||
Consumers []consumerRow `json:"consumer_detail"`
|
||||
}
|
||||
|
||||
type consumerRow struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Filters []string `json:"filters"`
|
||||
Pending int64 `json:"pending"`
|
||||
Unacked int64 `json:"unacknowledged"`
|
||||
MaxDeliver int64 `json:"max_deliver"`
|
||||
}
|
||||
|
||||
// errNoStream is a stream the bus module says it does not hold.
|
||||
var errNoStream = errors.New("no such stream")
|
||||
|
||||
// streams asks the bus module for every stream, or for one with its consumers.
|
||||
func (c *Checker) streams(stream string) ([]streamRow, error) {
|
||||
args := map[string]any{}
|
||||
if stream != "" {
|
||||
args["stream"] = stream
|
||||
}
|
||||
raw, err := c.Ask(StreamsTool, args)
|
||||
if err != nil {
|
||||
if stream != "" && strings.Contains(err.Error(), "no stream called") {
|
||||
return nil, errNoStream
|
||||
}
|
||||
return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err)
|
||||
}
|
||||
var a streamsAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", StreamsTool, err)
|
||||
}
|
||||
return a.Detail, nil
|
||||
}
|
||||
|
||||
// consumerWords is a consumer as the mesh says it, with its name.
|
||||
func consumerWords(c buscompose.Consumer) string {
|
||||
return fmt.Sprintf("the consumer %s on %s", c.Name, c.Stream)
|
||||
}
|
||||
|
||||
// expectedConsumers are the durable consumers the mesh defines, by stream.
|
||||
func expectedConsumers() map[string][]buscompose.Consumer {
|
||||
out := map[string][]buscompose.Consumer{}
|
||||
for _, c := range buscompose.MeshConsumers() {
|
||||
out[c.Stream] = append(out[c.Stream], c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeConsumers is D6: every durable consumer the mesh defines exists as defined, and is near its stream's head.
|
||||
func probeConsumers(ctx context.Context, c *Checker) ([]Observation, error) {
|
||||
var out []Observation
|
||||
byStream := expectedConsumers()
|
||||
streams := make([]string, 0, len(byStream))
|
||||
for s := range byStream {
|
||||
streams = append(streams, s)
|
||||
}
|
||||
sort.Strings(streams)
|
||||
for _, stream := range streams {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
rows, err := c.streams(stream)
|
||||
held := map[string]consumerRow{}
|
||||
switch {
|
||||
case errors.Is(err, errNoStream):
|
||||
case err != nil:
|
||||
return nil, err
|
||||
default:
|
||||
for _, r := range rows {
|
||||
for _, cr := range r.Consumers {
|
||||
held[cr.Name] = cr
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, want := range byStream[stream] {
|
||||
who := want.Stream + "." + want.Name
|
||||
have, ok := held[want.Name]
|
||||
if !ok {
|
||||
out = append(out, Observation{Kind: "consumer-lost", Token: "missing", Severity: Warning,
|
||||
Subject: Subject{Scope: ScopeBus, ID: who},
|
||||
Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(want)),
|
||||
Said: "no consumer " + want.Name + " on " + want.Stream})
|
||||
continue
|
||||
}
|
||||
if differs := consumerDiffers(want, have); differs != "" {
|
||||
out = append(out, Observation{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who},
|
||||
Summary: fmt.Sprintf("%s is not as the mesh defines it: %s", consumerWords(want), differs),
|
||||
Said: differs})
|
||||
}
|
||||
if want.Stream != "NODES" && have.Pending > behindBound {
|
||||
// One look at a consumer a burst put behind is not one that stays behind: held for the next run,
|
||||
// so healer H4 resets nothing on a single sample (novox/hq issue 277).
|
||||
out = append(out, Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true,
|
||||
Subject: Subject{Scope: ScopeBus, ID: who},
|
||||
Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(want), have.Pending),
|
||||
Said: fmt.Sprintf("%d pending, %d handed out and not settled", have.Pending, have.Unacked)})
|
||||
}
|
||||
}
|
||||
}
|
||||
return sorted(out), nil
|
||||
}
|
||||
|
||||
// consumerDiffers is what about a consumer on the bus is not as defined, of what the bus module says of it: its
|
||||
// filters, its shape (push or pull) and how often it hands a message over. Empty when nothing is.
|
||||
func consumerDiffers(want buscompose.Consumer, have consumerRow) string {
|
||||
var differs []string
|
||||
haveFilters := slices.Clone(have.Filters)
|
||||
wantFilters := slices.Clone(want.Filters)
|
||||
sort.Strings(haveFilters)
|
||||
sort.Strings(wantFilters)
|
||||
if !slices.Equal(haveFilters, wantFilters) {
|
||||
differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters))
|
||||
}
|
||||
if want.MaxDeliver != 0 && have.MaxDeliver != int64(want.MaxDeliver) {
|
||||
differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, want.MaxDeliver))
|
||||
}
|
||||
if wantPush, havePush := want.Push || want.Queue != "", have.Kind == "push"; wantPush != havePush {
|
||||
differs = append(differs, "delivers by the other shape (push or pull) than defined")
|
||||
}
|
||||
return strings.Join(differs, "; ")
|
||||
}
|
||||
|
||||
// probeStreams is D7: every stream the mesh defines exists as defined.
|
||||
func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) {
|
||||
rows, err := c.streams("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held := map[string]streamRow{}
|
||||
for _, r := range rows {
|
||||
held[r.Name] = r
|
||||
}
|
||||
var out []Observation
|
||||
for _, s := range buscompose.MeshStreams() {
|
||||
have, ok := held[s.Name]
|
||||
if !ok {
|
||||
out = append(out, Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name},
|
||||
Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)),
|
||||
Said: "no stream " + s.Name})
|
||||
continue
|
||||
}
|
||||
if differs := streamDiffers(s, have); differs != "" {
|
||||
out = append(out, Observation{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name},
|
||||
Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs),
|
||||
Said: differs})
|
||||
}
|
||||
}
|
||||
return sorted(out), nil
|
||||
}
|
||||
|
||||
// streamDiffers is what about a stream on the bus is not as defined, of what the bus module says of it: its
|
||||
// subjects, its retention and how many it keeps per subject. Empty when nothing is.
|
||||
func streamDiffers(want buscompose.Stream, have streamRow) string {
|
||||
var differs []string
|
||||
haveSubjects := slices.Clone(have.Subjects)
|
||||
wantSubjects := slices.Clone(want.Subjects)
|
||||
sort.Strings(haveSubjects)
|
||||
sort.Strings(wantSubjects)
|
||||
if !slices.Equal(haveSubjects, wantSubjects) {
|
||||
differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects))
|
||||
}
|
||||
retention, perSubject := "limits", int64(want.MaxMsgsPerSubject)
|
||||
switch want.Retention {
|
||||
case buscompose.RetentionWorkQueue:
|
||||
retention = "workqueue"
|
||||
case buscompose.RetentionLastPerSubject:
|
||||
perSubject = 1
|
||||
}
|
||||
if have.Retention != "" && have.Retention != retention {
|
||||
differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention))
|
||||
}
|
||||
if perSubject != 0 && have.MaxPerSubject != perSubject {
|
||||
differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxPerSubject, perSubject))
|
||||
}
|
||||
return strings.Join(differs, "; ")
|
||||
}
|
||||
|
||||
// --- D15 -----------------------------------------------------------------------------------------------------
|
||||
|
||||
// closedAnswer is what nats_closed_connections answers.
|
||||
type closedAnswer struct {
|
||||
Hours float64 `json:"hours"`
|
||||
Reaches bool `json:"reaches"`
|
||||
Users []struct {
|
||||
User string `json:"user"`
|
||||
Closed int `json:"closed"`
|
||||
Dropped int `json:"dropped"`
|
||||
Names []struct {
|
||||
Name string `json:"name"`
|
||||
Dropped int `json:"dropped"`
|
||||
Reasons map[string]int `json:"reasons"`
|
||||
} `json:"names"`
|
||||
} `json:"users"`
|
||||
}
|
||||
|
||||
// probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour.
|
||||
func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) {
|
||||
raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err)
|
||||
}
|
||||
var read closedAnswer
|
||||
if err := json.Unmarshal(raw, &read); err != nil {
|
||||
return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", ClosedTool, err)
|
||||
}
|
||||
hours := read.Hours
|
||||
if hours <= 0 {
|
||||
hours = 1
|
||||
}
|
||||
var out []Observation
|
||||
for _, u := range read.Users {
|
||||
if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax {
|
||||
continue // refused before it logged in is nobody's reconnect loop
|
||||
}
|
||||
reasons := map[string]int{}
|
||||
var names []string
|
||||
for _, n := range u.Names {
|
||||
if n.Dropped == 0 {
|
||||
continue
|
||||
}
|
||||
names = append(names, fmt.Sprintf("%q ×%d", n.Name, n.Dropped))
|
||||
for r, k := range n.Reasons {
|
||||
if r != "Client Closed" {
|
||||
reasons[r] += k
|
||||
}
|
||||
}
|
||||
}
|
||||
var why []string
|
||||
for r, k := range reasons {
|
||||
why = append(why, fmt.Sprintf("%s ×%d", r, k))
|
||||
}
|
||||
sort.Strings(why)
|
||||
partial := ""
|
||||
if !read.Reaches {
|
||||
partial = " (at least: the server's record of closed connections does not reach back the whole hour)"
|
||||
}
|
||||
who, machine := busUserWords(u.User)
|
||||
out = append(out, Observation{Kind: "bus-reconnects", Severity: Warning,
|
||||
Subject: Subject{Scope: ScopeBus, ID: u.User, Machine: machine},
|
||||
Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+
|
||||
"reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax),
|
||||
Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped, u.Closed,
|
||||
hours, strings.Join(names, ", "), strings.Join(why, ", ")),
|
||||
Headline: capital(who) + " keeps losing the bus",
|
||||
Explanation: capital(fmt.Sprintf("%s lost its connection to the bus %d times in the last hour and connected "+
|
||||
"again each time. While it reconnects, what it says and what it is asked waits.", who, u.Dropped)),
|
||||
Resolved: "Resolved: " + who + " stays connected"})
|
||||
}
|
||||
return sorted(out), nil
|
||||
}
|
||||
|
||||
// busUserWords is a bus user as the operator says it, and the machine it is on.
|
||||
func busUserWords(user string) (string, string) {
|
||||
switch {
|
||||
case user == "controller":
|
||||
return "the controller", ""
|
||||
case strings.HasPrefix(user, "node."):
|
||||
m := strings.TrimPrefix(user, "node.")
|
||||
return "the node-engine on " + m, m
|
||||
}
|
||||
if m, module, ok := strings.Cut(user, "."); ok {
|
||||
if module == "node-tools" {
|
||||
return "the tool runner on " + m, m
|
||||
}
|
||||
return module + " on " + m, m
|
||||
}
|
||||
return "a client of the bus", ""
|
||||
}
|
||||
|
||||
// --- DW and H-bus --------------------------------------------------------------------------------------------
|
||||
|
||||
// probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the
|
||||
// watchman's pulse on `checked`, is the other half.
|
||||
func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) {
|
||||
if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil {
|
||||
return []Observation{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"},
|
||||
Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " +
|
||||
"late signal is being said",
|
||||
Said: firstLine(err.Error())}}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a request crosses it
|
||||
// to the bus module's machine and back. The question to the bus module is that request: an answer is its round trip,
|
||||
// and no answer is no pass.
|
||||
func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) {
|
||||
rows, err := c.streams("")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err)
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, r := range rows {
|
||||
held[r.Name] = true
|
||||
}
|
||||
var problems []string
|
||||
for _, s := range buscompose.MeshStreams() {
|
||||
if !held[s.Name] {
|
||||
problems = append(problems, "the stream "+s.Name+" is missing")
|
||||
}
|
||||
}
|
||||
byStream := expectedConsumers()
|
||||
streams := make([]string, 0, len(byStream))
|
||||
for s := range byStream {
|
||||
streams = append(streams, s)
|
||||
}
|
||||
sort.Strings(streams)
|
||||
for _, stream := range streams {
|
||||
if !held[stream] {
|
||||
continue // said above
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
detail, err := c.streams(stream)
|
||||
if err != nil && !errors.Is(err, errNoStream) {
|
||||
return nil, err
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, r := range detail {
|
||||
for _, cr := range r.Consumers {
|
||||
have[cr.Name] = true
|
||||
}
|
||||
}
|
||||
for _, want := range byStream[stream] {
|
||||
if !have[want.Name] {
|
||||
problems = append(problems, consumerWords(want)+" is missing")
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return []Observation{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"},
|
||||
Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil
|
||||
}
|
||||
|
||||
// sorted is findings in a stated order, so two runs over one mesh say the same.
|
||||
func sorted(out []Observation) []Observation {
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
|
||||
return out
|
||||
}
|
||||
|
||||
func capital(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
return strings.ToUpper(s[:1]) + s[1:]
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
)
|
||||
|
||||
// bus is the bus module's answers as a probe asks them: every mesh stream and consumer as defined, unless changed.
|
||||
type bus struct {
|
||||
drop map[string]bool // stream or stream.consumer missing
|
||||
change func(rows map[string]*streamRow)
|
||||
closed string
|
||||
signals error
|
||||
asksFail error
|
||||
}
|
||||
|
||||
func (b bus) rows() map[string]*streamRow {
|
||||
rows := map[string]*streamRow{}
|
||||
for _, s := range buscompose.MeshStreams() {
|
||||
if b.drop[s.Name] {
|
||||
continue
|
||||
}
|
||||
retention := "limits"
|
||||
per := int64(s.MaxMsgsPerSubject)
|
||||
switch s.Retention {
|
||||
case buscompose.RetentionWorkQueue:
|
||||
retention = "workqueue"
|
||||
case buscompose.RetentionLastPerSubject:
|
||||
per = 1
|
||||
}
|
||||
rows[s.Name] = &streamRow{Name: s.Name, Subjects: s.Subjects, Retention: retention, MaxPerSubject: per}
|
||||
}
|
||||
for _, c := range buscompose.MeshConsumers() {
|
||||
r := rows[c.Stream]
|
||||
if r == nil || b.drop[c.Stream+"."+c.Name] {
|
||||
continue
|
||||
}
|
||||
kind := "pull"
|
||||
if c.Push || c.Queue != "" {
|
||||
kind = "push"
|
||||
}
|
||||
r.Consumers = append(r.Consumers, consumerRow{Name: c.Name, Kind: kind, Filters: c.Filters, MaxDeliver: int64(c.MaxDeliver)})
|
||||
}
|
||||
if b.change != nil {
|
||||
b.change(rows)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func (b bus) ask(key string, body any) (json.RawMessage, error) {
|
||||
if b.asksFail != nil {
|
||||
return nil, b.asksFail
|
||||
}
|
||||
switch key {
|
||||
case StreamsTool:
|
||||
stream, _ := body.(map[string]any)["stream"].(string)
|
||||
var detail []streamRow
|
||||
for name, r := range b.rows() {
|
||||
if stream == "" {
|
||||
row := *r
|
||||
row.Consumers = nil
|
||||
detail = append(detail, row)
|
||||
} else if name == stream {
|
||||
detail = append(detail, *r)
|
||||
}
|
||||
}
|
||||
if stream != "" && len(detail) == 0 {
|
||||
return nil, errors.New("no stream called " + stream)
|
||||
}
|
||||
return json.Marshal(map[string]any{"detail": detail})
|
||||
case ClosedTool:
|
||||
if b.closed == "" {
|
||||
return json.RawMessage(`{"hours":1,"reaches":true,"users":[]}`), nil
|
||||
}
|
||||
return json.RawMessage(b.closed), nil
|
||||
case SignalsVerb:
|
||||
if b.signals != nil {
|
||||
return nil, b.signals
|
||||
}
|
||||
return json.RawMessage(`{"signals":[]}`), nil
|
||||
}
|
||||
return nil, errors.New("nats: no responders available for request")
|
||||
}
|
||||
|
||||
func keysOf(found []Observation) []string {
|
||||
var out []string
|
||||
for _, f := range found {
|
||||
out = append(out, f.Key())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Each probe that moved judges the bus module's answers with its assertion unchanged (ADR 0564 §6; issue 1952 SC2).
|
||||
func TestTheBusProbes(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
probe func(context.Context, *Checker) ([]Observation, error)
|
||||
bus bus
|
||||
want []string
|
||||
err string
|
||||
}{
|
||||
{"D6 all as defined", probeConsumers, bus{}, nil, ""},
|
||||
{"D6 a consumer gone", probeConsumers, bus{drop: map[string]bool{"DEAD_LETTER_NOTICES.controller": true}},
|
||||
[]string{"bus.DEAD_LETTER_NOTICES.controller.missing"}, ""},
|
||||
{"D6 a stream gone takes its consumers", probeConsumers, bus{drop: map[string]bool{"CONTROL": true}},
|
||||
[]string{"bus.CONTROL.controller.missing"}, ""},
|
||||
{"D6 redefined and far behind", probeConsumers, bus{change: func(rows map[string]*streamRow) {
|
||||
for i, c := range rows["EVENTS"].Consumers {
|
||||
if c.Name == "controller" {
|
||||
rows["EVENTS"].Consumers[i].MaxDeliver, rows["EVENTS"].Consumers[i].Pending = 9, 5000
|
||||
}
|
||||
}
|
||||
}}, []string{"bus.EVENTS.controller.behind", "bus.EVENTS.controller.redefined"}, ""},
|
||||
{"D6 the bus module silent", probeConsumers, bus{asksFail: errors.New("no responders")}, nil, "did not answer"},
|
||||
{"D7 all as defined", probeStreams, bus{}, nil, ""},
|
||||
{"D7 missing and redefined", probeStreams, bus{drop: map[string]bool{"NODES": true}, change: func(rows map[string]*streamRow) {
|
||||
rows["EVENTS"].Retention = "workqueue"
|
||||
}}, []string{"bus.EVENTS.stream-redefined", "bus.NODES.stream-missing"}, ""},
|
||||
{"D15 quiet", probeReconnects, bus{}, nil, ""},
|
||||
{"D15 a loop", probeReconnects, bus{closed: `{"hours":1,"reaches":true,"users":[
|
||||
{"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]},
|
||||
{"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`},
|
||||
[]string{"bus.g14.node-tools.bus-reconnects"}, ""},
|
||||
{"DW answers", probeWatchdogs, bus{}, nil, ""},
|
||||
{"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.silent"}, ""},
|
||||
{"H-bus healthy", probeBusHealth, bus{}, nil, ""},
|
||||
{"H-bus a stream and a consumer missing", probeBusHealth,
|
||||
bus{drop: map[string]bool{"NODES": true, "EVENTS.controller-merges": true}}, []string{"bus.mesh.unhealthy"}, ""},
|
||||
{"H-bus no round trip", probeBusHealth, bus{asksFail: errors.New("timeout")}, nil, "no request crossed"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
c := &Checker{Ask: tc.bus.ask}
|
||||
found, err := tc.probe(context.Background(), c)
|
||||
if tc.err != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.err) {
|
||||
t.Errorf("%s: error %v, want %q", tc.name, err, tc.err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", tc.name, err)
|
||||
continue
|
||||
}
|
||||
if got := keysOf(found); !slices.Equal(got, tc.want) {
|
||||
t.Errorf("%s: found %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The registry holds exactly the probes ADR 0564's table moves here, each with a kind, and a deferred one names what
|
||||
// it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4).
|
||||
func TestTheRegistryIsADR0564sTable(t *testing.T) {
|
||||
moved := []string{"D2", "D3", "D4", "D6", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard",
|
||||
"DW", "H-bus"}
|
||||
var ids []string
|
||||
for _, p := range Registry() {
|
||||
ids = append(ids, p.ID)
|
||||
if p.Kind == "" || p.Asserts == "" {
|
||||
t.Errorf("%s has no kind or assertion", p.ID)
|
||||
}
|
||||
if (p.Run == nil) == (p.Deferred == "") {
|
||||
t.Errorf("%s is neither run nor deferred naming what it lacks", p.ID)
|
||||
}
|
||||
}
|
||||
slices.Sort(ids)
|
||||
slices.Sort(moved)
|
||||
if !slices.Equal(ids, moved) {
|
||||
t.Errorf("the registry holds %v, ADR 0564 moves %v", ids, moved)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
# What these tests read outside their package, as paths from the repository's root (issue 1936).
|
||||
modules/mesh-self-check/module.json
|
||||
@@ -0,0 +1,394 @@
|
||||
package main
|
||||
|
||||
// The self-check (novox/hq ADR 0564 §6): the registry of probes, each a live invariant of a design, run every five
|
||||
// minutes, each probe within thirty seconds. What a probe finds is written as a finding in this module's bucket
|
||||
// `findings`, which the condition keeper raises and clears conditions from (ADR 0564 §2); every run is kept in the
|
||||
// bucket `verdicts` for a week and the last as the key `last` of `verdict`, and is said as `checked`, which the
|
||||
// watchman and mesh-watcher hear (S10, ADR 0227 §6). A probe that errors or does not finish is never a pass: it is
|
||||
// said in the verdict at once and written as a finding of its own when the next run cannot run it either.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// Module and Seat are this module and the seat it holds once for the mesh.
|
||||
const (
|
||||
Module = "mesh-self-check"
|
||||
Seat = "mesh-self-check"
|
||||
)
|
||||
|
||||
// The states it keeps, and the event that ends every run.
|
||||
const (
|
||||
FindingsState = health.FindingsState
|
||||
VerdictState = "verdict"
|
||||
VerdictsState = "verdicts"
|
||||
LastKey = "last"
|
||||
CheckedEvent = health.EventChecked
|
||||
// CheckedSubject is where the seat says it: a seat's events are on mesh.seat.<seat>.event.<name>, which its
|
||||
// consumers hear (catalogue.SeatSays).
|
||||
CheckedSubject = "mesh.seat." + Seat + ".event." + CheckedEvent
|
||||
)
|
||||
|
||||
// The clocks: the self-check's run is the one timer among the four holders (ADR 0564 §1).
|
||||
var (
|
||||
Every = 5 * time.Minute
|
||||
FirstAfter = time.Minute
|
||||
Within = 30 * time.Second
|
||||
)
|
||||
|
||||
// The verdicts a probe can have, as mesh-sdk's health names them.
|
||||
const (
|
||||
VerdictPass = health.VerdictPassed
|
||||
VerdictFail = health.VerdictFailed
|
||||
VerdictFailedToRun = health.VerdictFailedToRun
|
||||
VerdictDeferred = health.VerdictDeferred
|
||||
)
|
||||
|
||||
// sourceSelf is what writes a probe's failure to run: the self-check itself.
|
||||
const sourceSelf = "doctor"
|
||||
|
||||
// Store is the bus's buckets as this module reaches them, through its runtime.
|
||||
type Store interface {
|
||||
Get(state, key string) ([]byte, bool, error)
|
||||
Put(state, key string, value any) error
|
||||
Delete(state, key string) error
|
||||
Keys(state string) ([]string, error)
|
||||
}
|
||||
|
||||
// Probe is one live invariant: its id, what it asserts, the kind it raises, and the run; or why it waits.
|
||||
type Probe struct {
|
||||
ID string
|
||||
Asserts string
|
||||
From string
|
||||
Kind string
|
||||
Raises []string
|
||||
// Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6):
|
||||
// answered deferred, never a pass.
|
||||
Deferred string
|
||||
Run func(ctx context.Context, c *Checker) ([]Observation, error)
|
||||
}
|
||||
|
||||
// ProbeVerdict, Counts and Run are a run as mesh-sdk's health holds it: the body of `checked`, the controller's
|
||||
// doctor-heartbeat body unchanged (ADR 0564 §6), Controller naming the machine that ran it.
|
||||
type (
|
||||
ProbeVerdict = health.ProbeVerdict
|
||||
Counts = health.CheckedCounts
|
||||
Run = health.Checked
|
||||
)
|
||||
|
||||
// Checker is the self-check: the registry and what its probes reach.
|
||||
type Checker struct {
|
||||
Store Store
|
||||
Ask func(key string, body any) (json.RawMessage, error)
|
||||
// Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish).
|
||||
Publish func(subject string, body any, id string) error
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
Host string
|
||||
Registry []Probe
|
||||
|
||||
running sync.Mutex
|
||||
mu sync.Mutex
|
||||
n uint64
|
||||
last *Run
|
||||
// seen are the keys the run before saw, confirmed or not: a finding a single look can be wrong about is
|
||||
// written only when two runs in a row see it, or while it is already written (novox/hq issue 277).
|
||||
seen map[string]bool
|
||||
// unsaid counts the runs whose `checked` could not be said since the start.
|
||||
unsaid int
|
||||
}
|
||||
|
||||
// LastRun is the last run, nil before the first.
|
||||
func (c *Checker) LastRun() *Run {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.last
|
||||
}
|
||||
|
||||
type result struct {
|
||||
found []Observation
|
||||
err error
|
||||
took time.Duration
|
||||
}
|
||||
|
||||
// RunOnce runs every probe of the registry, writes what changed in `findings`, keeps the verdict and says `checked`.
|
||||
// One run at a time: a person's run during a scheduled one waits for it.
|
||||
func (c *Checker) RunOnce(ctx context.Context, why string) Run {
|
||||
c.running.Lock()
|
||||
defer c.running.Unlock()
|
||||
c.mu.Lock()
|
||||
c.n++
|
||||
n := c.n
|
||||
c.mu.Unlock()
|
||||
started := c.Now()
|
||||
run := Run{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
|
||||
IntervalSeconds: int(Every / time.Second), Controller: c.Host, Why: why}
|
||||
|
||||
results := make([]result, len(c.Registry))
|
||||
var wg sync.WaitGroup
|
||||
for i, p := range c.Registry {
|
||||
if p.Run == nil {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(i int, p Probe) {
|
||||
defer wg.Done()
|
||||
results[i] = c.probe(ctx, p)
|
||||
}(i, p)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
written, readErr := c.written()
|
||||
seen := map[string]bool{}
|
||||
want := map[string]Observation{} // every finding this run writes, by key
|
||||
ran := map[string]bool{} // the sources whose findings this run judged whole
|
||||
var blind []Observation
|
||||
for i, p := range c.Registry {
|
||||
v := ProbeVerdict{ID: p.ID}
|
||||
r := results[i]
|
||||
switch {
|
||||
case p.Run == nil:
|
||||
v.Verdict = VerdictDeferred
|
||||
v.Error = "deferred: " + p.Deferred
|
||||
run.Counts.Deferred++
|
||||
case r.err != nil:
|
||||
v.Verdict, v.Error = VerdictFailedToRun, r.err.Error()
|
||||
run.Counts.FailedToRun++
|
||||
blind = append(blind, Observation{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true,
|
||||
Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf,
|
||||
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
|
||||
Said: firstLine(r.err.Error())})
|
||||
default:
|
||||
ran[p.ID] = true
|
||||
for _, f := range r.found {
|
||||
if f.Kind == "" {
|
||||
f.Kind = p.Kind
|
||||
}
|
||||
f.Source = p.ID
|
||||
f = worded(f)
|
||||
key := f.Key()
|
||||
seen[key] = true
|
||||
if f.Confirm && !c.seenBefore(key) && !written.has(key) {
|
||||
v.Unconfirmed = append(v.Unconfirmed, f.Summary)
|
||||
continue
|
||||
}
|
||||
want[key] = f
|
||||
v.Found = append(v.Found, f.Summary)
|
||||
}
|
||||
if len(v.Found) == 0 {
|
||||
v.Verdict = VerdictPass
|
||||
run.Counts.Passed++
|
||||
} else {
|
||||
v.Verdict = VerdictFail
|
||||
run.Counts.Failed++
|
||||
}
|
||||
}
|
||||
if p.Run != nil {
|
||||
v.Took = r.took.Round(time.Millisecond).String()
|
||||
}
|
||||
run.Probes = append(run.Probes, v)
|
||||
}
|
||||
ran[sourceSelf] = true
|
||||
for _, f := range blind {
|
||||
f = worded(f)
|
||||
key := f.Key()
|
||||
seen[key] = true
|
||||
if c.seenBefore(key) || written.has(key) {
|
||||
want[key] = f
|
||||
}
|
||||
}
|
||||
if readErr != nil {
|
||||
// Blind to its own bucket: nothing is written or deleted (ADR 0227 §4), and the log says so.
|
||||
c.Logf("[%s] its findings cannot be read, so none is written or cleared this run: %v", Module, readErr)
|
||||
} else if err := c.write(written, want, ran, run.Run); err != nil {
|
||||
c.Logf("[%s] what this run found is not all written: %v", Module, err)
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.seen = seen
|
||||
c.mu.Unlock()
|
||||
|
||||
run.At = c.Now().UTC()
|
||||
run.Took = run.At.Sub(started).Round(time.Millisecond).String()
|
||||
c.mu.Lock()
|
||||
run.Unsaid = c.unsaid
|
||||
c.mu.Unlock()
|
||||
c.keep(run)
|
||||
return run
|
||||
}
|
||||
|
||||
// probe runs one probe within its bound; a panic or an overrun is an error, never a pass.
|
||||
func (c *Checker) probe(ctx context.Context, p Probe) result {
|
||||
probing, cancel := context.WithTimeout(ctx, Within)
|
||||
defer cancel()
|
||||
began := c.Now()
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
|
||||
}
|
||||
}()
|
||||
found, err := p.Run(probing, c)
|
||||
done <- result{found: found, err: err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
r.took = c.Now().Sub(began)
|
||||
return r
|
||||
case <-probing.Done():
|
||||
return result{err: fmt.Errorf("it did not finish within %s", Within), took: c.Now().Sub(began)}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Checker) seenBefore(key string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.seen[key]
|
||||
}
|
||||
|
||||
// writtenFindings are the findings in the bucket, by key.
|
||||
type writtenFindings map[string]health.Finding
|
||||
|
||||
func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok }
|
||||
|
||||
// written reads every finding this module has written.
|
||||
func (c *Checker) written() (writtenFindings, error) {
|
||||
keys, err := c.Store.Keys(FindingsState)
|
||||
if err != nil {
|
||||
return writtenFindings{}, err
|
||||
}
|
||||
out := writtenFindings{}
|
||||
for _, k := range keys {
|
||||
raw, found, err := c.Store.Get(FindingsState, k)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
f, err := health.DecodeFinding(raw)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the finding %s is not readable: %w", k, err)
|
||||
}
|
||||
out[k] = f
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding
|
||||
// of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4).
|
||||
func (c *Checker) write(written writtenFindings, want map[string]Observation, ran map[string]bool, run string) error {
|
||||
var errs []string
|
||||
keys := make([]string, 0, len(want))
|
||||
for k := range want {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
now := c.Now()
|
||||
for _, k := range keys {
|
||||
f := want[k].Finding(run, now)
|
||||
if was, ok := written[k]; ok && says(was) == says(f) {
|
||||
continue
|
||||
}
|
||||
raw, err := health.EncodeFinding(f)
|
||||
if err == nil {
|
||||
err = c.Store.Put(FindingsState, k, json.RawMessage(raw))
|
||||
}
|
||||
if err != nil {
|
||||
errs = append(errs, k+": "+err.Error())
|
||||
}
|
||||
}
|
||||
for k, f := range written {
|
||||
if _, still := want[k]; still || !ran[f.Source] {
|
||||
continue
|
||||
}
|
||||
if err := c.Store.Delete(FindingsState, k); err != nil {
|
||||
errs = append(errs, k+": "+err.Error())
|
||||
}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(errs, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keep keeps the run as the last verdict and in the week's verdicts, and says it as `checked`. Not said is said in
|
||||
// the log: the watchman's S10 says the self-check silent when `checked` stops.
|
||||
func (c *Checker) keep(run Run) {
|
||||
c.mu.Lock()
|
||||
c.last = &run
|
||||
c.mu.Unlock()
|
||||
if err := c.Store.Put(VerdictState, LastKey, run); err != nil {
|
||||
c.Logf("[%s] the run %s is not kept as the last verdict: %v", Module, run.Run, err)
|
||||
}
|
||||
if err := c.Store.Put(VerdictsState, run.Run, Overview(run)); err != nil {
|
||||
c.Logf("[%s] the run %s is not kept among the week's verdicts: %v", Module, run.Run, err)
|
||||
}
|
||||
if c.Publish == nil {
|
||||
return
|
||||
}
|
||||
if err := c.Publish(CheckedSubject, run, run.Run); err != nil {
|
||||
c.mu.Lock()
|
||||
c.unsaid++
|
||||
c.mu.Unlock()
|
||||
c.Logf("[%s] the run %s could NOT be said as checked, so the watchers will say the self-check is silent: %v",
|
||||
Module, run.Run, err)
|
||||
}
|
||||
}
|
||||
|
||||
// overviewFindings is how many findings of one probe an overview carries (novox/hq issue 314).
|
||||
const overviewFindings = 20
|
||||
|
||||
// Overview is a run with each probe's findings at most overviewFindings, with their count and how to read them all.
|
||||
func Overview(run Run) Run {
|
||||
probes := make([]ProbeVerdict, len(run.Probes))
|
||||
for i, p := range run.Probes {
|
||||
total := len(p.Found) + len(p.Unconfirmed)
|
||||
if total > overviewFindings {
|
||||
p.FoundCount = len(p.Found)
|
||||
if len(p.Found) > overviewFindings {
|
||||
p.Found = p.Found[:overviewFindings:overviewFindings]
|
||||
}
|
||||
if room := overviewFindings - len(p.Found); len(p.Unconfirmed) > room {
|
||||
p.Unconfirmed = p.Unconfirmed[:room:room]
|
||||
}
|
||||
p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `%s.doctor probe=%s` gives them all",
|
||||
p.FoundCount, total-p.FoundCount, len(p.Found)+len(p.Unconfirmed), Seat, p.ID)
|
||||
}
|
||||
probes[i] = p
|
||||
}
|
||||
run.Probes = probes
|
||||
return run
|
||||
}
|
||||
|
||||
// Keep runs the registry on its schedule until ctx ends: the first run FirstAfter the start, then every Every.
|
||||
func (c *Checker) Keep(ctx context.Context) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(FirstAfter):
|
||||
}
|
||||
tick := time.NewTicker(Every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
c.RunOnce(ctx, "the schedule")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
line, _, _ := strings.Cut(s, "\n")
|
||||
return line
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// memStore is the bus's buckets in memory, counting the writes.
|
||||
type memStore struct {
|
||||
mu sync.Mutex
|
||||
kv map[string]map[string][]byte
|
||||
puts map[string]int
|
||||
keysErr error
|
||||
}
|
||||
|
||||
func newStore() *memStore {
|
||||
return &memStore{kv: map[string]map[string][]byte{}, puts: map[string]int{}}
|
||||
}
|
||||
|
||||
func (s *memStore) Get(state, key string) ([]byte, bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v, ok := s.kv[state][key]
|
||||
return v, ok, nil
|
||||
}
|
||||
|
||||
func (s *memStore) Put(state, key string, value any) error {
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.kv[state] == nil {
|
||||
s.kv[state] = map[string][]byte{}
|
||||
}
|
||||
s.kv[state][key] = raw
|
||||
s.puts[state+"/"+key]++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) Delete(state, key string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
delete(s.kv[state], key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *memStore) Keys(state string) ([]string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.keysErr != nil && state == FindingsState {
|
||||
return nil, s.keysErr
|
||||
}
|
||||
var out []string
|
||||
for k := range s.kv[state] {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *memStore) findingKeys() []string {
|
||||
keys, _ := s.Keys(FindingsState)
|
||||
return keys
|
||||
}
|
||||
|
||||
// checker is a self-check over memory, with a registry the test drives.
|
||||
func checker(store *memStore, registry ...Probe) (*Checker, *[]string) {
|
||||
var said []string
|
||||
now := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
|
||||
c := &Checker{Store: store, Registry: registry, Host: "novox", Logf: func(string, ...any) {},
|
||||
Now: func() time.Time { now = now.Add(time.Second); return now },
|
||||
Publish: func(subject string, body any, id string) error {
|
||||
raw, _ := json.Marshal(body)
|
||||
said = append(said, subject+" "+id+" "+string(raw))
|
||||
return nil
|
||||
}}
|
||||
return c, &said
|
||||
}
|
||||
|
||||
// A run writes what a probe finds, writes nothing it already wrote, and deletes what it no longer sees; a probe that
|
||||
// could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can
|
||||
// be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1).
|
||||
func TestARunWritesFindingsAsTheyChange(t *testing.T) {
|
||||
var found []Observation
|
||||
var fail error
|
||||
steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { return found, fail }}
|
||||
deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"}
|
||||
missing := Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"},
|
||||
Summary: "the stream EVENTS is not on the bus"}
|
||||
behind := Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true,
|
||||
Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"}
|
||||
store := newStore()
|
||||
c, said := checker(store, steady, deferred)
|
||||
|
||||
steps := []struct {
|
||||
name string
|
||||
found []Observation
|
||||
fail error
|
||||
want []string // finding keys written after the run
|
||||
verdict string
|
||||
missingPs int // puts of the missing finding so far
|
||||
}{
|
||||
{"a finding is written", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"the same again is no write", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"one look can be wrong: held", []Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"seen twice: written", []Observation{missing, behind}, nil,
|
||||
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1},
|
||||
{"blind once: clears nothing, held", nil, errors.New("timed out"),
|
||||
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1},
|
||||
{"blind twice: its own finding", nil, errors.New("timed out"),
|
||||
[]string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing", "probe.P1.failed"}, VerdictFailedToRun, 1},
|
||||
{"no longer seen: cleared", nil, nil, nil, VerdictPass, 1},
|
||||
}
|
||||
for _, s := range steps {
|
||||
found, fail = s.found, s.fail
|
||||
run := c.RunOnce(context.Background(), "the test")
|
||||
if got := store.findingKeys(); !slices.Equal(got, s.want) {
|
||||
t.Errorf("%s: findings %v, want %v", s.name, got, s.want)
|
||||
}
|
||||
if run.Probes[0].Verdict != s.verdict {
|
||||
t.Errorf("%s: P1 %s, want %s", s.name, run.Probes[0].Verdict, s.verdict)
|
||||
}
|
||||
if run.Probes[1].Verdict != VerdictDeferred || !strings.Contains(run.Probes[1].Error, "overlay") {
|
||||
t.Errorf("%s: a deferred probe answered %+v", s.name, run.Probes[1])
|
||||
}
|
||||
if n := store.puts[FindingsState+"/bus.EVENTS.stream-missing"]; n != s.missingPs {
|
||||
t.Errorf("%s: the unchanged finding was written %d times", s.name, n)
|
||||
}
|
||||
}
|
||||
raw, _, _ := store.Get(FindingsState, "probe.P1.failed")
|
||||
if raw != nil {
|
||||
t.Errorf("the blind finding outlived the probe running again: %s", raw)
|
||||
}
|
||||
// Every run is kept, the last as `last`, and said as checked with the controller's heartbeat body.
|
||||
if keys, _ := store.Keys(VerdictsState); len(keys) != len(steps) {
|
||||
t.Errorf("verdicts kept %d runs, want %d", len(keys), len(steps))
|
||||
}
|
||||
// Said on the seat's subject, where its consumers hear it, de-duplicated by the run's id.
|
||||
if len(*said) != len(steps) || !strings.HasPrefix((*said)[0], "mesh.seat.mesh-self-check.event.checked doctor-") {
|
||||
t.Fatalf("said %v", *said)
|
||||
}
|
||||
_, rest, _ := strings.Cut((*said)[0], " ")
|
||||
_, rest, _ = strings.Cut(rest, " ")
|
||||
var body map[string]any
|
||||
_ = json.Unmarshal([]byte(rest), &body)
|
||||
for _, field := range []string{"run", "at", "started", "took", "interval-seconds", "counts", "probes", "controller", "why"} {
|
||||
if _, ok := body[field]; !ok {
|
||||
t.Errorf("checked's body lacks %s, which doctor-heartbeat carries: %v", field, body)
|
||||
}
|
||||
}
|
||||
last, _, _ := store.Get(VerdictState, LastKey)
|
||||
var r Run
|
||||
_ = json.Unmarshal(last, &r)
|
||||
if r.Counts.Passed != 1 || r.Counts.Deferred != 1 {
|
||||
t.Errorf("the last verdict counts %+v", r.Counts)
|
||||
}
|
||||
// A finding carries the operator's words of its kind, the probe that found it and who observed it.
|
||||
found, fail = []Observation{missing}, nil
|
||||
c.RunOnce(context.Background(), "the test")
|
||||
raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing")
|
||||
f, err := health.DecodeFinding(raw)
|
||||
if err != nil || f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" ||
|
||||
f.Explanation == "" || f.Token != "stream-missing" || len(f.Evidence) != 1 || f.Run == "" {
|
||||
t.Errorf("the written finding is %+v", f)
|
||||
}
|
||||
// Blind to its own bucket, a run writes and clears nothing.
|
||||
store.keysErr = errors.New("the bus is away")
|
||||
found = nil
|
||||
c.RunOnce(context.Background(), "the test")
|
||||
store.keysErr = nil
|
||||
if got := store.findingKeys(); !slices.Equal(got, []string{"bus.EVENTS.stream-missing"}) {
|
||||
t.Errorf("a run blind to its findings changed them: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A probe that panics or overruns its bound could not run: never a pass.
|
||||
func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) {
|
||||
within := Within
|
||||
Within = 50 * time.Millisecond
|
||||
t.Cleanup(func() { Within = within })
|
||||
for name, run := range map[string]func(context.Context, *Checker) ([]Observation, error){
|
||||
"panic": func(context.Context, *Checker) ([]Observation, error) { panic("boom") },
|
||||
"overrun": func(ctx context.Context, _ *Checker) ([]Observation, error) { time.Sleep(time.Second); return nil, nil },
|
||||
} {
|
||||
c, _ := checker(newStore(), Probe{ID: "P", Kind: "k", Run: run})
|
||||
c.Now = time.Now
|
||||
r := c.RunOnce(context.Background(), "the test")
|
||||
if r.Probes[0].Verdict != VerdictFailedToRun || r.Counts.Passed != 0 {
|
||||
t.Errorf("%s: %+v", name, r.Probes[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The verb answers the last run, one probe whole, a run now, and before any run says when one comes; the reads answer
|
||||
// every state the module keeps (ADR 0564 §12).
|
||||
func TestTheVerbsAndReads(t *testing.T) {
|
||||
store := newStore()
|
||||
many := make([]Observation, 25)
|
||||
for i := range many {
|
||||
many[i] = Observation{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"}
|
||||
}
|
||||
c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong",
|
||||
Run: func(context.Context, *Checker) ([]Observation, error) { return many, nil }})
|
||||
if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") {
|
||||
t.Fatalf("before a run: %v", err)
|
||||
}
|
||||
if _, err := c.Doctor(true, "", "asked by person.jochen"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
over, _ := c.Doctor(false, "", "")
|
||||
if v := over["run"].(Run).Probes[0]; len(v.Found) != overviewFindings || v.More == "" {
|
||||
t.Errorf("the overview carries %d findings, more %q", len(v.Found), v.More)
|
||||
}
|
||||
whole, _ := c.Doctor(false, "d7", "")
|
||||
if v := whole["run"].(Run).Probes[0]; len(v.Found) != 25 {
|
||||
t.Errorf("one probe whole carries %d findings", len(v.Found))
|
||||
}
|
||||
if _, err := c.Doctor(false, "D99", ""); err == nil {
|
||||
t.Error("an unknown probe answered")
|
||||
}
|
||||
// A fresh checker over the same bucket answers the last verdict from it.
|
||||
fresh, _ := checker(store)
|
||||
if _, err := fresh.Doctor(false, "", ""); err != nil {
|
||||
t.Errorf("the last verdict is not read from its bucket: %v", err)
|
||||
}
|
||||
fs, err := c.Findings("")
|
||||
if err != nil || fs.(map[string]any)["count"] != 25 {
|
||||
t.Errorf("findings answered %v, %v", fs, err)
|
||||
}
|
||||
if one, err := c.Findings("bus.S03.stream-wrong"); err != nil || !strings.Contains(string(one.(json.RawMessage)), "S03") {
|
||||
t.Errorf("one finding answered %v, %v", one, err)
|
||||
}
|
||||
vs, err := c.Verdicts("", 0)
|
||||
if err != nil || len(vs.(map[string]any)["verdicts"].([]map[string]any)) != 1 {
|
||||
t.Errorf("verdicts answered %v, %v", vs, err)
|
||||
}
|
||||
if _, err := c.Verdicts("doctor-1-1", 0); err == nil {
|
||||
t.Error("an unknown run answered")
|
||||
}
|
||||
if p := c.Probes()["probes"].([]map[string]any); len(p) != 1 || p[0]["id"] != "D7" {
|
||||
t.Errorf("probes answered %v", p)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
package main
|
||||
|
||||
// The seat's verbs and the module's own tools (novox/hq ADR 0564 §6, §12): `doctor` answers the last verdict at once,
|
||||
// runs now, or gives one probe's verdict whole; `probes` is the registry; and a read of each state it keeps — the
|
||||
// findings, the last verdict and the week's verdicts — so a person sees all it holds and a recovery is a call.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// Verbs are the seat's verbs, as the manifest claims them.
|
||||
var Verbs = []string{"doctor", "probes"}
|
||||
|
||||
// Tool names: the seat's verbs, and the module's own reads.
|
||||
const (
|
||||
DoctorTool = Seat + ".doctor"
|
||||
ProbesTool = Seat + ".probes"
|
||||
FindingsTool = "self_check_findings"
|
||||
VerdictsTool = "self_check_verdicts"
|
||||
)
|
||||
|
||||
func obj(props map[string]any) map[string]any {
|
||||
return map[string]any{"type": "object", "properties": props}
|
||||
}
|
||||
|
||||
// Tools are what the runtime serves.
|
||||
func Tools(c *Checker) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: DoctorTool,
|
||||
Description: "The mesh's self-check (novox/hq ADR 0564 §6): the last run's verdict at once, each probe passed, " +
|
||||
"failed with what it found, could not run (never a pass) or deferred with the facts it lacks; run=true runs " +
|
||||
"every probe now and answers that run; probe=<id> gives one probe's verdict whole, every finding.",
|
||||
Input: obj(map[string]any{
|
||||
"run": map[string]any{"type": "boolean", "description": "run every probe now"},
|
||||
"probe": map[string]any{"type": "string", "description": "one probe's verdict, whole, e.g. D6"}}),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return c.Doctor(boolArg(a, "run"), strArg(a, "probe"), "asked by "+stdio.CallerOf(a))
|
||||
}},
|
||||
{Name: ProbesTool,
|
||||
Description: "The self-check's registry: each probe's id, what it asserts, where it came from, the condition kinds " +
|
||||
"it raises, and for one deferred the facts it lacks on the bus.",
|
||||
Run: func(map[string]any) (any, error) { return c.Probes(), nil }},
|
||||
{Name: FindingsTool,
|
||||
Description: "What the self-check finds wrong now, as the condition keeper reads it (its bucket findings): every " +
|
||||
"finding's key, kind, severity, the probe that found it and its summary; or one finding whole by key.",
|
||||
Input: obj(map[string]any{"key": map[string]any{"type": "string", "description": "one finding's key"}}),
|
||||
Run: func(a map[string]any) (any, error) { return c.Findings(strArg(a, "key")) }},
|
||||
{Name: VerdictsTool,
|
||||
Description: "The self-check's runs of the last week (its bucket verdicts, each kept a week) newest first, each " +
|
||||
"with its counts; or one run whole by its id. The last run is the doctor verb's.",
|
||||
Input: obj(map[string]any{
|
||||
"run": map[string]any{"type": "string", "description": "one run's id, e.g. doctor-1793284800-12"},
|
||||
"limit": map[string]any{"type": "number", "description": "at most this many, newest (default 50)"}}),
|
||||
Run: func(a map[string]any) (any, error) { return c.Verdicts(strArg(a, "run"), intArg(a, "limit", 50)) }},
|
||||
}
|
||||
}
|
||||
|
||||
// Doctor is the verb: the last verdict, a run now, or one probe of either.
|
||||
func (c *Checker) Doctor(run bool, probe, why string) (map[string]any, error) {
|
||||
var r Run
|
||||
switch {
|
||||
case run:
|
||||
r = c.RunOnce(context.Background(), why)
|
||||
case c.LastRun() != nil:
|
||||
r = *c.LastRun()
|
||||
default:
|
||||
raw, found, err := c.Store.Get(VerdictState, LastKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the last verdict cannot be read: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after it starts, then "+
|
||||
"every %s — run=true runs it now", FirstAfter, Every)
|
||||
}
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, fmt.Errorf("the last verdict is not readable: %w", err)
|
||||
}
|
||||
}
|
||||
if probe != "" {
|
||||
one, ok := oneProbe(r, probe)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the run %s has no probe %q — %s lists them", r.Run, probe, ProbesTool)
|
||||
}
|
||||
r = one
|
||||
} else {
|
||||
r = Overview(r)
|
||||
}
|
||||
return map[string]any{"run": r, "age": c.Now().Sub(r.At).Round(time.Second).String(),
|
||||
"note": "a probe that could not run is never a pass; each finding is a condition the keeper raises until a run no " +
|
||||
"longer sees it, and one a single look can be wrong about is written when two runs in a row see it"}, nil
|
||||
}
|
||||
|
||||
func oneProbe(r Run, id string) (Run, bool) {
|
||||
for _, p := range r.Probes {
|
||||
if strings.EqualFold(p.ID, id) {
|
||||
r.Probes = []ProbeVerdict{p}
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return Run{}, false
|
||||
}
|
||||
|
||||
// Probes is the registry as the verb answers it.
|
||||
func (c *Checker) Probes() map[string]any {
|
||||
var out []map[string]any
|
||||
for _, p := range c.Registry {
|
||||
row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind}
|
||||
if len(p.Raises) > 0 {
|
||||
row["raises"] = p.Raises
|
||||
}
|
||||
if p.Run == nil {
|
||||
row["deferred"] = p.Deferred
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return map[string]any{"probes": out, "every": Every.String(), "each within": Within.String()}
|
||||
}
|
||||
|
||||
// Findings are the findings written now, or one whole.
|
||||
func (c *Checker) Findings(key string) (any, error) {
|
||||
if key != "" {
|
||||
raw, found, err := c.Store.Get(FindingsState, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("no finding is written under %s now", key)
|
||||
}
|
||||
return json.RawMessage(raw), nil
|
||||
}
|
||||
written, err := c.written()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the findings cannot be read: %w", err)
|
||||
}
|
||||
rows := []map[string]any{}
|
||||
for k, f := range written {
|
||||
rows = append(rows, map[string]any{"key": k, "kind": f.Kind, "severity": f.Severity, "probe": f.Source,
|
||||
"summary": f.Summary})
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i]["key"].(string) < rows[j]["key"].(string) })
|
||||
return map[string]any{"findings": rows, "count": len(rows)}, nil
|
||||
}
|
||||
|
||||
// Verdicts are the week's runs newest first, or one whole.
|
||||
func (c *Checker) Verdicts(run string, limit int) (any, error) {
|
||||
if run != "" {
|
||||
raw, found, err := c.Store.Get(VerdictsState, run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("no run %s is kept: runs are kept a week", run)
|
||||
}
|
||||
return json.RawMessage(raw), nil
|
||||
}
|
||||
keys, err := c.Store.Keys(VerdictsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the week's verdicts cannot be read: %w", err)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool { return runOrder(keys[i]) > runOrder(keys[j]) })
|
||||
total := len(keys)
|
||||
if limit > 0 && len(keys) > limit {
|
||||
keys = keys[:limit]
|
||||
}
|
||||
rows := []map[string]any{}
|
||||
for _, k := range keys {
|
||||
raw, found, err := c.Store.Get(VerdictsState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
var r Run
|
||||
if json.Unmarshal(raw, &r) == nil {
|
||||
rows = append(rows, map[string]any{"run": r.Run, "at": r.At, "why": r.Why, "counts": r.Counts})
|
||||
}
|
||||
}
|
||||
return map[string]any{"verdicts": rows, "kept": total, "kept-for": "a week"}, nil
|
||||
}
|
||||
|
||||
// runOrder orders run ids `doctor-<unix>-<n>` by time, then number.
|
||||
func runOrder(id string) string {
|
||||
parts := strings.Split(id, "-")
|
||||
if len(parts) != 3 {
|
||||
return id
|
||||
}
|
||||
return fmt.Sprintf("%020s-%020s", parts[1], parts[2])
|
||||
}
|
||||
|
||||
func strArg(a map[string]any, k string) string {
|
||||
s, _ := a[k].(string)
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// boolArg reads a flag as a boolean or as the seat's "true".
|
||||
func boolArg(a map[string]any, k string) bool {
|
||||
switch v := a[k].(type) {
|
||||
case bool:
|
||||
return v
|
||||
case string:
|
||||
return strings.EqualFold(strings.TrimSpace(v), "true")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func intArg(a map[string]any, k string, otherwise int) int {
|
||||
if n, ok := a[k].(float64); ok && n > 0 {
|
||||
return int(n)
|
||||
}
|
||||
return otherwise
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
// The self-check whole against a real bus (novox/hq ADR 0564 §6, issue 1952 SC1): the built binary, played over its
|
||||
// real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a real
|
||||
// nats-server's key-value buckets, mesh/publish onto its subjects, and the bus module's nats_streams answered from
|
||||
// that server's own streams). One run asked through the seat's verb finds the mesh's streams missing on a bare bus,
|
||||
// writes them as findings into its bucket on the bus, keeps the verdict, and says checked.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
type wire struct {
|
||||
ID json.RawMessage `json:"id,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Params json.RawMessage `json:"params,omitempty"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// player is the node's tool runtime, as far as the self-check needs it, over a real bus.
|
||||
type player struct {
|
||||
ctx context.Context
|
||||
nc *nats.Conn
|
||||
js jetstream.JetStream
|
||||
mu sync.Mutex
|
||||
in io.Writer
|
||||
next int
|
||||
pending map[string]chan wire
|
||||
}
|
||||
|
||||
func (p *player) send(m map[string]any) {
|
||||
raw, _ := json.Marshal(m)
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
_, _ = p.in.Write(append(raw, '\n'))
|
||||
}
|
||||
|
||||
func (p *player) answer(id json.RawMessage, result any, err error) {
|
||||
if err != nil {
|
||||
p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
|
||||
return
|
||||
}
|
||||
p.send(map[string]any{"id": id, "result": result})
|
||||
}
|
||||
|
||||
func (p *player) serve(out io.Reader) {
|
||||
scanner := bufio.NewScanner(out)
|
||||
scanner.Buffer(make([]byte, 1<<20), 1<<24)
|
||||
for scanner.Scan() {
|
||||
var m wire
|
||||
if json.Unmarshal(scanner.Bytes(), &m) != nil {
|
||||
continue
|
||||
}
|
||||
if m.Method == "" {
|
||||
var key string
|
||||
_ = json.Unmarshal(m.ID, &key)
|
||||
p.mu.Lock()
|
||||
ch := p.pending[key]
|
||||
delete(p.pending, key)
|
||||
p.mu.Unlock()
|
||||
if ch != nil {
|
||||
ch <- m
|
||||
}
|
||||
continue
|
||||
}
|
||||
go p.handle(m)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *player) kv(state string) (jetstream.KeyValue, error) {
|
||||
return p.js.KeyValue(p.ctx, Module+"_"+state)
|
||||
}
|
||||
|
||||
func (p *player) handle(m wire) {
|
||||
var a struct {
|
||||
State string `json:"state"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &a)
|
||||
switch m.Method {
|
||||
case "mesh/state.get":
|
||||
kv, err := p.kv(a.State)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
e, err := kv.Get(p.ctx, a.Key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
p.answer(m.ID, nil, nil)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
|
||||
case "mesh/state.put":
|
||||
kv, err := p.kv(a.State)
|
||||
if err == nil {
|
||||
var rev uint64
|
||||
if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil {
|
||||
p.answer(m.ID, map[string]any{"revision": rev}, nil)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.answer(m.ID, nil, err)
|
||||
case "mesh/state.keys":
|
||||
kv, err := p.kv(a.State)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
return
|
||||
}
|
||||
keys, err := kv.Keys(p.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
keys, err = []string{}, nil
|
||||
}
|
||||
p.answer(m.ID, keys, err)
|
||||
case "mesh/state.delete":
|
||||
kv, err := p.kv(a.State)
|
||||
if err == nil {
|
||||
err = kv.Delete(p.ctx, a.Key)
|
||||
}
|
||||
p.answer(m.ID, map[string]any{}, err)
|
||||
case "mesh/seat.publish":
|
||||
var e struct {
|
||||
Subject string `json:"subject"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &e)
|
||||
p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body))
|
||||
case "mesh/publish":
|
||||
var e struct {
|
||||
Key string `json:"key"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &e)
|
||||
p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body))
|
||||
case "mesh/ask":
|
||||
var ask struct {
|
||||
Key string `json:"key"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &ask)
|
||||
if ask.Key != StreamsTool {
|
||||
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
|
||||
return
|
||||
}
|
||||
// The bus module's answer, from the server's own streams.
|
||||
var detail []map[string]any
|
||||
for name := range p.js.StreamNames(p.ctx).Name() {
|
||||
detail = append(detail, map[string]any{"name": name})
|
||||
}
|
||||
p.answer(m.ID, map[string]any{"detail": detail}, nil)
|
||||
default:
|
||||
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *player) call(name string, args map[string]any) (json.RawMessage, error) {
|
||||
p.mu.Lock()
|
||||
p.next++
|
||||
id := fmt.Sprintf("c-%d", p.next)
|
||||
ch := make(chan wire, 1)
|
||||
p.pending[id] = ch
|
||||
p.mu.Unlock()
|
||||
p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args,
|
||||
"_meta": map[string]any{"mesh/caller": "person.jochen"}}})
|
||||
select {
|
||||
case m := <-ch:
|
||||
if m.Error != nil {
|
||||
return nil, errors.New(m.Error.Message)
|
||||
}
|
||||
var r struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
}
|
||||
if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 {
|
||||
return nil, fmt.Errorf("%s answered %s", name, m.Result)
|
||||
}
|
||||
if r.IsError {
|
||||
return nil, errors.New(r.Content[0].Text)
|
||||
}
|
||||
return json.RawMessage(r.Content[0].Text), nil
|
||||
case <-time.After(60 * time.Second):
|
||||
return nil, fmt.Errorf("%s did not answer in 60s", name)
|
||||
}
|
||||
}
|
||||
|
||||
// 1952/SC1: one run of the self-check whole ends with checked and a verdict, its findings on the bus.
|
||||
func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
dir := t.TempDir()
|
||||
bin := filepath.Join(dir, Module)
|
||||
build := exec.Command("go", "build", "-o", bin, ".")
|
||||
build.Stderr = os.Stderr
|
||||
if err := build.Run(); err != nil {
|
||||
t.Fatalf("the self-check does not build: %v", err)
|
||||
}
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range []string{FindingsState, VerdictState, VerdictsState} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: Module + "_" + s}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
checked, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + CheckedEvent)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, bin)
|
||||
cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module)
|
||||
stdin, _ := cmd.StdinPipe()
|
||||
stdout, _ := cmd.StdoutPipe()
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
|
||||
p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
|
||||
go p.serve(stdout)
|
||||
|
||||
raw, err := p.call(DoctorTool, map[string]any{"run": true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var answer struct {
|
||||
Run Run `json:"run"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verdicts := map[string]string{}
|
||||
for _, v := range answer.Run.Probes {
|
||||
verdicts[v.ID] = v.Verdict
|
||||
}
|
||||
if verdicts["D7"] != VerdictFail || verdicts["DW"] != VerdictFail || verdicts["D15"] != VerdictFailedToRun ||
|
||||
verdicts["D2"] != VerdictDeferred || answer.Run.Controller != "ace" || !strings.Contains(answer.Run.Why, "person.jochen") {
|
||||
t.Errorf("the run answered %+v", answer.Run)
|
||||
}
|
||||
msg, err := checked.NextMsg(10 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("checked was not said: %v", err)
|
||||
}
|
||||
var said Run
|
||||
if err := json.Unmarshal(msg.Data, &said); err != nil || said.Run != answer.Run.Run {
|
||||
t.Errorf("checked said %s", msg.Data)
|
||||
}
|
||||
findings, _ := js.KeyValue(ctx, Module+"_"+FindingsState)
|
||||
e, err := findings.Get(ctx, "bus.EVENTS.stream-missing")
|
||||
if err != nil {
|
||||
t.Fatalf("the missing stream is not a finding on the bus: %v", err)
|
||||
}
|
||||
f, err := health.DecodeFinding(e.Value())
|
||||
if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" {
|
||||
t.Errorf("the finding is %s", e.Value())
|
||||
}
|
||||
verdict, _ := js.KeyValue(ctx, Module+"_"+VerdictState)
|
||||
if _, err := verdict.Get(ctx, LastKey); err != nil {
|
||||
t.Errorf("the last verdict is not kept: %v", err)
|
||||
}
|
||||
if raw, err := p.call(VerdictsTool, map[string]any{}); err != nil || !strings.Contains(string(raw), answer.Run.Run) {
|
||||
t.Errorf("the week's verdicts answered %s, %v", raw, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
module mesh-self-check
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"module": "mesh-self-check",
|
||||
"version": "1",
|
||||
"runs-as": "mesh-self-check",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-self-check",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"doctor",
|
||||
"probes"
|
||||
]
|
||||
}
|
||||
],
|
||||
"invokes": [
|
||||
"nats.nats_streams",
|
||||
"nats.nats_closed_connections",
|
||||
"seat:mesh-watchdogs.signals"
|
||||
],
|
||||
"state": [
|
||||
"findings",
|
||||
"verdict",
|
||||
{
|
||||
"name": "verdicts",
|
||||
"ttl-seconds": 604800
|
||||
}
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-self-check",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-self-check",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-self-check"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-self-check",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-self-check",
|
||||
"binary": "mesh-self-check",
|
||||
"loads": [
|
||||
"mesh-self-check"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user